diff --git a/README.md b/README.md index 316cf8cd7..30d9c857b 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,8 @@ cve-search ========== +[![Join the chat at https://gitter.im/cve-search/cve-search](https://badges.gitter.im/cve-search/cve-search.svg)](https://gitter.im/cve-search/cve-search?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) + ![cve-search logo](https://avatars3.githubusercontent.com/u/15033728?v=3&s=200) [![Build Status](https://travis-ci.org/cve-search/cve-search.svg?branch=master)](https://travis-ci.org/cve-search/cve-search) @@ -152,11 +154,11 @@ or department within your organization or any meaningful name for you. As an example, you can add a partial CPE name like "sap:netweaver" which is very critical for your accounting department. - ./python3.3 sbin/db_ranking.py -c "sap:netweaver" -g "accounting" -r 3 + ./sbin/db_ranking.py -c "sap:netweaver" -g "accounting" -r 3 and then you can lookup the ranking (-r option) for a specific CVE-ID: - ./python3.3 bin/search.py -c CVE-2012-4341 -r -n + ./bin/search.py -c CVE-2012-4341 -r -n Advanced usage -------------- @@ -192,15 +194,15 @@ Fulltext indexing If you want to index all the CVEs from your current MongoDB collection: - ./python3.3 sbin/db_fulltext.py + ./sbin/db_fulltext.py and you query the fulltext index (to get a list of matching CVE-ID): - ./python3.3 bin/search_fulltext.py -q NFS -q Linux + ./bin/search_fulltext.py -q NFS -q Linux or to query the fulltext index and output the JSON object for each CVE-ID: - ./python3.3 bin/search_fulltext.py -q NFS -q Linux -j + ./bin/search_fulltext.py -q NFS -q Linux -j Fulltext visualization ---------------------- @@ -211,7 +213,7 @@ required to generate the keywords with the most common English stopwords and lemmatize the output. [NTLK for Python 3](http://nltk.org/nltk3-alpha/) exists but you need to use the alpha version of NLTK. - ./python3.3 bin/search_fulltext.py -g -s >cve.json + ./bin/search_fulltext.py -g -s >cve.json ![cve-search visualization](https://farm9.staticflickr.com/8109/8603509755_c7690c2de4_n.jpg "CVE Keywords Visualization Using Data From cve-search") @@ -225,7 +227,7 @@ query a specific CVE. You'll need flask in order to run the website and [Flask-P the web interface: cd ./web - ./python3.3 index.py + ./index.py Then you can connect on http://127.0.0.1:5000/ to browser the last CVE. diff --git a/bin/cve_doc.py b/bin/cve_doc.py old mode 100644 new mode 100755 index 2b450f3cf..391a8b66b --- a/bin/cve_doc.py +++ b/bin/cve_doc.py @@ -17,7 +17,7 @@ from optparse import OptionParser -from lib.Query import lastentries, apigetcve, apibrowse, apisearch +from lib.Query import apigetcve optp = OptionParser() optp.add_option('-c', '--cve', dest='cve', default='CVE-2015-0001', help='CVE id to convert') diff --git a/bin/db_dump.py b/bin/db_dump.py index 53e55d107..8550319db 100755 --- a/bin/db_dump.py +++ b/bin/db_dump.py @@ -37,4 +37,8 @@ if 'cvss' in item: if type(item['cvss']) == str: item['cvss'] = float(item['cvss']) - print (json.dumps(item, sort_keys=True, default=json_util.default)) + date_fields = ['cvss-time', 'Modified', 'Published'] + for field in date_fields: + if field in item: + item[field] = str(item[field]) + print(json.dumps(item, sort_keys=True, default=json_util.default)) diff --git a/bin/dump_last.py b/bin/dump_last.py index 9395ac986..20ea85c91 100755 --- a/bin/dump_last.py +++ b/bin/dump_last.py @@ -69,7 +69,7 @@ print ("") print ("" + str(x['id']) + " - " + x['summary'][:90] + "...") print ("") - print ("CVSS: " + str(x['cvss']) + " Published: " + x['Published'] + "") + print ("CVSS: " + str(x['cvss']) + " Published: " + str(x['Published']) + "") print ("") print (" Summary: " + x['summary'] + "") print ("") diff --git a/bin/search.py b/bin/search.py index c2685a37e..82b80226b 100755 --- a/bin/search.py +++ b/bin/search.py @@ -94,9 +94,13 @@ sorttype = -1 -def printCVE(item): +def printCVE(item, indent=None): + date_fields = ['cvss-time', 'Modified', 'Published'] + for field in date_fields: + if field in item: + item[field] = str(item[field]) if not namelookup and not rankinglookup and not capeclookup: - print(json.dumps(item, sort_keys=True, default=json_util.default)) + print(json.dumps(item, sort_keys=True, default=json_util.default, indent=indent)) else: if "vulnerable_configuration" in item: vulconf = [] @@ -115,7 +119,7 @@ def printCVE(item): if "cwe" in item and capeclookup: if item['cwe'].lower() != 'unknown': item['capec'] = cves.getcapec(cweid=(item['cwe'].split('-')[1])) - print(json.dumps(item, sort_keys=True, default=json_util.default)) + print(json.dumps(item, sort_keys=True, default=json_util.default, indent=indent)) if cveSearch: for cveid in db.getCVEs(cves=cveSearch): @@ -126,7 +130,7 @@ def printCVE(item): if vFreeSearch: try: for item in db.getFreeText(vFreeSearch): - print(item) + printCVE(item, indent=2) except: sys.exit("Free text search not enabled on the database!") sys.exit(0) @@ -150,11 +154,11 @@ def printCVE(item): nl = " ".join(item['vulnerable_configuration']) csvoutput = csv.writer(sys.stdout, delimiter='|', quotechar='|', quoting=csv.QUOTE_MINIMAL) if not namelookup: - csvoutput.writerow([item['id'], item['Published'], item['cvss'], item['summary'], refs]) + csvoutput.writerow([item['id'], str(item['Published']), item['cvss'], item['summary'], refs]) else: - csvoutput.writerow([item['id'], item['Published'], item['cvss'], item['summary'], refs, nl]) + csvoutput.writerow([item['id'], str(item['Published']), item['cvss'], item['summary'], refs, nl]) elif htmlOutput: - print("

" + item['id'] + "

CVSS score: " + str(item['cvss']) + "
" + "" + item['Published'] + "
" + item['summary'] + "
") + print("

" + item['id'] + "

CVSS score: " + str(item['cvss']) + "
" + "" + str(item['Published']) + "
" + item['summary'] + "
") print("References:
") for entry in item['references']: print(entry + "
") @@ -167,7 +171,7 @@ def printCVE(item): c = SubElement(r, 'id') c.text = item['id'] c = SubElement(r, 'Published') - c.text = item['Published'] + c.text = str(item['Published']) c = SubElement(r, 'cvss') c.text = str(item['cvss']) c = SubElement(r, 'summary') @@ -182,7 +186,7 @@ def printCVE(item): print(item['id']) else: print("CVE\t: " + item['id']) - print("DATE\t: " + item['Published']) + print("DATE\t: " + str(item['Published'])) print("CVSS\t: " + str(item['cvss'])) print(item['summary']) print("\nReferences:") diff --git a/bin/search_cpe.py b/bin/search_cpe.py old mode 100644 new mode 100755 diff --git a/bin/search_irc.py b/bin/search_irc.py old mode 100644 new mode 100755 diff --git a/lib/DatabaseLayer.py b/lib/DatabaseLayer.py index 9eafeb439..7cc7e129d 100644 --- a/lib/DatabaseLayer.py +++ b/lib/DatabaseLayer.py @@ -89,7 +89,7 @@ def bulkvFeedUpdate(dbpath, vfeedmap): else: icveid = names.index("cveid") except Exception as ex: - sys.exit('Exeption in %s: %s' % (vmap, ex)) + print('Exeption in %s: %s' % (vmap, ex)) continue mapArray={} for i in range(0,len(r)): diff --git a/lib/PluginManager.py b/lib/PluginManager.py index b7538f041..2e7165763 100644 --- a/lib/PluginManager.py +++ b/lib/PluginManager.py @@ -18,13 +18,11 @@ import lib.DatabaseLayer as db from lib.Config import Configuration as conf from lib.Config import ConfigReader -from lib.Plugins import Plugin, WebPlugin -from flask.ext.login import current_user class PluginManager(): def __init__(self): self.plugins = {} - + def loadPlugins(self): settingsReader = ConfigReader(conf.getPluginsettings()) if not os.path.exists(conf.getPluginLoadSettings()): @@ -129,23 +127,27 @@ def cvePluginInfo(self, cve, **args): print("[!] -> %s"%e) return cveInfo - def getSearchResults(self, text): + def getSearchResults(self, text, **args): result = {'data':[]} results = [] # Get all data for plugin in self.plugins.values(): - data = plugin.search(text) + data = plugin.search(text, **args) # Validate format + if type(data) == dict: data = [data] if type(data) == list and all([(type(x) == dict and 'n' in x and 'd' in x) for x in data]): results.extend(data) # Sort through data for collection in results: for item in collection['d']: # Check if already in result data - if not any(item==entry['id'] for entry in result['data']): - entry=db.getCVE(item) - entry['reason']=collection['n'] - result['data'].append(entry) + try: + if not any(item==entry['id'] for entry in result['data']): + entry=db.getCVE(item) + entry['reason']=collection['n'] + result['data'].append(entry) + except: + pass return result # Actions @@ -176,7 +178,7 @@ def openPage(self, name, **args): return ("error.html", {'status': {'except': 'plugin-not-webplugin'}}) return ("error.html", {'status': {'except': 'plugin-not-loaded'}}) - def openSubpage(self, subpage, **args): + def openSubpage(self, name, subpage, **args): if name.strip() in self.plugins.keys(): # Check if plugin exists if self.plugins[name].isWebPlugin(): # Check if plugin is web plugin pageInfo = self.plugins[name].getSubpage(subpage, **args) diff --git a/lib/Plugins.py b/lib/Plugins.py index 1efa402dc..4a7fc4ff4 100644 --- a/lib/Plugins.py +++ b/lib/Plugins.py @@ -28,7 +28,7 @@ def isWebPlugin(self): return False def loadSettings(self, reader): pass def onDatabaseUpdate(self): pass # To override with returns - def search(self, text): pass + def search(self, text, **args): pass class WebPlugin(Plugin): diff --git a/lib/Toolkit.py b/lib/Toolkit.py index 5606c4ac8..1f5279270 100644 --- a/lib/Toolkit.py +++ b/lib/Toolkit.py @@ -10,7 +10,6 @@ # Imports from dateutil import tz import dateutil.parser -import time import re # Note of warning: CPEs like cpe:/o:microsoft:windows_8:-:-:x64 are given to us by Mitre @@ -61,8 +60,8 @@ def impactScore(cve): I=((cve['impact'])['integrity']).upper() A=((cve['impact'])['availability']).upper() res = 10.41*(1-(1-score[C])*(1-score[I])*(1-score[A])) - return 10.0 if res > 10.0 else res - except Exception as ex: + return 10.0 if res > 10.0 else res + except: return '-' def exploitabilityScore(cve): @@ -100,28 +99,6 @@ def vFeedName(string): string=string.replace('cve_','') return string.title() -def convertDateToDBFormat(string): - result = None - try: - result = time.strptime(string, "%d-%m-%Y") - except: - pass - try: - result = time.strptime(string, "%d-%m-%y") - except: - pass - try: - result = time.strptime(string, "%d/%m/%Y") - except: - pass - try: - result = time.strptime(string, "%d/%m/%y") - except: - pass - if result is not None: - result = time.strftime('%Y-%m-%d', result) - return result - def mergeSearchResults(database, plugins): if 'errors' in database: results = {'data':[], 'errors':database['errors']} diff --git a/lib/User.py b/lib/User.py index 057d9585e..6df273460 100644 --- a/lib/User.py +++ b/lib/User.py @@ -15,7 +15,7 @@ import os runPath = os.path.dirname(os.path.realpath(__file__)) -from flask.ext.login import UserMixin +from flask_login import UserMixin from lib.Config import Configuration import lib.DatabaseLayer as db @@ -35,7 +35,7 @@ def __init__(self, id): USERS = {} for user in db.getUsers(): USERS[user['username']] = user['password'] - + if not id in USERS: raise UserNotFoundError() self.id = id diff --git a/lib/cpelist.py b/lib/cpelist.py index 0bb507c72..c32acff7a 100644 --- a/lib/cpelist.py +++ b/lib/cpelist.py @@ -16,7 +16,6 @@ sys.path.append(os.path.join(runPath, "..")) import json -import re from lib.Toolkit import toStringFormattedCPE import lib.DatabaseLayer as db diff --git a/sbin/db_blacklist.py b/sbin/db_blacklist.py old mode 100644 new mode 100755 index 2471512d8..2ef5f4610 --- a/sbin/db_blacklist.py +++ b/sbin/db_blacklist.py @@ -16,7 +16,6 @@ import argparse -from lib.Config import Configuration from lib.cpelist import CPEList # parse command line arguments diff --git a/sbin/db_mgmt.py b/sbin/db_mgmt.py index d49bce6a8..790d4aab8 100755 --- a/sbin/db_mgmt.py +++ b/sbin/db_mgmt.py @@ -19,6 +19,8 @@ from xml.sax import make_parser from xml.sax.handler import ContentHandler +from dateutil.parser import parse as parse_datetime + from lib.ProgressBar import progressbar from lib.Toolkit import toStringFormattedCPE from lib.Config import Configuration @@ -173,16 +175,16 @@ def endElement(self, name): self.cves[-1]['impact']['availability'] = self.impacta if name == 'cvss:generated-on-datetime': self.inCVSSgenElem = 0 - self.cves[-1]['cvss-time'] = self.cvssgen + self.cves[-1]['cvss-time'] = parse_datetime(self.cvssgen, ignoretz=True) if name == 'vuln:summary': self.inSUMMElem = 0 self.cves[-1]['summary'] = self.SUMM if name == 'vuln:published-datetime': self.inDTElem = 0 - self.cves[-1]['Published'] = self.DT + self.cves[-1]['Published'] = parse_datetime(self.DT, ignoretz=True) if name == 'vuln:last-modified-datetime': self.inPUBElem = 0 - self.cves[-1]['Modified'] = self.PUB + self.cves[-1]['Modified'] = parse_datetime(self.PUB, ignoretz=True) if __name__ == '__main__': parser = make_parser() @@ -197,11 +199,12 @@ def endElement(self, name): except: sys.exit("Cannot open url %s. Bad URL or not connected to the internet?"%(Configuration.getCVEDict() + getfile)) i = db.getInfo("cve") + last_modified = parse_datetime(r.headers['last-modified'], ignoretz=True) if i is not None: - if r.headers['last-modified'] == i['last-modified']: + if last_modified == i['last-modified']: print("Not modified") sys.exit(0) - db.setColUpdate("cve", r.headers['last-modified']) + db.setColUpdate("cve", last_modified) # get your parser on !! parser = make_parser() @@ -275,8 +278,7 @@ def endElement(self, name): item['cvss'] = float(item['cvss']) # check if year is not cve-free if len(ch.cves) != 0: + print("Importing CVEs for year " + str(x)) ret = db.insertCVE(ch.cves) - if ret: - print ("Year " + str(x) + " imported.") else: print ("Year " + str(x) + " has no CVE's.") diff --git a/sbin/db_mgmt_admin.py b/sbin/db_mgmt_admin.py old mode 100644 new mode 100755 index 910870d03..46e28a9df --- a/sbin/db_mgmt_admin.py +++ b/sbin/db_mgmt_admin.py @@ -23,7 +23,6 @@ import getpass from passlib.hash import pbkdf2_sha256 -from lib.Config import Configuration import lib.DatabaseLayer as dbLayer # args diff --git a/sbin/db_mgmt_capec.py b/sbin/db_mgmt_capec.py index 7e9fef910..ec0ca1f35 100755 --- a/sbin/db_mgmt_capec.py +++ b/sbin/db_mgmt_capec.py @@ -14,6 +14,8 @@ from xml.sax import make_parser from xml.sax.handler import ContentHandler +from dateutil.parser import parse as parse_datetime + from lib.ProgressBar import progressbar from lib.Config import Configuration import lib.DatabaseLayer as db @@ -167,8 +169,9 @@ def endElement(self, name): except: sys.exit("Cannot open url %s. Bad URL or not connected to the internet?"%(capecurl)) i = db.getLastModified('capec') +last_modified = parse_datetime(f.headers['last-modified'], ignoretz=True) if i is not None: - if f.headers['last-modified'] == i: + if last_modified == i: print("Not modified") sys.exit(0) # parse xml and store in database @@ -179,4 +182,4 @@ def endElement(self, name): db.bulkUpdate("capec", attacks) #update database info after successful program-run -db.setColUpdate('capec', f.headers['last-modified']) +db.setColUpdate('capec', last_modified) diff --git a/sbin/db_mgmt_cpe_dictionary.py b/sbin/db_mgmt_cpe_dictionary.py index 9541ccd3a..45b1ca2f6 100755 --- a/sbin/db_mgmt_cpe_dictionary.py +++ b/sbin/db_mgmt_cpe_dictionary.py @@ -25,6 +25,8 @@ from xml.sax import make_parser from xml.sax.handler import ContentHandler +from dateutil.parser import parse as parse_datetime + from lib.ProgressBar import progressbar from lib.Toolkit import toStringFormattedCPE from lib.Config import Configuration @@ -81,8 +83,9 @@ def endElement(self, name): except: sys.exit("Cannot open url %s. Bad URL or not connected to the internet?"%(cpedict)) i = db.getLastModified('cpe') +last_modified = parse_datetime(f.headers['last-modified'], ignoretz=True) if i is not None: - if f.headers['last-modified'] == i: + if last_modified == i: print("Not modified") sys.exit(0) # parse xml and store in database @@ -97,4 +100,4 @@ def endElement(self, name): db.bulkUpdate("cpe", cpeList) #update database info after successful program-run -db.setColUpdate('cpe', f.headers['last-modified']) +db.setColUpdate('cpe', last_modified) diff --git a/sbin/db_mgmt_cpe_other_dictionary.py b/sbin/db_mgmt_cpe_other_dictionary.py old mode 100644 new mode 100755 index 96070859e..f69295d5b --- a/sbin/db_mgmt_cpe_other_dictionary.py +++ b/sbin/db_mgmt_cpe_other_dictionary.py @@ -34,7 +34,6 @@ import urllib from lib.ProgressBar import progressbar -from lib.Config import Configuration import lib.DatabaseLayer as db # get dates diff --git a/sbin/db_mgmt_create_index.py b/sbin/db_mgmt_create_index.py old mode 100644 new mode 100755 index bc36a1d01..40c54bfc0 --- a/sbin/db_mgmt_create_index.py +++ b/sbin/db_mgmt_create_index.py @@ -16,7 +16,6 @@ from pymongo import TEXT -from lib.Config import Configuration import lib.DatabaseLayer as dbLayer def setIndex(col, field): diff --git a/sbin/db_mgmt_cwe.py b/sbin/db_mgmt_cwe.py index d89b13058..01fbd60cc 100755 --- a/sbin/db_mgmt_cwe.py +++ b/sbin/db_mgmt_cwe.py @@ -25,6 +25,8 @@ runPath = os.path.dirname(os.path.realpath(__file__)) sys.path.append(os.path.join(runPath, "..")) +from dateutil.parser import parse as parse_datetime + from xml.sax import make_parser from xml.sax.handler import ContentHandler import argparse @@ -82,7 +84,7 @@ def endElement(self, name): f = Configuration.getFile(cwedict) except: sys.exit("Cannot open url %s. Bad URL or not connected to the internet?"%(cwedict)) -lastmodified = f.headers['last-modified'] +lastmodified = parse_datetime(f.headers['last-modified'], ignoretz=True) i = db.getLastModified('cwe') if i is not None: if lastmodified == i: diff --git a/sbin/db_mgmt_d2sec.py b/sbin/db_mgmt_d2sec.py index e321b312b..21d2313fb 100755 --- a/sbin/db_mgmt_d2sec.py +++ b/sbin/db_mgmt_d2sec.py @@ -13,6 +13,8 @@ runPath = os.path.dirname(os.path.realpath(__file__)) sys.path.append(os.path.join(runPath, "..")) +from dateutil.parser import parse as parse_datetime + from xml.sax import make_parser from xml.sax.handler import ContentHandler import argparse @@ -101,9 +103,10 @@ def endElement(self, name): f = Configuration.getFile(d2securl) except: sys.exit("Cannot open url %s. Bad URL or not connected to the internet?"%(d2securl)) +last_modified = parse_datetime(f.headers['last-modified'], ignoretz=True) i = db.getLastModified("d2sec") if i is not None: - if f.headers['last-modified'] == i: + if last_modified == i: print("Not modified") sys.exit(0) # parse xml and store in database @@ -117,4 +120,4 @@ def endElement(self, name): db.bulkUpdate("d2sec", exploitList) #update database info after successful program-run -db.setColUpdate('d2sec', f.headers['last-modified']) +db.setColUpdate('d2sec', last_modified) diff --git a/sbin/db_mgmt_exploitdb.py b/sbin/db_mgmt_exploitdb.py old mode 100644 new mode 100755 diff --git a/sbin/db_mgmt_ms.py b/sbin/db_mgmt_ms.py old mode 100644 new mode 100755 diff --git a/sbin/db_mgmt_vendorstatements.py b/sbin/db_mgmt_vendorstatements.py index 1a3c8ed5c..dcdd7f798 100755 --- a/sbin/db_mgmt_vendorstatements.py +++ b/sbin/db_mgmt_vendorstatements.py @@ -18,6 +18,8 @@ runPath = os.path.dirname(os.path.realpath(__file__)) sys.path.append(os.path.join(runPath, "..")) +from dateutil.parser import parse as parse_datetime + from xml.sax import make_parser from xml.sax.handler import ContentHandler import argparse @@ -69,9 +71,10 @@ def endElement(self, name): (f, r) = Configuration.getFile(vendordict, compressed = True) except: sys.exit("Cannot open url %s. Bad URL or not connected to the internet?"%(vendordict)) +last_modified = parse_datetime(r.headers['last-modified'], ignoretz=True) i = db.getLastModified('vendor') if i is not None: - if r.headers['last-modified'] == i: + if last_modified == i: print("Not modified") sys.exit(0) # parse xml and store in database @@ -84,4 +87,4 @@ def endElement(self, name): db.bulkUpdate('vendor', statements) #update database info after successful program-run -db.setColUpdate('vendor', r.headers['last-modified']) +db.setColUpdate('vendor', last_modified) diff --git a/sbin/db_mgmt_vfeed.py b/sbin/db_mgmt_vfeed.py index fa7da490e..53eb37ffc 100755 --- a/sbin/db_mgmt_vfeed.py +++ b/sbin/db_mgmt_vfeed.py @@ -14,11 +14,11 @@ runPath = os.path.dirname(os.path.realpath(__file__)) sys.path.append(os.path.join(runPath, "..")) +from dateutil.parser import parse as parse_datetime + import tarfile import shutil -import sqlite3 -from lib.ProgressBar import progressbar from lib.Config import Configuration import lib.DatabaseLayer as db @@ -31,9 +31,10 @@ u = Configuration.getFile(vFeedurl) except: sys.exit("Cannot open url %s. Bad URL or not connected to the internet?"%(vFeedurl)) +last_modified = parse_datetime(u.headers['last-modified'], ignoretz=True) i = db.getLastModified('vfeed') if i is not None: - if u.headers['last-modified'] == i: + if last_modified == i: print("Not modified") sys.exit(0) # create temp file and download and unpack database @@ -43,7 +44,7 @@ shutil.copyfileobj(u, fp) t = tarfile.open(name=tmppath+'/vfeed.db.tgz', mode='r') t.extract('vfeed.db', path=tmppath) -t.close +t.close() # excluded map_cve_milw0rm because it moved to a different domain, thus the id is irrelevant. # Talked about this with Toolswatch dev, he's going to take a look, so leave this comment in until further notice @@ -59,4 +60,4 @@ db.bulkvFeedUpdate(tmppath, vfeedmap) #update database info after successful program-run -db.setColUpdate('vfeed', u.headers['last-modified']) +db.setColUpdate('vfeed', last_modified) diff --git a/sbin/db_ranking.py b/sbin/db_ranking.py index 51327e007..65a9076e4 100755 --- a/sbin/db_ranking.py +++ b/sbin/db_ranking.py @@ -30,7 +30,6 @@ import argparse -from lib.Config import Configuration import lib.DatabaseLayer as db diff --git a/sbin/db_updater.py b/sbin/db_updater.py index 2dfdba485..16dc011b9 100755 --- a/sbin/db_updater.py +++ b/sbin/db_updater.py @@ -46,23 +46,23 @@ if not args.m: sources.extend([{'name': 'vfeed', - 'updater': "python3 " + os.path.join(runPath, "db_mgmt_vfeed.py")}, + 'updater': "{} {}".format(sys.executable, os.path.join(runPath, "db_mgmt_vfeed.py"))}, {'name': 'vendor', - 'updater': "python3 " + os.path.join(runPath, "db_mgmt_vendorstatements.py")}, + 'updater': "{} {}".format(sys.executable, os.path.join(runPath, "db_mgmt_vendorstatements.py"))}, {'name': 'cwe', - 'updater': "python3 " + os.path.join(runPath, "db_mgmt_cwe.py")}, + 'updater': "{} {}".format(sys.executable, os.path.join(runPath, "db_mgmt_cwe.py"))}, {'name': 'capec', - 'updater': "python3 " + os.path.join(runPath, "db_mgmt_capec.py")}, + 'updater': "{} {}".format(sys.executable, os.path.join(runPath, "db_mgmt_capec.py"))}, {'name': 'redis-cache-cpe', - 'updater': "python3 " + os.path.join(runPath, "db_cpe_browser.py")}, + 'updater': "{} {}".format(sys.executable, os.path.join(runPath, "db_cpe_browser.py"))}, {'name': 'd2sec', - 'updater': "python3 " + os.path.join(runPath, "db_mgmt_d2sec.py")}, + 'updater': "{} {}".format(sys.executable, os.path.join(runPath, "db_mgmt_d2sec.py"))}, {'name': 'ms', - 'updater': "python3 " + os.path.join(runPath, "db_mgmt_ms.py")}, + 'updater': "{} {}".format(sys.executable, os.path.join(runPath, "db_mgmt_ms.py"))}, {'name': 'redis-nist-ref', - 'updater': "python3 " + os.path.join(runPath, "db_mgmt_ref.py")}, + 'updater': "{} {}".format(sys.executable, os.path.join(runPath, "db_mgmt_ref.py"))}, {'name': 'exploitdb', - 'updater': "python3 " + os.path.join(runPath, "db_mgmt_exploitdb.py")}]) + 'updater': "{} {}".format(sys.executable, os.path.join(runPath, "db_mgmt_exploitdb.py"))}]) if not args.v: logging.basicConfig(stream=sys.stdout, level=logging.DEBUG) @@ -119,7 +119,7 @@ def log(message=""): log('Starting ' + source['name']) before = nbelement(collection=source['name']) if args.f and source['name'] is "cves": - updater = "python3 " + os.path.join(runPath, "db_mgmt.py -p") + updater = "{} {}".format(sys.executable, os.path.join(runPath, "db_mgmt.py -p")) subprocess.Popen((shlex.split(updater))).wait() else: subprocess.Popen((shlex.split(source['updater']))).wait() diff --git a/sbin/db_whitelist.py b/sbin/db_whitelist.py index 0439cb6d3..8a7dfd5b4 100755 --- a/sbin/db_whitelist.py +++ b/sbin/db_whitelist.py @@ -16,7 +16,6 @@ import argparse -from lib.Config import Configuration from lib.cpelist import CPEList # parse command line arguments diff --git a/web/api.py b/web/api.py index 64e9cc1a7..5f82676b3 100644 --- a/web/api.py +++ b/web/api.py @@ -31,7 +31,7 @@ from logging.handlers import RotatingFileHandler from lib.Config import Configuration -from lib.Toolkit import toStringFormattedCPE, toOldCPE, convertDateToDBFormat +from lib.Toolkit import toStringFormattedCPE, toOldCPE import lib.CVEs as cves import lib.DatabaseLayer as dbLayer diff --git a/web/index.py b/web/index.py index 20a0c5ffb..57143fe09 100644 --- a/web/index.py +++ b/web/index.py @@ -20,14 +20,13 @@ from tornado.httpserver import HTTPServer from tornado.ioloop import IOLoop from flask import Flask, render_template, request, redirect, jsonify, send_file, abort -from flask.ext.login import LoginManager, current_user, login_user, logout_user, login_required +from flask_login import LoginManager, current_user, login_user, logout_user, login_required from passlib.hash import pbkdf2_sha256 from redis import exceptions as redisExceptions import jinja2 import json import re -import argparse import time import urllib import random @@ -36,11 +35,11 @@ import subprocess from io import TextIOWrapper, BytesIO from logging.handlers import RotatingFileHandler +from dateutil.parser import parse as parse_datetime from lib.User import User -from lib.Config import Configuration from lib.PluginManager import PluginManager -from lib.Toolkit import toStringFormattedCPE, toOldCPE, currentTime, isURL, vFeedName, convertDateToDBFormat, mergeSearchResults +from lib.Toolkit import toStringFormattedCPE, toOldCPE, isURL, vFeedName, mergeSearchResults import lib.CVEs as cves import lib.DatabaseLayer as db from sbin.db_whitelist import * @@ -183,17 +182,17 @@ def filter_logic(f, limit, skip): query.append({'summary': re.compile(exp)}) # plugin filters - query.extend(plugManager.doFilter(f, current_user=current_user)) + query.extend(plugManager.doFilter(f, **pluginArgs())) # cvss logic if f['cvssSelect'] == "above": query.append({'cvss': {'$gt': float(f['cvss'])}}) elif f['cvssSelect'] == "equals": query.append({'cvss': float(f['cvss'])}) elif f['cvssSelect'] == "below": query.append({'cvss': {'$lt': float(f['cvss'])}}) - + # date logic if f['timeSelect'] != "all": - startDate = convertDateToDBFormat(f['startDate']) - endDate = convertDateToDBFormat(f['endDate']) + startDate = parse_datetime(f['startDate'], ignoretz=True, dayfirst=True) + endDate = parse_datetime(f['endDate'], ignoretz=True, dayfirst=True) if f['timeSelect'] == "from": query.append({f['timeTypeSelect']: {'$gt': startDate}}) if f['timeSelect'] == "until": @@ -206,7 +205,7 @@ def filter_logic(f, limit, skip): # marking relevant records if f['whitelistSelect'] == "on": cve = whitelist_mark(cve) if f['blacklistSelect'] == "mark": cve = blacklist_mark(cve) - plugManager.mark(cve, current_user=current_user) + plugManager.mark(cve, **pluginArgs()) cve = list(cve) return cve @@ -239,6 +238,11 @@ def getFilterSettingsFromPost(r): cve = db.getCVEs(limit=pageLength, skip=r) return(filters,cve) +def pluginArgs(): + args = {"current_user": current_user, + "plugin_manager": plugManager} + return args + @login_manager.user_loader def load_user(id): return User.get(id) @@ -247,18 +251,18 @@ def load_user(id): @app.route('/') def index(): # get default page on HTTP get (navigating to page) - filters={'blacklistSelect': 'on', 'whitelistSelect': 'on', - 'unlistedSelect': 'show', 'timeSelect': 'all', + filters={'blacklistSelect': 'on', 'whitelistSelect': 'on', + 'unlistedSelect': 'show', 'timeSelect': 'all', 'startDate': '', 'endDate': '', 'timeTypeSelect': 'Modified', 'cvssSelect': 'all', 'cvss': '', 'rejectedSelect': 'hide'} - + cve = filter_logic(filters, pageLength, 0) - return render_template('index.html', cve=cve, r=0, pageLength=pageLength, filters=plugManager.getFilters(current_user=current_user)) + return render_template('index.html', cve=cve, r=0, pageLength=pageLength, filters=plugManager.getFilters(**pluginArgs())) @app.route('/', methods=['POST']) def filterPost(): settings,cve = getFilterSettingsFromPost(0) - return render_template('index.html', settings=settings, cve=cve, r=0, pageLength=pageLength, filters=plugManager.getFilters(current_user=current_user)) + return render_template('index.html', settings=settings, cve=cve, r=0, pageLength=pageLength, filters=plugManager.getFilters(**pluginArgs())) @app.route('/r/', methods=['POST']) @@ -266,24 +270,24 @@ def filterLast(r): if not r: r = 0 settings,cve = getFilterSettingsFromPost(r) - return render_template('index.html', settings=settings, cve=cve, r=r, pageLength=pageLength, filters=plugManager.getFilters(current_user=current_user)) + return render_template('index.html', settings=settings, cve=cve, r=r, pageLength=pageLength, filters=plugManager.getFilters(**pluginArgs())) # Plugins @app.route('/_get_plugins', methods=['GET']) def get_plugins(): if not current_user.is_authenticated(): # Don't show plugins requiring auth if not authenticated - plugins = [{"name": x.getName(), "link": x.getUID()} for x in plugManager.getWebPluginsWithPage(current_user=current_user) if not x.requiresAuth] + plugins = [{"name": x.getName(), "link": x.getUID()} for x in plugManager.getWebPluginsWithPage(**pluginArgs()) if not x.requiresAuth] else: - plugins = [{"name": x.getName(), "link": x.getUID()} for x in plugManager.getWebPluginsWithPage(current_user=current_user)] + plugins = [{"name": x.getName(), "link": x.getUID()} for x in plugManager.getWebPluginsWithPage(**pluginArgs())] return jsonify({"plugins": plugins}) @app.route('/plugin/_get_cve_actions', methods=['GET']) def get_cve_actions(): cve = request.args.get('cve', type=str) if not current_user.is_authenticated(): # Don't show actions requiring auth if not authenticated - actions = [x for x in plugManager.getCVEActions(cve, current_user=current_user) if not x['auth']] + actions = [x for x in plugManager.getCVEActions(cve, **pluginArgs()) if not x['auth']] else: - actions = plugManager.getCVEActions(cve, current_user=current_user) + actions = plugManager.getCVEActions(cve, **pluginArgs()) return jsonify({"actions": actions}) @app.route('/plugin/', methods=['GET']) @@ -291,7 +295,7 @@ def openPlugin(plugin): if plugManager.requiresAuth(plugin) and not current_user.is_authenticated(): return render_template("requiresAuth.html") else: - page, args = plugManager.openPage(plugin, current_user=current_user) + page, args = plugManager.openPage(plugin, **pluginArgs()) if page: try: return render_template(page, **args) @@ -304,7 +308,7 @@ def openPluginSubpage(plugin, page): if plugManager.requiresAuth(plugin) and not current_user.is_authenticated(): return render_template("requiresAuth.html") else: - page, args = plugManager.openSubpage(plugin, page, current_user=current_user) + page, args = plugManager.openSubpage(plugin, page, **pluginArgs()) if page: try: return render_template(page, **args) @@ -315,10 +319,13 @@ def openPluginSubpage(plugin, page): @app.route('/plugin//_cve_action/', methods=['GET']) def jsonCVEAction(plugin, action): cve = request.args.get('cve', type=str) - if plugManager.onCVEAction(cve, plugin, action, current_user=current_user, fields=dict(request.args)): + response = plugManager.onCVEAction(cve, plugin, action, fields=dict(request.args), **pluginArgs()) + if type(response) is bool and response is True: return jsonify({'status': 'plugin_action_complete'}) - else: + elif type(response) is bool and response is False or response is None: return jsonify({'status': 'plugin_action_failed'}) + elif type(response) is dict: + return jsonify(response) # API @app.route('/api/cpe2.3/', methods=['GET']) @@ -382,9 +389,10 @@ def cve(cveid): if cve is None: return render_template('error.html',status={'except':'cve-not-found','info':{'cve':cveid}}) cve = markCPEs(cve) - - plugManager.onCVEOpen(cveid, current_user=current_user) - pluginData = plugManager.cvePluginInfo(cveid, current_user=current_user) + + plugManager.onCVEOpen(cveid, **pluginArgs()) + pluginData = plugManager.cvePluginInfo(cveid, **pluginArgs()) + return render_template('cve.html', cve=cve, plugins=pluginData) @app.route('/browse/') @@ -422,7 +430,7 @@ def capec(capecid): def searchText(): search = request.form.get('search') dbResults = db.getSearchResults(search) - plugResults = plugManager.getSearchResults(search) + plugResults = plugManager.getSearchResults(search, **pluginArgs()) result = mergeSearchResults(dbResults, plugResults) cve=result['data'] errors=result['errors'] if 'errors' in result else [] @@ -445,7 +453,7 @@ def link(vFeedMap=None,field=None,value=None): # marking relevant records cve = whitelist_mark(cve) cve = blacklist_mark(cve) - plugManager.mark(cve, current_user=current_user) + plugManager.mark(cve, **pluginArgs()) cve = list(cve) cvssList=[float(x['cvss']) for x in cve if 'cvss' in x] if cvssList: @@ -488,7 +496,7 @@ def change_pass(): @app.route('/admin/updatedb') @login_required def updatedb(): - process = subprocess.Popen(["python3", os.path.join(_runPath, "../sbin/db_updater.py"), "-civ"], stdout=subprocess.PIPE, stderr=subprocess.PIPE) + process = subprocess.Popen([sys.executable, os.path.join(_runPath, "../sbin/db_updater.py"), "-civ"], stdout=subprocess.PIPE, stderr=subprocess.PIPE) out, err = process.communicate() output="%s\n\nErrors:\n%s"%(str(out,'utf-8'),str(err,'utf-8')) if err else str(out,'utf-8') return jsonify({"updateOutput": output, "status": "db_updated"}) @@ -531,7 +539,7 @@ def whitelistView(): @app.route('/admin/addToList') @login_required -def listAdd(): +def listAdd(): cpe = request.args.get('cpe') cpeType = request.args.get('type') lst = request.args.get('list') @@ -716,10 +724,6 @@ def page_not_found(e): # filters -@app.template_filter('currentTime') -def currentTimeFilter(utc): - return currentTime(utc) - @app.template_filter('htmlDecode') def htmlDedode(string): return urllib.parse.unquote_plus(string) diff --git a/web/minimal-web.py b/web/minimal-web.py index b53678abc..1871de4a6 100644 --- a/web/minimal-web.py +++ b/web/minimal-web.py @@ -31,9 +31,10 @@ import signal import logging from logging.handlers import RotatingFileHandler +from dateutil.parser import parse as parse_datetime from lib.Config import Configuration -from lib.Toolkit import toStringFormattedCPE, toOldCPE, currentTime, isURL, vFeedName, convertDateToDBFormat +from lib.Toolkit import toStringFormattedCPE, toOldCPE, isURL, vFeedName import lib.CVEs as cves import lib.DatabaseLayer as dbLayer @@ -93,8 +94,8 @@ def filter_logic(unlisted, timeSelect, startDate, endDate, query.append({'cvss': {'$lt': float(cvss)}}) # date logic if timeSelect != "all": - startDate = convertDateToDBFormat(startDate) - endDate = convertDateToDBFormat(endDate) + startDate = parse_datetime(startDate, ignoretz=True, dayfirst=True) + endDate = parse_datetime(endDate, ignoretz=True, dayfirst=True) if timeSelect == "from": query.append({timeTypeSelect: {'$gt': startDate}}) if timeSelect == "until": @@ -303,11 +304,6 @@ def page_not_found(e): # filters -@app.template_filter('currentTime') -def currentTimeFilter(utc): - return currentTime(utc) - - @app.template_filter('htmlDecode') def htmlDedode(string): return urllib.parse.unquote_plus(string) diff --git a/web/static/js/custom/cve.js b/web/static/js/custom/cve.js index dd507f705..85951a982 100644 --- a/web/static/js/custom/cve.js +++ b/web/static/js/custom/cve.js @@ -1,12 +1,12 @@ function copyToClipboard() { var oTable = document.getElementById('cveInfo'); var rowLength = oTable.rows.length; - var text = "" + var text = ""; for (i = 0; i < rowLength; i++){ var oCells = oTable.rows.item(i).cells; var attribute = oCells.item(0).innerHTML; var value = oCells.item(1).innerHTML; - + var toAdd = attribute + ": \t" + value; text = text + toAdd + "\n"; } diff --git a/web/static/js/custom/pager.js b/web/static/js/custom/pager.js new file mode 100644 index 000000000..9ffd55f19 --- /dev/null +++ b/web/static/js/custom/pager.js @@ -0,0 +1,18 @@ +// Requires filter.js, where setSetings() is located +function postURL(url) { + var form = document.getElementById("filter"); + form.action = url; + form.submit(); +} +function next(n){ + setSettings(); + var url = "/r/"+n; + postURL(url); +} +function previous(n){ + setSettings(); + if(n < 0){ + n = 0;} + var url = "/r/" + n; + postURL(url); +} diff --git a/web/static/js/custom/scripts.js b/web/static/js/custom/scripts.js index ea022da4f..3d5551c63 100644 --- a/web/static/js/custom/scripts.js +++ b/web/static/js/custom/scripts.js @@ -15,7 +15,7 @@ function redirect() { document.body.appendChild(form); form.submit(); } - + } //Bootstrap tooltip @@ -43,7 +43,7 @@ jQuery(document).ready(function() { //Temporary undo class (function($){ - $.fn.extend({ + $.fn.extend({ removeTemporaryClass: function(className, duration) { var elements = this; setTimeout(function() { diff --git a/web/templates/admin.html b/web/templates/admin.html index a041f58fa..a4b8a079f 100644 --- a/web/templates/admin.html +++ b/web/templates/admin.html @@ -22,19 +22,19 @@ CVES {{stats['cveA']}} - {% if stats['cveU'] is not none %}{{stats['cveU']|currentTime}} {% else %}Not updated{% endif %} + {% if stats['cveU'] is not none %}{{stats['cveU'].strftime('%d-%m-%Y - %H:%M')}} {% else %}Not updated{% endif %} CPE {{stats['cpeA']}} - {% if stats['cpeU'] is not none %}{{stats['cpeU']|currentTime}} {% else %}Not updated{% endif %} + {% if stats['cpeU'] is not none %}{{stats['cpeU'].strftime('%d-%m-%Y - %H:%M')}} {% else %}Not updated{% endif %} CPE-other {{stats['cpeOtherA']}} - {% if stats['cpeOtherU'] is not none %}{{stats['cpeOtherU']|currentTime}} {% else %}Not updated{% endif %} + {% if stats['cpeOtherU'] is not none %}{{stats['cpeOtherU'].strftime('%d-%m-%Y - %H:%M')}} {% else %}Not updated{% endif %} Capec {{stats['capecA']}} - {% if stats['capecU'] is not none %}{{stats['capecU']|currentTime}} {% else %}Not updated{% endif %} + {% if stats['capecU'] is not none %}{{stats['capecU'].strftime('%d-%m-%Y - %H:%M')}} {% else %}Not updated{% endif %} d2sec {{stats['d2secA']}} - {% if stats['d2secU'] is not none %}{{stats['d2secU']|currentTime}} {% else %}Not updated{% endif %} + {% if stats['d2secU'] is not none %}{{stats['d2secU'].strftime('%d-%m-%Y - %H:%M')}} {% else %}Not updated{% endif %} Vendor statements{{stats['vendorA']}} - {% if stats['vendorU'] is not none %}{{stats['vendorU']|currentTime}} {% else %}Not updated{% endif %} + {% if stats['vendorU'] is not none %}{{stats['vendorU'].strftime('%d-%m-%Y - %H:%M')}} {% else %}Not updated{% endif %} vFeed info {{stats['vfeedA']}} - {% if stats['vfeedU'] is not none %}{{stats['vfeedU']|currentTime}} {% else %}Not updated{% endif %} + {% if stats['vfeedU'] is not none %}{{stats['vfeedU'].strftime('%d-%m-%Y - %H:%M')}} {% else %}Not updated{% endif %} Whitelist: {{stats['wlA']}} rules
@@ -52,7 +52,7 @@ {% for plug in plugins|sort(attribute="name") %} {% set ptype = "Web" if plug.isWebPlugin() else "Backend" %} - {{plug.name}}{{plug.uid}}{{ptype}}{{plug.loadstate}} + {{plug.name}}{{plug.uid}}{{ptype}}{{plug.loadstate}} {% endfor %} @@ -88,7 +88,7 @@ - +
Current
New
Repeat
Repeat
diff --git a/web/templates/cve.html b/web/templates/cve.html index 0be6d4259..1ef9932fd 100644 --- a/web/templates/cve.html +++ b/web/templates/cve.html @@ -74,7 +74,7 @@ CVSS - +
Base: {{ cve['cvss'] }} {% if 'cvss-time' in cve %}(as of {{ cve['cvss-time']|currentTime }}){% endif %}
Base: {{ cve['cvss'] }} {% if 'cvss-time' in cve %}(as of {{ cve['cvss-time'].strftime('%d-%m-%Y - %H:%M') }}){% endif %}
Impact: {{ cve['impactCVSS'] }}
Exploitability:{{ cve['exploitCVSS'] }}
@@ -93,7 +93,7 @@ CAPEC -
+
    {% for c in cve['capec'] %}
  • @@ -175,16 +175,16 @@ {% endfor %} Last major update - {{ cve['Modified']|currentTime }} + {{ cve['Modified'].strftime('%d-%m-%Y - %H:%M') }} Published - {{ cve['Published']|currentTime }} + {{ cve['Published'].strftime('%d-%m-%Y - %H:%M') }} {% if 'last-modified' in cve%} Last modified - {{ cve['last-modified']|currentTime }} + {{ cve['last-modified'].strftime('%d-%m-%Y - %H:%M') }} {% endif %} diff --git a/web/templates/index-minimal.html b/web/templates/index-minimal.html index b5d64580a..bc5303993 100644 --- a/web/templates/index-minimal.html +++ b/web/templates/index-minimal.html @@ -6,6 +6,7 @@ + {% endblock %} {% block content %} - -
    - -
    + {% include 'subpages/filters.html' %} {% include 'subpages/pager.html' %} diff --git a/web/templates/index.html b/web/templates/index.html index d5140e012..8e4471466 100644 --- a/web/templates/index.html +++ b/web/templates/index.html @@ -6,6 +6,7 @@ + {% endblock %} {% block content %} - - -
    - -
    + {% include 'subpages/filters.html' %} {% include 'subpages/pager.html' %} diff --git a/web/templates/subpages/filters.html b/web/templates/subpages/filters.html new file mode 100644 index 000000000..1f2b25f5e --- /dev/null +++ b/web/templates/subpages/filters.html @@ -0,0 +1,94 @@ + + +
    + +
    diff --git a/web/templates/subpages/pager.html b/web/templates/subpages/pager.html index 500464b85..d4dccd17b 100644 --- a/web/templates/subpages/pager.html +++ b/web/templates/subpages/pager.html @@ -24,13 +24,13 @@
    • {% for n in range(pre,0,-1) %} -
  • {{page - n+1}}
    • +
  • {{page - n+1}}
    • {% endfor %}
  • {{page+1}} (current)
    • {% for n in range(1,post) %} -
  • {{page + n+1}}
    • +
  • {{page + n+1}}
    • {% endfor %}
  • diff --git a/web/templates/subpages/table.html b/web/templates/subpages/table.html index c5842e46d..16b350c93 100644 --- a/web/templates/subpages/table.html +++ b/web/templates/subpages/table.html @@ -35,10 +35,10 @@
- {{ c['Modified']|currentTime }} + {{ c['Modified'].strftime('%d-%m-%Y - %H:%M') }} - {{ c['Published']|currentTime }} + {{ c['Published'].strftime('%d-%m-%Y - %H:%M') }} {% endfor %}