Initial import restructure

.gitignore

@@ -1,143 +1,20 @@
-          # Created by .ignore support plugin (hsz.mobi)
-### Python template
-# Byte-compiled / optimized / DLL files
 __pycache__/
-*.py[cod]
-*$py.class
+bin/__pycache__/
+lib/__pycache__/
+sbin/__pycache__/
 
-# C extensions
-*.so
+log/
+ssl/
+tmp/
+plugins/
+indexdir/
+web/templates/plugins/
 
-# Distribution / packaging
-.Python
-build/
-develop-eggs/
-dist/
-downloads/
-eggs/
-.eggs/
-lib/
-lib64/
-parts/
-sdist/
-var/
-wheels/
-pip-wheel-metadata/
-share/python-wheels/
-*.egg-info/
-.installed.cfg
-*.egg
-MANIFEST
+etc/configuration.ini
+etc/plugins.ini
+etc/plugins.txt
 
-# PyInstaller
-#  Usually these files are written by a python script from a template
-#  before PyInstaller builds the exe, so as to inject date/other infos into it.
-*.manifest
-*.spec
-
-# Installer logs
-pip-log.txt
-pip-delete-this-directory.txt
-
-# Unit test / coverage reports
-htmlcov/
-.tox/
-.nox/
-.coverage
-.coverage.*
-.cache
-nosetests.xml
-coverage.xml
-*.cover
-*.py,cover
-.hypothesis/
-.pytest_cache/
-cover/
-
-# Translations
-*.mo
-*.pot
-
-# Django stuff:
-*.log
-local_settings.py
-db.sqlite3
-db.sqlite3-journal
-
-# Flask stuff:
-instance/
-.webassets-cache
-
-# Scrapy stuff:
-.scrapy
-
-# Sphinx documentation
-docs/_build/
-
-# PyBuilder
-.pybuilder/
-target/
-
-# Jupyter Notebook
-.ipynb_checkpoints
-
-# IPython
-profile_default/
-ipython_config.py
-
-# pyenv
-#   For a library or package, you might want to ignore these files since the code is
-#   intended to run in multiple environments; otherwise, check them in:
-# .python-version
-
-# pipenv
-#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
-#   However, in case of collaboration, if having platform-specific dependencies or dependencies
-#   having no cross-platform support, pipenv may install dependencies that don't work, or not
-#   install all needed dependencies.
-#Pipfile.lock
-
-# PEP 582; used by e.g. github.com/David-OConnor/pyflow
-__pypackages__/
-
-# Celery stuff
-celerybeat-schedule
-celerybeat.pid
-
-# SageMath parsed files
-*.sage.py
-
-# Environments
-.env
-.venv
-env/
-venv/
-ENV/
-env.bak/
-venv.bak/
-
-# Spyder project settings
-.spyderproject
-.spyproject
-
-# Rope project settings
-.ropeproject
-
-# mkdocs documentation
-/site
-
-# mypy
-.mypy_cache/
-.dmypy.json
-dmypy.json
-
-# Pyre type checker
-.pyre/
-
-# pytype static type analyzer
-.pytype/
-
-# Cython debug symbols
-cython_debug/
+.gitignore
 
 .idea
+venv

.travis.yml

@@ -17,8 +17,6 @@ jobs:
           - ./db_mgmt_json.py -p
           - ./db_mgmt_cpe_dictionary.py -p
           - popd
-      after_success:
-        - codecov
     - stage: unit tests
       python: nightly
       script: pytest --cov-report term --cov=./ test/unit -v
@@ -57,3 +55,6 @@ notifications:
     email:
         on_success: change
         on_failure: change
+
+after_success:
+  - codecov

doc/html/Installation.html

@@ -50,6 +50,8 @@ <h2>Requirements</h2>
   <li>Jinja2</li>
   <li>itsdangerous</li>
   <li>click</li>
+  <li>ijson</li>
+  <li>tqdm</li>
 </ul>
 </p>
 <h2>Setting up CVE-Search</h2>

doc/markdown/installation.md

@@ -41,6 +41,8 @@ After installing these packages, you need to install some python modules
  * Jinja2
  * itsdangerous
  * click
+ * ijson
+ * tqdm
 
 ## Setting up CVE-Search
 Before setting up CVE-Search, you have to make sure the scripts are

etc/configuration.ini.sample

@@ -2,6 +2,7 @@
 Host: localhost
 Port: 6379
 Password: RedisPassword
+redisQ: 9
 VendorsDB: 10
 NotificationsDB: 11
 RefDB: 12

lib/Config.py

@@ -32,6 +32,7 @@ class Configuration:
     default = {
         "redisHost": "localhost",
         "redisPort": 6379,
+        "redisQ": 9,
         "redisVendorDB": 10,
         "redisNotificationsDB": 11,
         "redisRefDB": 12,
@@ -186,6 +187,21 @@ def getRedisRefConnection(cls):
             decode_responses=True,
         )
 
+    @classmethod
+    def getRedisQConnection(cls):
+        redisHost = cls.getRedisHost()
+        redisPort = cls.getRedisPort()
+        redisDB = cls.readSetting("Redis", "redisQ", cls.default["redisQ"])
+        redisPass = cls.readSetting("Redis", "Password", cls.default["redisPass"])
+        return redis.StrictRedis(
+            host=redisHost,
+            port=redisPort,
+            db=redisDB,
+            password=redisPass,
+            charset="utf-8",
+            decode_responses=True,
+        )
+
     # Flask
     @classmethod
     def getFlaskHost(cls):

lib/DatabaseLayer.py

@@ -94,6 +94,7 @@ def updateCVE(cve):
                 "last-modified": cve["Modified"],
             }
         },
+        upsert=True,
     )
 
 
@@ -105,24 +106,21 @@ def bulkInsert(collection, data):
         bulk.execute()
 
 
-def bulkUpdate(collection, data):
-    if len(data) < 1:
-        return
-
+def bulkUpdate(collection, data, total):
     batch = []
-    count = 1000
-    print("Beginning bulk update of {} items".format(str(len(data))))
+    count = 0
+    print("Beginning bulk update of {} items".format(total))
     for d in data:
-        batch.append(UpdateOne({"id": d["id"]}, {"$set": d}, upsert=True))
+        batch.append(d)
         if len(batch) == 1000:
+            count += 1000
             db[collection].bulk_write(batch)
-            print("Updated {}/{} items".format(str(count), str(len(data))))
+            print("Updated {}/{} items".format(count, total))
             batch = []
-            count += 1000
     if len(batch) > 0:
         db[collection].bulk_write(batch)
         count += len(batch)
-        print("Updated {}/{} items.".format(str(count), str(len(data))))
+        print("Updated {}/{} items".format(count, total))
     print("Finished bulk update.")
 
 
@@ -137,9 +135,9 @@ def dropCollection(col):
 
 
 def getTableNames():
-    return db.collection_names()
+    # return db.collection_names()
     # jdt_NOTE: collection_names() is depreated, list_collection_names() should be used instead
-    # return db.list_collection_names()
+    return db.list_collection_names()
 
 
 # returns True if 'target_version' is less or equal than
@@ -169,9 +167,7 @@ def cvesForCPE(cpe, lax=False, vulnProdSearch=False):
         return []
 
     cpe_regex = cpe
-
     final_cves = []
-
     cpe_searchField = (
         "vulnerable_product" if vulnProdSearch else "vulnerable_configuration"
     )
@@ -205,7 +201,6 @@ def cvesForCPE(cpe, lax=False, vulnProdSearch=False):
         cves = colCVE.find(
             {cpe_searchField: {"$regex": cpe_regex, "$options": "i"}}
         ).sort("Modified", -1)
-
         i = 0
 
         for cve in cves:
@@ -240,6 +235,7 @@ def cvesForCPE(cpe, lax=False, vulnProdSearch=False):
 
         # default strict search
         cves = colCVE.find({cpe_searchField: {"$regex": cpe_regex}})
+
         final_cves = cves
 
     final_cves = sanitize(final_cves)