Merge pull request #1455 from p-l-/lint-bandit

Linting: add Bandit, severity high for now

.github/workflows/linting.yml

@@ -44,6 +44,8 @@ jobs:
 
     - run: black -t py36 --check ./doc/conf.py ./setup.py ./bin/ivre ./tests/tests.py ./ivre_bak/ ./pkg/stubs/
 
+    - run: bandit --severity-level high -r ./doc/conf.py ./setup.py ./bin/ivre ./tests/tests.py ./ivre_bak/ ./pkg/stubs/
+
     - run: codespell  --ignore-words=pkg/codespell_ignore `git ls-files | grep -vE '^web/static/(doc|an|bs|d3|jq|lk)/|^data/|\.(png|gif|svg)$'`
 
     - run: pylint -e all -d abstract-method,arguments-differ,attribute-defined-outside-init,broad-except,duplicate-code,fixme,function-redefined,global-statement,import-error,invalid-name,locally-disabled,missing-docstring,no-member,protected-access,super-init-not-called,suppressed-message,too-few-public-methods,too-many-ancestors,too-many-arguments,too-many-boolean-expressions,too-many-branches,too-many-instance-attributes,too-many-lines,too-many-locals,too-many-nested-blocks,too-many-public-methods,too-many-return-statements,too-many-statements,unsubscriptable-object,unused-argument,line-too-long,consider-using-f-string ivre ./doc/conf.py ./setup.py ./bin/ivre

doc/dev/linting.rst

@@ -7,8 +7,9 @@ to do so).
 
 So far, only the Python code uses such linters (`Flake8
 <https://flake8.pycqa.org>`_, `Pylint <https://pylint.org/>`_, `Mypy
-<http://mypy-lang.org/>`_ and `Black
-<https://github.com/psf/black>`_). Adding similar code linting
+<http://mypy-lang.org/>`_, `Bandit <https://github.com/PyCQA/bandit>`_
+and `Black <https://github.com/psf/black>`_). Adding similar code
+linting capabilities to the Zeek scripts (`zeek/`), LUA
 capabilities to the Zeek scripts (`zeek/`), LUA
 (`patches/nmap/scripts/`) and JavaScript / HTML (`web/static/`) could
 be a good PR idea!

ivre/db/__init__.py

@@ -779,7 +779,9 @@ def _ja3keyvalue(value_or_hash):
         if key == "raw":
             return (
                 "md5",
-                utils.hashlib.new("md5", value_or_hash.encode()).hexdigest(),
+                utils.hashlib.new(
+                    "md5", data=value_or_hash.encode(), usedforsecurity=False
+                ).hexdigest(),
             )
         return (key, value_or_hash.lower())
 

ivre/db/mongo.py

@@ -4814,7 +4814,9 @@ def _fix_sizes(cls, spec):
                 value = spec[key]
                 if not isinstance(value, bytes):
                     value = value.encode()
-                spec[key] = hashlib.sha1(value).hexdigest()
+                spec[key] = hashlib.new(
+                    "sha1", data=value, usedforsecurity=False
+                ).hexdigest()
         # We enforce a utils.MAXVALLEN // 10 size limits for subkey values in
         # infos; this is because MongoDB cannot index values longer than 1024
         # bytes.

ivre/passive.py

@@ -357,22 +357,27 @@ def _prepare_rec(spec, ignorenets, neverignore):
     ):
         value = spec["value"]
         spec.setdefault("infos", {})["raw"] = value
-        spec["value"] = hashlib.new("md5", value.encode()).hexdigest()
+        spec["value"] = hashlib.new(
+            "md5", data=value.encode(), usedforsecurity=False
+        ).hexdigest()
         if spec["recontype"] == "SSL_SERVER":
             clientvalue = spec["source"][4:]
             spec["infos"].setdefault("client", {})["raw"] = clientvalue
             spec["source"] = (
                 "ja3-%s"
                 % hashlib.new(
                     "md5",
-                    clientvalue.encode(),
+                    data=clientvalue.encode(),
+                    usedforsecurity=False,
                 ).hexdigest()
             )
     # SSH_{CLIENT,SERVER}_HASSH
     elif spec["recontype"] in ["SSH_CLIENT_HASSH", "SSH_SERVER_HASSH"]:
         value = spec["value"]
         spec.setdefault("infos", {})["raw"] = value
-        spec["value"] = hashlib.new("md5", value.encode()).hexdigest()
+        spec["value"] = hashlib.new(
+            "md5", data=value.encode(), usedforsecurity=False
+        ).hexdigest()
     # SSH_SERVER_HOSTKEY
     elif spec["recontype"] == "SSH_SERVER_HOSTKEY":
         spec["value"] = utils.encode_b64(utils.nmap_decode_data(spec["value"])).decode()

ivre/tools/passivereconworker.py

@@ -22,6 +22,7 @@
 
 import os
 import re
+import shlex
 import shutil
 import signal
 import subprocess
@@ -34,7 +35,6 @@
 SENSORS: Dict[str, str] = {}  # shortname: fullname
 FILEFORMAT = "^(?P<sensor>%s)[.-](?P<datetime>[0-9-]+)\\.log(?:\\.(?:gz|bz2))?$"
 SLEEPTIME = 2
-CMDLINE = "%(progname)s -s %(sensor)s"
 WANTDOWN = False
 
 
@@ -73,8 +73,7 @@ def create_process(progname: str, sensor: str) -> subprocess.Popen:
 
     """
     return subprocess.Popen(
-        CMDLINE % {"progname": progname, "sensor": SENSORS.get(sensor, sensor)},
-        shell=True,
+        shlex.split(progname) + ["-s", SENSORS.get(sensor, sensor)],
         stdin=subprocess.PIPE,
     )
 

ivre/web/utils.py

@@ -94,11 +94,10 @@ def get_notepad_pages_mediawiki(
         page exists.
 
         """
-        ipaddr_page = "^" + re.escape(base) + "\\/\\d+\\.\\d+\\.\\d+\\.\\d+$"
         cur = MySQLdb.Connect(server, username, password, dbname).cursor()
         cur.execute(
-            "SELECT `page_title` FROM `wiki_page` WHERE `page_title` "
-            "REGEXP '%s'" % ipaddr_page
+            "SELECT `page_title` FROM `wiki_page` WHERE `page_title` REGEXP %s",
+            ("^" + re.escape(base) + "\\/\\d+\\.\\d+\\.\\d+\\.\\d+$",),
         )
         return [page[0][len(base) + 1 :] for page in cur]
 

pkg/runchecks

@@ -16,12 +16,14 @@
 # You should have received a copy of the GNU General Public License
 # along with IVRE. If not, see <http://www.gnu.org/licenses/>.
 
-# pip install -U black flake8 mypy codespell pylint
+# pip install -U -r requirements-linting.txt
 
 black -t py36 --check ./doc/conf.py ./setup.py ./bin/ivre ./tests/tests.py ./ivre/ ./pkg/stubs/ && echo "black OK"
 
 flake8 --ignore=E402,E501,F401 ./doc/conf.py && flake8 --ignore=E501,W503 ./setup.py ./bin/ivre && flake8 --ignore=E203,E402,E501,W503 ./tests/tests.py && flake8 --ignore=E203,E501,W503 ./ivre/ && flake8 --ignore=E302,E305,E701,E704 ./pkg/stubs/*.pyi && echo "flake8 OK"
 
+bandit --severity-level high -r ./doc/conf.py ./setup.py ./bin/ivre ./tests/tests.py ./ivre/ ./pkg/stubs/ && echo "bandit OK"
+
 # TODO: remove --follow-imports=skip
 MYPYPATH=./pkg/stubs/ mypy --follow-imports=skip --disallow-untyped-calls --disallow-untyped-decorators --disallow-untyped-defs --disallow-incomplete-defs --no-implicit-optional --warn-redundant-casts --warn-unused-ignores --warn-return-any ./ivre/{active,analyzer,data,parser,tools,types}/*.py ./ivre/{__init__,activecli,agent,config,flow,geoiputils,graphroute,keys,nmapopt,utils,zgrabout}.py && echo "mypy OK"
 # partial sub-modules

requirements-all.txt

@@ -1,3 +1,4 @@
+bandit
 black
 bottle
 codespell

requirements-linting.txt

@@ -1,3 +1,4 @@
+bandit
 black
 codespell
 flake8

web/static/doc/_sources/dev/linting.rst.txt

@@ -7,8 +7,9 @@ to do so).
 
 So far, only the Python code uses such linters (`Flake8
 <https://flake8.pycqa.org>`_, `Pylint <https://pylint.org/>`_, `Mypy
-<http://mypy-lang.org/>`_ and `Black
-<https://github.com/psf/black>`_). Adding similar code linting
+<http://mypy-lang.org/>`_, `Bandit <https://github.com/PyCQA/bandit>`_
+and `Black <https://github.com/psf/black>`_). Adding similar code
+linting capabilities to the Zeek scripts (`zeek/`), LUA
 capabilities to the Zeek scripts (`zeek/`), LUA
 (`patches/nmap/scripts/`) and JavaScript / HTML (`web/static/`) could
 be a good PR idea!