Merge pull request #1470 from p-l-/passive-get-md5-ja3

Passive: accept MD5 JA3 values rather than raw
ivre · Jan 11, 2023 · c1d13bd · c1d13bd
2 parents aa06929 + 3c45295
commit c1d13bd
Show file tree

Hide file tree

Showing 3 changed files with 61 additions and 52 deletions.
diff --git a/ivre/active/data.py b/ivre/active/data.py
@@ -838,8 +838,8 @@ def is_server(script_id: str) -> bool:
         return script_id == "ssl-ja3-server"
 
     def ja3_equals(a: Dict[str, Any], b: Dict[str, Any], script_id: str) -> bool:
-        return a["raw"] == b["raw"] and (
-            not is_server(script_id) or a["client"]["raw"] == b["client"]["raw"]
+        return a["md5"] == b["md5"] and (
+            not is_server(script_id) or a["client"]["md5"] == b["client"]["md5"]
         )
 
     def ja3_output(ja3: Dict[str, Any], script_id: str) -> str:

diff --git a/ivre/passive.py b/ivre/passive.py
@@ -42,6 +42,7 @@
     ")",
     re.I,
 )
+MD5 = re.compile("^[0-9a-f]{32}$", re.I)
 
 
 # Zeek specific
@@ -356,21 +357,25 @@ def _prepare_rec(spec, ignorenets, neverignore):
         spec["recontype"] == "SSL_SERVER" and spec["source"].startswith("ja3-")
     ):
         value = spec["value"]
-        spec.setdefault("infos", {})["raw"] = value
-        spec["value"] = hashlib.new(
-            "md5", data=value.encode(), usedforsecurity=False
-        ).hexdigest()
+        if MD5.search(value) is None:
+            spec.setdefault("infos", {})["raw"] = value
+            spec["value"] = hashlib.new(
+                "md5", data=value.encode(), usedforsecurity=False
+            ).hexdigest()
         if spec["recontype"] == "SSL_SERVER":
             clientvalue = spec["source"][4:]
-            spec["infos"].setdefault("client", {})["raw"] = clientvalue
-            spec["source"] = (
-                "ja3-%s"
-                % hashlib.new(
-                    "md5",
-                    data=clientvalue.encode(),
-                    usedforsecurity=False,
-                ).hexdigest()
-            )
+            if MD5.search(clientvalue) is None:
+                spec["infos"].setdefault("client", {})["raw"] = clientvalue
+                spec["source"] = (
+                    "ja3-%s"
+                    % hashlib.new(
+                        "md5",
+                        data=clientvalue.encode(),
+                        usedforsecurity=False,
+                    ).hexdigest()
+                )
+            else:
+                spec["source"] = f"ja3-{clientvalue}"
     # SSH_{CLIENT,SERVER}_HASSH
     elif spec["recontype"] in ["SSH_CLIENT_HASSH", "SSH_SERVER_HASSH"]:
         value = spec["value"]
@@ -556,26 +561,35 @@ def _getinfos_cert(spec):
 
 def _getinfos_ja3_hassh(spec):
     """Extract hashes from JA3 & HASSH fingerprint strings."""
-    value = spec["infos"]["raw"]
-    data = value.encode()
-
-    info = dict(
-        (
-            (hashtype, hashlib.new(hashtype, data).hexdigest())
-            for hashtype in ["sha1", "sha256"]
-        ),
-        **spec["infos"],
-    )
+    try:
+        value = spec["infos"]["raw"]
+    except KeyError:
+        info = dict(spec.get("infos", {}))
+    else:
+        data = value.encode()
+        info = dict(
+            (
+                (hashtype, hashlib.new(hashtype, data).hexdigest())
+                for hashtype in ["sha1", "sha256"]
+            ),
+            **spec["infos"],
+        )
 
     if spec.get("recontype") == "SSL_SERVER":
-        clientvalue = spec["infos"]["client"]["raw"]
-        clientdata = clientvalue.encode()
-        info["client"].update(
-            (hashtype, hashlib.new(hashtype, clientdata).hexdigest())
-            for hashtype in ["sha1", "sha256"]
-        )
+        try:
+            clientvalue = spec["infos"]["client"]["raw"]
+        except KeyError:
+            pass
+        else:
+            clientdata = clientvalue.encode()
+            info["client"].update(
+                (hashtype, hashlib.new(hashtype, clientdata).hexdigest())
+                for hashtype in ["sha1", "sha256"]
+            )
 
-    return {"infos": info}
+    if info:
+        return {"infos": info}
+    return {}
 
 
 def _getinfos_from_banner(banner, proto="tcp", probe="NULL"):

diff --git a/ivre/view.py b/ivre/view.py
@@ -455,18 +455,15 @@ def _extract_passive_SSL_SERVER_ja3(rec):
         "protocol": rec.get("protocol", "tcp"),
     }
     script["output"] = rec["value"] + " - " + rec["source"][4:]
-    info = {
-        "raw": rec["infos"]["raw"],
-        "sha256": rec["infos"]["sha256"],
-        "sha1": rec["infos"]["sha1"],
-        "md5": rec["value"],
-        "client": {
-            "raw": rec["infos"]["client"]["raw"],
-            "sha256": rec["infos"]["client"]["sha256"],
-            "sha1": rec["infos"]["client"]["sha1"],
-            "md5": rec["source"][4:],
-        },
-    }
+    info = {"md5": rec["value"], "client": {"md5": rec["source"][4:]}}
+    if "infos" in rec:
+        for k in ["raw", "sha256", "sha1"]:
+            if k in rec["infos"]:
+                info[k] = rec["infos"][k]
+        if "client" in rec["infos"]:
+            for k in ["raw", "sha256", "sha1"]:
+                if k in rec["infos"]["client"]:
+                    info["client"][k] = rec["infos"]["client"][k]
     script["ssl-ja3-server"] = [info]
     port["scripts"] = [script]
     return {"ports": [port]}
@@ -523,14 +520,12 @@ def _extract_passive_SSL_CLIENT_ja3(rec):
     """Handle SSL client ja3 extraction."""
     script = {"id": "ssl-ja3-client"}
     script["output"] = rec["value"]
-    script["ssl-ja3-client"] = [
-        {
-            "raw": rec["infos"]["raw"],
-            "sha256": rec["infos"]["sha256"],
-            "sha1": rec["infos"]["sha1"],
-            "md5": rec["value"],
-        }
-    ]
+    info = {"md5": rec["value"]}
+    if "infos" in rec:
+        for k in ["raw", "sha256", "sha1"]:
+            if k in rec["infos"]:
+                info[k] = rec["infos"][k]
+    script["ssl-ja3-client"] = [info]
     port = {"port": -1, "scripts": [script]}
     if rec["value"] in scanners.JA3_CLIENT_VALUES:
         scanner, probe = scanners.JA3_CLIENT_VALUES[rec["value"]]