From 2063b150ca942d300b7e664fe60d7b9e975eee88 Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Thu, 11 Jan 2024 15:04:16 -0500
Subject: [PATCH 1/2] misc tweaks

---
 bbot/core/helpers/web.py         | 2 +-
 bbot/modules/base.py             | 9 ++++++++-
 bbot/modules/output/websocket.py | 9 ++++++---
 bbot/modules/sslcert.py          | 4 +---
 bbot/scanner/manager.py          | 2 +-
 5 files changed, 17 insertions(+), 9 deletions(-)

diff --git a/bbot/core/helpers/web.py b/bbot/core/helpers/web.py
index daec5d330..983a88c96 100644
--- a/bbot/core/helpers/web.py
+++ b/bbot/core/helpers/web.py
@@ -596,7 +596,7 @@ async def _acatch(self, url, raise_error):
             if raise_error:
                 raise
         except httpx.ConnectError:
-            log.verbose(f"HTTP connect failed to URL: {url}")
+            log.debug(f"HTTP connect failed to URL: {url}")
             if raise_error:
                 raise
         except httpx.RequestError as e:
diff --git a/bbot/modules/base.py b/bbot/modules/base.py
index 36d973783..7eb88ac6d 100644
--- a/bbot/modules/base.py
+++ b/bbot/modules/base.py
@@ -1152,7 +1152,14 @@ def log_table(self, *args, **kwargs):
         return table
 
     def _is_graph_important(self, event):
-        return self._preserve_graph and getattr(event, "_graph_important", False)
+        return self.preserve_graph and getattr(event, "_graph_important", False)
+
+    @property
+    def preserve_graph(self):
+        preserve_graph = self.config.get("preserve_graph", None)
+        if preserve_graph is None:
+            preserve_graph = self._preserve_graph
+        return preserve_graph
 
     def stdout(self, *args, **kwargs):
         """Writes log messages directly to standard output.
diff --git a/bbot/modules/output/websocket.py b/bbot/modules/output/websocket.py
index c6ff1e49a..76edcba4b 100644
--- a/bbot/modules/output/websocket.py
+++ b/bbot/modules/output/websocket.py
@@ -8,9 +8,12 @@
 class Websocket(BaseOutputModule):
     watched_events = ["*"]
     meta = {"description": "Output to websockets"}
-    options = {"url": "", "token": ""}
-    options_desc = {"url": "Web URL", "token": "Authorization Bearer token"}
-    _preserve_graph = True
+    options = {"url": "", "token": "", "preserve_graph": True}
+    options_desc = {
+        "url": "Web URL",
+        "token": "Authorization Bearer token",
+        "preserve_graph": "Preserve full chains of events in the graph (prevents orphans)",
+    }
 
     async def setup(self):
         self.url = self.config.get("url", "")
diff --git a/bbot/modules/sslcert.py b/bbot/modules/sslcert.py
index 4aeb7fbfb..9b55e8962 100644
--- a/bbot/modules/sslcert.py
+++ b/bbot/modules/sslcert.py
@@ -57,10 +57,8 @@ async def handle_event(self, event):
 
         if event.scope_distance == 0:
             abort_threshold = self.in_scope_abort_threshold
-            log_fn = self.info
         else:
             abort_threshold = self.out_of_scope_abort_threshold
-            log_fn = self.verbose
 
         tasks = [self.visit_host(host, port) for host in hosts]
         async for task in self.helpers.as_completed(tasks):
@@ -70,7 +68,7 @@ async def handle_event(self, event):
             dns_names, emails, (host, port) = result
             if len(dns_names) > abort_threshold:
                 netloc = self.helpers.make_netloc(host, port)
-                log_fn(
+                self.verbose(
                     f"Skipping Subject Alternate Names (SANs) on {netloc} because number of hostnames ({len(dns_names):,}) exceeds threshold ({abort_threshold})"
                 )
                 dns_names = dns_names[:1] + [n for n in dns_names[1:] if self.scan.in_scope(n)]
diff --git a/bbot/scanner/manager.py b/bbot/scanner/manager.py
index c594da6a6..0fda46469 100644
--- a/bbot/scanner/manager.py
+++ b/bbot/scanner/manager.py
@@ -411,7 +411,7 @@ async def distribute_event(self, event):
             for mod in self.scan.modules.values():
                 acceptable_dup = (not is_outgoing_duplicate) or mod.accept_dupes
                 # graph_important = mod._type == "output" and event._graph_important == True
-                graph_important = mod._preserve_graph and event._graph_important
+                graph_important = mod._is_graph_important(event)
                 if acceptable_dup or graph_important:
                     await mod.queue_event(event)
 

From 9ec0342e4ff803100a5ccbbca42bbe3ba868bcbc Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Thu, 11 Jan 2024 15:09:28 -0500
Subject: [PATCH 2/2] fix censys parsing bug

---
 bbot/modules/censys.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/bbot/modules/censys.py b/bbot/modules/censys.py
index 339f10bf7..b8609adf8 100644
--- a/bbot/modules/censys.py
+++ b/bbot/modules/censys.py
@@ -57,7 +57,10 @@ async def query(self, query):
             if resp is None:
                 break
 
-            d = resp.json()
+            try:
+                d = resp.json()
+            except Exception as e:
+                self.warning(f"Failed to parse JSON from {url} (response: {resp}): {e}")
 
             if resp.status_code < 200 or resp.status_code >= 400:
                 if isinstance(d, dict):