Merge pull request #605 from blacklanternsecurity/per_host_only

Per host only

bbot/modules/base.py

@@ -32,6 +32,8 @@ class BaseModule:
     accept_dupes = False
     # Whether to block outgoing duplicate events
     suppress_dupes = True
+    # Limit the module to only scanning once per host. By default, defined by event.host, but can be customized by overriding
+    per_host_only = False
 
     # Scope distance modifier - accept/deny events based on scope distance
     # None == accept all events
@@ -102,6 +104,9 @@ def __init__(self, scan):
         self._event_queued = asyncio.Condition()
         self._event_dequeued = asyncio.Condition()
 
+        # used for optional "per host" tracking
+        self._per_host_tracker = set()
+
     async def setup(self):
         """
         Perform setup functions at the beginning of the scan.
@@ -427,6 +432,12 @@ async def _event_postcheck(self, event):
             if not filter_result:
                 return False, msg
 
+        if self.per_host_only:
+            if self.get_per_host_hash(event) in self._per_host_tracker:
+                return False, "per_host_only enabled and already seen host"
+            else:
+                self._per_host_tracker.add(self.get_per_host_hash(event))
+
         if self._type == "output" and not event._stats_recorded:
             event._stats_recorded = True
             self.scan.stats.event_produced(event)
@@ -501,6 +512,15 @@ def set_error_state(self, message=None):
                 # if there are leftover objects in the queue, the scan will hang.
                 self._incoming_event_queue = False
 
+    # override in the module to define different values to comprise the hash
+    def get_per_host_hash(self, event):
+        parsed = getattr(event, "parsed", None)
+        if parsed is None:
+            to_hash = self.helpers.make_netloc(event.host, event.port)
+        else:
+            to_hash = f"{parsed.scheme}://{parsed.netloc}/"
+        return hash(to_hash)
+
     @property
     def name(self):
         return str(self._name)

bbot/modules/host_header.py

@@ -9,12 +9,11 @@ class host_header(BaseModule):
     meta = {"description": "Try common HTTP Host header spoofing techniques"}
 
     in_scope_only = True
+    per_host_only = True
 
     deps_apt = ["curl"]
 
     async def setup(self):
-        self.scanned_hosts = set()
-
         self.subdomain_tags = {}
         if self.scan.config.get("interactsh_disable", False) == False:
             try:
@@ -74,14 +73,6 @@ async def cleanup(self):
                 self.warning(f"Interactsh failure: {e}")
 
     async def handle_event(self, event):
-        host = f"{event.parsed.scheme}://{event.parsed.netloc}/"
-        host_hash = hash(host)
-        if host_hash in self.scanned_hosts:
-            self.debug(f"Host {host} was already scanned, exiting")
-            return
-        else:
-            self.scanned_hosts.add(host_hash)
-
         # get any set-cookie responses from the response and add them to the request
 
         added_cookies = {}

bbot/modules/iis_shortnames.py

@@ -1,5 +1,4 @@
 import re
-from threading import Lock
 
 from bbot.modules.base import BaseModule
 
@@ -47,7 +46,6 @@ async def detect(self, target):
         return detections
 
     async def setup(self):
-        self.scanned_tracker_lock = Lock()
         self.scanned_tracker = set()
         return True
 
@@ -145,8 +143,7 @@ async def solve_shortname_recursive(
 
     async def handle_event(self, event):
         normalized_url = self.normalize_url(event.data)
-        with self.scanned_tracker_lock:
-            self.scanned_tracker.add(normalized_url)
+        self.scanned_tracker.add(normalized_url)
 
         detections = await self.detect(normalized_url)
 
@@ -213,8 +210,7 @@ async def handle_event(self, event):
 
     async def filter_event(self, event):
         if "dir" in event.tags:
-            with self.scanned_tracker_lock:
-                if self.normalize_url(event.data) not in self.scanned_tracker:
-                    return True
-                return False
+            if self.normalize_url(event.data) not in self.scanned_tracker:
+                return True
+            return False
         return False

bbot/modules/robots.py

@@ -15,21 +15,13 @@ class robots(BaseModule):
     }
 
     in_scope_only = True
+    per_host_only = True
 
     async def setup(self):
-        self.scanned_hosts = set()
         return True
 
     async def handle_event(self, event):
-        parsed_host = event.parsed
-        host = f"{parsed_host.scheme}://{parsed_host.netloc}/"
-        host_hash = hash(host)
-        if host_hash in self.scanned_hosts:
-            self.debug(f"Host {host} was already scanned, exiting")
-            return
-        else:
-            self.scanned_hosts.add(host_hash)
-
+        host = f"{event.parsed.scheme}://{event.parsed.netloc}/"
         result = None
         url = f"{host}robots.txt"
         result = await self.helpers.request(url)

bbot/modules/smuggler.py

@@ -15,6 +15,7 @@ class smuggler(BaseModule):
     meta = {"description": "Check for HTTP smuggling"}
 
     in_scope_only = True
+    per_host_only = True
 
     deps_ansible = [
         {
@@ -23,19 +24,7 @@ class smuggler(BaseModule):
         }
     ]
 
-    async def setup(self):
-        self.scanned_hosts = set()
-        return True
-
     async def handle_event(self, event):
-        host = f"{event.parsed.scheme}://{event.parsed.netloc}/"
-        host_hash = hash(host)
-        if host_hash in self.scanned_hosts:
-            self.debug(f"Host {host} was already scanned, exiting")
-            return
-        else:
-            self.scanned_hosts.add(host_hash)
-
         command = [
             sys.executable,
             f"{self.scan.helpers.tools_dir}/smuggler/smuggler.py",

bbot/modules/telerik.py

@@ -139,6 +139,7 @@ class telerik(BaseModule):
     options_desc = {"exploit_RAU_crypto": "Attempt to confirm any RAU AXD detections are vulnerable"}
 
     in_scope_only = True
+    per_host_only = True
 
     deps_pip = ["pycryptodome~=3.17"]
 
@@ -159,19 +160,10 @@ class telerik(BaseModule):
     max_event_handlers = 5
 
     async def setup(self):
-        self.scanned_hosts = set()
         self.timeout = self.scan.config.get("httpx_timeout", 5)
         return True
 
     async def handle_event(self, event):
-        host = f"{event.parsed.scheme}://{event.parsed.netloc}/"
-        host_hash = hash(host)
-        if host_hash in self.scanned_hosts:
-            self.debug(f"Host {host} was already scanned, exiting")
-            return
-        else:
-            self.scanned_hosts.add(host_hash)
-
         webresource = "Telerik.Web.UI.WebResource.axd?type=rau"
         result, _ = await self.test_detector(event.data, webresource)
         if result:

bbot/modules/wafw00f.py

@@ -18,21 +18,10 @@ class wafw00f(BaseModule):
     options_desc = {"generic_detect": "When no specific WAF detections are made, try to peform a generic detect"}
 
     in_scope_only = True
-
-    async def setup(self):
-        self.scanned_hosts = set()
-        return True
+    per_host_only = True
 
     async def handle_event(self, event):
-        parsed_host = event.parsed
-        host = f"{parsed_host.scheme}://{parsed_host.netloc}/"
-        host_hash = hash(host)
-        if host_hash in self.scanned_hosts:
-            self.debug(f"Host {host} was already scanned, exiting")
-            return
-        else:
-            self.scanned_hosts.add(host_hash)
-
+        host = f"{event.parsed.scheme}://{event.parsed.netloc}/"
         WW = await self.scan.run_in_executor(wafw00f_main.WAFW00F, host)
         waf_detections = await self.scan.run_in_executor(WW.identwaf)
         if waf_detections:

bbot/test/test_step_1/test_modules_basic.py

@@ -2,6 +2,11 @@
 
 from ..bbot_fixtures import *
 
+from bbot.modules.base import BaseModule
+from bbot.modules.output.base import BaseOutputModule
+from bbot.modules.report.base import BaseReportModule
+from bbot.modules.internal.base import BaseInternalModule
+
 
 @pytest.mark.asyncio
 async def test_modules_basic(scan, helpers, events, bbot_config, bbot_scanner, httpx_mock):
@@ -12,12 +17,6 @@ async def test_modules_basic(scan, helpers, events, bbot_config, bbot_scanner, h
     for http_method in ("GET", "CONNECT", "HEAD", "POST", "PUT", "TRACE", "DEBUG", "PATCH", "DELETE", "OPTIONS"):
         httpx_mock.add_response(method=http_method, url=re.compile(r".*"), json={"test": "test"})
 
-    # event filtering
-    from bbot.modules.base import BaseModule
-    from bbot.modules.output.base import BaseOutputModule
-    from bbot.modules.report.base import BaseReportModule
-    from bbot.modules.internal.base import BaseInternalModule
-
     # output module specific event filtering tests
     base_output_module = BaseOutputModule(scan)
     base_output_module.watched_events = ["IP_ADDRESS"]
@@ -167,3 +166,42 @@ async def test_modules_basic(scan, helpers, events, bbot_config, bbot_scanner, h
         assert flag in flag_descriptions, f'Flag "{flag}" not listed in bbot/core/flags.py'
         description = flag_descriptions.get(flag, "")
         assert description, f'Flag "{flag}" has no description in bbot/core/flags.py'
+
+
+@pytest.mark.asyncio
+async def test_modules_basic_perhostonly(scan, helpers, events, bbot_config, bbot_scanner, httpx_mock, monkeypatch):
+    per_host_scan = bbot_scanner(
+        "evilcorp.com",
+        modules=list(set(available_modules + available_internal_modules)),
+        config=bbot_config,
+    )
+
+    await per_host_scan.load_modules()
+    await per_host_scan.setup_modules()
+    per_host_scan.status = "RUNNING"
+
+    # ensure that multiple events to the same "host" (schema + host) are blocked and check the per host tracker
+    for module_name, module in sorted(per_host_scan.modules.items()):
+        #    module.filter_event = base_module.filter_event
+        monkeypatch.setattr(module, "filter_event", BaseModule(per_host_scan).filter_event)
+
+        if "URL" in module.watched_events:
+            url_1 = per_host_scan.make_event(
+                "http://evilcorp.com/1", event_type="URL", source=per_host_scan.root_event, tags=["status-200"]
+            )
+            url_1.set_scope_distance(0)
+            url_2 = per_host_scan.make_event(
+                "http://evilcorp.com/2", event_type="URL", source=per_host_scan.root_event, tags=["status-200"]
+            )
+            url_2.set_scope_distance(0)
+            valid_1, reason_1 = await module._event_postcheck(url_1)
+            valid_2, reason_2 = await module._event_postcheck(url_2)
+
+            if module.per_host_only == True:
+                assert valid_1 == True
+                assert valid_2 == False
+                assert hash("http://evilcorp.com/") in module._per_host_tracker
+
+            else:
+                assert valid_1 == True
+                assert valid_2 == True