Merge pull request #569 from blacklanternsecurity/material-docs

Material docs

.github/ISSUE_TEMPLATE/bug_report.md

@@ -1,9 +1,9 @@
 ---
 name: Bug Report
 about: Create a report to help us improve
-title: ''
+title: ""
 labels: bug
-assignees: ''
+assignees: ""
 ---
 
 **Describe the bug**

.github/ISSUE_TEMPLATE/feature_request.md

@@ -1,9 +1,9 @@
 ---
 name: Feature Request
 about: Request a new feature
-title: ''
+title: ""
 labels: enhancement
-assignees: ''
+assignees: ""
 ---
 
 **Description**

.github/workflows/docs.yml

@@ -0,0 +1,47 @@
+name: docs
+on:
+  push:
+    branches:
+      - material-docs
+      - dev
+permissions:
+  contents: write
+jobs:
+  docgen:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.10"
+      - name: Install dependencies
+        run: |
+          pip install poetry
+          poetry install
+      - name: Generate docs
+        run: |
+          poetry run bbot/scripts/docs.py
+      - name: Commit docs
+        uses: EndBug/add-and-commit@v9
+        with:
+          add: "*.md"
+          default_author: github_actions
+          message: "Refresh module docs"
+  deploy:
+    runs-on: ubuntu-latest
+    needs: docgen
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v4
+        with:
+          python-version: 3.x
+      - run: echo "cache_id=$(date --utc '+%V')" >> $GITHUB_ENV
+      - uses: actions/cache@v3
+        with:
+          key: mkdocs-material-${{ env.cache_id }}
+          path: .cache
+          restore-keys: |
+            mkdocs-material-
+      - run: pip install mkdocs-material mkdocs-extra-sass-plugin livereload
+      - run: mkdocs gh-deploy --force

bbot/cli.py

@@ -228,6 +228,14 @@ async def _main():
                 if options.help_all:
                     log_fn(parser.format_help())
 
+                if options.list_flags:
+                    log.stdout("")
+                    log.stdout("### FLAGS ###")
+                    log.stdout("")
+                    for row in module_loader.flags_table(flags=options.flags).splitlines():
+                        log.stdout(row)
+                    return
+
                 log_fn("")
                 log_fn("### MODULES ###")
                 log_fn("")
@@ -241,7 +249,7 @@ async def _main():
                     for row in module_loader.modules_options_table(modules=help_modules).splitlines():
                         log_fn(row)
 
-                if options.list_modules or options.help_all:
+                if options.list_modules or options.list_flags or options.help_all:
                     return
 
                 module_list = module_loader.filter_modules(modules=modules)

bbot/core/configurator/args.py

@@ -25,6 +25,9 @@ def parse_args(self, *args, **kwargs):
         For targets, also allow input files containing additional targets
         """
         ret = super().parse_args(*args, **kwargs)
+        # silent implies -y
+        if ret.silent:
+            ret.yes = True
         ret.modules = chain_lists(ret.modules)
         ret.output_modules = chain_lists(ret.output_modules)
         ret.targets = chain_lists(ret.targets, try_files=True, msg="Reading targets from file: {filename}")
@@ -55,30 +58,57 @@ def error(self, message):
         pass
 
 
-epilog = """EXAMPLES
-
-    Subdomains:
-        bbot -t evilcorp.com -f subdomain-enum
-
-    Subdomains (passive only):
-        bbot -t evilcorp.com -f subdomain-enum -rf passive
-
-    Subdomains + port scan + web screenshots:
-        bbot -t evilcorp.com -f subdomain-enum -m naabu gowitness -n my_scan -o .
-
-    Subdomains + basic web scan (wappalyzer, robots.txt, iis shortnames, etc.):
-        bbot -t evilcorp.com -f subdomain-enum web-basic
-
-    Subdomains + web spider (search for emails, etc.):
-        bbot -t evilcorp.com -f subdomain-enum -c web_spider_distance=2 web_spider_depth=2
-
-    Subdomains + emails + cloud + port scan + non-intrusive web + web screenshots + nuclei:
-        bbot -t evilcorp.com -f subdomain-enum email-enum cloud-enum web-basic -m naabu gowitness nuclei --allow-deadly
-
-    List modules:
-        bbot -l
-
-"""
+scan_examples = [
+    (
+        "Subdomains",
+        "Perform a full subdomain enumeration on evilcorp.com",
+        "bbot -t evilcorp.com -f subdomain-enum",
+    ),
+    (
+        "Subdomains (passive only)",
+        "Perform a passive-only subdomain enumeration on evilcorp.com",
+        "bbot -t evilcorp.com -f subdomain-enum -rf passive",
+    ),
+    (
+        "Subdomains + port scan + web screenshots",
+        "Port-scan every subdomain, screenshot every webpage, output to current directory",
+        "bbot -t evilcorp.com -f subdomain-enum -m nmap gowitness -n my_scan -o .",
+    ),
+    (
+        "Subdomains + basic web scan",
+        "A basic web scan includes wappalyzer, robots.txt, and other non-intrusive web modules",
+        "bbot -t evilcorp.com -f subdomain-enum web-basic",
+    ),
+    (
+        "Web spider",
+        "Crawl www.evilcorp.com up to a max depth of 2, automatically extracting emails, secrets, etc.",
+        "bbot -t www.evilcorp.com -m httpx robots badsecrets secretsdb -c web_spider_distance=2 web_spider_depth=2",
+    ),
+    (
+        "Everything everywhere all at once",
+        "Subdomains, emails, cloud buckets, port scan, basic web, web screenshots, nuclei",
+        "bbot -t evilcorp.com -f subdomain-enum email-enum cloud-enum web-basic -m nmap gowitness nuclei --allow-deadly",
+    ),
+]
+
+usage_examples = [
+    (
+        "List modules",
+        "",
+        "bbot -l",
+    ),
+    (
+        "List flags",
+        "",
+        "bbot -lf",
+    ),
+]
+
+
+epilog = "EXAMPLES\n"
+for example in (scan_examples, usage_examples):
+    for title, description, command in example:
+        epilog += f"\n    {title}:\n        {command}\n"
 
 
 parser = BBOTArgumentParser(
@@ -125,6 +155,7 @@ def error(self, message):
         help=f'Enable modules by flag. Choices: {",".join(sorted(flag_choices))}',
         metavar="FLAG",
     )
+    modules.add_argument("-lf", "--list-flags", action="store_true", help=f"List available flags.")
     modules.add_argument(
         "-rf",
         "--require-flags",

bbot/core/configurator/environ.py

@@ -114,12 +114,13 @@ def prepare_environment(bbot_config):
 
     log = logging.getLogger()
     if bbot_config.get("debug", False):
-        global _log_level_override
         bbot_config["silent"] = False
-        _log_level_override = logging.DEBUG
         log = logging.getLogger("bbot")
         log.setLevel(logging.DEBUG)
         logging.getLogger("asyncio").setLevel(logging.DEBUG)
+    elif bbot_config.get("silent", False):
+        log = logging.getLogger("bbot")
+        log.setLevel(logging.CRITICAL)
 
     # copy config to environment
     bbot_environ = flatten_config(bbot_config)

bbot/core/configurator/files.py

@@ -1,3 +1,4 @@
+import sys
 from pathlib import Path
 from omegaconf import OmegaConf
 
@@ -13,7 +14,10 @@
 default_config = None
 
 
-def _get_config(filename, name="config", notify=True):
+def _get_config(filename, name="config"):
+    notify = False
+    if sys.argv and sys.argv[0].endswith("bbot") and not any(x in sys.argv for x in ("-s", "--silent")):
+        notify = True
     filename = Path(filename).resolve()
     try:
         conf = OmegaConf.load(str(filename))

bbot/core/flags.py

@@ -0,0 +1,22 @@
+flag_descriptions = {
+    "active": "Makes active connections to target systems",
+    "affiliates": "Discovers affiliated hostnames/domains",
+    "aggressive": "Generates a large amount of network traffic",
+    "cloud-enum": "Enumerates cloud resources",
+    "deadly": "Highly aggressive",
+    "email-enum": "Enumerates email addresses",
+    "iis-shortnames": "Scans for IIS Shortname vulnerability",
+    "passive": "Never connects to target systems",
+    "portscan": "Discovers open ports",
+    "report": "Generates a report at the end of the scan",
+    "safe": "Non-intrusive, safe to run",
+    "service-enum": "Identifies protocols running on open ports",
+    "slow": "May take a long time to complete",
+    "social-enum": "Enumerates social media",
+    "subdomain-enum": "Enumerates subdomains",
+    "subdomain-hijack": "Detects hijackable subdomains",
+    "web-basic": "Basic, non-intrusive web scan functionality",
+    "web-paramminer": "Discovers HTTP parameters through brute-force",
+    "web-screenshots": "Takes screenshots of web pages",
+    "web-thorough": "More advanced web scanning functionality",
+}

bbot/core/helpers/misc.py

@@ -910,10 +910,18 @@ def make_table(*args, **kwargs):
     # fix IndexError: list index out of range
     if args and not args[0]:
         args = ([[]],) + args[1:]
-    defaults = {"tablefmt": "grid", "disable_numparse": True, "maxcolwidths": 40}
+    tablefmt = os.environ.get("BBOT_TABLE_FORMAT", None)
+    defaults = {"tablefmt": "grid", "disable_numparse": True, "maxcolwidths": None}
+    if tablefmt is None:
+        defaults.update({"maxcolwidths": 40})
+    else:
+        defaults.update({"tablefmt": tablefmt})
     for k, v in defaults.items():
         if k not in kwargs:
             kwargs[k] = v
+    # don't wrap columns in markdown
+    if tablefmt in ("github", "markdown"):
+        kwargs.pop("maxcolwidths")
     return tabulate(*args, **kwargs)
 
 

bbot/core/helpers/modules.py

@@ -6,6 +6,7 @@
 from omegaconf import OmegaConf
 from contextlib import suppress
 
+from ..flags import flag_descriptions
 from .misc import list_files, sha1, search_dict_by_key, search_format_dict, make_table, os_platform
 
 
@@ -269,19 +270,27 @@ def add_or_create(d, k, *items):
 
     def modules_table(self, modules=None, mod_type=None):
         table = []
-        header = ["Module", "Type", "Needs\nAPI\nKey", "Description", "Flags", "Produced Events"]
-        maxcolwidths = [20, 20, 5, 40, 40, 40]
+        header = ["Module", "Type", "Needs API Key", "Description", "Flags", "Consumed Events", "Produced Events"]
+        maxcolwidths = [20, 10, 5, 30, 30, 20, 20]
         for module_name, preloaded in self.filter_modules(modules, mod_type):
             module_type = preloaded["type"]
+            consumed_events = sorted(preloaded.get("watched_events", []))
             produced_events = sorted(preloaded.get("produced_events", []))
             flags = sorted(preloaded.get("flags", []))
             api_key_required = ""
             meta = preloaded.get("meta", {})
-            if meta.get("auth_required", False):
-                api_key_required = "X"
+            api_key_required = "Yes" if meta.get("auth_required", False) else "No"
             description = meta.get("description", "")
             table.append(
-                [module_name, module_type, api_key_required, description, ",".join(flags), ",".join(produced_events)]
+                [
+                    module_name,
+                    module_type,
+                    api_key_required,
+                    description,
+                    ", ".join(flags),
+                    ", ".join(consumed_events),
+                    ", ".join(produced_events),
+                ]
             )
         return make_table(table, header, maxcolwidths=maxcolwidths)
 
@@ -302,16 +311,69 @@ def modules_options(self, modules=None, mod_type=None):
                 option_name = f"{module_key}.{module_name}.{k}"
                 option_type = type(v).__name__
                 option_description = module_options_desc[k]
-                modules_options[module_name].append((option_name, option_type, str(v), option_description))
+                modules_options[module_name].append((option_name, option_type, option_description, str(v)))
         return modules_options
 
     def modules_options_table(self, modules=None, mod_type=None):
         table = []
-        header = ["Option", "Type", "Default", "Description"]
+        header = ["Config Option", "Type", "Description", "Default"]
         for module_name, module_options in self.modules_options(modules, mod_type).items():
             table += module_options
         return make_table(table, header)
 
+    def flags(self, flags=None):
+        _flags = {}
+        for module_name, preloaded in self.preloaded().items():
+            for flag in preloaded.get("flags", []):
+                if not flags or flag in flags:
+                    try:
+                        _flags[flag].add(module_name)
+                    except KeyError:
+                        _flags[flag] = {module_name}
+
+        _flags = sorted(_flags.items(), key=lambda x: x[0])
+        _flags = sorted(_flags, key=lambda x: len(x[-1]), reverse=True)
+        return _flags
+
+    def flags_table(self, flags=None):
+        table = []
+        header = ["Flag", "# Modules", "Description", "Modules"]
+        maxcolwidths = [20, 5, 40, 80]
+        _flags = self.flags(flags=flags)
+        for flag, modules in _flags:
+            description = flag_descriptions.get(flag, "")
+            table.append([flag, f"{len(modules)}", description, ", ".join(sorted(modules))])
+        return make_table(table, header, maxcolwidths=maxcolwidths)
+
+    def events(self):
+        consuming_events = {}
+        producing_events = {}
+        for module_name, preloaded in self.preloaded().items():
+            consumed = preloaded.get("watched_events", [])
+            produced = preloaded.get("produced_events", [])
+            for c in consumed:
+                try:
+                    consuming_events[c].add(module_name)
+                except KeyError:
+                    consuming_events[c] = {module_name}
+            for c in produced:
+                try:
+                    producing_events[c].add(module_name)
+                except KeyError:
+                    producing_events[c] = {module_name}
+        return consuming_events, producing_events
+
+    def events_table(self):
+        table = []
+        header = ["Event Type", "# Consuming Modules", "# Producing Modules", "Consuming Modules", "Producing Modules"]
+        consuming_events, producing_events = self.events()
+        all_event_types = sorted(set(consuming_events).union(set(producing_events)))
+        for e in all_event_types:
+            consuming = sorted(consuming_events.get(e, []))
+            producing = sorted(producing_events.get(e, []))
+            table.append([e, len(consuming), len(producing), ", ".join(consuming), ", ".join(producing)])
+        return make_table(table, header)
+
     def filter_modules(self, modules=None, mod_type=None):
         if modules is None:
             module_list = list(self.preloaded(type=mod_type).items())