From f41aa867465aefdc61204028536adbc60437289d Mon Sep 17 00:00:00 2001 From: GangGreenTemperTatum <104169244+GangGreenTemperTatum@users.noreply.github.com> Date: Wed, 22 Jul 2026 11:18:11 -0400 Subject: [PATCH] feat(web-security): add measured ProjectDiscovery recon --- .../web-security/agents/web-security.md | 3 +- capabilities/web-security/capability.yaml | 24 +++- .../web-security/scripts/install_tools.sh | 12 +- capabilities/web-security/scripts/pd-tool | 48 ++++++++ .../skills/initial-recon/SKILL.md | 47 ++++++++ .../scripts/write_coverage_ledger.py | 96 +++++++++++++++ .../tests/test_coverage_ledger.py | 110 ++++++++++++++++++ .../tests/test_initial_recon_contract.py | 44 +++++++ .../test_interrupted_tool_result_hook.py | 2 +- .../tests/test_pd_tool_resolver.py | 83 +++++++++++++ 10 files changed, 463 insertions(+), 6 deletions(-) create mode 100755 capabilities/web-security/scripts/pd-tool create mode 100644 capabilities/web-security/skills/initial-recon/SKILL.md create mode 100755 capabilities/web-security/skills/initial-recon/scripts/write_coverage_ledger.py create mode 100644 capabilities/web-security/tests/test_coverage_ledger.py create mode 100644 capabilities/web-security/tests/test_initial_recon_contract.py create mode 100644 capabilities/web-security/tests/test_pd_tool_resolver.py diff --git a/capabilities/web-security/agents/web-security.md b/capabilities/web-security/agents/web-security.md index 389eadd..dc56c79 100644 --- a/capabilities/web-security/agents/web-security.md +++ b/capabilities/web-security/agents/web-security.md @@ -16,6 +16,7 @@ You are relentless but methodical. You operate in continuous OODA loops — obse Before attacking, understand the target: +- **Use the initial-recon skill for broad scopes**: When starting from a domain, wildcard, organization, ASN, IP list, or CIDR, load `initial-recon`. Prefer the provisioned ProjectDiscovery tools (`subfinder`, `dnsx`, `naabu`, `httpx`, `tlsx`, `katana`) through `scripts/pd-tool`; establish measurable breadth before selecting targets for depth. - **Map the surface**: Crawl or enumerate all endpoints, parameters, forms, APIs, and static resources. Identify the technology stack from headers, error pages, URL patterns, and JavaScript. - **Understand authentication**: How does the application manage sessions? Cookies, JWTs, API keys, OAuth? What roles exist? Where are the privilege boundaries? See the **Authentication Context** section below for how to handle credentials the operator provides vs. credentials you discover. - **Probe OAuth/OIDC surface**: Check `/.well-known/openid-configuration` and `/.well-known/oauth-authorization-server`. If `registration_endpoint` exists, test for unauthenticated Dynamic Client Registration (load `mcp-auth-exploitation` skill). If OAuth flows use PKCE, test enforcement by stripping `code_challenge` (load `oauth-flow-hijack` skill, Section 5). Fingerprint the OAuth library/framework for known CVEs (django-allauth, oauth2-proxy, Cloudflare Workers — see `oauth-flow-hijack` Section 6). @@ -91,7 +92,7 @@ Any tool that scans, fuzzes, or floods runs on shared local hardware. Cap concur ### Built-in tools (always available) - Use `execute_http` for standard HTTP work: reconnaissance, payload delivery, session-based testing, and response analysis. `reset_http_session` clears cookies/state; `get_http_cookies` inspects the jar. -- For fuzzing, wordlist-based attacks, complex encoding chains, multi-request scripting, or any task requiring shell pipelines — use `bash` with `curl`, `python`, `ffuf`, or other CLI tools directly. `execute_http` is not suited for high-volume or programmatic testing. +- For broad reconnaissance, use the ProjectDiscovery workflow in `initial-recon` when those binaries are available. For targeted fuzzing, complex encoding chains, custom multi-request logic, or shell pipelines, use `bash` with the smallest appropriate CLI. `execute_http` is not suited for high-volume or programmatic testing. - Use browser automation only when a real browser is required: DOM behavior, client-side execution, login flows, clickjacking, screenshots, or JavaScript-driven state changes. Prefer the `agent-browser` CLI when it is available on the current `PATH`; use the `agent_browser_*` MCP tools as the fallback. - Use Protoscope when inspecting or crafting protobuf payloads. Prefer the local `protoscope` CLI when it is available on the current `PATH`; use the `protoscope_*` MCP tools as the fallback. - Use `store_credential` and `get_credential` to preserve auth state for the current session instead of manually re-entering secrets or tokens. When the credential was operator-sourced, also persist the auth *flow* to project memory (see **Authentication Context** above). Also supports TOTP/MFA via `add_totp_credential` and `generate_mfa_code`. diff --git a/capabilities/web-security/capability.yaml b/capabilities/web-security/capability.yaml index 4829699..0cfcb75 100644 --- a/capabilities/web-security/capability.yaml +++ b/capabilities/web-security/capability.yaml @@ -1,6 +1,6 @@ schema: 1 name: web-security -version: "1.2.0" +version: "1.3.0" description: > Web application penetration testing with 70+ attack technique playbooks covering request smuggling, cache poisoning, SSRF, SSTI, DOM @@ -120,10 +120,28 @@ dependencies: - scripts/install_tools.sh checks: + - name: pd-tool-resolver + command: test -x scripts/pd-tool - name: nuclei - command: 'command -v nuclei >/dev/null 2>&1 || test -x "$HOME/.pdtm/go/bin/nuclei"' + command: scripts/pd-tool nuclei -version >/dev/null 2>&1 - name: httpx - command: 'command -v httpx >/dev/null 2>&1 || test -x "$HOME/.pdtm/go/bin/httpx"' + command: scripts/pd-tool httpx -version >/dev/null 2>&1 + - name: subfinder + command: scripts/pd-tool subfinder -version >/dev/null 2>&1 + - name: naabu + command: scripts/pd-tool naabu -version >/dev/null 2>&1 + - name: dnsx + command: scripts/pd-tool dnsx -version >/dev/null 2>&1 + - name: katana + command: scripts/pd-tool katana -version >/dev/null 2>&1 + - name: tlsx + command: scripts/pd-tool tlsx -version >/dev/null 2>&1 + - name: alterx + command: scripts/pd-tool alterx -version >/dev/null 2>&1 + - name: asnmap + command: scripts/pd-tool asnmap -version >/dev/null 2>&1 + - name: uncover + command: scripts/pd-tool uncover -version >/dev/null 2>&1 - name: interactsh-client command: 'command -v interactsh-client >/dev/null 2>&1 || test -x "$HOME/go/bin/interactsh-client"' - name: protoscope diff --git a/capabilities/web-security/scripts/install_tools.sh b/capabilities/web-security/scripts/install_tools.sh index 79c8f81..006114d 100755 --- a/capabilities/web-security/scripts/install_tools.sh +++ b/capabilities/web-security/scripts/install_tools.sh @@ -4,6 +4,15 @@ set -euo pipefail ARCH="$(uname -m)" +OS="$(uname -s)" + +case "$OS" in + Linux) ;; + *) + echo "web-security sandbox provisioning supports Linux; manage local CLI dependencies separately on $OS" >&2 + exit 0 + ;; +esac # -- Go toolchain (needed for pdtm, protoscope, interactsh, surf) --------- if ! command -v go &>/dev/null; then @@ -18,7 +27,8 @@ fi # -- PDTM + ProjectDiscovery tools ---------------------------------------- go install github.com/projectdiscovery/pdtm/cmd/pdtm@latest -pdtm -install nuclei,httpx,subfinder,naabu,dnsx,uncover,alterx,tlsx,asnmap +PDTM_BIN="$(go env GOPATH)/bin/pdtm" +"$PDTM_BIN" -install nuclei,httpx,subfinder,naabu,dnsx,uncover,alterx,tlsx,asnmap export PATH="$HOME/.pdtm/go/bin:$PATH" # -- katana (pre-built binary, go-tree-sitter build issue) ----------------- diff --git a/capabilities/web-security/scripts/pd-tool b/capabilities/web-security/scripts/pd-tool new file mode 100755 index 0000000..32dd132 --- /dev/null +++ b/capabilities/web-security/scripts/pd-tool @@ -0,0 +1,48 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [[ $# -lt 1 ]]; then + echo "usage: pd-tool [args...]" >&2 + exit 2 +fi + +tool="$1" +shift + +case "$tool" in + subfinder|httpx|naabu|dnsx|katana|tlsx|alterx|asnmap|uncover|nuclei) ;; + *) + echo "unsupported ProjectDiscovery tool: $tool" >&2 + exit 2 + ;; +esac + +candidates=() +if [[ -n "${PDTM_BIN_DIR:-}" ]]; then + candidates+=("${PDTM_BIN_DIR%/}/$tool") +fi +candidates+=("$HOME/.pdtm/go/bin/$tool") +if command -v go >/dev/null 2>&1; then + candidates+=("$(go env GOPATH 2>/dev/null)/bin/$tool") +fi + +for candidate in "${candidates[@]}"; do + if [[ -x "$candidate" ]]; then + exec "$candidate" "$@" + fi +done + +path_candidate="$(command -v "$tool" 2>/dev/null || true)" +if [[ -n "$path_candidate" ]]; then + if [[ "$tool" == "httpx" ]]; then + version_output="$($path_candidate -version 2>&1 || true)" + if ! grep -Eqi 'projectdiscovery|current version|httpx version' <<<"$version_output"; then + echo "refusing ambiguous httpx binary at $path_candidate; expected ProjectDiscovery httpx" >&2 + exit 126 + fi + fi + exec "$path_candidate" "$@" +fi + +echo "$tool not found; expected it under \$PDTM_BIN_DIR, ~/.pdtm/go/bin, or PATH" >&2 +exit 127 diff --git a/capabilities/web-security/skills/initial-recon/SKILL.md b/capabilities/web-security/skills/initial-recon/SKILL.md new file mode 100644 index 0000000..7144d74 --- /dev/null +++ b/capabilities/web-security/skills/initial-recon/SKILL.md @@ -0,0 +1,47 @@ +--- +name: initial-recon +description: Use when beginning a web assessment from a domain, wildcard, ASN, organization, IP list, or CIDR and measurable attack-surface coverage is needed. +allowed-tools: bash +--- + +# Initial Reconnaissance + +Establish a bounded breadth baseline before choosing deep tests. Keep this phase short for a single URL; use the full funnel for wildcards, organizations, ASNs, IP lists, and CIDRs. + +## Funnel + +1. Confirm scope and separate inherited assets from newly discovered assets. +2. For domain scope, enumerate with `subfinder`, resolve with `dnsx`, then probe with ProjectDiscovery `httpx`. +3. For IP or CIDR scope, discover ports with `naabu`, then probe responsive hosts with ProjectDiscovery `httpx`. Add `tlsx` when certificate names can expand the map. +4. Use `katana` only on responsive web applications. Use `nuclei` only on confirmed, high-signal targets. +5. Prefer JSON/JSONL output and preserve it as an artifact. Orient on the results before adding breadth or beginning exploitation. + +Use the ProjectDiscovery binary resolver because `httpx` may otherwise resolve to the unrelated Python CLI: + +```bash +PD="scripts/pd-tool" # capability root is the tool working directory +"$PD" subfinder -d example.com -json -silent +"$PD" dnsx -json -silent +"$PD" naabu -host 192.0.2.0/24 -json -silent -rate 500 +"$PD" httpx -json -silent -title -tech-detect -status-code +``` + +If a dedicated binary is unavailable, use the smallest available fallback and record that substitution. Do not rebuild high-volume enumeration with sequential `curl` or `dig` loops when the provisioned ProjectDiscovery tools are healthy. + +## Coverage ledger + +Write a ledger with `scripts/write_coverage_ledger.py` after each completed phase. Record facts, not intent: + +- scope and tool +- status: `completed`, `partial`, or `failed` +- addresses or hosts scheduled and completed +- ports scheduled +- responsive hosts and facts produced +- artifact path +- deferred scope and reason + +Never describe a scan as comprehensive without scheduled/completed denominators. A timeout is partial coverage unless the saved artifact proves completion. + +## Handoff + +Rank responsive assets by exposed identity/admin/API surfaces, non-standard ports, unusual technology, and exploitable behavior. Hand the highest-signal assets back to the normal web-security OODA loop; focused testing remains more valuable than indiscriminate scanning. diff --git a/capabilities/web-security/skills/initial-recon/scripts/write_coverage_ledger.py b/capabilities/web-security/skills/initial-recon/scripts/write_coverage_ledger.py new file mode 100755 index 0000000..4e2367a --- /dev/null +++ b/capabilities/web-security/skills/initial-recon/scripts/write_coverage_ledger.py @@ -0,0 +1,96 @@ +#!/usr/bin/env python3 +"""Append one measured reconnaissance phase to a JSON coverage ledger.""" + +from __future__ import annotations + +import argparse +import json +from datetime import UTC, datetime +from pathlib import Path +from typing import Any + + +def _non_negative(value: str) -> int: + parsed = int(value) + if parsed < 0: + raise argparse.ArgumentTypeError("must be non-negative") + return parsed + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--scope", action="append", required=True) + parser.add_argument("--phase", required=True) + parser.add_argument("--tool", required=True) + parser.add_argument( + "--status", choices=("completed", "partial", "failed"), required=True + ) + parser.add_argument("--targets-scheduled", type=_non_negative) + parser.add_argument("--targets-completed", type=_non_negative) + parser.add_argument("--ports-scheduled", type=_non_negative) + parser.add_argument("--responsive-hosts", type=_non_negative) + parser.add_argument("--facts-produced", type=_non_negative) + parser.add_argument("--artifact") + parser.add_argument("--deferred", action="append", default=[]) + parser.add_argument("--note") + return parser + + +def _compact(data: dict[str, Any]) -> dict[str, Any]: + return {key: value for key, value in data.items() if value not in (None, "", [])} + + +def main() -> None: + args = _parser().parse_args() + if ( + args.targets_scheduled is not None + and args.targets_completed is not None + and args.targets_completed > args.targets_scheduled + ): + raise SystemExit("targets-completed cannot exceed targets-scheduled") + + if args.output.exists(): + ledger = json.loads(args.output.read_text()) + else: + ledger = { + "schema_version": "web-security-coverage-v1", + "scope": args.scope, + "started_at": datetime.now(UTC).isoformat(), + "phases": [], + } + + if sorted(ledger.get("scope", [])) != sorted(args.scope): + raise SystemExit("scope does not match the existing ledger") + + phase = _compact( + { + "name": args.phase, + "tool": args.tool, + "status": args.status, + "recorded_at": datetime.now(UTC).isoformat(), + "targets_scheduled": args.targets_scheduled, + "targets_completed": args.targets_completed, + "ports_scheduled": args.ports_scheduled, + "address_port_checks": ( + args.targets_completed * args.ports_scheduled + if args.targets_completed is not None + and args.ports_scheduled is not None + else None + ), + "responsive_hosts": args.responsive_hosts, + "facts_produced": args.facts_produced, + "artifact": args.artifact, + "deferred": args.deferred, + "note": args.note, + } + ) + ledger.setdefault("phases", []).append(phase) + ledger["updated_at"] = phase["recorded_at"] + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(ledger, indent=2, sort_keys=True) + "\n") + print(args.output) + + +if __name__ == "__main__": + main() diff --git a/capabilities/web-security/tests/test_coverage_ledger.py b/capabilities/web-security/tests/test_coverage_ledger.py new file mode 100644 index 0000000..a40166e --- /dev/null +++ b/capabilities/web-security/tests/test_coverage_ledger.py @@ -0,0 +1,110 @@ +from __future__ import annotations + +import json +import subprocess +import sys +from pathlib import Path + + +SCRIPT = ( + Path(__file__).parents[1] + / "skills" + / "initial-recon" + / "scripts" + / "write_coverage_ledger.py" +) + + +def test_writes_measured_phase(tmp_path: Path) -> None: + output = tmp_path / "coverage.json" + result = subprocess.run( + [ + sys.executable, + str(SCRIPT), + "--output", + str(output), + "--scope", + "192.0.2.0/24", + "--phase", + "port-scan", + "--tool", + "naabu", + "--status", + "completed", + "--targets-scheduled", + "256", + "--targets-completed", + "256", + "--ports-scheduled", + "7", + "--responsive-hosts", + "12", + "--artifact", + "naabu.jsonl", + ], + text=True, + capture_output=True, + check=False, + ) + + assert result.returncode == 0 + ledger = json.loads(output.read_text()) + phase = ledger["phases"][0] + assert ledger["schema_version"] == "web-security-coverage-v1" + assert phase["address_port_checks"] == 1792 + assert phase["responsive_hosts"] == 12 + + +def test_rejects_completed_count_above_scheduled(tmp_path: Path) -> None: + result = subprocess.run( + [ + sys.executable, + str(SCRIPT), + "--output", + str(tmp_path / "coverage.json"), + "--scope", + "example.com", + "--phase", + "http-probe", + "--tool", + "httpx", + "--status", + "partial", + "--targets-scheduled", + "2", + "--targets-completed", + "3", + ], + text=True, + capture_output=True, + check=False, + ) + + assert result.returncode != 0 + assert "cannot exceed" in result.stderr + + +def test_appends_only_when_scope_matches(tmp_path: Path) -> None: + output = tmp_path / "coverage.json" + base = [ + sys.executable, + str(SCRIPT), + "--output", + str(output), + "--scope", + "example.com", + "--phase", + "passive", + "--tool", + "subfinder", + "--status", + "completed", + ] + assert subprocess.run(base, check=False).returncode == 0 + mismatch = base.copy() + mismatch[mismatch.index("example.com")] = "example.org" + + result = subprocess.run(mismatch, text=True, capture_output=True, check=False) + + assert result.returncode != 0 + assert "scope does not match" in result.stderr diff --git a/capabilities/web-security/tests/test_initial_recon_contract.py b/capabilities/web-security/tests/test_initial_recon_contract.py new file mode 100644 index 0000000..952e0b7 --- /dev/null +++ b/capabilities/web-security/tests/test_initial_recon_contract.py @@ -0,0 +1,44 @@ +from __future__ import annotations + +from pathlib import Path + +import yaml + + +ROOT = Path(__file__).parents[1] + + +def test_agent_routes_broad_scopes_to_initial_recon() -> None: + agent = (ROOT / "agents" / "web-security.md").read_text() + + assert "load `initial-recon`" in agent + assert "`scripts/pd-tool`" in agent + assert "with `curl`, `python`, `ffuf`" not in agent + + +def test_initial_recon_names_measured_funnel_and_ledger() -> None: + skill = (ROOT / "skills" / "initial-recon" / "SKILL.md").read_text() + + for tool in ("subfinder", "dnsx", "naabu", "httpx", "tlsx", "katana", "nuclei"): + assert f"`{tool}`" in skill + assert "Coverage ledger" in skill + assert "scheduled/completed denominators" in skill + + +def test_manifest_checks_the_recon_toolchain_through_resolver() -> None: + manifest = yaml.safe_load((ROOT / "capability.yaml").read_text()) + checks = {entry["name"]: entry["command"] for entry in manifest["checks"]} + + for tool in ( + "subfinder", + "httpx", + "naabu", + "dnsx", + "katana", + "tlsx", + "alterx", + "asnmap", + "uncover", + "nuclei", + ): + assert checks[tool].startswith(f"scripts/pd-tool {tool} ") diff --git a/capabilities/web-security/tests/test_interrupted_tool_result_hook.py b/capabilities/web-security/tests/test_interrupted_tool_result_hook.py index 82cad39..ffc1151 100644 --- a/capabilities/web-security/tests/test_interrupted_tool_result_hook.py +++ b/capabilities/web-security/tests/test_interrupted_tool_result_hook.py @@ -128,7 +128,7 @@ async def test_manifest_wires_hook_file() -> None: manifest_path = Path(__file__).resolve().parents[1] / "capability.yaml" manifest = yaml.safe_load(manifest_path.read_text(encoding="utf-8")) - assert manifest["version"] == "1.1.5" + assert manifest["version"] == "1.3.0" assert manifest["hooks"] == ["hooks/interrupted_tool_result.py"] diff --git a/capabilities/web-security/tests/test_pd_tool_resolver.py b/capabilities/web-security/tests/test_pd_tool_resolver.py new file mode 100644 index 0000000..a4a1eb5 --- /dev/null +++ b/capabilities/web-security/tests/test_pd_tool_resolver.py @@ -0,0 +1,83 @@ +from __future__ import annotations + +import os +import subprocess +from pathlib import Path + +import pytest + + +RESOLVER = Path(__file__).parents[1] / "scripts" / "pd-tool" + + +def _binary(path: Path, body: str) -> None: + path.write_text(f"#!/usr/bin/env bash\n{body}\n") + path.chmod(0o755) + + +def _run(tmp_path: Path, tool: str, *args: str) -> subprocess.CompletedProcess[str]: + env = os.environ.copy() + env["HOME"] = str(tmp_path / "home") + env["PATH"] = f"{tmp_path / 'path'}:/usr/bin:/bin" + return subprocess.run( + [str(RESOLVER), tool, *args], + env=env, + text=True, + capture_output=True, + check=False, + ) + + +def test_prefers_pdtm_httpx_over_path_collision(tmp_path: Path) -> None: + pdtm = tmp_path / "home" / ".pdtm" / "go" / "bin" + path_dir = tmp_path / "path" + pdtm.mkdir(parents=True) + path_dir.mkdir() + _binary(pdtm / "httpx", 'echo "projectdiscovery:$*"') + _binary(path_dir / "httpx", 'echo "python-httpx:$*"') + + result = _run(tmp_path, "httpx", "-version") + + assert result.returncode == 0 + assert result.stdout.strip() == "projectdiscovery:-version" + + +def test_rejects_ambiguous_path_httpx(tmp_path: Path) -> None: + path_dir = tmp_path / "path" + path_dir.mkdir() + _binary(path_dir / "httpx", 'echo "The httpx command line client"') + + result = _run(tmp_path, "httpx", "-version") + + assert result.returncode == 126 + assert "refusing ambiguous httpx binary" in result.stderr + + +def test_runs_non_httpx_path_binary(tmp_path: Path) -> None: + path_dir = tmp_path / "path" + path_dir.mkdir() + _binary(path_dir / "subfinder", 'echo "subfinder:$*"') + + result = _run(tmp_path, "subfinder", "-version") + + assert result.returncode == 0 + assert result.stdout.strip() == "subfinder:-version" + + +def test_rejects_unknown_tool(tmp_path: Path) -> None: + result = _run(tmp_path, "definitely-not-pd") + + assert result.returncode == 2 + assert "unsupported ProjectDiscovery tool" in result.stderr + + +@pytest.mark.parametrize( + "tool", ["subfinder", "httpx", "naabu", "dnsx", "katana", "nuclei"] +) +def test_supported_tool_without_binary_is_clear(tmp_path: Path, tool: str) -> None: + (tmp_path / "path").mkdir() + + result = _run(tmp_path, tool, "-version") + + assert result.returncode == 127 + assert f"{tool} not found" in result.stderr