diff --git a/dementor/__init__.py b/dementor/__init__.py index c9e0c17..ae86be5 100755 --- a/dementor/__init__.py +++ b/dementor/__init__.py @@ -18,5 +18,5 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. -__version__ = "1.0.0.dev18" +__version__ = "1.0.0.dev19" __author__ = "MatrixEditor" diff --git a/dementor/__main__.py b/dementor/__main__.py index 0ab02c4..0244083 100644 --- a/dementor/__main__.py +++ b/dementor/__main__.py @@ -1,3 +1,22 @@ +# Copyright (c) 2025-Present MatrixEditor +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. from dementor.standalone import run_from_cli run_from_cli() \ No newline at end of file diff --git a/dementor/config/__init__.py b/dementor/config/__init__.py index 5a47a88..d718b88 100644 --- a/dementor/config/__init__.py +++ b/dementor/config/__init__.py @@ -17,37 +17,73 @@ # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. +# pyright: reportAny=false, reportExplicitAny=false import sys import pathlib import tomllib +from typing import Any + from dementor.paths import CONFIG_PATH, DEFAULT_CONFIG_PATH -# global configuration values -dm_config: dict +# --------------------------------------------------------------------------- # +# Global configuration storage +# --------------------------------------------------------------------------- # +dm_config: dict[str, Any] + -def _get_global_config() -> dict: +def get_global_config() -> dict[str, Any]: + """Return the current global configuration dictionary. + + :return: The configuration mapping, empty if ``init_from_file`` has not + been called yet. + :rtype: dict + """ return getattr(sys.modules[__name__], "dm_config", {}) -def _set_global_config(config: dict) -> None: +def _set_global_config(config: dict[str, Any]) -> None: + """Replace the current global configuration with *config*. + + The helper mirrors :func:`get_global_config` and writes the value back to + the module namespace. + + :param config: New configuration dictionary. + :type config: dict + """ setattr(sys.modules[__name__], "dm_config", config) def init_from_file(path: str) -> None: + """Load a TOML configuration file and merge it into the global config. + + The function follows a *replace-then-overwrite* strategy: the file at + *path* is parsed with :mod:`tomllib`. If the file exists and can be + read, its content completely replaces the previous ``dm_config`` value. + The caller is responsible for ordering calls to obtain the desired + precedence. + + If the file does not exist or is not a regular file the function returns + silently. + + :param path: Filesystem path to a TOML file. + :type path: str + :raises tomllib.TOMLDecodeError: Propagated if the file exists but contains + invalid TOML. + """ target = pathlib.Path(path) if not target.exists() or not target.is_file(): return - # by default, we just replace the global config + # By default we completely replace the existing configuration. with target.open("rb") as f: new_config = tomllib.load(f) _set_global_config(new_config) -# Default initialization procedure is: -# 1. use default config -# 2. use config file if it exists -# 3. use custom config if specified via CLI -init_from_file(DEFAULT_CONFIG_PATH) -init_from_file(CONFIG_PATH) +# --------------------------------------------------------------------------- # +# Default initialisation - performed on import so that the rest of the +# package can rely on ``dementor.config.dm_config`` being available. +# --------------------------------------------------------------------------- # +init_from_file(DEFAULT_CONFIG_PATH) # 1. bundled defaults +init_from_file(CONFIG_PATH) # 2. user-provided overrides diff --git a/dementor/config/session.py b/dementor/config/session.py index 89a08d9..d6259b9 100644 --- a/dementor/config/session.py +++ b/dementor/config/session.py @@ -21,7 +21,7 @@ import asyncio import typing -from typing import Any +from typing import Any, override from pathlib import Path from dementor.config.toml import TomlConfig, Attribute @@ -153,7 +153,7 @@ class SessionConfig(TomlConfig): upnp_enabled: bool def __init__(self) -> None: - super().__init__(config._get_global_config().get("Dementor", {})) + super().__init__(config.get_global_config().get("Dementor", {})) # global options that are not loaded from configuration self.ipv6 = None self.ipv4 = None @@ -189,3 +189,17 @@ def resolve_path(self, path: str | Path) -> Path: return Path(raw_path).resolve() return (Path(self.workspace_path) / raw_path).resolve() + + @override + def __getitem__(self, key: str) -> Any: + section, *parts = key.split(".") + attr = f"{section.lower()}_config" + if not hasattr(self, attr): + raise KeyError(f"unknown protocol: {attr}") + + config: TomlConfig = getattr(self, attr) + if not parts: + return config + + return config[".".join(parts)] + diff --git a/dementor/config/toml.py b/dementor/config/toml.py index 3f9915b..815c972 100644 --- a/dementor/config/toml.py +++ b/dementor/config/toml.py @@ -17,15 +17,45 @@ # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. -from typing import NamedTuple, Callable, Any, TypeVar +# pyright: reportAny=false, reportExplicitAny=false +from typing import ClassVar, NamedTuple, Callable, Any, TypeVar from dementor.config.util import get_value -_LOCAL = object() _T = TypeVar("_T", bound="TomlConfig") +# --------------------------------------------------------------------------- # +# Helper sentinel used to differentiate “no default supplied” from “None”. +# --------------------------------------------------------------------------- # +_LOCAL = object() + class Attribute(NamedTuple): + """ + Metadata describing a single configuration attribute. + + The :class:`TomlConfig` base class uses a list of ``Attribute`` objects + (``_fields_``) to know how to populate its instance attributes from a TOML + configuration dictionary. + + :param attr_name: Name of the instance attribute that will receive the value. + :type attr_name: str + :param qname: Qualified name of the configuration key. May contain a dot + (``"."``) to indicate that the key lives in a *different* configuration + section. + :type qname: str + :param default_val: Default value to fall back to when the key is missing. + ``_LOCAL`` (a private sentinel) means “no default - the key is required”. + :type default_val: Any | None, optional + :param section_local: If ``True`` the key is looked for only in the section + defined by the concrete subclass (``self._section_``). If ``False`` the + ``Globals`` section is also consulted. + :type section_local: bool, optional + :param factory: Optional callable that post-processes the raw value (e.g. + converting a string to ``bytes``). + :type factory: Callable[[Any], Any] | None, optional + """ + attr_name: str qname: str default_val: Any | None = _LOCAL @@ -34,10 +64,42 @@ class Attribute(NamedTuple): class TomlConfig: - _section_: str | None - _fields_: list[Attribute] + """ + Base class for configuration objects that are built from a TOML-derived + ``dict`` structure. + + Sub-classes must define two class attributes: + + * ``_section_`` - the name of the top-level configuration section that + contains the values for this type. + * ``_fields_`` - a list of :class:`Attribute` objects describing how each + instance attribute is resolved. + + Example + ------- + >>> class MyConfig(TomlConfig): + ... _section_ = "my" + ... _fields_ = [ + ... Attribute("host", "host", default_val="localhost"), + ... Attribute("port", "port", default_val=8080, factory=int), + ... ] + >>> cfg = MyConfig({"host": "example.com"}) + >>> cfg.host, cfg.port + ('example.com', 8080) + """ + + # Sub-classes are expected to provide these attributes. + _section_: ClassVar[str] + _fields_: ClassVar[list[Attribute]] def __init__(self, config: dict[str, Any] | None = None) -> None: + """ + Initialise the configuration object. + + :param config: Raw configuration dictionary that originates from the + TOML file; may be ``None`` to indicate an empty configuration. + :type config: dict[str, Any] | None, optional + """ for field in self._fields_: self._set_field( config or {}, @@ -49,6 +111,18 @@ def __init__(self, config: dict[str, Any] | None = None) -> None: ) def __getitem__(self, key: str) -> Any: + """ + Dictionary-style access to configuration attributes. + + The lookup first tries the real attribute name, then falls back to the + short name extracted from the qualified ``qname`` of each field. + + :param key: Attribute name to retrieve. + :type key: str + :raises KeyError: If *key* does not match any known attribute. + :return: The stored value. + :rtype: Any + """ if hasattr(self, key): return getattr(self, key) @@ -56,7 +130,6 @@ def __getitem__(self, key: str) -> Any: name = attr.qname if "." in name: _, name = name.rsplit(".", 1) - if key == name: return getattr(self, attr.attr_name) @@ -64,44 +137,83 @@ def __getitem__(self, key: str) -> Any: @staticmethod def build_config(cls_ty: type[_T], section: str | None = None) -> _T: + """ + Factory that builds a concrete ``TomlConfig`` subclass from the global + configuration. + + :param cls_ty: Concrete subclass of :class:`TomlConfig` to instantiate. + :type cls_ty: type[_T] + :param section: Override the subclass' ``_section_`` attribute. If not + supplied the subclass' own ``_section_`` is used. + :type section: str | None, optional + :raises ValueError: If *section* resolves to ``None``. + :return: An instantiated configuration object. + :rtype: _T + """ section_name = section or cls_ty._section_ - if section_name is None: + if not section_name: raise ValueError("section cannot be None") return cls_ty(get_value(section_name, key=None, default={})) + # --------------------------------------------------------------------- # + # Internal helper - resolves a single field according to the rules + # --------------------------------------------------------------------- # def _set_field( self, - config: dict, + config: dict[str, Any], field_name: str, qname: str, - default_val=None, - section_local=False, - factory=None, + default_val: Any | None = None, + section_local: bool = False, + factory: Callable[[Any], Any] | None = None, ) -> None: - # Behaviour: - # 1. resolve default value: - # - If default_val is _LOCAL, there will be no default value - # - If self._section_ is present, it will be used to fetch the - # defualt value. The name may contain "." - # 2. Retrieve value from target section - # 3. Apply value by either - # - Calling a function with 'set_', or - # - using setattr directly - + """ + Resolve and assign a single configuration attribute. + + The resolution algorithm follows three steps: + + 1. **Default value** - if ``default_val`` is not the sentinel ``_LOCAL`` the + method looks for a value in the configuration hierarchy: + + * first in the subclass' ``_section_``, + * then in an *alternative* section encoded in ``qname`` (everything + before the last ``"."``), and + * finally in the ``Globals`` section when ``section_local`` is + ``False``. + + 2. **Actual value** - the value from the caller-provided *config* dict + overrides the default. + + 3. **Post-processing** - optionally run a ``factory`` on the value and + finally store the value either via a custom ``set_`` method + or directly with ``setattr``. + + :param config: The user-supplied configuration dictionary. + :type config: dict + :param field_name: Instance attribute that will receive the value. + :type field_name: str + :param qname: Qualified configuration key (may contain a section prefix). + :type qname: str + :param default_val: Default value or ``_LOCAL`` sentinel. + :type default_val: Any, optional + :param section_local: When ``False`` also search the ``Globals`` section. + :type section_local: bool, optional + :param factory: Callable that transforms the raw value. + :type factory: Callable[[Any], Any] | None, optional + :raises Exception: If the key is required (``_LOCAL``) but missing. + """ section = getattr(self, "_section_", None) if "." in qname: - # REVISIT: section will be overwritten here - # get section path and target property name alt_section, qname = qname.rsplit(".", 1) else: alt_section = None + # --------------------------------------------------------------- # + # Resolve the default value (if any) by walking the hierarchy. + # --------------------------------------------------------------- # if default_val is not _LOCAL: - # PRIOROTY list: - # 1. _section_ - # 2. alternative section in qname - # 3. variable in dm_config.Globals + # Priority: own section > alternative section > Globals (if allowed) sections = [ get_value(section or "", key=None, default={}), get_value(alt_section or "", key=None, default={}), @@ -114,21 +226,27 @@ def _set_field( default_val = section_config[qname] break + # ----------------------------------------------------------------- # + # Pull the actual value from the caller-supplied ``config`` dict, + # falling back to the default we just resolved. + # ----------------------------------------------------------------- # value = config.get(qname, default_val) if value is _LOCAL: + # ``_LOCAL`` means “required but not supplied”. raise Exception( - f"Expected '{qname}' in config or section({section}) for {self.__class__.__name__}!" + f"Expected '{qname}' in config or section({section}) for " + + f"{self.__class__.__name__}!" ) if value is default_val and isinstance(value, type): - # use factory instead of return value value = value() + # Apply any user-supplied conversion ``factory``. if factory: value = factory(value) - func = getattr(self, f"set_{field_name}", None) - if func: - func(value) + setter = getattr(self, f"set_{field_name}", None) + if setter: + setter(value) else: setattr(self, field_name, value) diff --git a/dementor/config/util.py b/dementor/config/util.py index 0f81c9e..29f144e 100644 --- a/dementor/config/util.py +++ b/dementor/config/util.py @@ -17,6 +17,7 @@ # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. +# pyright: reportAny=false, reportExplicitAny=false import datetime import random import string @@ -25,30 +26,60 @@ from typing import Any from jinja2.sandbox import SandboxedEnvironment -from dementor.config import _get_global_config - +from dementor.config import get_global_config +# --------------------------------------------------------------------------- # +# Jinja2 sandbox used for safe templating of configuration strings. +# --------------------------------------------------------------------------- # _SANDBOX = SandboxedEnvironment() -def get_value(section: str, key: str | None, default=None) -> Any: - sections = section.split(".") - config = _get_global_config() +def get_value(section: str, key: str | None, default: Any | None = None) -> Any: + """ + Retrieve a value from the *global* configuration. + + The function walks a dotted ``section`` path (e.g. ``"http.server"``) and + returns either the sub-dictionary (when ``key`` is ``None``) or the concrete + value for ``key``. + + :param section: Section name; may contain ``"."`` to indicate nested tables. + :type section: str + :param key: Specific key inside the section, or ``None`` to obtain the whole + section dictionary. + :type key: str | None, optional + :param default: Value returned when *key* is missing. + :type default: Any, optional + :return: The requested configuration value or ``default``. + :rtype: Any + """ + sections: list[str] = section.split(".") + config = get_global_config() if len(sections) == 1: target = config.get(sections[0], {}) else: target = config - for section in sections: - target = target.get(section, {}) - + for sec in sections: + target = target.get(sec, {}) if key is None: return target - return target.get(key, default) -# --- factory methods for attributes --- +# --------------------------------------------------------------------------- # +# Simple factories used by :class:`Attribute` definitions. +# --------------------------------------------------------------------------- # def is_true(value: str) -> bool: + """ + Convert a string to a boolean using a loose interpretation. + + Recognised truthy values are ``"true"``, ``"1"``, ``"on"``, ``"yes"`` + (case-insensitive). Anything else evaluates to ``False``. + + :param value: Raw string value. + :type value: str + :return: ``True`` for truthy strings, ``False`` otherwise. + :rtype: bool + """ return str(value).lower() in ("true", "1", "on", "yes") @@ -67,13 +98,25 @@ class BytesValue: """ def __init__(self, length: int | None = None) -> None: + """ + :param length: Desired length for randomly generated tokens when the + input is ``None``. If omitted a single byte is generated. + :type length: int | None, optional + """ self.length: int | None = length def __call__(self, value: Any) -> bytes: + """ + Convert *value* to ``bytes``. + + :param value: Input to be converted. + :type value: Any + :return: ``bytes`` representation. + :rtype: bytes + """ match value: case None: return secrets.token_bytes(self.length or 1) - case str(): result = self._parse_str(value) if self.length is not None and len(result) != self.length: @@ -88,7 +131,6 @@ def __call__(self, value: Any) -> bytes: f"Expected {self.length} bytes, got {len(value)}" ) return value - case _: return self(str(value)) @@ -128,18 +170,53 @@ def _parse_str(self, value: str) -> bytes: def random_value(size: int) -> str: + """ + Produce a random alphabetic string of *size* characters. + + :param size: Number of characters. + :type size: int + :return: Random string. + :rtype: str + """ return "".join(random.choice(string.ascii_letters) for _ in range(size)) def format_string(value: str, locals: dict[str, Any] | None = None) -> str: - config = _get_global_config() + """ + Render a Jinja2 template against the global configuration. + + The function creates a sandboxed Jinja2 environment (see + :mod:`jinja2.sandbox`) and renders *value* with the following global + variables available: + + * ``config`` - the complete global configuration dictionary. + * ``random`` - a helper that calls :func:`random_value`. + * any key/value pairs supplied via the optional *locals* mapping. + + Errors during rendering are caught; the original *value* is returned + unchanged. + + :param value: Template string to render. + :type value: str + :param locals: Additional context variables for the template. + :type locals: dict[str, Any] | None, optional + :return: Rendered string or the original *value* on failure. + :rtype: str + """ + config = get_global_config() try: template = _SANDBOX.from_string(value) return template.render(config=config, random=random_value, **(locals or {})) - except Exception as e: - # TODO: log that + except Exception: # pragma: no cover - defensive fallback + # TODO: replace with proper logging once the logging subsystem is ready. return value def now() -> str: + """ + Return the current time formatted as ``YYYY-MM-DD-HH-MM-SS``. + + :return: Formatted timestamp. + :rtype: str + """ return datetime.datetime.now().strftime("%Y-%m-%d-%H-%M-%S") diff --git a/dementor/db/__init__.py b/dementor/db/__init__.py index f2f8fa2..4555a7d 100644 --- a/dementor/db/__init__.py +++ b/dementor/db/__init__.py @@ -18,10 +18,32 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. +# --------------------------------------------------------------------------- # +# Public constants +# --------------------------------------------------------------------------- # _CLEARTEXT = "Cleartext" +"""Constant indicating plaintext credentials (as opposed to hashes).""" + _NO_USER = "" +"""Placeholder string used when username is absent or invalid in credential logging.""" +_HOST_INFO = "_host_info" +"""Key used in extras dict to store host information for credential logging.""" def normalize_client_address(client: str) -> str: - return client.removeprefix("::ffff:") + """Normalize IPv6-mapped IPv4 addresses by stripping IPv6 prefix. + + Converts addresses like `::ffff:192.168.1.1` to `192.168.1.1` for consistent storage and display. + :param client: Raw client address string (e.g., from socket). + :type client: str + :return: Normalized address without IPv6 mapping prefix. + :rtype: str + + Example: + >>> normalize_client_address("::ffff:192.168.1.1") + '192.168.1.1' + >>> normalize_client_address("2001:db8::1") + '2001:db8::1' + """ + return client.removeprefix("::ffff:") diff --git a/dementor/db/connector.py b/dementor/db/connector.py index 20a6982..e27dfb3 100644 --- a/dementor/db/connector.py +++ b/dementor/db/connector.py @@ -19,6 +19,7 @@ # SOFTWARE. # pyright: reportUninitializedInstanceVariable=false import typing + from sqlalchemy import Engine, create_engine from dementor.config.session import SessionConfig @@ -28,8 +29,16 @@ class DatabaseConfig(TomlConfig): - _section_ = "DB" - _fields_ = [ + """ + Configuration mapping for the ``[DB]`` TOML section. + + The attributes correspond to the most common SQLAlchemy connection + parameters. All fields are optional - sensible defaults are applied + when a key is missing. + """ + + _section_: typing.ClassVar[str] = "DB" + _fields_: typing.ClassVar[list[A]] = [ A("db_raw_path", "Url", None), A("db_path", "Path", "Dementor.db"), A("db_duplicate_creds", "DuplicateCreds", False), @@ -37,7 +46,7 @@ class DatabaseConfig(TomlConfig): A("db_driver", "Driver", None), ] - if typing.TYPE_CHECKING: + if typing.TYPE_CHECKING: # pragma: no cover - only for static analysis db_raw_path: str | None db_path: str db_duplicate_creds: bool @@ -46,6 +55,16 @@ class DatabaseConfig(TomlConfig): def init_dementor_db(session: SessionConfig) -> Engine | None: + """ + Initialise the database engine and create all tables. + + :param session: The active :class:`~dementor.config.session.SessionConfig` + containing the ``db_config`` attribute. + :type session: SessionConfig + :return: The created SQLAlchemy ``Engine`` or ``None`` if an error + prevented initialisation. + :rtype: Engine | None + """ engine = init_engine(session) if engine is not None: ModelBase.metadata.create_all(engine) @@ -53,30 +72,48 @@ def init_dementor_db(session: SessionConfig) -> Engine | None: def init_engine(session: SessionConfig) -> Engine | None: - # based on dialect and driver configuration + """ + Build a SQLAlchemy ``Engine`` from a :class:`DatabaseConfig`. + + The logic follows the rules laid out in the SQLAlchemy documentation + (see https://docs.sqlalchemy.org/en/20/core/engines.html#database-urls). + + * If ``db_raw_path`` is supplied it is used verbatim. + * Otherwise a URL is composed from ``dialect``, ``driver`` and ``path``. + For SQLite the path is resolved relative to the session's + ``resolve_path`` helper; missing directories are created on the fly. + + Sensitive information (user/password) is hidden in the debug output. + + :param session: Current session configuration. + :type session: SessionConfig + :return: Configured ``Engine`` instance or ``None`` on failure. + :rtype: Engine | None + """ + # --------------------------------------------------------------- # + # 1. Resolve "raw" URL - either provided by the user or built. + # --------------------------------------------------------------- # raw_path = session.db_config.db_raw_path if raw_path is None: - # fall back to constructing the path manually + # Build the URL manually when the user didn't provide a full DSN. dialect = session.db_config.db_dialect or "sqlite" driver = session.db_config.db_driver or "pysqlite" path = session.db_config.db_path if not path: return dm_logger.error("Database path not specified!") - if dialect == "sqlite": + # :memory: is a special SQLite in-memory database. if path != ":memory:": real_path = session.resolve_path(path) if not real_path.parent.exists(): dm_logger.debug(f"Creating database directory {real_path.parent}") real_path.parent.mkdir(parents=True, exist_ok=True) - path = f"/{real_path}" - - # see https://docs.sqlalchemy.org/en/20/core/engines.html#database-urls raw_path = f"{dialect}+{driver}://{path}" else: + # Decompose the user-supplied URL to obtain dialect and driver. sql_type, path = raw_path.split("://") - if sql_type.count("+") > 0: + if "+" in sql_type: dialect, driver = sql_type.split("+") else: dialect = sql_type @@ -85,6 +122,7 @@ def init_engine(session: SessionConfig) -> Engine | None: if dialect != "sqlite": first_element, *parts = path.split("/") if "@" in first_element: + # keep only the “host:port” part, replace user:pass with stars first_element = first_element.split("@")[1] path = "***:***@" + "/".join([first_element] + list(parts)) @@ -93,7 +131,15 @@ def init_engine(session: SessionConfig) -> Engine | None: def create_db(session: SessionConfig) -> DementorDB: - # TODO: add support for custom database implementations + """ + High-level helper that returns a fully-initialised :class:`DementorDB`. + + :param session: Current session configuration. + :type session: SessionConfig + :return: Ready-to-use :class:`DementorDB` instance. + :rtype: DementorDB + :raises Exception: If the engine cannot be created. + """ engine = init_engine(session) if not engine: raise Exception("Failed to create database engine") diff --git a/dementor/db/model.py b/dementor/db/model.py index 436c913..863eee9 100644 --- a/dementor/db/model.py +++ b/dementor/db/model.py @@ -17,35 +17,67 @@ # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. -# pyright: reportUnusedCallResult=false +# pyright: reportUnusedCallResult=false, reportAny=false, reportExplicitAny=false, reportPrivateUsage=false import datetime +import json import threading -from typing import Any +from typing import Any, TypeVar + from rich import markup -from sqlalchemy.exc import NoInspectionAvailable, NoSuchTableError, OperationalError +from sqlalchemy import Engine, ForeignKey, MetaData, ScalarResult, Text, sql +from sqlalchemy.exc import ( + NoInspectionAvailable, + NoSuchTableError, + OperationalError, +) from sqlalchemy.orm import ( DeclarativeBase, Mapped, mapped_column, scoped_session, sessionmaker, + Session, ) -from sqlalchemy import Engine, ForeignKey, Text, sql - +from sqlalchemy.sql.selectable import TypedReturnsRows -from dementor.db import _CLEARTEXT, _NO_USER, normalize_client_address +from dementor.config.session import SessionConfig +from dementor.db import _CLEARTEXT, _NO_USER, normalize_client_address, _HOST_INFO from dementor.log.logger import dm_logger from dementor.log import dm_console_lock from dementor.log.stream import log_to +_T = TypeVar("_T") + + class ModelBase(DeclarativeBase): + """ + Base class for all ORM models. + + It exists solely to give a common ``metadata`` object that can be used + for ``create_all`` / ``drop_all`` calls. + """ + pass class HostInfo(ModelBase): - __tablename__ = "hosts" + """Stores basic host information from network scans. + + Each row represents a unique IP address with optional hostname and domain. + + :param id: Primary key (auto-incremented). + :type id: int + :param ip: IPv4/IPv6 address in normalized form (e.g., `192.168.1.1` or `2001:db8::1`). + :type ip: str + :param hostname: Resolved hostname (if available). + :type hostname: str | None + :param domain: Domain name associated with the host (e.g., `corp.local`). + :type domain: str | None + """ + + __tablename__: str = "hosts" id: Mapped[int] = mapped_column(primary_key=True) ip: Mapped[str] = mapped_column(Text, nullable=False) @@ -54,7 +86,21 @@ class HostInfo(ModelBase): class HostExtra(ModelBase): - __tablename__ = "extras" + """Stores additional metadata about hosts (key-value pairs). + + Used for storing OS fingerprints, open ports, services, etc., associated with a `HostInfo`. + + :param id: Primary key. + :type id: int + :param host: Foreign key to `HostInfo.id`. + :type host: int + :param key: Metadata key (e.g., "os", "service"). + :type key: str + :param value: Metadata value. + :type value: str + """ + + __tablename__: str = "extras" id: Mapped[int] = mapped_column(primary_key=True) host: Mapped[int] = mapped_column(ForeignKey("hosts.id")) @@ -63,7 +109,33 @@ class HostExtra(ModelBase): class Credential(ModelBase): - __tablename__ = "credentials" + """Stores captured authentication credentials. + + Each row represents a unique credential (username/password or hash) captured during a session. + + :param id: Primary key. + :type id: int + :param timestamp: ISO-formatted datetime string of capture. + :type timestamp: str + :param protocol: Protocol used (e.g., `smb`, `rdp`, `ssh`). + :type protocol: str + :param credtype: Type of credential (`"Cleartext"` or hash type like `"ntlm"`, `"sha256"`). + :type credtype: str + :param client: Client address and port as `IP:PORT`. + :type client: str + :param host: Foreign key to `HostInfo.id`. + :type host: int + :param hostname: Hostname associated with credential (denormalized for performance). + :type hostname: str | None + :param domain: Domain name associated with credential. + :type domain: str | None + :param username: Username (lowercased for case-insensitive matching). + :type username: str + :param password: Plaintext password or hash value. + :type password: str | None + """ + + __tablename__: str = "credentials" id: Mapped[int] = mapped_column(primary_key=True) timestamp: Mapped[str] = mapped_column(Text, nullable=False) @@ -78,28 +150,45 @@ class Credential(ModelBase): class DementorDB: - def __init__(self, engine: Engine, config) -> None: - self.db_engine = engine - self.db_path = engine.url.database - self.metadata = ModelBase.metadata - self.config = config + """Thread-safe wrapper around SQLAlchemy engine for Dementor's database operations. + + Manages ORM sessions, locks, and schema initialization. Provides high-level methods + for adding hosts, extras, and credentials while handling duplicates and logging. + """ + + def __init__(self, engine: Engine, config: "SessionConfig") -> None: + self.db_engine: Engine = engine + self.db_path: str = str(engine.url.database) + self.metadata: MetaData = ModelBase.metadata + self.config: "SessionConfig" = config + + # Ensure tables exist; any problem is reported immediately. with self.db_engine.connect(): try: self.metadata.create_all(self.db_engine, checkfirst=True) - except (NoSuchTableError, NoInspectionAvailable): - dm_logger.error(f"Failed to connect to database {self.db_path}!") + except (NoSuchTableError, NoInspectionAvailable) as exc: + dm_logger.error(f"Failed to connect to database {self.db_path}! {exc}") raise session_factory = sessionmaker(bind=self.db_engine, expire_on_commit=True) - session_ty = scoped_session(session_factory) - - self.session = session_ty() - self.lock = threading.Lock() + self.session: Session = scoped_session(session_factory)() + self.lock: threading.Lock = threading.Lock() + # --------------------------------------------------------------------- # + # Low-level helpers + # --------------------------------------------------------------------- # def close(self) -> None: + """Close the underlying SQLAlchemy session.""" self.session.close() - def _execute(self, q): + def _execute(self, q: TypedReturnsRows[tuple[_T]]) -> ScalarResult[_T] | None: + """Execute a SQLAlchemy query and handle common operational errors. + + :param q: SQLAlchemy query object. + :type q: Select | Insert | Update | Delete + :return: Query result or `None` if error occurred. + :rtype: Any + """ try: return self.session.scalars(q) except OperationalError as e: @@ -111,6 +200,7 @@ def _execute(self, q): raise e def commit(self): + """Commit the current transaction and handle schema-related errors.""" try: self.session.commit() except OperationalError as e: @@ -121,6 +211,9 @@ def commit(self): else: raise e + # --------------------------------------------------------------------- # + # Public CRUD-style helpers + # --------------------------------------------------------------------- # def add_host( self, ip: str, @@ -128,18 +221,36 @@ def add_host( domain: str | None = None, extras: dict[str, str] | None = None, ) -> HostInfo | None: + """ + Insert a host row if it does not already exist. + + The method is *idempotent*: calling it repeatedly with the same + ``ip`` will never create duplicate rows; instead the existing row + is updated with any newly supplied ``hostname``/``domain`` values. + + :param ip: IPv4/IPv6 address of the host. + :type ip: str + :param hostname: Optional human-readable hostname. + :type hostname: str | None, optional + :param domain: Optional DNS domain. + :type domain: str | None, optional + :param extras: Optional mapping of extra key/value attributes. + :type extras: Mapping[str, str] | None, optional + :return: The persisted :class:`HostInfo` object or ``None`` on failure. + :rtype: HostInfo | None + """ with self.lock: q = sql.select(HostInfo).where(HostInfo.ip == ip) result = self._execute(q) if result is None: return None - host = result.one_or_none() if not host: host = HostInfo(ip=ip, hostname=hostname, domain=domain) self.session.add(host) self.commit() else: + # Preserve existing values; only fill missing data. host.domain = host.domain or domain or "" host.hostname = host.hostname or hostname or "" self.commit() @@ -152,24 +263,43 @@ def add_host( def add_host_extra( self, host_id: int, key: str, value: str, no_lock: bool = False ) -> None: + """ + Store an arbitrary extra attribute for a host. + + ``extras`` are stored in a separate table to keep the ``hosts`` row + small and to allow multiple values per host. + + :param host_id: Primary key of the target ``HostInfo``. + :type host_id: int + :param key: Attribute name. + :type key: str + :param value: Attribute value. + :type value: str + :param no_lock: Skip acquiring lock if `True` (internal use). + :type no_lock: bool, optional + """ if not no_lock: self.lock.acquire() - - q = sql.select(HostExtra).where(HostExtra.host == host_id, HostExtra.key == key) - result = self._execute(q) - if result is None: - return - - extra = result.one_or_none() - if not extra: - extra = HostExtra(host=host_id, key=key, value=value) - self.session.add(extra) - self.commit() - else: - extra.value = f"{extra.value}\0{extra.value}" - - if not no_lock: - self.lock.release() + try: + q = sql.select(HostExtra).where( + HostExtra.host == host_id, HostExtra.key == key + ) + result = self._execute(q) + if result is None: + return + extra = result.one_or_none() + if not extra: + extra = HostExtra(host=host_id, key=key, value=json.dumps([str(value)])) + self.session.add(extra) + self.commit() + else: + # REVISIT: + values: list[str] = json.loads(extra.value) + values.append(value) + extra.value = json.dumps(values) + finally: + if not no_lock: + self.lock.release() def add_auth( self, @@ -181,9 +311,41 @@ def add_auth( protocol: str | None = None, domain: str | None = None, hostname: str | None = None, - extras: dict | None = None, + extras: dict[str, str] | None = None, custom: bool = False, ) -> None: + """ + Store a captured credential in the database and emit user-friendly logs. + + The method performs a duplicate-check (unless the global config + ``db_duplicate_creds`` is ``True``) and respects read-only database + mode. + + :param client: ``(ip, port)`` tuple of the remote endpoint. + :type client: tuple[str, int] + :param credtype: ``_CLEARTEXT`` for passwords or a hash algorithm name. + :type credtype: str + :param username: Username that was observed. + :type username: str + :param password: Password or hash value. + :type password: str + :param logger: Optional logger that provides a ``debug``/``success``/… + interface; defaults to the global ``dm_logger``. + :type logger: Any, optional + :param protocol: Protocol name (e.g. ``"ssh"``); if omitted it is taken + from ``logger.extra["protocol"]``. + :type protocol: str | None, optional + :param domain: Optional domain name associated with the credential. + :type domain: str | None, optional + :param hostname: Optional host name for the remote system. + :type hostname: str | None, optional + :param extras: Optional additional key/value data to store alongside + the credential. + :type extras: Mapping[str, str] | None, optional + :param custom: When ``True`` the output omits the standard “Captured …” + prefix (used for artificial credentials). + :type custom: bool, optional + """ if not logger and not protocol: dm_logger.error( f"Failed to add {credtype} for {username} on {client[0]}:{client[1]}: " @@ -192,16 +354,21 @@ def add_auth( return target_logger = logger or dm_logger - protocol = str(protocol or logger.extra["protocol"]) + protocol = str(protocol or getattr(logger, "extra", {}).get("protocol", "")) client_address, port, *_ = client client_address = normalize_client_address(client_address) target_logger.debug( f"Adding {credtype} for {username} on {client_address}: " - + f"{logger} | {protocol} | {domain} | {hostname} | {username} | {password}" + + f"{target_logger} | {protocol} | {domain} | {hostname} | {username} | {password}" ) + # Ensure the host exists (or create it) before linking the cred. host = self.add_host(client_address, hostname, domain) + if host is None: + return + + # Build the duplicate-check query (case-insensitive). q = sql.select(Credential).filter( sql.func.lower(Credential.domain) == sql.func.lower(domain or ""), sql.func.lower(Credential.username) == sql.func.lower(username), @@ -209,29 +376,31 @@ def add_auth( sql.func.lower(Credential.protocol) == sql.func.lower(protocol), ) result = self._execute(q) - if result is None or host is None: + if result is None: return results = result.all() text = "Password" if credtype == _CLEARTEXT else "Hash" username_text = markup.escape(username) - is_blank = len(str(username).strip()) == 0 - if is_blank: + if len(str(username).strip()) == 0: username_text = "(blank)" + # Human-readable part used in log messages. full_name = ( f" for [b]{markup.escape(domain)}[/]/[b]{username_text}[/]" if domain else f" for [b]{username_text}[/]" ) - if is_blank: - full_name = "" + host_info: str | None = extras.pop(_HOST_INFO, None) if extras else None + if host_info: + full_name += f" on [b]{markup.escape(host_info)}[/]" if not results or self.config.db_config.db_duplicate_creds: if credtype != _CLEARTEXT: log_to("hashes", type=credtype, value=password) - # just insert a new row + cred = Credential( + # REVISIT: replace with util.now() timestamp=datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"), protocol=protocol.lower(), credtype=credtype.lower(), @@ -247,8 +416,8 @@ def add_auth( self.session.add(cred) self.session.commit() except OperationalError as e: - # attempt to write on a read-only database - if "readonly database" in str(e): + # Special handling for read-only SQLite databases. + if "readonly database" in str(e).lower(): dm_logger.fail( f"Failed to add {credtype} for {username} on {client_address}: " + "Database is read-only! (maybe restart in sudo mode?)" @@ -258,43 +427,41 @@ def add_auth( with dm_console_lock: head_text = text if not custom else "" - credtype = markup.escape(credtype) + credtype_esc = markup.escape(credtype) target_logger.success( - f"Captured {credtype} {head_text}{full_name} from {client_address}:", + f"Captured {credtype_esc} {head_text}{full_name} from {client_address}:", host=hostname or client_address, locked=True, ) if username != _NO_USER: target_logger.highlight( - f"{credtype} Username: {username_text}", + f"{credtype_esc} Username: {username_text}", host=hostname or client_address, locked=True, ) - target_logger.highlight( ( - f"{credtype} {text}: {markup.escape(password)}" + f"{credtype_esc} {text}: {markup.escape(password)}" if not custom - else f"{credtype}: {markup.escape(password)}" + else f"{credtype_esc}: {markup.escape(password)}" ), host=hostname or client_address, locked=True, ) if extras: target_logger.highlight( - f"{credtype} Extras:", + f"{credtype_esc} Extras:", host=hostname or client_address, locked=True, ) - - for name, value in (extras or {}).items(): - target_logger.highlight( - f" {name}: {markup.escape(value)}", - host=hostname or client_address, - locked=True, - ) - + for name, value in extras.items(): + target_logger.highlight( + f" {name}: {markup.escape(value)}", + host=hostname or client_address, + locked=True, + ) else: + # Credential already present - only emit a short notice. target_logger.highlight( f"Skipping previously captured {credtype} {text} for {full_name} from {client_address}", host=hostname or client_address, diff --git a/dementor/filters.py b/dementor/filters.py index 37ae4b3..3acacd5 100755 --- a/dementor/filters.py +++ b/dementor/filters.py @@ -17,6 +17,7 @@ # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. +# pyright: reportAny=false, reportExplicitAny=false import re import sys import glob @@ -24,20 +25,40 @@ import warnings from typing import Any + from dementor.config.toml import Attribute class FilterObj: - def __init__(self, target: str, extra: Any | None = None) -> None: - self.target: str = target - self.extra = extra or {} + """Represents a filter pattern for matching strings (e.g., hostnames, IPs). + + Supports three pattern types: + - Literal string match + - Regular expression (prefixed with `re:`) + - Glob pattern (prefixed with `g:`), translated to regex (Python 3.13+) + + :param target: Pattern string. May be literal, `re:...`, or `g:...`. + :type target: str + :param extra: Optional metadata associated with this filter. + :type extra: Any, optional + """ + + def __init__(self, target: str, extra: dict[str, Any] | None = None) -> None: + """Initialize a filter object with pattern and optional metadata. - # Patterns can be either regex directly or glob-style - # pre compute pattern + Automatically detects pattern type and compiles regex if applicable. + + :param target: Pattern string (literal, `re:regex`, or `g:glob`). + :type target: str + :param extra: Additional context or metadata (e.g., source file, rule ID). + :type extra: Any, optional + """ + self.target: str = target + self.extra: dict[str, Any] = extra or {} + # Determine the concrete matching strategy (regex, glob or plain) if self.target.startswith("re:"): - self.pattern = re.compile(self.target[3:]) + self.pattern: re.Pattern[str] | None = re.compile(self.target[3:]) self.target = self.target[3:] - elif self.target.startswith("g:"): self.target = self.target[2:] # glob.translate is only available since 3.13 @@ -49,11 +70,25 @@ def __init__(self, target: str, extra: Any | None = None) -> None: self.pattern = None else: self.pattern = re.compile(glob.translate(self.target)) - else: self.pattern = None def matches(self, source: str) -> bool: + """Check if the source string matches this filter. + + :param source: String to test against the filter. + :type source: str + :return: `True` if match, `False` otherwise. + :rtype: bool + + Example: + >>> f = FilterObj("re:.*\\.example\\.com") + >>> f.matches("api.example.com") + True + >>> f = FilterObj("host1") + >>> f.matches("host1") + True + """ return ( self.pattern.match(source) is not None if self.pattern @@ -61,22 +96,48 @@ def matches(self, source: str) -> bool: ) @staticmethod - def from_string(target: str, extra: Any | None = None): + def from_string(target: str, extra: Any | None = None) -> "FilterObj": + """Create a `FilterObj` from a string pattern. + + :param target: Pattern string. + :type target: str + :param extra: Optional metadata. + :type extra: Any, optional + :return: Filter object. + :rtype: FilterObj + """ return FilterObj(target, extra) @staticmethod def from_file(source: str, extra: Any | None) -> list["FilterObj"]: + """Load multiple filters from a text file (one per line). + + :param source: Path to file containing filter patterns. + :type source: str + :param extra: Metadata to attach to each filter. + :type extra: Any, optional + :return: List of `FilterObj` instances. + :rtype: list[FilterObj] + """ filters = [] path = pathlib.Path(source) if path.exists() and path.is_file(): filters = [ FilterObj(t, extra) for t in path.read_text("utf-8").splitlines() ] - return filters def _optional_filter(value: list[str | dict[str, Any]] | None) -> "Filters | None": + """Factory function to convert optional config list into `Filters` instance. + + Used with `Attribute` to auto-convert config values. Returns `None` if input is `None`. + + :param value: List of filter specs or `None`. + :type value: list[str | dict[str, Any]] | None + :return: `Filters` instance or `None`. + :rtype: Filters | None + """ return None if value is None else Filters(value) @@ -87,6 +148,18 @@ def _optional_filter(value: list[str | dict[str, Any]] | None) -> "Filters | Non section_local=False, factory=_optional_filter, ) +"""Attribute definition for blacklist filters. + +Maps TOML key `Ignore` to `ignored` attribute. Accepts list of strings or files. +Used to exclude matching targets from processing. + +Example TOML: +```toml +[Globals] +Ignore = ["re:.*\\.internal\\.", "g:*.test.*"] +``` +""" + ATTR_WHITELIST = Attribute( "targets", @@ -95,31 +168,94 @@ def _optional_filter(value: list[str | dict[str, Any]] | None) -> "Filters | Non section_local=False, factory=_optional_filter, ) +"""Attribute definition for whitelist filters. + +Maps TOML key `Targets` to `targets` attribute. Accepts list of strings or files. +Used to restrict processing to only matching targets. + +Example TOML: +```toml +[Globals] +Targets = ["192.168.1.100"] +``` +""" def in_scope(value: str, config: Any) -> bool: + """Determine if a value is allowed based on whitelist and blacklist filters. + + Evaluates filters in order: + 1. If `targets` exists and value is not in it -> `False` + 2. If `ignored` exists and value is in it -> `False` + 3. Otherwise -> `True` + + :param value: String to test (e.g., hostname, IP). + :type value: str + :param config: Object with optional `targets` and `ignored` attributes (`Filters`). + :type config: Any + :return: `True` if value is in scope, `False` otherwise. + :rtype: bool + + Example: + >>> class C: pass + >>> cfg = C() + >>> cfg.targets = Filters(["host1", "host2"]) + >>> in_scope("host1", cfg) + True + >>> in_scope("host3", cfg) + False + >>> cfg.ignored = Filters(["host1"]) + >>> in_scope("host1", cfg) + False + """ if hasattr(config, "targets"): is_target = value in config.targets if config.targets else True if not is_target: return False - if hasattr(config, "ignored"): is_ignored = value in config.ignored if config.ignored else False if is_ignored: return False - return True class Filters: + """Collection of `FilterObj` instances for matching against multiple patterns. + + Supports loading filters from: + - Direct string patterns + - File paths containing patterns (one per line) + - Config dictionaries with `Target` or `File` keys + + Implements `__contains__` for easy membership testing. + + :ivar filters: List of compiled `FilterObj` instances. + :vartype filters: list[FilterObj] + """ + def __init__(self, config: list[str | dict[str, Any]]) -> None: + """Initialize filters from a configuration list. + + Each item can be: + - A string: treated as literal or pattern (`re:...`, `g:...`) + - A dict: with `Target` (pattern) or `File` (path to patterns) + + :param config: List of filter specifications. + :type config: list[str | dict[str, Any]] + + Example: + >>> filters = Filters([ + ... "re:.*\\.example\\.com", + ... {"File": "targets.txt"}, + ... {"Target": "host1", "reason": "admin"} + ... ]) + """ self.filters: list[FilterObj] = [] for filter_config in config: if isinstance(filter_config, str): # String means simple filter expression without extra config if not filter_config: continue - self.filters.append(FilterObj.from_string(filter_config)) else: # must be a dictionary @@ -132,19 +268,47 @@ def __init__(self, config: list[str | dict[str, Any]]) -> None: # 2. source file with list of targets source = filter_config.get("File") if source is None: - # silently continue + # TODO: add logging here + # If "File" is missing we silently skip the entry. continue - self.filters.extend(FilterObj.from_file(source, filter_config)) def __contains__(self, host: str) -> bool: + """Check if a host matches any filter. + + :param host: String to test. + :type host: str + :return: `True` if any filter matches. + :rtype: bool + """ return self.has_match(host) - def get_machted(self, host: str) -> list[FilterObj]: + def get_matched(self, host: str) -> list[FilterObj]: + """Return all filters that match the host. + + :param host: String to test. + :type host: str + :return: List of matching `FilterObj` instances. + :rtype: list[FilterObj] + """ return list(filter(lambda x: x.matches(host), self.filters)) def get_first_match(self, host: str) -> FilterObj | None: - return next(iter(self.get_machted(host)), None) + """Return the first matching filter, or `None` if none match. + + :param host: String to test. + :type host: str + :return: First matching filter or `None`. + :rtype: FilterObj | None + """ + return next(iter(self.get_matched(host)), None) def has_match(self, host: str) -> bool: - return len(self.get_machted(host)) > 0 + """Check if at least one filter matches the host. + + :param host: String to test. + :type host: str + :return: `True` if any filter matches. + :rtype: bool + """ + return len(self.get_matched(host)) > 0 diff --git a/dementor/loader.py b/dementor/loader.py index 7fbf143..eb4f510 100755 --- a/dementor/loader.py +++ b/dementor/loader.py @@ -17,42 +17,125 @@ # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. +import os import threading import types -import os -import dementor import typing from importlib.machinery import SourceFileLoader + +import dementor from dementor.config.session import SessionConfig from dementor.paths import DEMENTOR_PATH - +# --------------------------------------------------------------------------- # +# Type aliases for the optional protocol entry‑points +# --------------------------------------------------------------------------- # ApplyConfigFunc = typing.Callable[[SessionConfig], None] +"""Type alias for function that applies protocol configuration. + +Signature: `apply_config(session: SessionConfig) -> None` + +Used by protocol modules to customize global configuration based on protocol-specific needs. +""" + CreateServersFunc = typing.Callable[[SessionConfig], list[threading.Thread]] +"""Type alias for function that creates server threads for a protocol. + +Signature: `create_server_threads(session: SessionConfig) -> list[threading.Thread]` +Returns a list of `threading.Thread` instances configured to run protocol servers. +""" + +# --------------------------------------------------------------------------- # +# Structural protocol used for static type checking +# --------------------------------------------------------------------------- # class ProtocolModule(typing.Protocol): + """Protocol defining the expected interface for a Dementor protocol module. + + Modules must expose at least one of `apply_config` or `create_server_threads`. + Optionally, may define a nested `config` submodule for hierarchical configuration. + + :cvar config: Optional submodule containing additional configuration logic. + :vartype config: ProtocolModule | None + :cvar apply_config: Function to apply protocol-specific config to session. + :vartype apply_config: ApplyConfigFunc | None + :cvar create_server_threads: Function to spawn protocol server threads. + :vartype create_server_threads: CreateServersFunc | None + """ + config: "ProtocolModule | None" apply_config: ApplyConfigFunc | None create_server_threads: CreateServersFunc | None class ProtocolLoader: + """Loads and manages protocol modules from filesystem. + + Searches for `.py` protocol files in predefined paths and optionally user-supplied directories. + Provides methods to load modules, apply configuration, and spawn server threads. + + :ivar rs_path: Path to built-in protocol directory (`DEMENTOR_PATH/protocols`). + :vartype rs_path: str + :ivar search_path: List of directories to scan for protocol modules. + :vartype search_path: list[str] + """ + def __init__(self) -> None: + """Initialize loader with default protocol search paths. + + Searches: + 1. Dementor package's internal `protocols/` directory + 2. External `DEMENTOR_PATH/protocols/` directory (for user extensions) + """ self.rs_path: str = os.path.join(DEMENTOR_PATH, "protocols") self.search_path: list[str] = [ os.path.join(os.path.dirname(dementor.__file__), "protocols"), self.rs_path, ] + # --------------------------------------------------------------------- # + # Loading helpers + # --------------------------------------------------------------------- # def load_protocol(self, protocol_path: str) -> types.ModuleType: + """Dynamically load a protocol module from a Python file. + + Uses `SourceFileLoader` to import the module without requiring it to be in `sys.path`. + + :param protocol_path: Absolute path to the `.py` protocol file. + :type protocol_path: str + :return: Loaded module object. + :rtype: types.ModuleType + :raises ImportError: If module cannot be loaded. + """ loader = SourceFileLoader("protocol", protocol_path) protocol = types.ModuleType(loader.name) loader.exec_module(protocol) return protocol - def get_protocols(self, session: SessionConfig | None = None) -> dict[str, str]: + # --------------------------------------------------------------------- # + # Discovery helpers + # --------------------------------------------------------------------- # + def get_protocols( + self, + session: SessionConfig | None = None, + ) -> dict[str, str]: + """Discover all available protocol modules in search paths. + + Scans directories and files for `.py` files (excluding `__init__.py`). + Optionally extends search paths with `session.extra_modules`. + + :param session: Optional session to extend search paths with custom modules. + :type session: SessionConfig | None + :return: Dict mapping protocol name (without `.py`) to full file path. + :rtype: dict[str, str] + + Example: + >>> loader = ProtocolLoader() + >>> protocols = loader.get_protocols() + >>> protocols["smb"] # -> "/path/to/dementor/protocols/smb.py" + """ protocols: dict[str, str] = {} protocol_paths: list[str] = list(self.search_path) @@ -61,40 +144,50 @@ def get_protocols(self, session: SessionConfig | None = None) -> dict[str, str]: for path in protocol_paths: if not os.path.exists(path): + # Missing entries are ignored – they may be optional. continue if os.path.isfile(path): if not path.endswith(".py"): continue - - protocol_path = path - name = os.path.basename(path)[:-3] - protocols[name] = protocol_path + name = os.path.basename(path)[:-3] # strip .py + protocols[name] = path continue - # TODO: check for directory for filename in os.listdir(path): if not filename.endswith(".py") or filename == "__init__.py": continue - protocol_path = os.path.join(path, filename) - name = filename[:-3] + name = filename[:-3] # strip extension protocols[name] = protocol_path return protocols - def apply_config(self, protocol: ProtocolModule, session: SessionConfig): - # if config is defined, apply it + # --------------------------------------------------------------------- # + # Hook dispatchers + # --------------------------------------------------------------------- # + def apply_config(self, protocol: ProtocolModule, session: SessionConfig) -> None: + """Apply protocol-specific configuration to the session. + + Looks for `apply_config(session)` function. If not found, checks for a nested `config` submodule + and recursively applies its config. + + :param protocol: Loaded protocol module. + :type protocol: ProtocolModule + :param session: Session configuration to modify. + :type session: SessionConfig + """ apply_config_fn: ApplyConfigFunc | None = getattr( protocol, "apply_config", None ) + if apply_config_fn is not None: - # sgnature is: apply_config(session: SessionConfig) + # signature is: apply_config(session: SessionConfig) apply_config_fn(session) else: - # maybe another submodule? + # Fallback to a nested config module, if present. if hasattr(protocol, "config"): - config_mod = protocol.config + config_mod: ProtocolModule | None = protocol.config if config_mod is not None: self.apply_config(config_mod, session) @@ -103,11 +196,26 @@ def create_servers( protocol: ProtocolModule, session: SessionConfig, ) -> list[threading.Thread]: - # creates servers for the given protocol (if defined) + """Create and return server threads for the given protocol. + + Looks for `create_server_threads(session)` function. Returns empty list if not defined. + + :param protocol: Loaded protocol module. + :type protocol: ProtocolModule + :param session: Session configuration for server setup. + :type session: SessionConfig + :return: List of thread objects ready to be started. + :rtype: list[threading.Thread] + """ create_server_threads: CreateServersFunc | None = getattr( - protocol, "create_server_threads", None + protocol, + "create_server_threads", + None, ) + if create_server_threads is None: return [] - return create_server_threads(session) + # Defensive conversion to list in case the protocol returns a tuple, + # generator or other iterable. + return list(create_server_threads(session)) diff --git a/dementor/log/__init__.py b/dementor/log/__init__.py index 4fdf6ad..3675254 100644 --- a/dementor/log/__init__.py +++ b/dementor/log/__init__.py @@ -1,19 +1,68 @@ +# Copyright (c) 2025-Present MatrixEditor +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# pyright: reportAny=false, reportExplicitAny=false import threading + +from typing import Any from rich.console import Console -dm_console = Console( +dm_console: Console = Console( soft_wrap=True, tab_size=4, highlight=False, highlighter=None, ) +"""Rich Console instance for thread-safe terminal output. + +Used globally for formatted logging output. Disables automatic highlighting and word wrapping +to ensure consistent rendering across platforms and loggers. + +Note: All output should go through `dm_print` to ensure thread safety. +""" + +dm_console_lock: threading.Lock = threading.Lock() +"""Threading lock to serialize console output. + +Prevents interleaved or corrupted log messages when multiple threads write to `dm_console` +simultaneously (e.g., during concurrent protocol execution). + +All `dm_print` calls respect this lock unless explicitly marked `locked=True`. +""" + + +def dm_print(msg: str, *args: Any, **kwargs: Any) -> None: + """Thread-safe wrapper for `dm_console.print()`. -dm_console_lock = threading.Lock() + Ensures log messages are printed atomically. If `locked=True` is passed, + bypasses the lock for internal use (e.g., when already holding the lock). + :param msg: Message to print (supports Rich markup). + :type msg: str + :param args: Positional arguments passed to `Console.print()`. + :param kwargs: Keyword arguments passed to `Console.print()`. + :keyword locked: If `True`, skips acquiring `dm_console_lock` (internal use). + :type locked: bool, optional -def dm_print(msg: str, *args, **kwargs) -> None: - # If someone has a better idea I'll be open for it. This is just - # here to synchronize the logging output + Example: + >>> dm_print("[bold green]Success![/]", locked=True) + """ if kwargs.pop("locked", False): dm_console.print(msg, *args, **kwargs) else: diff --git a/dementor/log/logger.py b/dementor/log/logger.py index 73832f5..8512a18 100644 --- a/dementor/log/logger.py +++ b/dementor/log/logger.py @@ -18,9 +18,7 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. # pyright: reportUninitializedInstanceVariable=false, reportUnusedCallResult=false -from typing import Any - - +# pyright: reportAny=false, reportExplicitAny=false import argparse import inspect import logging @@ -29,8 +27,9 @@ import sys import typing -from abc import abstractmethod +from typing import Any, ClassVar from logging.handlers import RotatingFileHandler +from typing_extensions import override from rich.logging import RichHandler from rich.markup import render @@ -40,12 +39,36 @@ from dementor.config.toml import TomlConfig, Attribute as A from dementor.log import dm_print, dm_console +# ------------------------------------------------------------------------- +# Global constants +# ------------------------------------------------------------------------- LOG_DEFAULT_TIMEFMT = "%H:%M:%S" +# ------------------------------------------------------------------------- +# Configuration wrapper +# ------------------------------------------------------------------------- class LoggingConfig(TomlConfig): - _section_ = "Log" - _fields_ = [ + """ + Configuration holder for the ``[Log]`` section of the ``dementor`` TOML file. + + :ivar log_debug_loggers: Names of loggers that can be switched on when + ``--debug`` is used. + :type log_debug_loggers: list[str] + :ivar log_dir: Directory where rotating log files are created. + :type log_dir: str + :ivar log_enable: Master switch - when ``False`` no file handlers are added. + :type log_enable: bool + :ivar log_timestamps: If ``True`` prepend a timestamp to every formatted + message. + :type log_timestamps: bool + :ivar log_timestamp_fmt: ``datetime.strftime`` format used for the timestamp + prefix. + :type log_timestamp_fmt: str + """ + + _section_: ClassVar[str] = "Log" + _fields_: ClassVar[list[A]] = [ A("log_debug_loggers", "DebugLoggers", list), A("log_dir", "LogDir", "logs"), A("log_enable", "Enabled", True), @@ -53,7 +76,7 @@ class LoggingConfig(TomlConfig): A("log_timestamp_fmt", "TimestampFmt", LOG_DEFAULT_TIMEFMT), ] - if typing.TYPE_CHECKING: + if typing.TYPE_CHECKING: # pragma: no cover log_debug_loggers: list[str] log_dir: str log_enable: bool @@ -61,7 +84,13 @@ class LoggingConfig(TomlConfig): log_timestamp_fmt: str -def init(): +def init() -> None: + """ + Initialise the global logging configuration. + + Called once at application startup. + """ + debug_parser = argparse.ArgumentParser(add_help=False) debug_parser.add_argument("--debug", action="store_true") debug_parser.add_argument("--verbose", action="store_true") @@ -69,7 +98,6 @@ def init(): config = TomlConfig.build_config(LoggingConfig) loggers = {name: logging.getLogger(name) for name in config.log_debug_loggers} - for debug_logger in loggers.values(): debug_logger.disabled = True @@ -81,10 +109,11 @@ def init(): markup=False, keywords=[], omit_repeated_times=False, - # show_path=False, ) - # should be disabled - handler.highlighter = None + # Explicitly disable any highlighter - the ProtocolLogger performs its + # own colour handling. + handler.highlighter = None # pyright: ignore[reportAttributeAccessIssue] + logging.basicConfig( format="(%(name)s) %(message)s", datefmt="[%X]", @@ -99,7 +128,6 @@ def init(): elif argv.debug: dm_logger.logger.setLevel(logging.DEBUG) root_logger.setLevel(logging.DEBUG) - for debug_logger in loggers.values(): debug_logger.disabled = False else: @@ -107,92 +135,222 @@ def init(): root_logger.setLevel(logging.INFO) -class ProtocolLogger(logging.LoggerAdapter): - def __init__(self, extra=None) -> None: +# ------------------------------------------------------------------------- +# Protocol-aware logger +# ------------------------------------------------------------------------- +class ProtocolLogger(logging.LoggerAdapter[Any]): + """Custom logger adapter for protocol-specific context-aware logging. + + Enhances standard logs with protocol name, host, port, and color-coded prefixes. + Supports both console output (via `dm_print`) and file logging (via `RotatingFileHandler`). + + :ivar _log_config: Cached `LoggingConfig` instance. + :vartype _log_config: LoggingConfig + """ + + def __init__(self, extra: dict[str, Any] | None = None) -> None: + """ + Initialise the adapter. + + :param extra: Dictionary of contextual values that will be merged with + per-call ``extra`` mappings. Typical keys are + ``protocol``, ``protocol_color``, ``host`` and ``port``. + :type extra: dict | None + """ super().__init__(logging.getLogger("dementor"), extra or {}) - self._log_config = None + self._log_config: LoggingConfig | None = None + # ----------------------------------------------------------------- + # Helper properties + # ----------------------------------------------------------------- @property def log_config(self) -> LoggingConfig: + """Lazily load and cache the :class:`LoggingConfig`.""" if not self._log_config: self._log_config = TomlConfig.build_config(LoggingConfig) return self._log_config - def _get_extra(self, name: str, extra=None, default=None): + def _get_extra( + self, + name: str, + extra: dict[str, Any] | None = None, + default: Any = None, + ) -> Any: + """ + Fetch ``name`` from *extra* or from the adapter's default mapping. + + :param name: Key to look up. + :type name: str + :param extra: Per-call extra mapping (may be ``None``). + :type extra: dict | None + :param default: Fallback value if the key is missing. + :type default: Any + :return: Resolved value. + :rtype: Any + """ value = (self.extra or {}).get(name, default) return extra.pop(name, value) if extra else value - def get_protocol_name(self, extra=None) -> str: + # ----------------------------------------------------------------- + # Accessors used by the formatting helpers + # ----------------------------------------------------------------- + def get_protocol_name(self, extra: dict[str, Any] | None = None) -> str: + """ + Return the protocol name (or an empty string). + + :param extra: Optional per-call extra mapping. + :type extra: dict | None + :return: Protocol name. + :rtype: str + """ return str(self._get_extra("protocol", extra, "")) - def get_protocol_color(self, extra=None) -> str: + def get_protocol_color(self, extra: dict[str, Any] | None = None) -> str: + """ + Return the colour used for the protocol prefix - defaults to ``white``. + + :param extra: Optional per-call extra mapping. + :type extra: dict | None + :return: Colour name. + :rtype: str + """ return str(self._get_extra("protocol_color", extra, "white")) - def get_host(self, extra=None) -> str: + def get_host(self, extra: dict[str, Any] | None = None) -> str: + """ + Return the host string (or empty). + + :param extra: Optional per-call extra mapping. + :type extra: dict | None + :return: Host. + :rtype: str + """ return str(self._get_extra("host", extra, "")) - def get_port(self, extra=None) -> str: + def get_port(self, extra: dict[str, Any] | None = None) -> str: + """ + Return the port string (or empty). + + :param extra: Optional per-call extra mapping. + :type extra: dict | None + :return: Port. + :rtype: str + """ return str(self._get_extra("port", extra, "")) - def format( - self, msg: str, *args: typing.Any, **kwargs: typing.Any - ) -> tuple[str, dict[str, Any]]: + # ----------------------------------------------------------------- + # Message formatting ---------------------------------------------- + # ----------------------------------------------------------------- + def format(self, msg: str, **kwargs: Any) -> tuple[str, dict[str, Any]]: + """Format message with timestamp, protocol, host, and port prefixes. + + Uses `log_timestamps` and `log_timestamp_fmt` from config. + + :param msg: Log message. + :type msg: str + :param args: Unused positional args. + :param kwargs: Contextual metadata (e.g., `host`, `protocol`). + :return: Formatted message and modified kwargs. + :rtype: tuple[str, dict[str, Any]] + """ ts_prefix = "" if self.log_config.log_timestamps: + # [ is escaped because later the string is passed through rich. ts_prefix = r"\[" - time_now = datetime.datetime.now() + now = datetime.datetime.now() try: - ts_prefix = f"{ts_prefix}{time_now.strftime(self.log_config.log_timestamp_fmt)}] " - except Exception as e: - # TODO: log that exception - ts_prefix = f"{ts_prefix}{time_now.strftime(LOG_DEFAULT_TIMEFMT)}] " + ts_prefix = ( + f"{ts_prefix}{now.strftime(self.log_config.log_timestamp_fmt)}] " + ) + except Exception: # pragma: no cover - fallback to default format + ts_prefix = f"{ts_prefix}{now.strftime(LOG_DEFAULT_TIMEFMT)}] " if self.extra is None: + # No context - simply prepend the timestamp (if any) and return. return f"{ts_prefix}{msg}", kwargs - mod = self.get_protocol_name(kwargs) + # Build the rich-style prefix: ``[bold colour]PROTOCOL[/] host port``. + proto = self.get_protocol_name(kwargs) host = self.get_host(kwargs) or "" port = self.get_port(kwargs) or "" - color = self.get_protocol_color(kwargs) - return ( - f"{ts_prefix}[bold {color}]{mod:<10}[/] {host:<25} {port:<6} {msg}", - kwargs, + colour = self.get_protocol_color(kwargs) + + formatted = ( + f"{ts_prefix}[bold {colour}]{proto:<10}[/] {host:<25} {port:<6} {msg}" ) + return formatted, kwargs def format_inline( self, msg: str, kwargs: dict[str, Any] ) -> tuple[str, dict[str, Any]]: - mod = self.get_protocol_name(kwargs) + """ + Produce a compact inline representation used by the convenience + methods (``success``, ``display``, ...). + + The format resembles ``(PROTO) (host:port) message``. + + :param msg: The original log message. + :type msg: str + :param kwargs: Mapping that may contain ``protocol``, ``host``, + ``port``, ``is_server`` and ``is_client`` flags. + :type kwargs: dict[str, Any] + :return: Rendered line and the (potentially mutated) ``kwargs``. + :rtype: tuple[str, dict[str, Any]] + """ + proto = self.get_protocol_name(kwargs) host = self.get_host(kwargs) port = self.get_port(kwargs) or "-" is_server = kwargs.pop("is_server", False) is_client = kwargs.pop("is_client", False) - line = msg + line = msg if is_client: line = f"C: {line}" elif is_server: line = f"S: {line}" - if host: line = f"({host}:{port}) {line}" - - if mod: - line = f"({mod}) {line}" + if proto: + line = f"({proto}) {line}" return line, kwargs + @override def log( self, level: int, msg: str, - *args, + *args: Any, exc_info: Any | None = None, stack_info: bool = False, - **kwargs, + **kwargs: Any, ) -> None: + """ + Emit a log record. + + The method first formats the message for *inline* output, then: + + * If the logger is disabled for the supplied ``level`` we still write + the entry to the file handler(s) via :meth:`_emit_log_entry`. + * Otherwise the standard ``LoggerAdapter.log`` implementation is used. + + :param level: Logging level (e.g. ``logging.INFO``). + :type level: int + :param msg: Log message. + :type msg: str + :param args: Positional arguments forwarded to the underlying ``Logger``. + :type args: tuple + :param exc_info: Exception info, passed unchanged to ``Logger.log``. + :type exc_info: Any | None + :param stack_info: ``True`` to include stack information. + :type stack_info: bool + :param kwargs: Additional keyword arguments (e.g. ``protocol``, + ``host``) that influence formatting. + :type kwargs: dict + """ msg, kwargs = self.format_inline(msg, kwargs) + if not self.isEnabledFor(level): - # always emit log entries to the file handler + # Message filtered for console - still persist it. return self._emit_log_entry(msg, level, *args, **kwargs) return super().log( @@ -205,37 +363,102 @@ def log( **kwargs, ) - def success(self, msg: str, color: str | None = None, *args, **kwargs): - color = color or "green" - prefix = r"[bold %s]\[+][/bold %s]" % (color, color) + # ----------------------------------------------------------------- + # Convenience helpers + # ----------------------------------------------------------------- + def success( + self, msg: str, color: str | None = None, *args: Any, **kwargs: Any + ) -> None: + """ + Log a successful operation (green ``[+]`` prefix). + + :param msg: Message to display. + :type msg: str + :param color: Override the colour used for the ``[+]`` marker. + :type color: str | None + :param _args: Positional arguments forwarded to :func:`dm_print`. + :type _args: typing.Any + :param _kwargs: Keyword arguments forwarded to :func:`dm_print`. + :type _kwargs: dict + :example: + >>> logger = ProtocolLogger() + >>> logger.success("Handshake completed") + """ + colour = color or "green" + prefix = r"[bold %s]\[+][/bold %s]" % (colour, colour) msg, kwargs = self.format(f"{prefix} {msg}", **kwargs) dm_print(msg, *args, **kwargs) - self._emit_log_entry(msg, *args, **kwargs) - - def display(self, msg: str, *args, **kwargs): + self._emit_log_entry(msg, logging.INFO, *args) + + def display(self, msg: str, *args: Any, **kwargs: Any) -> None: + """ + Log a generic informational message (blue ``[*]`` prefix). + + :param msg: Message to display. + :type msg: str + :param _args: Positional arguments forwarded to :func:`dm_print`. + :type _args: typing.Any + :param _kwargs: Keyword arguments forwarded to :func:`dm_print`. + :type _kwargs: dict + :example: + >>> logger.display("Waiting for data...") + """ prefix = r"[bold %s]\[*][/bold %s]" % ("blue", "blue") msg, kwargs = self.format(f"{prefix} {msg}", **kwargs) dm_print(msg, *args, **kwargs) - self._emit_log_entry(msg, *args, **kwargs) - - def highlight(self, msg: str, *args, **kwargs): + self._emit_log_entry(msg, logging.INFO, *args) + + def highlight(self, msg: str, *args: Any, **kwargs: Any) -> None: + """ + Render a highlighted line (yellow, bold). + + :param msg: Message to highlight. + :type msg: str + :param _args: Positional arguments forwarded to :func:`dm_print`. + :type _args: typing.Any + :param _kwargs: Keyword arguments forwarded to :func:`dm_print`. + :type _kwargs: dict + """ msg, kwargs = self.format(f"[bold yellow]{msg}[/yellow bold]", **kwargs) dm_print(msg, *args, **kwargs) - self._emit_log_entry(msg, *args, **kwargs) + self._emit_log_entry(msg, logging.INFO, *args) - def fail(self, msg: str, color=None, *args, **kwargs): - color = color or "red" - prefix = r"[bold %s]\[-][/bold %s]" % (color, color) + def fail( + self, msg: str, color: str | None = None, *args: Any, **kwargs: Any + ) -> None: + """ + Log an error condition (red ``[-]`` prefix). + + :param msg: Error description. + :type msg: str + :param color: Override the colour of the ``[-]`` marker. + :type color: str | None + :param _args: Positional arguments forwarded to :func:`dm_print`. + :type _args: typing.Any + :param _kwargs: Keyword arguments forwarded to :func:`dm_print`. + :type _kwargs: dict + """ + colour = color or "red" + prefix = r"[bold %s]\[-][/bold %s]" % (colour, colour) msg, kwargs = self.format(f"{prefix} {msg}", **kwargs) dm_print(msg, *args, **kwargs) - self._emit_log_entry(msg, *args, **kwargs) + self._emit_log_entry(msg, logging.ERROR, *args) - def _emit_log_entry( - self, text: str, level: int = logging.INFO, *args, **kwargs - ) -> None: + def _emit_log_entry(self, text: str, level: int = logging.INFO, *args: Any) -> None: + """Emit log entry to file handler only. + + Strips Rich markup and writes raw text to all file handlers. + + :param text: Formatted message (rich markup may be present). + :type text: str + :param level: Logging level - defaults to ``logging.INFO``. + :type level: int + :param _args: Positional arguments (kept for compatibility). + :type _args: typing.Any + """ caller = inspect.currentframe().f_back.f_back.f_back - text = render(text).plain - if len(self.logger.handlers) > 0: # file handler + plain = render(text).plain + if self.logger.handlers and caller: for handler in self.logger.handlers: handler.handle( logging.LogRecord( @@ -243,38 +466,48 @@ def _emit_log_entry( level, pathname=caller.f_code.co_filename, lineno=caller.f_lineno, - msg=text, + msg=plain, args=args, - # kwargs=kwargs, exc_info=None, ) ) + # ----------------------------------------------------------------- + # Rotating file support + # ----------------------------------------------------------------- def add_logfile(self, log_file_path: str) -> None: + """Add a rotating file handler for persistent logging. + + Creates log directory if needed. Appends startup metadata to existing files. + + :param log_file_path: Path to log file. + :type log_file_path: str + """ formatter = logging.Formatter( "%(asctime)s | %(filename)s:%(lineno)s - %(levelname)s (%(name)s): %(message)s", datefmt="[%X]", ) + outfile = pathlib.Path(log_file_path) file_exists = outfile.exists() + if not file_exists: - if not outfile.parent.exists(): - outfile.parent.mkdir(parents=True, exist_ok=True) + outfile.parent.mkdir(parents=True, exist_ok=True) + # Create an empty file atomically open(str(outfile), "x").close() handler = RotatingFileHandler( outfile, - maxBytes=100000, + maxBytes=100_000, encoding="utf-8", ) - with handler._open() as fp: - time = datetime.datetime.now().strftime("%d-%m-%Y %H:%M:%S") + + # Write a small header the first time the file is created. + with handler._open() as fp: # pylint: disable=protected-access + now = datetime.datetime.now().strftime("%d-%m-%Y %H:%M:%S") args = " ".join(sys.argv[1:]) - line = f"[{time}]> LOG_START\n[{time}]> ARGS: {args}\n" - if not file_exists: - fp.write(line) - else: - fp.write(f"\n{line}") + header = f"[{now}]> LOG_START\n[{now}]> ARGS: {args}\n" + fp.write(header if not file_exists else f"\n{header}") handler.setFormatter(formatter) self.logger.addHandler(handler) @@ -282,23 +515,30 @@ def add_logfile(self, log_file_path: str) -> None: @staticmethod def init_logfile(session: SessionConfig) -> None: + """Initialize log file based on global config and session path resolution. + + Called during application startup if logging is enabled. + + :param session: Session configuration containing log directory. + :type session: SessionConfig + """ config = TomlConfig.build_config(LoggingConfig) if not config.log_enable: return log_dir: pathlib.Path = session.resolve_path(config.log_dir or "logs") log_dir.mkdir(parents=True, exist_ok=True) + log_name = f"dm_log-{util.now()}.log" dm_logger.add_logfile(str(log_dir / log_name)) -class ProtocolLoggerMixin: - def __init__(self) -> None: - self.logger: ProtocolLogger = self.proto_logger() - - @abstractmethod - def proto_logger(self) -> ProtocolLogger: - pass - - +# ------------------------------------------------------------------------- +# Global logger instance used throughout the package +# ------------------------------------------------------------------------- dm_logger = ProtocolLogger() +"""Global instance of `ProtocolLogger` for application-wide logging. + +Used by modules without explicit context (e.g., core startup, utilities). +Context should be added via `extra` or overridden in subclasses. +""" diff --git a/dementor/log/stream.py b/dementor/log/stream.py index b2e38b8..6797765 100644 --- a/dementor/log/stream.py +++ b/dementor/log/stream.py @@ -18,47 +18,102 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. # pyright: reportUninitializedInstanceVariable=false +# pyright: reportAny=false, reportExplicitAny=false +import typing + from collections import defaultdict from io import IOBase from pathlib import Path -from typing import Any, override +from typing import Any, ClassVar, Generic, TypeVar +from typing_extensions import override from dementor.config import util +from dementor.config.session import SessionConfig from dementor.config.toml import TomlConfig, Attribute as A from dementor.log.logger import dm_logger -dm_streams = {} +_T = TypeVar("_T", bound="TomlConfig") + +dm_streams: dict[str, "LoggingStream[Any]"] = {} +"""Global registry of active logging streams by name.""" + + +class LoggingStream(Generic[_T]): + """Abstract base class for streaming log output. + Defines interface for writing structured or plain log entries to an output stream. + Designed for both file and in-memory streams. Subclasses must implement `add()`. + + :ivar fp: File-like object for output (opened by subclass). + :vartype fp: IOBase + :cvar _name_: Unique identifier for this stream (e.g., `"hosts"`). + :vartype _name_: str + :cvar _config_cls_: Configuration class used to load stream settings. + :vartype _config_cls_: type[TomlConfig] + """ -class LoggingStream: _name_: str - _config_cls_: type[TomlConfig] + _config_cls_: type[_T] def __init__(self, stream: IOBase) -> None: - self.fp = stream + """Initialize stream with a file-like object. + + :param stream: Opened file-like object (e.g., `io.TextIOWrapper`). + :type stream: IOBase + """ + self.fp: IOBase = stream def close(self) -> None: + """Close the underlying stream and flush buffers.""" if not self.fp.closed: self.fp.flush() self.fp.close() def write(self, data: str) -> None: + """Write a line to the stream with newline and flush. + + Automatically encodes string to bytes for binary streams. + + :param data: Line to write (without newline). + :type data: str + """ line = f"{data}\n" self.fp.write(line.encode()) self.fp.flush() - def write_columns(self, *values: Any) -> None: - line = "\t".join(map(str, values)) + def write_columns(self, *values: Any, sep: str | None = None) -> None: + """Write tab-separated columns to stream. + + Useful for structured output (e.g., CSV-like logs). + + :param values: Values to write as columns. + :type values: Any + """ + line = (sep or "\t").join(map(str, values)) self.write(line) def add(self, **kwargs: Any) -> None: + """Add a structured log entry. + + Must be overridden by subclasses to handle specific data formats. + + :param kwargs: Contextual data (e.g., `ip`, `type`, `value`). + """ pass @classmethod - def start(cls, session) -> None: + def start(cls, session: SessionConfig) -> None: + """Initialize and register this stream type from config. + + Loads config from TOML, resolves path via session, creates directory if needed, + and registers instance in `dm_streams`. + + :param session: Session configuration for path resolution. + :type session: SessionConfig + """ config = TomlConfig.build_config(cls._config_cls_) - path = config.path - if path is not None: + path = getattr(config, "path", None) + if path is not None and issubclass(cls, LoggingFileStream): path = session.resolve_path(path) if not path.parent.exists(): dm_logger.debug(f"Creating log directory {path.parent}") @@ -67,80 +122,154 @@ def start(cls, session) -> None: dm_streams[cls._name_] = cls(path, config) -class LoggingFileStream(LoggingStream): - def __init__(self, path: str | Path) -> None: - self.path = Path(path) - if self.path.exists(): - mode = "ab" - else: - mode = "wb" +class LoggingFileStream(LoggingStream[_T]): + """ + Extension of :class:`LoggingStream` that opens a regular file. - # make sure not error is thrown + The file is opened in **append-binary** mode (``ab``) when it already + exists, otherwise in **write-binary** mode (``wb``). The parent directory + is created automatically. + + :param path: Destination file path. + :type path: str | pathlib.Path + """ + + def __init__(self, path: str | Path, config: _T) -> None: + """Initialize file stream. + + Opens file in binary append mode if exists, binary write mode otherwise. + + :param path: Path to log file. + :type path: str | Path + """ + self.path: Path = Path(path) + mode = "ab" if self.path.exists() else "wb" self.path.parent.mkdir(parents=True, exist_ok=True) + self.config: _T = config super().__init__(self.path.open(mode)) def reopen(self) -> None: + """Close the current file handle (if open) and reopen it in ``wb`` mode.""" if not self.fp.closed: self.fp.close() - self.fp = self.path.open("wb") + self.fp: IOBase = self.path.open("wb") class HostsStreamConfig(TomlConfig): - _section_ = "Log.Stream.Hosts" - _fields_ = [ + """Configuration for host IP logging stream. + + Controls whether IPv4 and IPv6 addresses are logged. + """ + + _section_: ClassVar[str] = "Log.Stream.Hosts" + _fields_: ClassVar[list[A]] = [ A("path", "Path", default_val=None), A("log_ipv4", "IPv4", default_val=True), A("log_ipv6", "IPv6", default_val=True), ] + if typing.TYPE_CHECKING: + path: str + log_ipv4: bool + log_ipv6: bool + -class HostsStream(LoggingFileStream): - _name_ = "hosts" - _config_cls_ = HostsStreamConfig +class HostsStream(LoggingFileStream[HostsStreamConfig]): + """Log unique host IP addresses to a file. + + Filters by IPv4/IPv6 based on config. Prevents duplicates. + + :cvar _name_: Stream identifier (`"hosts"`). + :cvar _config_cls_: Config class (`HostsStreamConfig`). + :ivar hosts: Set of already logged IPs. + :vartype hosts: set[str] + :ivar ipv4: Whether to log IPv4 addresses. + :vartype ipv4: bool + :ivar ipv6: Whether to log IPv6 addresses. + :vartype ipv6: bool + """ + + _name_: str = "hosts" + _config_cls_: type[HostsStreamConfig] = HostsStreamConfig def __init__(self, path: str | Path, config: HostsStreamConfig) -> None: - super().__init__(path) - self.hosts = set() - self.ipv4 = config.log_ipv4 - self.ipv6 = config.log_ipv6 + """Initialize host IP logger. + + :param path: Path to output file. + :type path: str | Path + :param config: Loaded configuration. + :type config: HostsStreamConfig + """ + super().__init__(path, config) + self.hosts: set[str] = set() + self.ipv4: bool = config.log_ipv4 + self.ipv6: bool = config.log_ipv6 dm_logger.info( f"Logging host IPs to {path} (IPv4={self.ipv4}, IPv6={self.ipv6})" ) @override def add(self, **kwargs: Any) -> None: + """ + Add a new IP address to the log. + + The address is written only once; subsequent attempts are ignored. + IPv4/IPv6 filtering follows the configuration. + + :param ip: IP address to log. + :type ip: str, optional + """ ip = kwargs.get("ip") if ip and ip not in self.hosts: if not self.ipv4 and "." in ip: return - if not self.ipv6 and ":" in ip: return - self.write_columns(ip) self.hosts.add(ip) class DNSNamesStreamConfig(TomlConfig): - _section_ = "Log.Stream.DNS" - _fields_ = [ + """Configuration for DNS query logging stream.""" + + _section_: ClassVar[str] = "Log.Stream.DNS" + _fields_: ClassVar[list[A]] = [ A("path", "Path", default_val=None), # reserved for future extensions ] -class DNSNamesStream(LoggingFileStream): - _name_ = "dns" - _config_cls_ = DNSNamesStreamConfig +class DNSNamesStream(LoggingFileStream[DNSNamesStreamConfig]): + """Log unique DNS query names by type. + + Stores queries in a nested dict: `{record_type: {query}}`. + + :ivar hosts: Nested dict of DNS records by type. + :vartype hosts: defaultdict[set] + """ + + _name_: str = "dns" + _config_cls_: type[DNSNamesStreamConfig] = DNSNamesStreamConfig def __init__(self, path: str | Path, config: DNSNamesStreamConfig) -> None: - super().__init__(path) - self.hosts = defaultdict(set) + """Initialize DNS query logger. + + :param path: Path to output file. + :type path: str | Path + :param config: Loaded configuration. + :type config: DNSNamesStreamConfig + """ + super().__init__(path, config) + self.hosts: dict[str, set[str]] = defaultdict(set) dm_logger.info(f"Logging DNS names to {path}") @override def add(self, **kwargs: Any) -> None: + """Add a DNS query if not previously logged. + + :param kwargs: Must contain `type` (e.g., `"A"`, `"AAAA"`) and `name` (domain). + """ name = kwargs.get("type") query = kwargs.get("name") if name and query: @@ -150,29 +279,65 @@ def add(self, **kwargs: Any) -> None: class HashesStreamConfig(TomlConfig): - _section_ = "Log.Stream.Hashes" - _fields_ = [ + """Configuration for hash value logging stream. + + Supports single-file or per-hash-type split output with custom prefixes/suffixes. + """ + + _section_: ClassVar[str] = "Log.Stream.Hashes" + _fields_: ClassVar[list[A]] = [ A("path", "Path", default_val=None), A("split", "Split", default_val=None), A("prefix", "FilePrefix", default_val=None), A("suffix", "FileSuffix", default_val=".txt"), ] + if typing.TYPE_CHECKING: + path: str + split: bool + prefix: str + suffix: str + + +class HashStreams(LoggingFileStream[HashesStreamConfig]): + """Log credential hashes to file(s), optionally split by hash type. -class HashStreams(LoggingFileStream): - _name_ = "hashes" - _config_cls_ = HashesStreamConfig + If `split=True`, creates separate files per hash type (e.g., `ntlm_2024-06-15-14-30-22.txt`). + Uses `util.format_string` for dynamic prefix generation. + + :cvar _name_: Stream identifier (`"hashes"`). + :cvar _config_cls_: Config class (`HashesStreamConfig`). + :ivar config: Loaded configuration. + :vartype config: HashesStreamConfig + :ivar path: Base output path. + :vartype path: Path + :ivar start_time: Timestamp used for dynamic prefixes. + :vartype start_time: str + """ + + _name_: str = "hashes" + _config_cls_: type[HashesStreamConfig] = HashesStreamConfig def __init__(self, path: str | Path, config: HashesStreamConfig) -> None: - super().__init__(path if not config.split else "/dev/null") - self.config = config - self.path = Path(path) - self.start_time = util.now() + """Initialize hash logger. + + :param path: Base path (or directory for split mode). + :type path: str | Path + :param config: Loaded configuration. + :type config: HashesStreamConfig + """ + super().__init__(path if not config.split else "/dev/null", config) + self.path: Path = Path(path) + self.start_time: str = util.now() dm_logger.info(f"Logging hashes to {path} (split files: {config.split})") @override def add(self, **kwargs: Any) -> None: - hash_type = kwargs.get("type").upper() + """Add a hash value, optionally to a split file. + + :param kwargs: Must contain `type` (hash type, e.g., `"ntlm"`) and `value` (hash string). + """ + hash_type = str(kwargs.get("type")).upper() hash_value = kwargs.get("value") if hash_type and hash_value: if not self.config.split: @@ -190,45 +355,94 @@ def add(self, **kwargs: Any) -> None: "time": self.start_time, }, ) - target_path = Path(self.path) / f"{prefix}{suffix}" if not target_path.exists(): # create a new logging stream for that hash type target_path.parent.mkdir(parents=True, exist_ok=True) - dm_streams[f"HASH_{hash_type}"] = LoggingFileStream(target_path) - + dm_streams[f"HASH_{hash_type}"] = LoggingFileStream( + path=target_path, + config=TomlConfig(), + ) write_to(f"HASH_{hash_type}", str(hash_value)) -def init_streams(session): +def init_streams(session: SessionConfig): + """Initialize all configured logging streams at startup. + + Calls `.start()` on each stream class to load config and register instances. + + :param session: Session configuration for path resolution. + :type session: SessionConfig + """ HostsStream.start(session) DNSNamesStream.start(session) HashStreams.start(session) session.streams = dm_streams -def add_stream(name: str, stream: LoggingStream): +def add_stream(name: str, stream: LoggingStream[_T]): + """Manually register a stream instance. + + Useful for dynamic or custom streams. + + :param name: Unique stream identifier. + :type name: str + :param stream: Stream instance. + :type stream: LoggingStream + """ dm_streams[name] = stream -def get_stream(name: str) -> LoggingStream | None: +def get_stream(name: str) -> LoggingStream[_T] | None: + """Retrieve a stream by name. + + :param name: Stream identifier. + :type name: str + :return: Stream instance or `None` if not found. + :rtype: LoggingStream | None + """ return dm_streams.get(name) -def close_streams(session): +def close_streams(session: SessionConfig): + """Close all active streams. + + Called during graceful shutdown. + + :param session: Session containing streams registry. + :type session: SessionConfig + """ for stream in session.streams.values(): stream.close() -def log_to(__name: str, /, **kwargs): +def log_to(__name: str, /, **kwargs: Any): + """Write structured data to a registered stream. + + :param __name: Stream name (e.g., `"hosts"`, `"hashes"`). + :type __name: str + :param kwargs: Data to pass to stream's `add()` method. + """ if __name in dm_streams: dm_streams[__name].add(**kwargs) def write_to(name: str, line: str): + """Write a raw line to a stream (no formatting). + + :param name: Stream name. + :type name: str + :param line: Raw string to write. + :type line: str + """ if name in dm_streams: dm_streams[name].write(line) def log_host(ip: str): + """Convenience function to log a host IP to the `"hosts"` stream. + + :param ip: Normalized IP address. + :type ip: str + """ log_to("hosts", ip=ip) diff --git a/dementor/protocols/ftp.py b/dementor/protocols/ftp.py index c672055..4fb8780 100755 --- a/dementor/protocols/ftp.py +++ b/dementor/protocols/ftp.py @@ -18,95 +18,235 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. # pyright: reportUninitializedInstanceVariable=false +# pyright: reportAny=false, reportExplicitAny=false import typing +from socket import socket +from typing import ClassVar + +from typing_extensions import override from dementor.config.session import SessionConfig from dementor.config.toml import TomlConfig, Attribute as A from dementor.config.util import get_value from dementor.log.logger import ProtocolLogger from dementor.servers import BaseProtoHandler, ThreadingTCPServer, ServerThread -from dementor.db import _CLEARTEXT - -ReplyCodes = { - 220: b"220 Service ready for new user.", - 331: b"331 User name okay, need password.", +from dementor.db import _CLEARTEXT # pyright: ignore[reportPrivateUsage] + +# --------------------------------------------------------------------------- # +# RFC-959 reply codes used by this minimal implementation. +# --------------------------------------------------------------------------- # +ReplyCodes: dict[int, bytes] = { + 220: b"220 Service ready for new user.", # Service ready + 230: b"230 User logged in, proceed.", # Successful login + 331: b"331 User name okay, need password.", # Username accepted 501: b"501 Syntax error in parameters or arguments.", 502: b"502 Command not implemented.", 530: b"530 Not logged in.", + 221: b"221 Service closing control connection.", } +# --------------------------------------------------------------------------- # +# Configuration handling +# --------------------------------------------------------------------------- # class FTPServerConfig(TomlConfig): - _section_ = "FTP" - _fields_ = [A("ftp_port", "Port")] + """ + Configuration container for a single FTP server instance. - if typing.TYPE_CHECKING: - ftp_port: int + The configuration is read from the ``[FTP]`` section of the TOML file. + Only the listening port is required. + :param ftp_port: TCP port on which the FTP server should listen, + defaults to the standard FTP port ``21``. + :type ftp_port: int + """ -def apply_config(session: SessionConfig) -> None: - session.ftp_config = [] - if session.ftp_enabled: - for server_config in get_value("FTP", "Server", default=[]): - session.ftp_config.append(FTPServerConfig(server_config)) + _section_: ClassVar[str] = "FTP" + _fields_: ClassVar[list[A]] = [A("ftp_port", "Port")] + + if typing.TYPE_CHECKING: # pragma: no cover + ftp_port: int -def create_server_threads(session) -> list[ServerThread]: +def apply_config(session: SessionConfig) -> None: + """ + Load FTP server configuration and store it on *session*. + + ``session.ftp_config`` becomes a list of :class:`FTPServerConfig` + objects - one per ``[FTP.Server]`` stanza in the TOML file. + + :param session: Current session object. + :type session: :class:`dementor.config.session.SessionConfig` + """ + session.ftp_config = [] # type: ignore[attr-defined] + if session.ftp_enabled: # pragma: no branch + for server_cfg in get_value("FTP", "Server", default=[]): + session.ftp_config.append(FTPServerConfig(server_cfg)) + + +def create_server_threads(session: SessionConfig) -> list[ServerThread]: + """ + Build a list of :class:`ServerThread` objects - one per configured FTP + server - and return it. The caller is responsible for starting each + thread. + + :param session: Session containing the ``ftp_config`` list. + :type session: :class:`dementor.config.session.SessionConfig` + :return: List of ready-to-start :class:`ServerThread` objects. + :rtype: list[ServerThread] + """ return [ - ServerThread(session, FTPServer, server_address=("", server_config.ftp_port)) - for server_config in session.ftp_config + ServerThread( + session, + FTPServer, + server_address=("", cfg.ftp_port), + ) + for cfg in session.ftp_config ] +# --------------------------------------------------------------------------- # +# FTP request handling +# --------------------------------------------------------------------------- # class FTPHandler(BaseProtoHandler): + """ + Minimal FTP request handler. + + The handler sends the initial ``220`` greeting, then processes a very + small login sequence (``USER`` → ``PASS``). All other commands result + in a ``501`` reply. + + :class:`ProtocolLogger` is used to attach FTP-specific metadata to log + records. + """ + + # ------------------------------------------------------------------- # + # Logging helper + # ------------------------------------------------------------------- # + @override def proto_logger(self) -> ProtocolLogger: + """ + Return a :class:`ProtocolLogger` pre-populated with FTP-specific fields. + + :return: Configured ``ProtocolLogger`` instance. + :rtype: ProtocolLogger + """ return ProtocolLogger( extra={ "protocol": "FTP", "protocol_color": "medium_purple2", "host": self.client_host, - "port": self.server.server_address[1], + "port": self.server_port, } ) - def handle_data(self, data, transport) -> None: - # Server ready for new user + # ------------------------------------------------------------------- # + # Core request processing + # ------------------------------------------------------------------- # + @override + def handle_data(self, data: bytes | None, transport: socket) -> None: + """ + Process client commands after a TCP connection is accepted. + """ + # ----------------------------------------------------------------- + # 1. Send the initial greeting as required by RFC-959. + # ----------------------------------------------------------------- self.reply(220) - data = transport.recv(1024) - if len(data) < 4: - # ignore short packets and return error - self.reply(502) - return - - if data[:4] == b"USER": - user_name = data[4:].decode(errors="replace").strip() - if not user_name: - self.reply(501) - return - - self.reply(331) - data = transport.recv(1024) - if len(data) >= 4 and data[:4] == b"PASS": - password = data[4:].decode(errors="replace").strip() + while True: + raw: bytes = self.request.recv(1024) + if not raw: + # Client closed the connection. + break + + # Strip CRLF and split on whitespace; FTP commands are case-insensitive. + parts = raw.strip().split(None, 1) # e.g. [b'USER', b'alice'] + cmd = parts[0].upper() if parts else b"" + + # ----------------------------------------------------------------- + # USER command handling + # ----------------------------------------------------------------- + if cmd == b"USER": + username = ( + parts[1].decode(errors="replace").strip() if len(parts) > 1 else "" + ) + if not username: + self.reply(501) # Empty username → syntax error + continue + + self.reply(331) # Password required + # ------------------------------------------------------------- + # Expect PASS command next; loop back to receive it. + # ------------------------------------------------------------- + raw = self.request.recv(1024) + if not raw: + break + + parts = raw.strip().split(None, 1) + if parts[0].upper() != b"PASS": + self.reply(501) + continue + + password = ( + parts[1].decode(errors="replace").strip() if len(parts) > 1 else "" + ) self.config.db.add_auth( client=self.client_address, - credtype=_CLEARTEXT, - username=user_name, + credtype=_CLEARTEXT, # intentional clear-text + username=username, password=password, logger=self.logger, ) - self.reply(502) # Command not implemented rather than error - return - - self.reply(501) - + # # Command not implemented rather than error + self.reply(502) + break + + # ----------------------------------------------------------------- + # Any other command is treated as a syntax error. + # ----------------------------------------------------------------- + self.reply(501) + + # Send a polite closing message before the socket is closed by the + # server framework (optional, but makes the dialogue look more genuine). + try: + self.reply(221) + except Exception: + pass # Silently ignore errors during shutdown. + + # ------------------------------------------------------------------- # + # Helper to send a reply line. + # ------------------------------------------------------------------- # def reply(self, code: int) -> None: - self.request.send(ReplyCodes[code] + b"\r\n") + """ + Send a one-line FTP reply identified by *code*. + + The reply string is taken from :data:`ReplyCodes` and terminated with + CRLF as required by the protocol. + + :param code: Numeric FTP reply code (e.g. ``220``). + :type code: int + :raises KeyError: If *code* is not present in :data:`ReplyCodes`. + """ + try: + self.request.sendall(ReplyCodes[code] + b"\r\n") + except OSError as exc: + # Logging the failure helps debugging but we do not re-raise, + # because the connection is likely already broken. + self.logger.debug(f"Failed to send FTP reply {code}: {exc}") + + +# --------------------------------------------------------------------------- # +# Server class - thin wrapper around ``ThreadingTCPServer``. +# --------------------------------------------------------------------------- # +class FTPServer(ThreadingTCPServer): + """ + TCP server that accepts FTP connections. + Only the class-level defaults are defined here; all functional behaviour + is provided by :class:`ThreadingTCPServer` and :class:`FTPHandler`. + """ -class FTPServer(ThreadingTCPServer): - default_port = 21 - default_handler_class = FTPHandler - service_name = "FTP" + default_port: ClassVar[int] = 21 + default_handler_class: ClassVar[type[FTPHandler]] = FTPHandler + service_name: str = "FTP" diff --git a/dementor/protocols/ldap.py b/dementor/protocols/ldap.py index d12389d..c9dae8b 100755 --- a/dementor/protocols/ldap.py +++ b/dementor/protocols/ldap.py @@ -50,7 +50,6 @@ from dementor.db import _CLEARTEXT from dementor.protocols.ntlm import ( NTLM_AUTH_CreateChallenge, - NTLM_AUTH_format_host, NTLM_report_auth, NTLM_split_fqdn, ATTR_NTLM_CHALLENGE, @@ -175,9 +174,6 @@ def handle_NTLM_Negotiate( negotiate = NTLMAuthNegotiate() negotiate.fromString(nego_token_raw) - host_format = NTLM_AUTH_format_host(negotiate) - self.logger.debug("Starting NTLM-auth: %s", host_format) - fqdn = self.server.server_config.ldap_fqdn if "." in fqdn: name, domain = fqdn.split(".", 1) diff --git a/dementor/protocols/ntlm.py b/dementor/protocols/ntlm.py index 46901eb..a0f82cd 100755 --- a/dementor/protocols/ntlm.py +++ b/dementor/protocols/ntlm.py @@ -17,7 +17,8 @@ # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. - +# pyright: reportAny=false, reportExplicitAny=false, reportUnknownVariableType=false +# pyright: reportUnknownArgumentType=false """NTLM authentication helper module for Dementor. Implements the server-side CHALLENGE_MESSAGE construction and @@ -42,7 +43,7 @@ |--- AUTHENTICATE_MESSAGE --------------> | <- capture & extract hashes | | -Variable names follow [MS-NLMP] specification terminology: +Variable names follow [MS-NLMP] specification terminology (terminology reference): ServerChallenge 8-byte nonce in the CHALLENGE_MESSAGE NegotiateFlags 32-bit flag field in any NTLM message @@ -63,12 +64,16 @@ import time import calendar import secrets +from typing import Any +from caterpillar.py import LittleEndian, uint16 from impacket import ntlm +from impacket.smb3 import WIN_VERSIONS from dementor.config.toml import Attribute from dementor.config.session import SessionConfig from dementor.config.util import is_true, get_value, BytesValue +from dementor.db import _HOST_INFO from dementor.log.logger import ProtocolLogger, dm_logger # =========================================================================== @@ -165,7 +170,14 @@ def NTLM_AUTH_classify( nt_response: bytes, lm_response: bytes, negotiate_flags: int ) -> str: - """Classify the hash type from an AUTHENTICATE_MESSAGE response.""" + """Classify the hash type from an AUTHENTICATE_MESSAGE response. + + :param bytes nt_response: The NtChallengeResponse field + :param bytes lm_response: The LmChallengeResponse field + :param int negotiate_flags: The NegotiateFlags from the message + :return: Classification label (NTLM_V1, NTLM_V1_ESS, NTLM_V2, or NTLM_V2_LM) + :rtype: str + """ # Fallback to NetNTLMv1 on TypeError (None or non-bytes input) rather than raising. try: nt_len = len(nt_response) @@ -272,10 +284,7 @@ def apply_config(session: SessionConfig) -> None: On any parsing error, safe defaults are kept so startup continues. - Parameters - ---------- - session : SessionConfig - Session object whose NTLM attributes will be populated. + :param SessionConfig session: Session object whose NTLM attributes will be populated """ # Safe defaults (session remains valid even if config parsing fails). session.ntlm_challenge = secrets.token_bytes(NTLM_CHALLENGE_LEN) @@ -321,9 +330,9 @@ def apply_config(session: SessionConfig) -> None: if session.ntlm_disable_ntlmv2: dm_logger.warning( "NTLM DisableNTLMv2 is enabled — Level 3+ clients (all modern Windows) " - "will FAIL authentication and NO hashes will be captured. " - "This only helps against pre-Vista / manually-configured Level 0-2 clients. " - "Use with caution." + + "will FAIL authentication and NO hashes will be captured. " + + "This only helps against pre-Vista / manually-configured Level 0-2 clients. " + + "Use with caution." ) @@ -344,23 +353,15 @@ def NTLM_AUTH_decode_string( ) -> str: """Decode an NTLM wire string into a Python str. - Parameters - ---------- - data : bytes or None - Raw bytes from the NTLM message field. - negotiate_flags : int - NegotiateFlags from the message. Determines encoding for - CHALLENGE_MESSAGE and AUTHENTICATE_MESSAGE fields. - is_negotiate_oem : bool - If True, forces OEM/ASCII decoding regardless of flags. Set this - when decoding fields from a NEGOTIATE_MESSAGE, where Unicode - negotiation has not yet occurred per [MS-NLMP section 2.2]. - - Returns - ------- - str - Decoded string. Returns "" for None or empty input. - Malformed bytes are replaced with U+FFFD rather than raising. + :param bytes|None data: Raw bytes from the NTLM message field + :param int negotiate_flags: NegotiateFlags from the message. Determines encoding for + CHALLENGE_MESSAGE and AUTHENTICATE_MESSAGE fields + :param bool is_negotiate_oem: If True, forces OEM/ASCII decoding regardless of flags. + Set this when decoding fields from a NEGOTIATE_MESSAGE, where Unicode + negotiation has not yet occurred per [MS-NLMP section 2.2] + :return: Decoded string. Returns "" for None or empty input. + Malformed bytes are replaced with U+FFFD rather than raising + :rtype: str """ if not data: return "" @@ -380,18 +381,11 @@ def NTLM_AUTH_decode_string( def NTLM_AUTH_encode_string(string: str | None, negotiate_flags: int) -> bytes: """Encode a Python str for inclusion in a CHALLENGE_MESSAGE. - Parameters - ---------- - string : str or None - The string to encode (server name, domain, etc.). - negotiate_flags : int - NegotiateFlags that determine encoding. - - Returns - ------- - bytes - UTF-16LE if Unicode is negotiated, cp437 (OEM) otherwise. - Returns b"" for None or empty input. + :param str|None string: The string to encode (server name, domain, etc.) + :param int negotiate_flags: NegotiateFlags that determine encoding + :return: UTF-16LE if Unicode is negotiated, cp437 (OEM) otherwise. + Returns b"" for None or empty input + :rtype: bytes """ if not string: return b"" @@ -414,16 +408,10 @@ def NTLM_AUTH_encode_string(string: str | None, negotiate_flags: int) -> bytes: def _compute_dummy_lm_responses(server_challenge: bytes) -> set[bytes]: """Compute the two known dummy LmChallengeResponse values (per §3.3.1). - Parameters - ---------- - server_challenge : bytes - 8-byte ServerChallenge from the CHALLENGE_MESSAGE. - - Returns - ------- - set of bytes - Two 24-byte DESL() outputs for the null and empty-string LM hashes. - Any LmChallengeResponse matching either contains no crackable material. + :param bytes server_challenge: 8-byte ServerChallenge from the CHALLENGE_MESSAGE + :return: Two 24-byte DESL() outputs for the null and empty-string LM hashes. + Any LmChallengeResponse matching either contains no crackable material + :rtype: set of bytes """ return { ntlm.ntlmssp_DES_encrypt(NTLM_ESS_ZERO_PAD, server_challenge), @@ -436,28 +424,16 @@ def _compute_dummy_lm_responses(server_challenge: bytes) -> set[bytes]: # =========================================================================== -def NTLM_AUTH_format_host(token: ntlm.NTLMAuthNegotiate) -> str: - """Extract a human-readable host description from a NEGOTIATE_MESSAGE. - - All string fields in a NEGOTIATE_MESSAGE are OEM-encoded per [MS-NLMP - section 2.2] -- Unicode has not been negotiated yet, so - is_negotiate_oem=True is passed to the decoder. - - Parameters - ---------- - token : NTLMAuthNegotiate - Parsed NEGOTIATE_MESSAGE from the client. +def NTLM_AUTH_format_host(token: ntlm.NTLMAuthChallengeResponse) -> str: + """Extract a human-readable host description from a CHALLENGE_MESSAGE. - Returns - ------- - str - "HOSTNAME (domain: DOMAIN) (OS: X.Y.Build)" - - Uses "" for any missing or unparseable field. Never raises. + :param ntlm.NTLMAuthChallengeResponse token: Parsed CHALLENGE_MESSAGE from the client + :return: "OS [ (name: HOSTNAME) ] [ (domain: DOMAIN) ]" Never raises + :rtype: str """ flags: int = 0 - hostname: str = "" - domain_name: str = "" + hostname: str = "" + domain_name: str = "" os_version: str = "0.0.0" try: @@ -468,7 +444,7 @@ def NTLM_AUTH_format_host(token: ntlm.NTLMAuthNegotiate) -> str: flags, is_negotiate_oem=True, ) - or "" + or "" ) domain_name = ( NTLM_AUTH_decode_string( @@ -476,7 +452,7 @@ def NTLM_AUTH_format_host(token: ntlm.NTLMAuthNegotiate) -> str: flags, is_negotiate_oem=True, ) - or "" + or "" ) except Exception: dm_logger.debug( @@ -487,19 +463,35 @@ def NTLM_AUTH_format_host(token: ntlm.NTLMAuthNegotiate) -> str: # Parse the OS VERSION structure separately so a version parse failure # does not discard the already-decoded hostname and domain. try: - ver = token["os_version"] - os_version = ( - f"{ver['ProductMajorVersion']}." - f"{ver['ProductMinorVersion']}." - f"{ver['ProductBuild']}" - ) + ver_raw: bytes = token["Version"] + major: int = ver_raw[0] + minor: int = ver_raw[1] + build: int = uint16.from_bytes(ver_raw[2:4], order=LittleEndian) + + os_version = f"{major}.{minor}" + if build in WIN_VERSIONS: + os_version = f"{WIN_VERSIONS[build]}" + + if build: + os_version = f"{os_version} Build {build}" + + if (major, minor, build) == (6, 1, 0): + os_version = "Unix - Samba" + except Exception: dm_logger.debug( "Failed to parse OS version from NEGOTIATE_MESSAGE; using 0.0.0", exc_info=True, ) - return f"{hostname} (domain: {domain_name}) (OS: {os_version})" + host_info = os_version + if hostname: + host_info += f" (name: {hostname})" + + if domain_name: + host_info += f" (domain: {domain_name})" + + return host_info # =========================================================================== @@ -547,48 +539,33 @@ def NTLM_AUTH_to_hashcat_formats( ) -> list[tuple[str, str]]: """Extract all crackable hashcat lines from an AUTHENTICATE_MESSAGE. - Returns up to two entries: the primary hash and, for NetNTLMv2, the NetLMv2 - companion. Callers must check for anonymous auth before invoking. - - Parameters - ---------- - server_challenge : bytes - 8-byte ServerChallenge from the CHALLENGE_MESSAGE Dementor sent. - user_name : bytes or str - UserName from the AUTHENTICATE_MESSAGE. - domain_name : bytes or str - DomainName from the AUTHENTICATE_MESSAGE. - lm_response : bytes or None - LmChallengeResponse from the AUTHENTICATE_MESSAGE. - nt_response : bytes or None - NtChallengeResponse from the AUTHENTICATE_MESSAGE. - negotiate_flags : int - NegotiateFlags from the NTLM exchange. - - Returns - ------- - list of (str, str) - (label, hashcat_line) tuples. Labels: NTLM_V2 ("NetNTLMv2"), - NTLM_V2_LM ("NetLMv2"), NTLM_V1_ESS ("NetNTLMv1-ESS"), NTLM_V1 ("NetNTLMv1"). - - Raises - ------ - ValueError - If server_challenge is not exactly NTLM_CHALLENGE_LEN bytes. - - Notes - ----- - - Hash type determined by NTLM_AUTH_classify() called once; no raw length - comparisons appear in the branches below. - - Dummy LM responses (DESL of null or empty-string LM hash) are discarded. - - Level 2 duplication (LM == NT) omits the LM slot. - - Per §3.3.2 rule 7: when MsvAvTimestamp is present, clients set - LmChallengeResponse to Z(24); this null NetLMv2 is detected and skipped. + Returns up to two entries: the primary hash and, for NetNTLMv2, the LMv2 + companion. Callers must check for anonymous auth before invoking. + + :param bytes server_challenge: 8-byte ServerChallenge from the CHALLENGE_MESSAGE Dementor sent + :param bytes|str user_name: UserName from the AUTHENTICATE_MESSAGE + :param bytes|str domain_name: DomainName from the AUTHENTICATE_MESSAGE + :param bytes|None lm_response: LmChallengeResponse from the AUTHENTICATE_MESSAGE + :param bytes|None nt_response: NtChallengeResponse from the AUTHENTICATE_MESSAGE + :param int negotiate_flags: NegotiateFlags from the NTLM exchange + :return: (label, hashcat_line) tuples. Labels: NTLM_V2 ("NetNTLMv2"), + NTLM_V2_LM ("LMv2"), NTLM_V1_ESS ("NetNTLMv1-ESS"), NTLM_V1 ("NetNTLMv1") + :rtype: list of (str, str) + :raises ValueError: If server_challenge is not exactly NTLM_CHALLENGE_LEN bytes + + .. note:: + + - Hash type determined by NTLM_AUTH_classify() called once; no raw length + comparisons appear in the branches below. + - Dummy LM responses (DESL of null or empty-string LM hash) are discarded. + - Level 2 duplication (LM == NT) omits the LM slot. + - Per §3.3.2 rule 7: when MsvAvTimestamp is present, clients set + LmChallengeResponse to Z(24); this null LMv2 is detected and skipped. """ if len(server_challenge) != NTLM_CHALLENGE_LEN: raise ValueError( f"server_challenge must be {NTLM_CHALLENGE_LEN} bytes, " - f"got {len(server_challenge)}" + + f"got {len(server_challenge)}" ) captures: list[tuple[str, str]] = [] @@ -658,9 +635,9 @@ def NTLM_AUTH_to_hashcat_formats( ( NTLM_V2, f"{user}::{domain}" - f":{server_challenge_hex}" - f":{nt_proof_str_hex}" - f":{blob_hex}", + + f":{server_challenge_hex}" + + f":{nt_proof_str_hex}" + + f":{blob_hex}", ) ) dm_logger.debug("Appended %s hash (nt_len=%d)", NTLM_V2, len(nt_response)) @@ -678,7 +655,7 @@ def NTLM_AUTH_to_hashcat_formats( if lm_response == b"\x00" * NTLMV1_RESPONSE_LEN: dm_logger.debug( "LmChallengeResponse is Z(%d) " - "(MsvAvTimestamp suppression or null LM); skipping %s", + + "(MsvAvTimestamp suppression or null LM); skipping %s", NTLMV1_RESPONSE_LEN, NTLM_V2_LM, ) @@ -691,9 +668,9 @@ def NTLM_AUTH_to_hashcat_formats( ( NTLM_V2_LM, f"{user}::{domain}" - f":{server_challenge_hex}" - f":{lm_proof_hex}" - f":{lm_cc_hex}", + + f":{server_challenge_hex}" + + f":{lm_proof_hex}" + + f":{lm_cc_hex}", ) ) dm_logger.debug("Appended %s companion hash", NTLM_V2_LM) @@ -723,9 +700,9 @@ def NTLM_AUTH_to_hashcat_formats( ( NTLM_V1_ESS, f"{user}::{domain}" - f":{lm_ess_hex}" - f":{nt_response_hex}" - f":{server_challenge_hex}", + + f":{lm_ess_hex}" + + f":{nt_response_hex}" + + f":{server_challenge_hex}", ) ) dm_logger.debug("Appended %s hash", NTLM_V1_ESS) @@ -749,7 +726,7 @@ def NTLM_AUTH_to_hashcat_formats( # Case 1: duplication — LM is a copy of NT, skip it dm_logger.debug( "LmChallengeResponse == NtChallengeResponse " - "(Level 2 duplication); omitting LM slot" + + "(Level 2 duplication); omitting LM slot" ) elif lm_response in _compute_dummy_lm_responses(server_challenge): # Case 2: dummy DESL output — no crackable credential material @@ -767,9 +744,9 @@ def NTLM_AUTH_to_hashcat_formats( ( NTLM_V1, f"{user}::{domain}" - f":{lm_slot_hex}" - f":{nt_response_hex}" - f":{server_challenge_hex}", + + f":{lm_slot_hex}" + + f":{nt_response_hex}" + + f":{server_challenge_hex}", ) ) dm_logger.debug( @@ -787,7 +764,11 @@ def NTLM_AUTH_to_hashcat_formats( def NTLM_new_timestamp() -> int: - """Return the current UTC time as a Windows FILETIME (100ns ticks since 1601-01-01).""" + """Return the current UTC time as a Windows FILETIME (100ns ticks since 1601-01-01). + + :return: Current UTC time in 100-nanosecond intervals since Windows epoch (1601-01-01) + :rtype: int + """ # calendar.timegm() → UTC seconds since 1970; scaled to 100ns ticks since 1601. return ( NTLM_FILETIME_EPOCH_OFFSET @@ -798,17 +779,11 @@ def NTLM_new_timestamp() -> int: def NTLM_split_fqdn(fqdn: str) -> tuple[str, str]: """Split a fully-qualified domain name into (hostname, domain). - Parameters - ---------- - fqdn : str - e.g. "SERVER1.corp.example.com" - - Returns - ------- - tuple of (str, str) - ("SERVER1", "corp.example.com") if dotted, or + :param str fqdn: Fully-qualified domain name, e.g. "SERVER1.corp.example.com" + :return: ("SERVER1", "corp.example.com") if dotted, or (fqdn, "WORKGROUP") if no dots present, or - ("WORKGROUP", "WORKGROUP") if empty. + ("WORKGROUP", "WORKGROUP") if empty + :rtype: tuple of (str, str) """ if not fqdn: return ("WORKGROUP", "WORKGROUP") @@ -831,14 +806,9 @@ def NTLM_AUTH_is_anonymous(token: ntlm.NTLMAuthChallengeResponse) -> bool: empty or Z(1). For capture-first operation, do not trust the anonymous flag alone, and do not fail-closed on parsing exceptions. - Parameters - ---------- - token : NTLMAuthChallengeResponse - Parsed AUTHENTICATE_MESSAGE from the client. - - Returns - ------- - bool + :param ntlm.NTLMAuthChallengeResponse token: Parsed AUTHENTICATE_MESSAGE from the client + :return: True if the message is structurally anonymous + :rtype: bool """ try: # Structural anonymous: all response fields empty or Z(1) @@ -856,16 +826,12 @@ def NTLM_AUTH_is_anonymous(token: ntlm.NTLMAuthChallengeResponse) -> bool: dm_logger.debug("Structurally anonymous AUTHENTICATE_MESSAGE detected") return True - if flags & ntlm.NTLMSSP_NEGOTIATE_ANONYMOUS: - dm_logger.debug( - "Anonymous flag set but responses present; treating as non-anonymous" - ) - return is_anon + return is_anon or bool(flags & ntlm.NTLMSSP_NEGOTIATE_ANONYMOUS) except Exception: dm_logger.debug( "Failed to check anonymous status in AUTHENTICATE_MESSAGE; " - "treating as non-anonymous to avoid dropping captures", + + "treating as non-anonymous to avoid dropping captures", exc_info=True, ) return False @@ -887,7 +853,7 @@ def NTLM_AUTH_is_anonymous(token: ntlm.NTLMAuthChallengeResponse) -> bool: def NTLM_AUTH_CreateChallenge( - token: ntlm.NTLMAuthNegotiate | dict, + token: ntlm.NTLMAuthNegotiate | dict[str, Any], name: str, domain: str, challenge: bytes, @@ -896,53 +862,38 @@ def NTLM_AUTH_CreateChallenge( ) -> ntlm.NTLMAuthChallenge: """Build a CHALLENGE_MESSAGE from the client's NEGOTIATE_MESSAGE flags. - Parameters - ---------- - token : NTLMAuthNegotiate or dict - Parsed NEGOTIATE_MESSAGE (must have a "flags" key). - name : str - Server NetBIOS computer name — the flat hostname label, e.g. - "DEMENTOR" or "SERVER1". Must not contain a dot; callers should - obtain this from NTLM_split_fqdn. - domain : str - Server DNS domain name or "WORKGROUP", e.g. "corp.example.com". + :param ntlm.NTLMAuthNegotiate|dict token: Parsed NEGOTIATE_MESSAGE (must have a "flags" key) + :param str name: Server NetBIOS computer name — the flat hostname label, e.g. + "DEMENTOR" or "SERVER1". Must not contain a dot; callers should + obtain this from NTLM_split_fqdn + :param str domain: Server DNS domain name or "WORKGROUP", e.g. "corp.example.com". A domain-joined machine supplies its full DNS domain; a standalone - machine supplies "WORKGROUP". Callers should obtain this from - NTLM_split_fqdn. - challenge : bytes - 8-byte ServerChallenge nonce. - disable_ess : bool - Strip NTLMSSP_NEGOTIATE_EXTENDED_SESSIONSECURITY from the response. - Produces NTLMv1 instead of NTLMv1-ESS. NTLMv1 with a fixed - ServerChallenge is vulnerable to rainbow table attacks. - disable_ntlmv2 : bool - Clear NTLMSSP_NEGOTIATE_TARGET_INFO and omit TargetInfoFields. + machine supplies "WORKGROUP". Callers should obtain this from + NTLM_split_fqdn + :param bytes challenge: 8-byte ServerChallenge nonce + :param bool disable_ess: Strip NTLMSSP_NEGOTIATE_EXTENDED_SESSIONSECURITY from the response. + Produces NTLMv1 instead of NTLMv1-ESS. NTLMv1 with a fixed + ServerChallenge is vulnerable to rainbow table attacks + :param bool disable_ntlmv2: Clear NTLMSSP_NEGOTIATE_TARGET_INFO and omit TargetInfoFields. Without TargetInfoFields the client cannot construct the NTLMv2 - Blob per [MS-NLMP section 3.3.2]. Level 0-2 clients fall back to - NTLMv1. Level 3+ clients will FAIL authentication. + Blob per [MS-NLMP section 3.3.2]. Level 0-2 clients fall back to + NTLMv1. Level 3+ clients will FAIL authentication + :return: Serialisable CHALLENGE_MESSAGE ready to send to the client + :rtype: ntlm.NTLMAuthChallenge + :raises ValueError: If challenge is not exactly 8 bytes - Returns - ------- - NTLMAuthChallenge - Serialisable CHALLENGE_MESSAGE ready to send to the client. + .. note:: - Raises - ------ - ValueError - If challenge is not exactly 8 bytes. - - Notes - ----- - Flag echoing per [MS-NLMP section 3.2.5.1.1]: + Flag echoing per [MS-NLMP section 3.2.5.1.1]: SIGN, SEAL, ALWAYS_SIGN, KEY_EXCH, 56, 128 are echoed when the - client requests them. This is mandatory -- failing to echo SIGN + client requests them. This is mandatory -- failing to echo SIGN causes some clients to drop the connection before sending the - AUTHENTICATE_MESSAGE, losing the capture. Dementor never computes + AUTHENTICATE_MESSAGE, losing the capture. Dementor never computes session keys; it only echoes these flags to keep the handshake alive through hash capture. - ESS / LM_KEY mutual exclusivity per [MS-NLMP section 2.2.2.5 flag P]: + ESS / LM_KEY mutual exclusivity per [MS-NLMP section 2.2.2.5 flag P]: If both are requested, only ESS is returned. """ @@ -1136,7 +1087,7 @@ def NTLM_AUTH_CreateChallenge( # MsvAvTimestamp (0x0007) is intentionally NOT included. # Per §3.3.2 rule 7: when the server sends MsvAvTimestamp, the # client MUST NOT send an LmChallengeResponse (sets it to Z(24)). - # Omitting it ensures clients still send a real NetLMv2 alongside the + # Omitting it ensures clients still send a real LMv2 alongside the # NetNTLMv2 response, maximising the number of captured hash types. challenge_message["TargetInfoFields_len"] = len(av_pairs) challenge_message["TargetInfoFields_max_len"] = len(av_pairs) @@ -1163,31 +1114,22 @@ def NTLM_report_auth( client: tuple[str, int], session: SessionConfig, logger: ProtocolLogger | None = None, - extras: dict | None = None, + extras: dict[str, Any] | None = None, transport: str = NTLM_TRANSPORT_NTLMSSP, ) -> None: """Extract all crackable hashes from an AUTHENTICATE_MESSAGE and log them. Top-level entry point called by protocol handlers (SMB, HTTP, LDAP). - Extracts every valid hashcat line (NetNTLMv2 + NetLMv2, or NetNTLMv1/NetNTLMv1-ESS) + Extracts every valid hashcat line (NetNTLMv2 + LMv2, or NetNTLMv1/NetNTLMv1-ESS) and writes each as a separate entry to the session capture database. - Parameters - ---------- - auth_token : NTLMAuthChallengeResponse - Parsed AUTHENTICATE_MESSAGE. - challenge : bytes - 8-byte ServerChallenge from the CHALLENGE_MESSAGE Dementor sent. - client : tuple[str, int] - Client connection context (passed through to db.add_auth). - session : SessionConfig - Session context with a .db attribute for capture storage. - logger : ProtocolLogger or None - Logger for capture output. - extras : dict or None - Additional metadata for db.add_auth. - transport : str - NTLM transport identifier (NTLM_TRANSPORT_*); used for logging only. + :param ntlm.NTLMAuthChallengeResponse auth_token: Parsed AUTHENTICATE_MESSAGE + :param bytes challenge: 8-byte ServerChallenge from the CHALLENGE_MESSAGE Dementor sent + :param tuple[str, int] client: Client connection context (passed through to db.add_auth) + :param SessionConfig session: Session context with a .db attribute for capture storage + :param ProtocolLogger|None logger: Logger for capture output + :param dict|None extras: Additional metadata for db.add_auth + :param str transport: NTLM transport identifier (NTLM_TRANSPORT_*); used for logging only """ # Use the protocol logger for session-linked messages; fall back to the # module logger when no protocol logger is provided. @@ -1200,7 +1142,8 @@ def NTLM_report_auth( len(auth_token["lanman"] or b""), ) if NTLM_AUTH_is_anonymous(auth_token): - log.info("Anonymous NTLM login attempt; skipping hash extraction") + method = log.display if logger else log.debug + method("Anonymous NTLM login attempt; skipping hash extraction") return try: @@ -1218,7 +1161,7 @@ def NTLM_report_auth( if not all_hashes: log.warning( "AUTHENTICATE_MESSAGE produced no crackable hashes " - "(user=%r flags=0x%08x)", + + "(user=%r flags=0x%08x)", auth_token["user_name"], negotiate_flags, ) @@ -1239,7 +1182,11 @@ def NTLM_report_auth( user_name, domain_name, ) - + host_info = NTLM_AUTH_format_host(auth_token) + extras = extras or {} + extras[_HOST_INFO] = host_info + # REVISIT: this should be added once SMB1 legacy commands are implemented + # "Transport": transport, for version_label, hashcat_line in all_hashes: session.db.add_auth( client=client, @@ -1254,7 +1201,7 @@ def NTLM_report_auth( except ValueError: log.exception( "Invalid data in AUTHENTICATE_MESSAGE (bad challenge length or " - "malformed response fields); skipping capture" + + "malformed response fields); skipping capture" ) except Exception: log.exception("Failed to extract NTLM hashes from AUTHENTICATE_MESSAGE") diff --git a/dementor/protocols/quic.py b/dementor/protocols/quic.py index fb94a62..9169c61 100755 --- a/dementor/protocols/quic.py +++ b/dementor/protocols/quic.py @@ -36,7 +36,7 @@ from dementor.config.toml import TomlConfig, Attribute as A from dementor.config.session import SessionConfig -from dementor.log.logger import ProtocolLogger, ProtocolLoggerMixin, dm_logger +from dementor.log.logger import ProtocolLogger, dm_logger class QuicServerConfig(TomlConfig): @@ -72,7 +72,7 @@ def create_server_threads(session: SessionConfig): return servers -class QuicHandler(QuicConnectionProtocol, ProtocolLoggerMixin): +class QuicHandler(QuicConnectionProtocol): def __init__( self, config: SessionConfig, @@ -85,7 +85,7 @@ def __init__( self.config = config # stream_id -> (w, r) self.conn_data = {} - ProtocolLoggerMixin.__init__(self) + self.logger = self.proto_logger() def proto_logger(self) -> ProtocolLogger: return ProtocolLogger( diff --git a/dementor/protocols/smb.py b/dementor/protocols/smb.py index e0c720d..9d676ee 100644 --- a/dementor/protocols/smb.py +++ b/dementor/protocols/smb.py @@ -455,9 +455,6 @@ def smb1_negotiate_protocol(handler: "SMBHandler", packet: smb.NewSMBPacket) -> def smb1_session_setup(handler: "SMBHandler", packet: smb.NewSMBPacket) -> None: command = smb.SMBCommand(packet["Data"][0]) - handler.log_client(f"session setup: {command.fields}", "SMB_COM_SESSION_SETUP_ANDX") - # handler.send_data(packet.getData()) - # From [MS-SMB] # When extended security is being used (see section 3.2.4.2.4), the # request MUST take the following form diff --git a/dementor/protocols/smtp.py b/dementor/protocols/smtp.py index f5d1649..cd7c9e9 100755 --- a/dementor/protocols/smtp.py +++ b/dementor/protocols/smtp.py @@ -41,7 +41,6 @@ ) from aiosmtpd.controller import Controller -from impacket import ntlm from impacket.ntlm import ( NTLMAuthChallengeResponse, NTLMAuthNegotiate, @@ -53,7 +52,6 @@ from dementor.log.logger import ProtocolLogger, dm_logger from dementor.protocols.ntlm import ( NTLM_AUTH_CreateChallenge, - NTLM_AUTH_format_host, NTLM_report_auth, ATTR_NTLM_CHALLENGE, ATTR_NTLM_DISABLE_ESS, @@ -252,7 +250,7 @@ async def chapture_ntlm_auth(self, server: SMTPServerBase, blob=None) -> Any: if blob is None: # 4. The server sends the SMTP_NTLM_Supported_Response message, indicating that it can perform # NTLM authentication. - blob = server.challenge_auth(SMTP_NTLM_Supported_Response_Message) + blob = await server.challenge_auth(SMTP_NTLM_Supported_Response_Message) if blob is MISSING: # authentication failed await server.push("501 5.7.0 Auth aborted") @@ -260,10 +258,6 @@ async def chapture_ntlm_auth(self, server: SMTPServerBase, blob=None) -> Any: negotiate_message = NTLMAuthNegotiate() negotiate_message.fromString(blob) - self.logger.debug( - "Starting NTLM-auth: %s", - NTLM_AUTH_format_host(negotiate_message), - ) if self.server_config.smtp_fqdn.count(".") > 0: name, domain = self.server_config.smtp_fqdn.split(".", 1) @@ -326,7 +320,7 @@ def create_logger(self): } ) - async def start_server(self, controller, config: SessionConfig, smtp_config): + async def start_server(self, controller: Controller, config: SessionConfig, smtp_config): controller.port = smtp_config.smtp_port # NOTE: hostname on the controller points to the local address that will be diff --git a/dementor/servers.py b/dementor/servers.py index a067a04..3e0cd6a 100755 --- a/dementor/servers.py +++ b/dementor/servers.py @@ -17,56 +17,102 @@ # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. +# pyright: reportAny=false, reportExplicitAny=false import traceback import pathlib import socket import socketserver import threading import struct -import abc import ssl +import errno +import sys from io import StringIO -from typing import Any, Tuple +from typing import Any, ClassVar from socketserver import BaseRequestHandler +from typing_extensions import override from dementor import db from dementor.log import hexdump -from dementor.log.logger import ProtocolLoggerMixin, dm_logger +from dementor.log.logger import ProtocolLogger, dm_logger from dementor.log.stream import log_host from dementor.config.session import SessionConfig class ServerThread(threading.Thread): - def __init__(self, config: SessionConfig, server_class: type, *args, **kwargs): + """ + A thread-based server wrapper for running network protocol handlers. + + Provides graceful startup/shutdown and proper error handling. + + :param config: Session configuration object + :type config: SessionConfig + :param server_class: The server class to instantiate + :type server_class: type + :param args: Additional positional arguments for server_class + :type args: tuple[Any, ...] + :param kwargs: Additional keyword arguments for server_class + :type kwargs: dict[str, Any] + """ + + def __init__( + self, config: SessionConfig, server_class: type, *args: Any, **kwargs: Any + ) -> None: self.config: SessionConfig = config self.server_class: type = server_class - self.args = args + self.args: tuple[Any, ...] = args self.kwargs: dict[str, Any] = kwargs - super().__init__() + self._server: socketserver.BaseServer | None = None + super().__init__(daemon=False) @property def service_name(self) -> str: + """Get the service name from server class or use class name as fallback. + + :return: Service name. + :rtype: str + """ return getattr( self.server_class, "service_name", self.server_class.__name__, ) + @property + def server(self) -> socketserver.BaseServer: + """Get the server instance if it has been created. + + :return: Server instance. + :rtype: socketserver.BaseServer + """ + if not self._server: + raise ValueError("Server has not been initialized yet") + return self._server + + @override def run(self) -> None: - address = "" - port = "" + """Start and run the server indefinitely until shutdown is requested.""" + address: str = "" + port: int = 0 try: - self.server = self.server_class(self.config, *self.args, **self.kwargs) - address, port, *_ = self.server.server_address + self._server = self.server_class(self.config, *self.args, **self.kwargs) + address, port = self.server.server_address[:2] dm_logger.debug(f"Starting {self.service_name} Service on {address}:{port}") + + # Run server with periodic stop checks instead of blocking forever self.server.serve_forever() except OSError as e: - if e.errno == 13: + if e.errno == errno.EACCES: # Permission denied dm_logger.error( f"Failed to start server for {self.service_name}: Permission Denied!" ) + elif e.errno == errno.EADDRINUSE: # Address already in use + dm_logger.error( + f"Failed to start server for {self.service_name}: " + f"Address {address}:{port} already in use" + ) else: dm_logger.error( f"Failed to start server for {self.service_name} ({address}:{port}): {e}" @@ -75,27 +121,87 @@ def run(self) -> None: dm_logger.exception( f"Failed to start server for {self.service_name} ({address}:{port}): {e}" ) + finally: + self.shutdown() + def shutdown(self) -> None: + """Gracefully shutdown the server thread.""" + dm_logger.debug(f"Shutting down {self.service_name} Service") + if self._server is not None: + try: + self.server.shutdown() + except Exception as e: + dm_logger.warning(f"Error during {self.service_name} shutdown: {e}") + + +class BaseProtoHandler(BaseRequestHandler): + """Base handler for protocol-specific request processing. + + Provides common functionality for TCP/UDP protocol handlers including + data sending/receiving, client tracking, and exception handling. + """ -class BaseProtoHandler(BaseRequestHandler, ProtocolLoggerMixin): class TerminateConnection(Exception): + """Exception to signal handler should terminate the connection.""" + pass - def __init__(self, config: SessionConfig, request, client_address, server) -> None: - self.client_address = client_address - self.server = server + def __init__( + self, + config: SessionConfig, + request: socket.socket | tuple[bytes, socket.socket], + client_address: tuple[str, int], + server: socketserver.BaseServer, + ) -> None: + """Initialize the protocol handler. + + :param config: Session configuration + :type config: SessionConfig + :param request: Socket or (data, transport) tuple for UDP + :type request: socket.socket | tuple[bytes, socket.socket] + :param client_address: Client address tuple (host, port) + :type client_address: tuple[str, int] + :param server: Parent server instance + :type server: socketserver.BaseServer + """ + self.client_address: tuple[str, int] = client_address + self.server: socketserver.BaseServer = server self.config: SessionConfig = config - ProtocolLoggerMixin.__init__(self) - super().__init__(request, client_address, server) + self.logger: ProtocolLogger = self.proto_logger() + super(BaseProtoHandler, self).__init__(request, client_address, server) log_host(self.client_host) - self.config.db.add_host(self.client_host) + _ = self.config.db.add_host(self.client_host) + + def handle_data(self, data: bytes | None, transport: socket.socket) -> None: + """Process incoming protocol data. Must be implemented by subclasses. + + :param data: Received data bytes (None for TCP) + :type data: bytes | None + :param transport: Socket object for sending responses + :type transport: socket.socket + """ + raise NotImplementedError( + "handle_data must be implemented by protocol handlers" + ) - @abc.abstractmethod - def handle_data(self, data, transport) -> None: - pass + def proto_logger(self) -> ProtocolLogger: + """Return the :class:`ProtocolLogger` instance that will be exposed as ``self.logger``. + + Concrete classes typically return ``dm_logger`` or a subclass + customised for a specific protocol. + + :return: Protocol logger instance. + :rtype: ProtocolLogger + """ + raise NotImplementedError + @override def handle(self) -> None: - data = None + """Main request handler. Retrieves data and dispatches to handle_data(). + + Handles common exceptions and logs errors appropriately. + """ + data: bytes | None = None try: if isinstance(self.request, tuple): data, transport = self.request @@ -111,7 +217,8 @@ def handle(self) -> None: except TimeoutError: pass except OSError as e: - if e.errno not in (32, 104): # EPIPE, ECONNRESET + # Only log unexpected OS errors (not broken pipe/connection reset) + if e.errno not in (errno.EPIPE, errno.ECONNRESET): self.logger.exception(e) except Exception as e: self.logger.fail( @@ -127,17 +234,33 @@ def handle(self) -> None: ) def recv(self, size: int) -> bytes: + """Receive data from client. + + Handles both TCP and UDP sockets appropriately. + + :param size: Maximum bytes to receive + :type size: int + :return: Received data bytes + :rtype: bytes + """ if isinstance(self.request, tuple): - # UDP can't receive a single packet - # REVISIT: should we return this here? + # UDP: data already received in tuple data, transport = self.request self.request = (b"", transport) else: + # TCP: receive from socket data = self.request.recv(size) return data def send(self, data: bytes) -> None: + """Send data to client. + + Handles both TCP and UDP sockets appropriately. + + :param data: Bytes to send + :type data: bytes + """ if isinstance(self.request, tuple): _, transport = self.request transport.sendto(data, self.client_address) @@ -147,35 +270,94 @@ def send(self, data: bytes) -> None: @property def client_host(self) -> str: + """Get normalized client host address. + + :return: Normalized client host address. + :rtype: str + """ return db.normalize_client_address(self.client_address[0]) @property def client_port(self) -> int: + """Get client port number. + + :return: Client port number. + :rtype: int + """ return self.client_address[1] + @property + def server_port(self) -> int: + """Get server port number. + + :return: Server port number. + :rtype: int + """ + return self.server.server_address[1] + class BaseServerProtoHandler(BaseProtoHandler): + """Extended handler for protocol servers with protocol-specific configuration. + + Adds support for per-protocol configuration objects in addition to + the session-level configuration. + """ + def __init__( - self, config: SessionConfig, server_config, request, client_address, server + self, + config: SessionConfig, + server_config: Any, + request: socket.socket | tuple[bytes, socket.socket], + client_address: tuple[str, int], + server: socketserver.BaseServer, ) -> None: - self.server_config = server_config + """Initialize the server protocol handler. + + :param config: Session configuration + :type config: SessionConfig + :param server_config: Protocol-specific server configuration + :type server_config: Any + :param request: Socket or (data, transport) tuple + :type request: socket.socket | tuple[bytes, socket.socket] + :param client_address: Client address tuple + :type client_address: tuple[str, int] + :param server: Parent server instance + :type server: socketserver.BaseServer + """ + self.server_config: Any = server_config super().__init__(config, request, client_address, server) class ThreadingUDPServer(socketserver.ThreadingMixIn, socketserver.UDPServer): - default_port: int - default_handler_class: type + """Threaded UDP server with IPv6 support and cross-platform binding. + + :var default_port: Default port to listen on + :var default_handler_class: Handler class for processing requests + :var ipv4_only: Whether to only use IPv4 (skip IPv6) + """ + + default_port: ClassVar[int] + default_handler_class: ClassVar[type] ipv4_only: bool - allow_reuse_address = True + allow_reuse_address: bool = True def __init__( self, config: SessionConfig, - server_address: Tuple[str, int] | None = None, + server_address: tuple[str, int] | None = None, RequestHandlerClass: type | None = None, ) -> None: - self.config = config + """Initialize the UDP server. + + :param config: Session configuration + :type config: SessionConfig + :param server_address: (host, port) tuple or None to use defaults + :type server_address: tuple[str, int] | None + :param RequestHandlerClass: Handler class or None to use default + :type RequestHandlerClass: type | None + """ + self.config: SessionConfig = config self.ipv4_only = getattr(config, "ipv4_only", False) if config.ipv6 and not self.ipv4_only: self.address_family = socket.AF_INET6 @@ -185,35 +367,89 @@ def __init__( RequestHandlerClass or self.default_handler_class, ) + @override def server_bind(self) -> None: + """Bind the server socket with interface and IPv6 settings.""" bind_server(self, self.config) socketserver.UDPServer.server_bind(self) - def finish_request(self, request, client_address) -> None: + @override + def finish_request( # pyright: ignore[reportIncompatibleMethodOverride] + self, + request: bytes, + client_address: tuple[str, int], + ) -> None: + """Finish a single request by instantiating the handler. + + :param request: The request data + :type request: bytes + :param client_address: Client address tuple + :type client_address: tuple[str, int] + """ self.RequestHandlerClass(self.config, request, client_address, self) -def bind_server(server, session): - interface = session.interface.encode("ascii") + b"\x00" - server.socket.setsockopt(socket.SOL_SOCKET, socket.SO_BINDTODEVICE, interface) +def bind_server( + server: socketserver.TCPServer | socketserver.UDPServer, session: SessionConfig +) -> None: + """Configure socket options for interface binding and IPv6 behavior. + + Handles platform-specific socket options safely: + - SO_BINDTODEVICE (Linux only) + - IPV6_V6ONLY (IPv6 dual-stack behavior) + + :param server: Server instance with socket to configure + :type server: socketserver.TCPServer | socketserver.UDPServer + :param session: Session configuration with interface and IPv6 settings + :type session: SessionConfig + """ + # Platform-specific: SO_BINDTODEVICE only available on Linux + if sys.platform == "linux" and hasattr(session, "interface"): + try: + interface = (session.interface or "").encode("ascii") + b"\x00" + server.socket.setsockopt( + socket.SOL_SOCKET, socket.SO_BINDTODEVICE, interface + ) + except (OSError, AttributeError) as e: + dm_logger.warning(f"Failed to bind to interface '{session.interface}': {e}") + + # Configure IPv6 dual-stack behavior (affects both IPv4 and IPv6 traffic) if session.ipv6 and not getattr(session, "ipv4_only", False): - server.socket.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_V6ONLY, False) + try: + server.socket.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_V6ONLY, False) + except OSError as e: + dm_logger.warning(f"Failed to set IPV6_V6ONLY: {e}") class ThreadingTCPServer(socketserver.ThreadingMixIn, socketserver.TCPServer): - default_port: int - default_handler_class: type - ipv4_only: bool + """Threaded TCP server with IPv6 support and cross-platform binding. - allow_reuse_address = True + :var default_port: Default port to listen on + :var default_handler_class: Handler class for processing requests + :var ipv4_only: Whether to only use IPv4 (skip IPv6) + """ + + default_port: ClassVar[int] + default_handler_class: ClassVar[type] + ipv4_only: bool + allow_reuse_address: bool = True def __init__( self, config: SessionConfig, - server_address: Tuple[str, int] | None = None, + server_address: tuple[str, int] | None = None, RequestHandlerClass: type | None = None, ) -> None: - self.config = config + """Initialize the TCP server. + + :param config: Session configuration + :type config: SessionConfig + :param server_address: (host, port) tuple or None to use defaults + :type server_address: tuple[str, int] | None + :param RequestHandlerClass: Handler class or None to use default + :type RequestHandlerClass: type | None + """ + self.config: SessionConfig = config self.ipv4_only = getattr(config, "ipv4_only", False) if config.ipv6 and not self.ipv4_only: self.address_family = socket.AF_INET6 @@ -222,44 +458,111 @@ def __init__( RequestHandlerClass or self.default_handler_class, ) + @override def server_bind(self) -> None: + """Bind the server socket with interface and IPv6 settings.""" bind_server(self, self.config) socketserver.TCPServer.server_bind(self) - def finish_request(self, request, client_address) -> None: + @override + def finish_request( # pyright: ignore[reportIncompatibleMethodOverride] + self, + request: socket.socket, + client_address: tuple[str, int], + ) -> None: + """Finish a single request by instantiating the handler. + + :param request: Connected socket + :type request: socket.socket + :param client_address: Client address tuple + :type client_address: tuple[str, int] + """ self.RequestHandlerClass(self.config, request, client_address, self) def create_tls_context( - server_config, server=None, force=False + server_config: Any, + server: socketserver.BaseServer | None = None, + force: bool = False, ) -> ssl.SSLContext | None: + """Create an SSL/TLS context from server configuration. + + :param server_config: Configuration object with use_ssl, certfile, keyfile attributes + :type server_config: Any + :param server: Optional server instance for logging service name + :type server: socketserver.BaseServer | None + :param force: Force SSL context creation even if use_ssl is False + :type force: bool + :return: Configured SSLContext or None if SSL not needed or files missing + :rtype: ssl.SSLContext | None + + .. note:: Logs errors if certificate or key files not found. + """ if getattr(server_config, "use_ssl", False) or force: # if defined use ssl cert_path = pathlib.Path(str(getattr(server_config, "certfile", None))) key_path = pathlib.Path(str(getattr(server_config, "keyfile", None))) if not cert_path.exists() or not key_path.exists(): - service_name = getattr(server, "service_name", "") + service_name = getattr(server, "service_name", "") dm_logger.error( f"({service_name}) Certificate or key file not found: " - f"Cert={cert_path} " - f"Key={key_path}" + + f"Cert={cert_path} " + + f"Key={key_path}" ) - return + return None ssl_context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) ssl_context.load_cert_chain(certfile=cert_path, keyfile=key_path) return ssl_context - -def add_mcast_membership(target, session, group4=None, group6=None, ttl=255): + return None + + +def add_mcast_membership( + target: socket.socket, + session: SessionConfig, + group4: str | None = None, + group6: str | None = None, + ttl: int = 255, +) -> None: + """Add multicast group memberships to a socket. + + Handles both IPv4 and IPv6 multicast with platform-specific behavior. + IPv6 multicast requires interface support (not available on Windows). + + :param target: Socket to configure + :type target: socket.socket + :param session: Session configuration with interface and IP info + :type session: SessionConfig + :param group4: IPv4 multicast group address (e.g., "224.0.0.1") + :type group4: str | None + :param group6: IPv6 multicast group address (e.g., "ff02::1") + :type group6: str | None + :param ttl: Time-to-live for multicast packets (default 255) + :type ttl: int + + .. note:: Logs warnings for IPv6 multicast failures on Windows. + """ + # Set TTL for all multicast packets target.setsockopt(socket.IPPROTO_IP, socket.IP_MULTICAST_TTL, 2) + # IPv4 multicast if session.ipv4 and group4: - mreq = socket.inet_aton(group4) + socket.inet_aton(session.ipv4) - target.setsockopt(socket.IPPROTO_IP, socket.IP_ADD_MEMBERSHIP, mreq) - target.setsockopt(socket.IPPROTO_IP, socket.IP_MULTICAST_LOOP, 1) + try: + mreq = socket.inet_aton(group4) + socket.inet_aton(session.ipv4) + target.setsockopt(socket.IPPROTO_IP, socket.IP_ADD_MEMBERSHIP, mreq) + target.setsockopt(socket.IPPROTO_IP, socket.IP_MULTICAST_LOOP, 1) + except OSError as e: + dm_logger.warning(f"Failed to join IPv4 multicast group {group4}: {e}") + # IPv6 multicast (requires if_nametoindex, not available on Windows) if session.ipv6 and group6: - mreq = socket.inet_pton(socket.AF_INET6, group6) - mreq += struct.pack("@I", socket.if_nametoindex(session.interface)) - target.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_JOIN_GROUP, mreq) - target.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_MULTICAST_LOOP, 1) + try: + mreq = socket.inet_pton(socket.AF_INET6, group6) + mreq += struct.pack("@I", socket.if_nametoindex(session.interface)) + target.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_JOIN_GROUP, mreq) + target.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_MULTICAST_LOOP, 1) + except (OSError, AttributeError) as e: + if sys.platform == "win32": + dm_logger.debug(f"IPv6 multicast not fully supported on Windows: {e}") + else: + dm_logger.warning(f"Failed to join IPv6 multicast group {group6}: {e}") diff --git a/docs/source/compat.rst b/docs/source/compat.rst index 023e7f7..bcc9555 100644 --- a/docs/source/compat.rst +++ b/docs/source/compat.rst @@ -27,7 +27,7 @@ in development. The legend for each symbol is as follows: Supported Protocols Responder (3.2.2.0) - Dementor (1.0.0.dev18) + Dementor (1.0.0.dev19) @@ -188,42 +188,6 @@ in development. The legend for each symbol is as follows: NetNTLMv2 - - Dummy LM filtering - - - LM dedup filtering - - - Anonymous detection - - - Flag mirroring - - - NetNTLMv2 threshold (≥ 48 B) - - - AV_PAIRS correctness - - - Hash label accuracy - - - Configurable challenge - - - SPNEGO unwrapping - - - Non-NTLM mech redirect - - - ESS configurable - - - NetNTLMv2 configurable - @@ -243,42 +207,6 @@ in development. The legend for each symbol is as follows: - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - @@ -298,42 +226,6 @@ in development. The legend for each symbol is as follows: - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - @@ -794,13 +686,13 @@ in development. The legend for each symbol is as follows:

SMB Features

- - - - - - - + + + + + + + @@ -836,3 +728,89 @@ in development. The legend for each symbol is as follows:
FeatureResponder (3.2.2.0)Dementor (1.0.0.dev18)
FeatureResponder (3.2.2.0)Dementor (1.0.0.dev19)
Tree Connect

[1]: Responder combines NetNTLMv1 and NetNTLMv1-ESS under a single "NTLMv1-SSP" label. This is not incorrect — hashcat -m 5500 handles both — but Dementor distinguishes them for more granular reporting. Applies to all NTLM-capable protocols (SMB, HTTP, MSSQL, LDAP, DCE/RPC).

+ +

NTLM Spcifics

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
FeatureResponder (3.2.2.0)Dementor (1.0.0.dev19)
Dummy LM filtering
LM dedup filtering
Anonymous detection
Flag mirroring
NetNTLMv2 threshold (≥ 48 B)
AV_PAIRS correctness
Hash label accuracy
Configurable challenge
SPNEGO unwrapping
Non-NTLM mech redirect
ESS configurable
NetNTLMv2 configurable
+ diff --git a/docs/source/conf.py b/docs/source/conf.py index e8d6e60..e73fe55 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -9,7 +9,7 @@ project = "Dementor" copyright = "2025-Present, MatrixEditor" author = "MatrixEditor" -release = "1.0.0.dev18" +release = "1.0.0.dev19" # -- General configuration --------------------------------------------------- # https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration @@ -53,6 +53,8 @@ "source_type": "github", "source_user": "MatrixEditor", "source_repo": "dementor", + "source_version": "master", + "source_docs_path": "/docs/source/", } html_theme_options = { diff --git a/docs/source/config/http.rst b/docs/source/config/http.rst index cbf8666..d8e8081 100644 --- a/docs/source/config/http.rst +++ b/docs/source/config/http.rst @@ -135,20 +135,31 @@ Section ``[HTTP]`` Determines whether access to the WPAD script requires authentication. .. py:attribute:: Server.ExtendedSessionSecurity - :type: bool :value: true + :type: bool + + .. versionremoved:: 1.0.0.dev19 + **Deprecated**: renamed to :attr:`DisableExtendedSessionSecurity` + + .. py:attribute:: Server.DisableExtendedSessionSecurity + :value: false + :type: bool - *Maps to* :attr:`http.HTTPServerConfig.http_ess`. *May also be set in* ``[HTTP]`` + *Linked to* :attr:`http.HTTPServerConfig.ntlm_disable_ess` .. versionchanged:: 1.0.0.dev5 Internal mapping changed from ``http_ess`` to ``ntlm_ess`` + .. versionchanged:: 1.0.0.dev19 + Renamed from ``ExtendedSessionSecurity`` to explicit ``DisableExtendedSessionSecurity`` + + Enables Extended Session Security (ESS) for NTLM authentication. With ESS, NetNTLMv1-ESS/NetNTLMv2 hashes are captured instead of raw NTLM hashes. Resolution precedence: - 1. :attr:`HTTP.Server.ExtendedSessionSecurity` (per-instance) - 2. :attr:`HTTP.ExtendedSessionSecurity` (global HTTP fallback) - 3. :attr:`NTLM.ExtendedSessionSecurity` (final fallback) + 1. :attr:`HTTP.Server.DisableExtendedSessionSecurity` (per-instance) + 2. :attr:`HTTP.DisableExtendedSessionSecurity` (global HTTP fallback) + 3. :attr:`NTLM.DisableExtendedSessionSecurity` (final fallback) .. py:attribute:: Server.Challenge :type: str diff --git a/docs/source/config/imap.rst b/docs/source/config/imap.rst index ba661d5..d586e3b 100644 --- a/docs/source/config/imap.rst +++ b/docs/source/config/imap.rst @@ -104,18 +104,28 @@ Section ``[IMAP]`` Specifies the path to the private key file associated with the TLS certificate. - .. py:attribute:: ExtendedSessionSecurity - :type: bool + .. py:attribute:: Server.ExtendedSessionSecurity :value: true + :type: bool + + .. versionremoved:: 1.0.0.dev19 + **Deprecated**: renamed to :attr:`DisableExtendedSessionSecurity` + + .. py:attribute:: Server.DisableExtendedSessionSecurity + :value: false + :type: bool + + *Linked to* :attr:`imap.IMAPServerConfig.ntlm_disable_ess` - *Maps to* :attr:`imap.IMAPServerConfig.ntlm_ess`. + .. versionchanged:: 1.0.0.dev19 + Renamed from ``ExtendedSessionSecurity`` to explicit ``DisableExtendedSessionSecurity`` Enables NTLM Extended Session Security (ESS). When enabled, NetNTLMv1-ESS/NetNTLMv2 hashes are captured instead of raw NTLM hashes. Resolution precedence: - 1. :attr:`IMAP.ExtendedSessionSecurity` - 2. :attr:`NTLM.ExtendedSessionSecurity` (fallback) + 1. :attr:`IMAP.DisableExtendedSessionSecurity` + 2. :attr:`NTLM.DisableExtendedSessionSecurity` (fallback) .. py:attribute:: Challenge :type: str diff --git a/docs/source/config/mssql.rst b/docs/source/config/mssql.rst index efc18ef..362c823 100644 --- a/docs/source/config/mssql.rst +++ b/docs/source/config/mssql.rst @@ -52,19 +52,29 @@ Section ``[MSSQL]`` via :attr:`SSRP.InstanceName`. .. py:attribute:: ExtendedSessionSecurity - :type: bool :value: true + :type: bool + + .. versionremoved:: 1.0.0.dev19 + **Deprecated**: renamed to :attr:`DisableExtendedSessionSecurity` - *Maps to* :attr:`mssql.MSSQLConfig.ntlm_ess` +.. py:attribute:: DisableExtendedSessionSecurity + :type: bool + :value: false + + *Maps to* :attr:`mssql.MSSQLConfig.ntlm_disable_ess` .. versionchanged:: 1.0.0.dev5 Internal mapping changed frmo ``mssql_ess`` to ``ntlm_ess`` + .. versionchanged:: 1.0.0.dev19 + Renamed from ``ExtendedSessionSecurity`` to explicit ``DisableExtendedSessionSecurity`` + Enables NTLM Extended Session Security (ESS). When enabled, NetNTLMv1-ESS/NetNTLMv2 hashes are captured instead of raw NTLM hashes. Resolution precedence: - 1. :attr:`MSSQL.ExtendedSessionSecurity` - 2. :attr:`NTLM.ExtendedSessionSecurity` (fallback) + 1. :attr:`MSSQL.DisableExtendedSessionSecurity` + 2. :attr:`NTLM.DisableExtendedSessionSecurity` (fallback) .. py:attribute:: Challenge :type: str diff --git a/docs/source/config/ntlm.rst b/docs/source/config/ntlm.rst index defbbb1..77fd1f8 100644 --- a/docs/source/config/ntlm.rst +++ b/docs/source/config/ntlm.rst @@ -14,6 +14,9 @@ Section ``[NTLM]`` *Linked to* :attr:`config.SessionConfig.ntlm_challenge` + .. versionchanged:: 1.0.0.dev19 + The challenge now accepts different configuration formats. + Specifies the NTLM ServerChallenge nonce sent in the ``CHALLENGE_MESSAGE``. The value must represent exactly ``8`` bytes and can be given in any of the following formats: @@ -62,12 +65,22 @@ Section ``[NTLM]`` Target Info Version 255.255 (Build 65535); NTLM Current Revision 255 +.. py:attribute:: ExtendedSessionSecurity + :value: true + :type: bool + + .. versionremoved:: 1.0.0.dev19 + **Deprecated**: renamed to :attr:`DisableExtendedSessionSecurity` + .. py:attribute:: DisableExtendedSessionSecurity :value: false :type: bool *Linked to* :attr:`config.SessionConfig.ntlm_disable_ess` + .. versionchanged:: 1.0.0.dev19 + Renamed from ``ExtendedSessionSecurity`` to explicit ``DisableExtendedSessionSecurity`` + When ``true``, strips the ``NTLMSSP_NEGOTIATE_EXTENDED_SESSIONSECURITY`` flag from the ``CHALLENGE_MESSAGE``, preventing ESS negotiation. @@ -86,7 +99,6 @@ Section ``[NTLM]`` .. note:: Dementor detects ESS from the ``LmChallengeResponse`` byte structure - (``len == 24`` and ``LM[8:24] == Z(16)`` per ``[MS-NLMP §3.3.1]``) rather than solely from the flag, so classification is accurate even when this setting is toggled. @@ -107,14 +119,14 @@ Section ``[NTLM]`` - ``true`` — ``TargetInfoFields`` is empty. Without it, clients cannot build the NTLMv2 blob per ``[MS-NLMP §3.3.2]``. - LmCompatibilityLevel 0–2 clients fall back to NTLMv1. + LmCompatibilityLevel 0-2 clients fall back to NTLMv1. Level 3+ clients (all modern Windows) will **fail authentication** and produce **zero captured hashes**. .. warning:: This setting is almost never needed. Clients at - ``LmCompatibilityLevel`` 0–2 already send **NTLMv1 unconditionally** + ``LmCompatibilityLevel`` 0-2 already send **NTLMv1 unconditionally** and will never send NTLMv2 regardless of whether ``TargetInfoFields`` is present. This option therefore only affects level 3+ clients (all modern Windows defaults), which **require** ``TargetInfoFields`` to @@ -126,9 +138,6 @@ Section ``[NTLM]`` Protocol Behaviour ------------------ -Three-Message Handshake -~~~~~~~~~~~~~~~~~~~~~~~~ - Dementor acts as a **capture server**, not an authentication server. Per ``[MS-NLMP §1.3.1.1]``, the handshake proceeds as follows: @@ -138,16 +147,13 @@ Dementor acts as a **capture server**, not an authentication server. Per | | |--- NEGOTIATE_MESSAGE ---------------► | inspect client flags |◄-- CHALLENGE_MESSAGE ---------------- | Dementor controls entirely - |--- AUTHENTICATE_MESSAGE -----------► | extract & store hashes + |--- AUTHENTICATE_MESSAGE ------------► | extract & store hashes | | Dementor does not verify responses, compute session keys, or participate in signing or sealing. The connection is terminated (or returned to the calling protocol handler) immediately after the ``AUTHENTICATE_MESSAGE`` is received. -Hash Type Classification -~~~~~~~~~~~~~~~~~~~~~~~~~ - Four hash types are extracted, classified from the ``AUTHENTICATE_MESSAGE`` using NT and LM response byte structure per ``[MS-NLMP §3.3]``. The ESS flag is cross-checked but the **byte structure is authoritative**: @@ -172,19 +178,12 @@ is cross-checked but the **byte structure is authoritative**: - > 24 bytes - n/a - ``-m 5600`` - * - ``NetLMv2`` + * - ``LMv2`` - > 24 bytes † - 24 bytes, non-null - ``-m 5600`` -† NetLMv2 is always paired with NetNTLMv2 and uses the same hashcat mode. - -ESS detection relies on ``len(LM) == 24 and LM[8:24] == Z(16)`` per §3.3.1. -This is reliable even when :attr:`DisableExtendedSessionSecurity` is toggled — -the flag is supplementary, not authoritative. - -Hashcat Output Formats -~~~~~~~~~~~~~~~~~~~~~~~ +† LMv2 is always paired with NetNTLMv2 and uses the same hashcat mode. Each captured hash is written in hashcat-compatible format: @@ -221,7 +220,7 @@ The ``CHALLENGE_MESSAGE`` is built directly from the client's content is not verified by clients per §2.2.2.10. AV_PAIRS (``TargetInfoFields``) -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ When :attr:`DisableNTLMv2` is ``false`` (the default), ``TargetInfoFields`` is populated with AV_PAIRs per @@ -291,7 +290,7 @@ response unless the client set ``LmChallengeResponse`` to ``Z(24)``. Clients only send ``Z(24)`` here when the server included ``MsvAvTimestamp`` (``0x0007``) in the ``CHALLENGE_MESSAGE``, which instructs them to suppress the LM slot. Dementor intentionally omits ``MsvAvTimestamp``, so this suppression -never occurs and both NetNTLMv2 and NetLMv2 are always captured. +never occurs and both NetNTLMv2 and LMv2 are always captured. Anonymous Authentication ~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -331,35 +330,11 @@ Default Configuration # Omit TargetInfoFields (AV_PAIRS) from CHALLENGE_MESSAGE. # false (default): NetNTLMv2 + NetLMv2 captured from all modern clients. - # true: Level 0–2 clients fall back to NTLMv1; level 3+ clients + # true: Level 0-2 clients fall back to NTLMv1; level 3+ clients # (all modern Windows) will refuse and produce NO captures. DisableNTLMv2 = false -Capture Prerequisites ---------------------- - -For a hash to be written to the session database, the following conditions -must all hold once Dementor receives the ``AUTHENTICATE_MESSAGE``: - -1. **Non-anonymous** — the token must not carry the - ``NTLMSSP_NEGOTIATE_ANONYMOUS`` flag and must include a non-empty - ``UserName`` and a non-null ``NtChallengeResponse``. -2. **Parseable** — the NTLM token must be structurally valid; any parse - error causes the token to be treated as anonymous and discarded. -3. **NT response present** — the ``NtChallengeResponse`` field must be - non-empty; a missing or zero-length NT response means there is nothing - to capture. - -The LM slot in a NetNTLMv1 hashcat line is additionally subject to the three -filtering conditions described under `LM Response Filtering`_. Filtering the -LM slot does **not** discard the capture — the NT response is still written; -only the LM field is omitted from the output line. - -When a capture is discarded Dementor logs the event but writes nothing to -disk. - - LmCompatibilityLevel Reference -------------------------------- @@ -402,7 +377,7 @@ to the hash type Dementor captures and the relevant hashcat mode. .. note:: - Windows Vista and later default to **level 3**. Levels 0–2 are only + Windows Vista and later default to **level 3**. Levels 0-2 are only found on legacy systems or when explicitly downgraded via Group Policy. Leave :attr:`DisableNTLMv2` at ``false`` (the default) to capture hashes from clients at any level. diff --git a/docs/source/config/smb.rst b/docs/source/config/smb.rst index 02e606e..45d180e 100644 --- a/docs/source/config/smb.rst +++ b/docs/source/config/smb.rst @@ -103,6 +103,12 @@ Section ``[SMB]`` .. seealso:: :attr:`NTLM.Challenge` for accepted formats and behaviour. + .. py:attribute:: Server.ExtendedSessionSecurity + :value: true + :type: bool + + .. versionremoved:: 1.0.0.dev19 + **Deprecated**: renamed to :attr:`DisableExtendedSessionSecurity` .. py:attribute:: Server.DisableExtendedSessionSecurity :type: bool @@ -110,6 +116,9 @@ Section ``[SMB]`` *Linked to* :attr:`smb.SMBServerConfig.ntlm_disable_ess`. *Can also be set in* ``[SMB]`` + .. versionchanged:: 1.0.0.dev19 + Renamed from ``ExtendedSessionSecurity`` to explicit ``DisableExtendedSessionSecurity`` + Per-SMB override for :attr:`NTLM.DisableExtendedSessionSecurity`. When set in ``[SMB]`` it applies to every ``[[SMB.Server]]`` instance; when set inside a single ``[[SMB.Server]]`` block it applies only to that port. Falls back to diff --git a/pyproject.toml b/pyproject.toml index 007ffaf..e1e3f9d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [project] name = "dementor" -version = "1.0.0.dev18" -license = { file = "LICENSE" } +version = "1.0.0.dev19" +license-files = ["LICENSE"] requires-python = ">=3.10" description = "LLMNR/NBT-NS/mDNS Poisoner and rogue service provider" @@ -12,7 +12,6 @@ classifiers = [ "Development Status :: 4 - Beta", "Intended Audience :: Developers", "Intended Audience :: Science/Research", - "License :: OSI Approved :: MIT License", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.13", "Programming Language :: Python :: 3 :: Only",