diff --git a/docs/design-notes.md b/docs/design-notes.md index baf0d095..e1c334e0 100644 --- a/docs/design-notes.md +++ b/docs/design-notes.md @@ -48,3 +48,33 @@ working while the new one is evaluated. The corollary is the reason it was needed: **an unpinned image is a scheduled outage.** `:latest` plus watchtower rolled Paperless from 2.x to 3.0.2 unattended and broke filing across the whole e2e suite. + +## Two rot vectors in one URL, and a hook that stopped halfway (2026-08-01) + +The Mac took a new DHCP lease. The vault clone's `origin` was +`http://@:42040/family/memory.git`, so every host-side +git call hung until its timeout. `on_start_ready` was mid-run when the +`TimeoutExpired` escaped, which is why `family/brain` was never created, +which is why the curator's brain push returned 403 on every cycle from +then on. The embedded token had separately expired. The container plane +never noticed any of it: its remote is `stack-code:3000`, a service name. + +What the fix encodes: host-side remotes use loopback and the published +port, remote URLs are re-derived on every start (both halves rot, and +nothing else refreshes them), git never raises a timeout at a caller, +and a wedged sync recovers by policy - source preserves local commits, +the projection may realign freely. + +**Not built, worth deciding: hook steps that create durable resources +should be independently re-runnable rather than sequential-and-abort.** +`on_start_ready` is one function where step 3 creating `family/brain` +depends on step 1 finishing, so an unrelated failure upstream silently +skips it and the only symptom is a line in `stack up` output. Hooks are +already required to be idempotent, which is most of the way there; what +is missing is that a hook is one all-or-nothing block. A shape worth +weighing: let a hook declare independent steps, run each, report per +step, and fail the hook without skipping the ones that would have +succeeded. The cost is a framework concept where today there is a plain +function, so it needs to earn its place - but "a resource nobody created +and nobody noticed" is the second time this pattern has cost a debugging +session. diff --git a/stacklets/memory/bot/curator.py b/stacklets/memory/bot/curator.py index eca6a98f..0c31f4c3 100644 --- a/stacklets/memory/bot/curator.py +++ b/stacklets/memory/bot/curator.py @@ -59,9 +59,15 @@ from loguru import logger # noqa: E402 from memory.lib import ( # noqa: E402 + PRESERVE_LOCAL, + RESET_LOCAL, + SyncResult, _parse_frontmatter, authenticated_remote, brain_remote_url, + is_auth_failure, + reconcile_with_remote, + run_git, vault_remote_url, ) @@ -420,15 +426,71 @@ def nightly_due(nightly: str, last_run_date: str, now_local: time.struct_time) - return (now_local.tm_hour, now_local.tm_min) >= (hour, minute) +# ── Remotes and sync reporting ─────────────────────────────────────────── + + +def curator_remote(build_url) -> str | None: + """Build a container-plane remote URL from the environment, or None. + + Read fresh on every call rather than captured at boot: this is also + the recovery path for a credential Forgejo has since rejected, and + a value cached at startup can only ever hand back the rejected one. + The service name in `CODE_URL` is what makes the container plane + immune to the DHCP lease that broke the host plane. + """ + code_url = os.environ.get("CODE_URL", "") + admin_user = os.environ.get("MATRIX_ADMIN_USER", "") + admin_password = os.environ.get("MATRIX_ADMIN_PASSWORD", "") + if not (code_url and admin_user and admin_password): + return None + return authenticated_remote(build_url(code_url), admin_user, admin_password) + + +# What a reconcile outcome means for whoever is reading the logs. The +# healthy statuses say nothing at all; everything else is the data +# plane failing to make progress, and this incident is what happens +# when that is filed under DEBUG: a sync that had not worked in weeks, +# retrying quietly every 30 seconds, with no line anywhere to show it. +_SYNC_REPORT = { + "up_to_date": (None, ""), + "fast_forwarded": (None, ""), + "ahead": (None, ""), + "pushed": ("info", "{label}: pushed local commits to Forgejo"), + "rebased": ("warning", "{label}: local commits replayed onto the remote and pushed"), + "rebased_unpushed": ("warning", "{label}: local commits replayed onto the remote but the push failed ({detail})"), + "push_failed": ("warning", "{label}: local commits could not be pushed ({detail})"), + "reset_to_remote": ("warning", "{label}: realigned to the remote (projection, rebuilt from source)"), + "preserved_and_reset": ("warning", "{label}: unrelated history, local history kept on branch {detail} and the working copy reset to the remote"), + "unreachable": ("warning", "{label}: Forgejo unreachable, sync is not making progress ({detail})"), + "auth_failed": ("error", "{label}: Forgejo rejected the credentials, sync is stuck until they are refreshed ({detail})"), + "failed": ("error", "{label}: git refused the recovery ({detail})"), +} + + +def report_sync(label: str, result: SyncResult) -> SyncResult: + """Log a reconcile outcome at a level that matches its severity.""" + level, template = _SYNC_REPORT.get( + result.status, ("warning", "{label}: unexpected sync status {detail}"), + ) + if level is not None: + getattr(logger, level)( + "[curator] " + + template.format(label=label, detail=result.detail or result.status), + ) + return result + + # ── Git plumbing ───────────────────────────────────────────────────────── class Vault: - """Read-only git view of the vault working copy. - - The wiki container owns the `git pull`; we only ever read. The - bind-mounted repo belongs to the host user, so git's dubious- - ownership check is satisfied via a private GIT_CONFIG_GLOBAL - rather than touching any shared config. + """The curator's git view of the vault working copy. + + Reads only, with one exception: `sync` may replay local commits + onto the remote to get out of a divergence. It never authors vault + content — memory's writers stay the archivist, the CLI, and humans + (ADR-011). The bind-mounted repo belongs to the host user, so git's + dubious-ownership check is satisfied via a private + GIT_CONFIG_GLOBAL rather than touching any shared config. """ def __init__(self, path: Path): @@ -452,28 +514,48 @@ async def head(self) -> str | None: out = await asyncio.to_thread(self._run, "rev-parse", "HEAD") return out.strip() if out else None - async def sync(self) -> None: - """Fast-forward the working copy from Forgejo. - - Same cheap shape the wiki entrypoint used before the curator - took ownership: an idle tick is one `ls-remote` ref query, the - fetch + ff happens only when the remote actually moved. Never - fatal — Forgejo briefly unreachable means the tick is skipped - and everything keeps serving what is on disk. + async def sync(self) -> SyncResult: + """Reconcile the working copy with Forgejo, recovering if wedged. + + The vault is the database (ADR-011), so its policy is + `PRESERVE_LOCAL`: a local-only commit may be a todo tick, which + exists nowhere else, and is replayed onto the remote rather + than dropped. Only a history with no merge base at all — the + remote repo re-created underneath us — makes the working copy + step aside, and then onto a branch that keeps every commit. + + One fetch per tick replaces the old `ls-remote` probe. It costs + the same ref exchange when there is nothing new, and having the + remote's objects already in hand is what lets a recovery decide + and act without a second round trip. + + Never fatal. Forgejo briefly unreachable means the tick is + skipped and everything keeps serving what is on disk. It is no + longer *silent*, though: anything short of progress is logged + where an operator sees it. """ - def _sync() -> None: - # Same tick shape as the old wiki entrypoint loop: skip on - # any read failure, compare refs, pull only on real change. - local = self._run("rev-parse", "HEAD") - if not local: - return - remote = self._run("ls-remote", CURATOR_REMOTE, "HEAD") - remote_head = remote.split()[0] if remote and remote.split() else "" - if not remote_head or local.strip() == remote_head: - return - self._run("pull", "--quiet", "--ff-only", CURATOR_REMOTE, "main") - - await asyncio.to_thread(_sync) + return await asyncio.to_thread(self._sync) + + def _sync(self) -> SyncResult: + result = self._reconcile() + if result.status == "auth_failed" and self._refresh_remote(): + # The rejected credential may simply be the one we cached at + # boot. Re-derive from the current environment and try once + # more, then stop: a second failure is a real one. + result = self._reconcile() + return report_sync("vault", result) + + def _reconcile(self) -> SyncResult: + return reconcile_with_remote( + self.path, CURATOR_REMOTE, recovery=PRESERVE_LOCAL, env=self._env, + ) + + def _refresh_remote(self) -> bool: + url = curator_remote(vault_remote_url) + if not url: + return False + self.ensure_remote(CURATOR_REMOTE, url) + return True def ensure_remote(self, name: str, url: str) -> None: """Idempotently point the named remote at `url` (add on first boot).""" @@ -532,17 +614,14 @@ def __init__(self, path: Path, source: Path): self.source = source self._env = {**os.environ, "GIT_CONFIG_GLOBAL": "/tmp/curator-gitconfig"} - def _run(self, *args: str) -> tuple[int, str]: - result = subprocess.run( - ["git", "-C", str(self.path), *args], - capture_output=True, text=True, env=self._env, - ) - if result.returncode != 0: - logger.debug("[curator] brain git {} failed: {}", args[0], result.stderr.strip()) - return result.returncode, result.stdout + def _run(self, *args: str) -> tuple[int, str, str]: + rc, out, err = run_git(self.path, *args, env=self._env) + if rc != 0: + logger.debug("[curator] brain git {} failed: {}", args[0], err) + return rc, out, err async def tracked_files(self) -> list[str]: - _, out = await asyncio.to_thread(self._run, "ls-files") + _, out, _ = await asyncio.to_thread(self._run, "ls-files") return [p for p in out.splitlines() if p.strip()] def frontmatter_at(self, path: str) -> dict: @@ -586,9 +665,9 @@ def _commit_push(self, message: str) -> bool: commit, which is still a success).""" self._run("add", "-A") # `diff --cached --quiet` exits 1 when staged changes exist. - code, _ = self._run("diff", "--cached", "--quiet") + code, _, _ = self._run("diff", "--cached", "--quiet") if code != 0: - rc, _ = self._run( + rc, _, _ = self._run( "-c", f"user.name={_BRAIN_AUTHOR_NAME}", "-c", f"user.email={_BRAIN_AUTHOR_EMAIL}", "commit", "-m", message, @@ -598,25 +677,65 @@ def _commit_push(self, message: str) -> bool: # Push even with nothing newly committed: a prior cycle may have # committed locally and lost the push (Forgejo briefly down), and # "nothing to commit" must not report that state as in-sync. - rc, _ = self._run("push", "--quiet", CURATOR_REMOTE, "main") - if rc != 0: - # Brain is a disposable, single-writer projection (ADR-011). - # A diverged remote is residue of a retired second writer or - # a wiped clone — the working copy is the truth, overwrite. - rc, _ = self._run("push", "--quiet", "--force", CURATOR_REMOTE, "main") + rc, _, err = self._run("push", "--quiet", CURATOR_REMOTE, "main") + if rc == 0: + return True + if is_auth_failure(err) and self._refresh_remote(): + rc, _, err = self._run("push", "--quiet", CURATOR_REMOTE, "main") if rc == 0: - logger.warning("[curator] brain remote diverged — overwrote (projection is disposable)") - return rc == 0 + return True + # No force-push here any more. Forcing was indiscriminate: it + # fired on every failure, including the 403 from a `family/brain` + # that had never been created, where it could not help and its + # "remote diverged" line actively hid the real cause. A refused + # push now says why, and a genuinely diverged remote is handled + # by `sync` at the top of the next cycle, which realigns and + # re-projects instead of overwriting history. + logger.error("[curator] brain push failed, projection is not reaching Forgejo: {}", err) + return False def ensure_remote(self, name: str, url: str) -> None: """Idempotently point the named remote at `url` (add on first boot).""" - code, _ = self._run("remote", "set-url", name, url) + code, _, _ = self._run("remote", "set-url", name, url) if code != 0: self._run("remote", "add", name, url) + def _refresh_remote(self) -> bool: + url = curator_remote(brain_remote_url) + if not url: + return False + self.ensure_remote(CURATOR_REMOTE, url) + return True + async def commit_push(self, message: str) -> bool: return await asyncio.to_thread(self._commit_push, message) + async def sync(self) -> SyncResult: + """Reconcile the projection with its remote before rebuilding it. + + Brain is machine-owned and regenerable (ADR-011), so its policy + is `RESET_LOCAL`: if the remote holds commits this copy does not, + or the repo was re-created and shares no history at all, the + remote is simply taken as the new base. Nothing is preserved + because nothing here is irreplaceable; the caller re-projects + from memory on top. + + Local commits that are merely ahead are left alone — those are + last cycle's projection waiting on a push, not a divergence. + """ + return await asyncio.to_thread(self._sync) + + def _sync(self) -> SyncResult: + result = self._reconcile() + if result.status == "auth_failed" and self._refresh_remote(): + result = self._reconcile() + return report_sync("brain", result) + + def _reconcile(self) -> SyncResult: + return reconcile_with_remote( + self.path, CURATOR_REMOTE, recovery=RESET_LOCAL, env=self._env, + ) + # ── Rebuild ────────────────────────────────────────────────────────────── @@ -681,14 +800,11 @@ async def main() -> None: # the host CLI set and use it); it is unreachable from inside this # container. Give the curator its own remote on the stack network. # Auth: the unified stack admin is also Forgejo's admin. - code_url = os.environ.get("CODE_URL", "") - admin_user = os.environ.get("MATRIX_ADMIN_USER", "") - admin_password = os.environ.get("MATRIX_ADMIN_PASSWORD", "") - if code_url and admin_user and admin_password: - vault.ensure_remote(CURATOR_REMOTE, authenticated_remote( - vault_remote_url(code_url), admin_user, admin_password)) - brain.ensure_remote(CURATOR_REMOTE, authenticated_remote( - brain_remote_url(code_url), admin_user, admin_password)) + vault_url = curator_remote(vault_remote_url) + brain_url = curator_remote(brain_remote_url) + if vault_url and brain_url: + vault.ensure_remote(CURATOR_REMOTE, vault_url) + brain.ensure_remote(CURATOR_REMOTE, brain_url) else: logger.warning("[curator] no CODE_URL/admin creds — remote sync disabled, serving local state") @@ -774,6 +890,17 @@ async def mirror_reconcile() -> bool: if not head: continue + # Brain has to start the cycle on top of the remote it is about + # to push to. Skip this and a remote that moved — or was + # re-created, which is how this failure actually arrives — turns + # every push from here on into a rejection. A realignment throws + # away the local projection, which is regenerable, so the mirror + # state is cleared with it and the block below rebuilds brain + # from memory in full. + if (await brain.sync()).status == "reset_to_remote": + mirror_sha = "" + mirror_file.unlink(missing_ok=True) + # ── Source mirror (memory -> brain) ─────────────────────────── # Data-plane like the pull: brain must always carry memory's # current source so Quartz renders fresh captures, even with LLM diff --git a/stacklets/memory/cli/ontology.py b/stacklets/memory/cli/ontology.py index 2f4d0f86..9b697e60 100644 --- a/stacklets/memory/cli/ontology.py +++ b/stacklets/memory/cli/ontology.py @@ -30,6 +30,7 @@ REPO_NAME, REPO_OWNER, SEED_ONTOLOGY_PATH, + host_code_url, pull_vault, vault_path_for, ) @@ -48,7 +49,7 @@ def run(args, stacklet, config): seed_text = SEED_ONTOLOGY_PATH.read_text(encoding="utf-8") secrets = config.get("secrets", {}) if config else {} - code_url = secrets.get("__code_url", "") or _code_url(config) + code_url = host_code_url(secrets.get("__code_url", "") or _code_url(config)) vault = vault_path_for(Path(data_dir)) # Two install paths leave creds in different places: the bot-token # install ships a `memory__MEMORY_BOT_TOKEN` secret; the admin-only diff --git a/stacklets/memory/cli/pull.py b/stacklets/memory/cli/pull.py index 5d8851a0..3f71059e 100644 --- a/stacklets/memory/cli/pull.py +++ b/stacklets/memory/cli/pull.py @@ -19,6 +19,7 @@ BOT_USERNAME, authenticated_remote, ensure_vault_cloned, + host_code_url, pull_vault, vault_path_for, vault_remote_url, @@ -39,7 +40,9 @@ def run(args, stacklet, config): # ever ran. if not (vault / ".git").exists(): token = config.get("secrets", {}).get("memory__MEMORY_BOT_TOKEN", "") - code_url = config.get("secrets", {}).get("__code_url", "") or _code_url(config) + code_url = host_code_url( + config.get("secrets", {}).get("__code_url", "") or _code_url(config) + ) if not (token and code_url): return {"error": "Vault not cloned and Forgejo credentials missing — run `stack up memory` first"} remote = authenticated_remote(vault_remote_url(code_url), BOT_USERNAME, token) diff --git a/stacklets/memory/hooks/on_start_ready.py b/stacklets/memory/hooks/on_start_ready.py index d0ce8efb..1690a92a 100644 --- a/stacklets/memory/hooks/on_start_ready.py +++ b/stacklets/memory/hooks/on_start_ready.py @@ -3,6 +3,13 @@ The install hook clones the vault once; this hook keeps it fresh on every restart. Best-effort — a failed pull never blocks startup. Readers fall back to whatever is already on disk, or to the seed. + +It also re-derives the vault's `origin` URL on every run. Both halves +of that URL rot on their own schedule: the host part when the Mac takes +a new DHCP lease, the embedded token when Forgejo expires it. Neither +is refreshed by anything else, so a clone made months ago quietly stops +working. Re-pointing it here costs one git config write and makes a +restart the fix. """ from __future__ import annotations @@ -17,6 +24,8 @@ brain_path_for, ensure_brain_projection_admin, ensure_vault_cloned, + host_code_url, + point_remote_at, pull_vault, purge_local_generated_memory_pages, vault_path_for, @@ -28,30 +37,38 @@ def run(ctx): vault = vault_path_for(ctx.stack.data) brain = brain_path_for(ctx.stack.data) + # Loopback, not the LAN address: this hook runs on the machine + # Forgejo is published from. See `host_code_url`. + code_url = host_code_url(ctx.env.get("CODE_URL", "")) + token = ctx.secret("MEMORY_BOT_TOKEN") + remote = ( + authenticated_remote(vault_remote_url(code_url), BOT_USERNAME, token) + if code_url and token else "" + ) + # If the vault never got cloned (install hook ran before code # stacklet was reachable, for example), try once more here. This # is the recovery path — same idempotent shape as install. if not (vault / ".git").exists(): - code_url = ctx.env.get("CODE_URL", "") - token = ctx.secret("MEMORY_BOT_TOKEN") - if not (code_url and token): + if not remote: ctx.step("Memory vault not cloned and Forgejo credentials missing; skipping") return - remote = authenticated_remote(vault_remote_url(code_url), BOT_USERNAME, token) if ensure_vault_cloned(vault, remote): ctx.step(f"Memory vault cloned to {vault}") else: ctx.step(f"Memory vault clone failed at {vault}") return - elif pull_vault(vault): - ctx.step("Memory vault pulled from Forgejo") else: - ctx.step("Memory vault pull skipped (Forgejo unreachable or non-FF)") + if remote: + point_remote_at(vault, remote) + if pull_vault(vault): + ctx.step("Memory vault pulled from Forgejo") + else: + ctx.step("Memory vault pull skipped (Forgejo unreachable or non-FF)") # Seamless B1 migration for existing installs. Those instances will # not rerun on_install_success, so create/clone brain here when # missing and purge legacy generated pages from the source repo. - code_url = ctx.env.get("CODE_URL", "") admin_user = ctx.env.get("ADMIN_USER", "") admin_password = ctx.env.get("ADMIN_PASSWORD", "") if not (code_url and admin_user and admin_password): diff --git a/stacklets/memory/lib.py b/stacklets/memory/lib.py index b55ab183..c600a74c 100644 --- a/stacklets/memory/lib.py +++ b/stacklets/memory/lib.py @@ -31,6 +31,7 @@ import re import subprocess +import time from dataclasses import dataclass, field from pathlib import Path from typing import Callable, List, Optional @@ -170,12 +171,78 @@ def brain_remote_url(code_url: str) -> str: return f"{code_url.rstrip('/')}/{REPO_OWNER}/{BRAIN_REPO_NAME}.git" +# The host's own address for its own published ports. `127.0.0.1` and +# not `localhost`, so the URL cannot be re-pointed by /etc/hosts or +# answered with an IPv6 address the port is not bound to. +LOOPBACK_HOST = "127.0.0.1" + +_IPV4_LITERAL = re.compile(r"^\d{1,3}(?:\.\d{1,3}){3}$") + + +def host_code_url(code_url: str) -> str: + """Rewrite a Forgejo URL so the *host* reaches it over loopback. + + In port mode `{code_url}` renders the LAN address + (`http://192.168.188.42:42040`), because it is also the URL a phone + on the couch clicks. Baked into a git remote or an API base on the + machine itself, that address is a time bomb: the next DHCP lease + moves it, and every host-side git call then hangs until its timeout + and fails. The host is local to itself, so it talks to the + published port on loopback, which no lease can move. + + Only a literal IPv4 address is rewritten. A hostname is left as it + is: in domain mode it is stable DNS, an operator-configured + `[core].host` name follows the machine, and on the container plane + `stack-code` is the service name that must survive this call + untouched. + """ + scheme, sep, rest = code_url.partition("://") + if not sep: + return code_url + authority, slash, path = rest.partition("/") + host, colon, port = authority.rpartition(":") + if not colon: # authority is a bare host + host, port = authority, "" + if not _IPV4_LITERAL.match(host): + return code_url + rebuilt = f"{LOOPBACK_HOST}:{port}" if port else LOOPBACK_HOST + return f"{scheme}://{rebuilt}{slash}{path}" + + # ─── Vault sync ────────────────────────────────────────────────────────── # # Clone-if-missing and best-effort pulls. Both shell out to `git` — # every Mac and Linux has it, and the framework already uses subprocess # liberally for Docker. No GitPython dep, no libgit2. +def _git(argv: List[str], *, timeout: int, env: Optional[dict] = None) -> tuple[int, str, str]: + """Run one git command and never raise. Returns (rc, stdout, stderr). + + A remote that stopped answering is the failure mode this whole + module exists to survive, and it arrives as a hang, not as an error + code: git sits on the socket until the timeout expires. Letting + `TimeoutExpired` escape is how a single unreachable remote took a + lifecycle hook down mid-run and left the resources after it + uncreated. So a timeout (and a missing git binary) comes back as a + failing command like any other. + """ + try: + result = subprocess.run( + argv, capture_output=True, text=True, timeout=timeout, env=env, + ) + except subprocess.TimeoutExpired: + return 124, "", f"git timed out after {timeout}s: {' '.join(argv[1:3])}" + except (FileNotFoundError, OSError) as e: + return 127, "", f"git unavailable: {e}" + return result.returncode, result.stdout.strip(), result.stderr.strip() + + +def run_git(repo_path: Path, *args: str, timeout: int = 60, + env: Optional[dict] = None) -> tuple[int, str, str]: + """Run a git command inside `repo_path`. Never raises — see `_git`.""" + return _git(["git", "-C", str(repo_path), *args], timeout=timeout, env=env) + + def ensure_vault_cloned( vault_path: Path, remote_url: str, @@ -193,11 +260,30 @@ def ensure_vault_cloned( return True vault_path.parent.mkdir(parents=True, exist_ok=True) - result = subprocess.run( - ["git", "clone", remote_url, str(vault_path)], - capture_output=True, text=True, timeout=timeout, + rc, _, _ = _git( + ["git", "clone", remote_url, str(vault_path)], timeout=timeout, ) - return result.returncode == 0 + return rc == 0 + + +def point_remote_at(repo_path: Path, remote_url: str, *, + name: str = "origin", timeout: int = 10) -> bool: + """Re-derive a working copy's remote URL, adding the remote if absent. + + A remote URL rots in two independent ways: the host part (a LAN IP + baked in at clone time, moved by the next DHCP lease) and the + embedded credential (a Forgejo token that expired). Both are held + in git's config, where nothing ever refreshes them. So every start + re-points the remote at the URL the current config says it should + be, and neither kind of rot survives a `stack up`. + """ + repo_path = Path(repo_path) + if not (repo_path / ".git").exists(): + return False + rc, _, _ = run_git(repo_path, "remote", "set-url", name, remote_url, timeout=timeout) + if rc != 0: + rc, _, _ = run_git(repo_path, "remote", "add", name, remote_url, timeout=timeout) + return rc == 0 def pull_vault(vault_path: Path, *, timeout: int = 30) -> bool: @@ -210,11 +296,8 @@ def pull_vault(vault_path: Path, *, timeout: int = 30) -> bool: vault_path = Path(vault_path) if not (vault_path / ".git").exists(): return False - result = subprocess.run( - ["git", "-C", str(vault_path), "pull", "--ff-only"], - capture_output=True, text=True, timeout=timeout, - ) - return result.returncode == 0 + rc, _, _ = run_git(vault_path, "pull", "--ff-only", timeout=timeout) + return rc == 0 def vault_remote_head(vault_path: Path, *, timeout: int = 5) -> Optional[str]: @@ -297,6 +380,214 @@ def refresh_vault_if_stale( return "pull_failed" +# ─── Divergence recovery ───────────────────────────────────────────────── +# +# `git pull --ff-only` has exactly one outcome when a working copy and +# its remote disagree: it fails, and it keeps failing, every cycle, +# forever. That is not a sync loop, it is a wedge with a retry timer on +# it. This section is the way out. +# +# Which way out depends on who owns the truth, and the two repos answer +# differently. `family/memory` is the database (ADR-011): records and +# state documents, and a local-only commit may be a todo tick, which is +# information that exists nowhere else. Its local commits are never +# discarded, only replayed or set aside on a branch. `family/brain` is +# a projection: machine-owned, regenerable from memory at any time, so +# it may take the remote as given and re-project on top. +# +# Both policies run through one function, because the difference +# between them is the interesting part and it should be readable in one +# place. + +PRESERVE_LOCAL = "preserve" +"""Recovery policy for source repos: local commits are irreplaceable.""" + +RESET_LOCAL = "reset" +"""Recovery policy for projections: the remote may be taken as given.""" + +# Prefix of the branch that holds a history we had to step away from. +# The name is the one the operator wrote by hand the night this broke: +# `git branch wedged-orphan- main` before resetting to the remote. +PRESERVED_BRANCH_PREFIX = "wedged-orphan" + +# Text git and Forgejo use when the credentials, not the network, are +# the problem. An expired token reads differently from an unreachable +# host, and the two need different responses: refresh and retry versus +# wait and retry. +_AUTH_FAILURE_MARKERS = ( + "authentication failed", + "invalid username or password", + "credentials are incorrect", # Forgejo's wording for an expired token + "could not read username", + "returned error: 401", + "returned error: 403", +) + + +def is_auth_failure(stderr: str) -> bool: + """True when git's error text says the credentials were rejected.""" + text = (stderr or "").lower() + return any(marker in text for marker in _AUTH_FAILURE_MARKERS) + + +def preserved_branch_name(head_sha: str, *, today: Optional[str] = None) -> str: + """Name of the branch that keeps a history we are about to leave. + + Dated so an operator can see when it happened, and suffixed with + the short SHA so preserving the same history twice reuses the same + branch instead of littering one per attempt. + """ + day = today or time.strftime("%Y%m%d", time.localtime()) + return f"{PRESERVED_BRANCH_PREFIX}-{day}-{head_sha[:7]}" + + +@dataclass +class SyncResult: + """Outcome of one reconcile, terse enough to log verbatim. + + `status` is one of: + + "up_to_date" — nothing to do. + "fast_forwarded" — the everyday pull. + "ahead" — local has commits the remote lacks and + the caller pushes them itself. + "pushed" — local commits delivered to the remote. + "rebased" — local commits replayed onto the remote + head and pushed. + "rebased_unpushed" — replayed, but the push did not land. + Nothing lost; the next cycle retries. + "preserved_and_reset" — histories could not be reconciled, so + the local one is kept on `detail`'s + branch and the working copy now matches + the remote. + "reset_to_remote" — projection realigned to the remote. + "unreachable" — the remote did not answer. + "auth_failed" — the remote rejected the credentials. + "failed" — git refused an operation; `detail` + carries its complaint. + + `detail` carries the preserved branch name or git's stderr, + whichever the status makes useful. + """ + + status: str + detail: str = "" + + +# Outcomes that mean the data plane made progress, or had nothing to +# do. Everything else is worth an operator's attention. +HEALTHY_SYNC_STATUSES = frozenset( + {"up_to_date", "fast_forwarded", "ahead", "pushed"} +) + + +def reconcile_with_remote( + repo_path: Path, + remote: str, + *, + recovery: str, + branch: str = "main", + env: Optional[dict] = None, + timeout: int = 60, +) -> SyncResult: + """Bring a working copy back into agreement with `remote`. + + The fast paths first: heads equal, or local is an ancestor of the + remote and a fast-forward does it. Local being *ahead* is not a + problem either, just commits that have not travelled yet. + + The two hard cases are the ones that wedge a `--ff-only` loop: + + - **Diverged with a shared merge base.** Both sides committed. + Under `PRESERVE_LOCAL` the local commits are replayed onto the + remote head and pushed, so the remote stays canonical and the + local work still exists and finally travels. Under + `RESET_LOCAL` the local side is dropped, because it can be + regenerated. + - **No merge base at all.** The remote repo was re-created, so + the two histories share nothing and no rebase can bridge them. + Under `PRESERVE_LOCAL` the local history is kept on a + `wedged-orphan-*` branch *before* the working copy is reset, and + the reset is skipped entirely if that branch cannot be created. + Under `RESET_LOCAL` the working copy simply resets. + + Never raises, and never resets a source repo without first putting + the history it is leaving somewhere a person can find it. + """ + def git(*args: str) -> tuple[int, str, str]: + return run_git(repo_path, *args, timeout=timeout, env=env) + + rc, _, err = git("fetch", "--quiet", remote, branch) + if rc != 0: + return SyncResult("auth_failed" if is_auth_failure(err) else "unreachable", err) + + rc, remote_head, err = git("rev-parse", "FETCH_HEAD") + if rc != 0 or not remote_head: + return SyncResult("unreachable", err) + rc, local_head, err = git("rev-parse", "HEAD") + if rc != 0 or not local_head: + return SyncResult("failed", err) + + if local_head == remote_head: + return SyncResult("up_to_date") + + if git("merge-base", "--is-ancestor", local_head, remote_head)[0] == 0: + rc, _, err = git("merge", "--ff-only", remote_head) + return SyncResult("fast_forwarded" if rc == 0 else "failed", err) + + if git("merge-base", "--is-ancestor", remote_head, local_head)[0] == 0: + if recovery == RESET_LOCAL: + return SyncResult("ahead") # the caller's own push delivers it + return _push_local(git, remote, branch) + + if recovery == RESET_LOCAL: + rc, _, err = git("reset", "--hard", remote_head) + return SyncResult("reset_to_remote" if rc == 0 else "failed", err) + + if git("merge-base", local_head, remote_head)[0] == 0: + rc, _, err = git( + "-c", f"user.name={BOT_USERNAME}", "-c", f"user.email={BOT_EMAIL}", + "rebase", remote_head, + ) + if rc == 0: + replayed = _push_local(git, remote, branch) + return SyncResult( + "rebased" if replayed.status == "pushed" else "rebased_unpushed", + replayed.detail, + ) + git("rebase", "--abort") # conflict: fall through to preserve + + return _preserve_and_reset(git, local_head, remote_head) + + +def _push_local(git, remote: str, branch: str) -> SyncResult: + """Deliver local commits to the remote. + + Local work that stays local diverges again on the next cycle, so + reconciling means pushing, not just re-ordering. A refused push + loses nothing: the commits are still here and the next cycle tries + again, loudly. + """ + rc, _, err = git("push", "--quiet", remote, f"HEAD:{branch}") + return SyncResult("pushed" if rc == 0 else "push_failed", err) + + +def _preserve_and_reset(git, local_head: str, remote_head: str) -> SyncResult: + """Keep the local history on a branch, then reset to the remote. + + The order matters and is the whole safety property: if the branch + cannot be created, nothing is reset. A wedged repo that stays + wedged is recoverable; a reset one is not. + """ + name = preserved_branch_name(local_head) + if git("rev-parse", "--verify", "--quiet", f"refs/heads/{name}")[0] != 0: + rc, _, err = git("branch", name, local_head) + if rc != 0: + return SyncResult("failed", f"could not preserve local history: {err}") + rc, _, err = git("reset", "--hard", remote_head) + return SyncResult("preserved_and_reset" if rc == 0 else "failed", name if rc == 0 else err) + + # ─── Vault writers ─────────────────────────────────────────────────────── def _code_url_from_config(config: dict | None) -> str: @@ -827,6 +1118,9 @@ def ensure_brain_projection_admin( `family/brain` if missing, seed its scaffold if missing, and clone it locally when the working copy is absent. """ + # Host plane: talk to Forgejo on loopback so neither the API calls + # nor the clone URL carry a LAN IP that the next lease invalidates. + code_url = host_code_url(code_url) admin = ForgejoClient( url=code_url, admin_user=admin_user, admin_password=admin_password, @@ -850,6 +1144,10 @@ def ensure_brain_projection_admin( admin_user, admin_token, ) cloned_brain = ensure_vault_cloned(brain_path, brain_remote) and not had_brain + if had_brain: + # Freshly issued token, current host URL — the clone made on + # an older lease with an older token gets both refreshed. + point_remote_at(brain_path, brain_remote) return { "created_brain_repo": brain_state["created_repo"], @@ -934,6 +1232,9 @@ def install_memory_to_forgejo( Returns a dict describing what changed. On `forgejo unreachable`, returns `{"skipped_reason": "..."}` and makes no further calls. """ + # Host plane: see `host_code_url`. The clone URL written here is + # the one the vault keeps as `origin` for the rest of its life. + code_url = host_code_url(code_url) admin = ForgejoClient( url=code_url, admin_user=admin_user, admin_password=admin_password, diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 00000000..dc2f1b4b --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,185 @@ +"""Git repositories in the states that break a sync loop. + +Every git test in this repo used to build the same thing: a healthy +temp repo, cloned once, asserted on the happy path. That is why a +remote which had stopped working — moved host, expired token, history +re-created underneath the clone — could fail every thirty seconds for +an unknown length of time without a single test noticing. + +These fixtures build the unhealthy states instead, and live here rather +than beside one stacklet's tests because nothing about them is specific +to memory: any stacklet that keeps a working copy in sync with Forgejo +(the docs mirror, anything that follows) can borrow them as they are. + + git_healthy_clone local and remote agree + git_diverged_clone both sides committed, shared merge base + git_unrelated_history_clone remote repo re-created, no merge base + git_unreachable_remote_clone remote URL points nowhere + +Each yields a `GitPair(remote, local)` of paths, and each is a +throwaway copy the test may wreck. `git_commit` is the callable they +are built from, exposed so a test can add its own commits. + +Every state is built once per session and copied per test. Spawning +git costs about 150ms a call here, so building four repositories from +scratch for each of a dozen tests is most of a minute of nothing; the +copies are milliseconds. `_build_*` below is still the readable +definition of each state, it just runs once. +""" + +from __future__ import annotations + +import shutil +import subprocess +from dataclasses import dataclass +from pathlib import Path + +import pytest + + +@dataclass +class GitPair: + """A bare "server" repo and a working copy cloned from it.""" + + remote: Path + local: Path + + +def _git(*args: str, cwd: Path | None = None) -> str: + """Run git, failing the test loudly with git's own words.""" + result = subprocess.run( + ["git", *args], cwd=str(cwd) if cwd else None, + capture_output=True, text=True, check=True, + ) + return result.stdout.strip() + + +def _init_working_copy(path: Path) -> None: + """Give a clone an identity and no signing. + + The developer running the suite may well have commit signing on + globally; the container these paths actually run in has no global + config at all. Pinning both here keeps the fixture the same repo on + either machine. + """ + _git("config", "user.email", "test@famstack.local", cwd=path) + _git("config", "user.name", "Test", cwd=path) + _git("config", "commit.gpgsign", "false", cwd=path) + + +def _commit(repo: Path, name: str, text: str, message: str) -> str: + target = Path(repo) / name + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(text, encoding="utf-8") + _git("add", ".", cwd=repo) + _git("commit", "-m", message, cwd=repo) + return _git("rev-parse", "HEAD", cwd=repo) + + +def _seed_bare(bare: Path, workdir: Path, text: str) -> None: + """Create a bare repo carrying one root commit on `main`.""" + _git("init", "--bare", "--initial-branch=main", str(bare)) + _git("clone", str(bare), str(workdir)) + _init_working_copy(workdir) + _commit(workdir, "README.md", text, "seed") + _git("push", "origin", "main", cwd=workdir) + + +def _build_healthy(root: Path) -> None: + """A working copy that agrees with its remote. The baseline.""" + _seed_bare(root / "remote.git", root / "seed", "# vault\n") + _git("clone", str(root / "remote.git"), str(root / "working-copy")) + _init_working_copy(root / "working-copy") + shutil.rmtree(root / "seed") + + +def _build_diverged(root: Path) -> None: + """Both sides committed since they last agreed. + + The shape a todo tick makes: someone edited the working copy while + someone else pushed to Forgejo. There is a merge base, so nothing + here is unrecoverable — but `pull --ff-only` fails on it forever. + """ + _build_healthy(root) + other = root / "other-writer" + _git("clone", str(root / "remote.git"), str(other)) + _init_working_copy(other) + _commit(other, "family/documents/filed.md", "filed upstream\n", "learn: filing") + _git("push", "origin", "main", cwd=other) + shutil.rmtree(other) + + _commit( + root / "working-copy", "family/todos.md", + "- [x] buy duff\n", "todo: tick buy duff", + ) + + +def _build_recreated_remote(root: Path) -> None: + """A remote repo re-created from nothing: no commit in common.""" + _seed_bare(root / "remote.git", root / "seed", "# new life\n") + shutil.rmtree(root / "seed") + + +@pytest.fixture(scope="session") +def _git_states(tmp_path_factory) -> dict[str, Path]: + root = tmp_path_factory.mktemp("git-states") + _build_healthy(root / "healthy") + _build_diverged(root / "diverged") + _build_recreated_remote(root / "recreated") + return { + "healthy": root / "healthy", + "diverged": root / "diverged", + "recreated": root / "recreated", + } + + +def _checkout(state: Path, into: Path) -> GitPair: + """Copy a prepared state into a test's own tmp dir.""" + remote, local = into / "remote.git", into / "working-copy" + shutil.copytree(state / "remote.git", remote) + shutil.copytree(state / "working-copy", local) + _git("-C", str(local), "remote", "set-url", "origin", str(remote)) + return GitPair(remote=remote, local=local) + + +@pytest.fixture +def git_commit(): + """`git_commit(repo, name, text, message=...)` -> the new commit SHA.""" + def _call(repo: Path, name: str, text: str, message: str = "edit") -> str: + return _commit(repo, name, text, message) + + return _call + + +@pytest.fixture +def git_healthy_clone(_git_states, tmp_path) -> GitPair: + return _checkout(_git_states["healthy"], tmp_path) + + +@pytest.fixture +def git_diverged_clone(_git_states, tmp_path) -> GitPair: + return _checkout(_git_states["diverged"], tmp_path) + + +@pytest.fixture +def git_unrelated_history_clone(_git_states, tmp_path) -> GitPair: + """The clone's remote was wiped and re-created: two histories, no + merge base, and no amount of rebasing can bridge them.""" + pair = _checkout(_git_states["healthy"], tmp_path) + recreated = tmp_path / "recreated.git" + shutil.copytree(_git_states["recreated"] / "remote.git", recreated) + _git("-C", str(pair.local), "remote", "set-url", "origin", str(recreated)) + return GitPair(remote=recreated, local=pair.local) + + +@pytest.fixture +def git_unreachable_remote_clone(_git_states, tmp_path) -> GitPair: + """A remote that no longer answers. + + A LAN address the machine gave up in a DHCP lease behaves like + this: the clone is intact, its remote is a dead end. + """ + pair = _checkout(_git_states["healthy"], tmp_path) + gone = tmp_path / "moved-away.git" + _git("-C", str(pair.local), "remote", "set-url", "origin", str(gone)) + return GitPair(remote=gone, local=pair.local) diff --git a/tests/stacklets/test_memory_curator_sync.py b/tests/stacklets/test_memory_curator_sync.py new file mode 100644 index 00000000..72e63882 --- /dev/null +++ b/tests/stacklets/test_memory_curator_sync.py @@ -0,0 +1,416 @@ +"""What the curator's sync does when the remote has stopped cooperating. + +A vault sync that cannot make progress used to have exactly one +behaviour: fail, log nothing anyone would read, and try again in +thirty seconds. Forever. These tests pin the way out of each wedge, +against real git repositories in the states that produce it (see +`tests/conftest.py`), because the failure being covered here is a real +remote misbehaving and a stubbed one cannot misbehave convincingly. + +The policies are the point, and they differ by who owns the truth: + + - **memory** is the database. Its local commits may be a todo tick, + which is information that exists nowhere else (ADR-011), so they + are replayed onto the remote or set aside on a branch, never + dropped. + - **brain** is a projection. It is regenerable from memory, so it may + take the remote as given and rebuild on top. +""" + +from __future__ import annotations + +import os +import shutil +import subprocess +import sys +from pathlib import Path + +import pytest + +_REPO_ROOT = Path(__file__).resolve().parent.parent.parent +sys.path.insert(0, str(_REPO_ROOT / "stacklets" / "memory")) +sys.path.insert(0, str(_REPO_ROOT / "stacklets" / "memory" / "bot")) +sys.path.insert(0, str(_REPO_ROOT / "stacklets" / "memory" / "bot" / "cli")) + +from lib import ( # noqa: E402 + PRESERVE_LOCAL, + PRESERVED_BRANCH_PREFIX, + RESET_LOCAL, + is_auth_failure, + preserved_branch_name, + reconcile_with_remote, +) + + +# ── Reading the repositories back ──────────────────────────────────────── + +def _git(repo: Path, *args: str) -> str: + result = subprocess.run( + ["git", "-C", str(repo), *args], + capture_output=True, text=True, check=True, + ) + return result.stdout.strip() + + +def _head(repo: Path, rev: str = "HEAD") -> str: + return _git(repo, "rev-parse", rev) + + +def _wedged_branches(repo: Path) -> list[str]: + out = _git(repo, "for-each-ref", "--format=%(refname:short)", "refs/heads/") + return [b for b in out.splitlines() if b.startswith(PRESERVED_BRANCH_PREFIX)] + + +def _log_subjects(repo: Path, rev: str = "HEAD") -> list[str]: + return _git(repo, "log", "--format=%s", rev).splitlines() + + +def _ff_only_pull_fails(repo: Path) -> bool: + """True when `git pull --ff-only` cannot resolve this state. + + The old sync was exactly this command, so a fixture that this + succeeds on is not reproducing the outage. Asserted alongside the + recovery so the test is pinned to the failure, not to our idea of it. + """ + return subprocess.run( + ["git", "-C", str(repo), "pull", "--ff-only"], + capture_output=True, text=True, + ).returncode != 0 + + +# ── The everyday outcomes ──────────────────────────────────────────────── + +class TestReconcileWhenNothingIsWrong: + """The healthy paths still have to be cheap and quiet, or the + recovery machinery has made the common case worse.""" + + def test_agreeing_copies_report_up_to_date(self, git_healthy_clone): + result = reconcile_with_remote( + git_healthy_clone.local, "origin", recovery=PRESERVE_LOCAL, + ) + assert result.status == "up_to_date" + + def test_a_remote_ahead_is_a_plain_fast_forward( + self, git_healthy_clone, git_commit, tmp_path, + ): + other = tmp_path / "other-writer" + subprocess.run( + ["git", "clone", str(git_healthy_clone.remote), str(other)], + capture_output=True, check=True, + ) + _git(other, "config", "user.email", "test@famstack.local") + _git(other, "config", "user.name", "Test") + git_commit(other, "family/notes/a.md", "note\n", "learn: a note") + _git(other, "push", "origin", "main") + + result = reconcile_with_remote( + git_healthy_clone.local, "origin", recovery=PRESERVE_LOCAL, + ) + + assert result.status == "fast_forwarded" + assert (git_healthy_clone.local / "family/notes/a.md").exists() + + +# ── memory: local commits survive, always ──────────────────────────────── + +class TestSourceRecovery: + """`PRESERVE_LOCAL` — the policy for `family/memory`.""" + + def test_diverged_history_keeps_the_local_commit_and_delivers_it( + self, git_diverged_clone, + ): + """The todo-tick case, and the worst possible outcome is losing it. + + Local ticked a todo, Forgejo took a filing. `pull --ff-only` + fails on this and would keep failing. Reconciling has to end + with both commits present and the divergence actually gone, not + merely re-ordered into the same wedge for the next cycle. + """ + local = git_diverged_clone.local + assert _ff_only_pull_fails(local) + + result = reconcile_with_remote(local, "origin", recovery=PRESERVE_LOCAL) + + assert result.status == "rebased" + assert (local / "family/todos.md").read_text() == "- [x] buy duff\n" + assert (local / "family/documents/filed.md").exists() + # Delivered, not just replayed: the remote carries the tick now. + assert _head(local) == _head(local, "origin/main") + assert "todo: tick buy duff" in _log_subjects(local) + + def test_local_only_commits_are_pushed_rather_than_left_to_diverge( + self, git_healthy_clone, git_commit, + ): + local = git_healthy_clone.local + git_commit(local, "family/todos.md", "- [x] call dentist\n", "todo: tick") + + result = reconcile_with_remote(local, "origin", recovery=PRESERVE_LOCAL) + + assert result.status == "pushed" + assert _head(local) == _head(local, "origin/main") + + def test_unrelated_history_is_preserved_on_a_branch_before_the_reset( + self, git_unrelated_history_clone, + ): + """The night this broke, by hand: `git branch wedged-orphan- + main`, then `git reset --hard `. Nothing else bridges two + histories that share no commit, and nothing else keeps the local + one findable afterwards.""" + local = git_unrelated_history_clone.local + stranded = _head(local) + assert _ff_only_pull_fails(local) + + result = reconcile_with_remote(local, "origin", recovery=PRESERVE_LOCAL) + + assert result.status == "preserved_and_reset" + preserved = _wedged_branches(local) + assert preserved == [preserved_branch_name(stranded)] + assert preserved[0].startswith(f"{PRESERVED_BRANCH_PREFIX}-") + assert _head(local, preserved[0]) == stranded + # And the working copy is now the remote, ready to move again. + assert _head(local) == _head(local, "origin/main") + assert (local / "README.md").read_text() == "# new life\n" + + def test_recovery_is_idempotent(self, git_unrelated_history_clone): + """A second cycle must find a healthy repo, not manufacture a + second orphan branch every thirty seconds.""" + local = git_unrelated_history_clone.local + reconcile_with_remote(local, "origin", recovery=PRESERVE_LOCAL) + after_first = _wedged_branches(local) + + result = reconcile_with_remote(local, "origin", recovery=PRESERVE_LOCAL) + + assert result.status == "up_to_date" + assert _wedged_branches(local) == after_first + assert len(after_first) == 1 + + +# ── brain: regenerable, so realignment is free ─────────────────────────── + +class TestProjectionRecovery: + """`RESET_LOCAL` — the policy for `family/brain`.""" + + def test_unrelated_history_resets_without_preserving_anything( + self, git_unrelated_history_clone, + ): + local = git_unrelated_history_clone.local + + result = reconcile_with_remote(local, "origin", recovery=RESET_LOCAL) + + assert result.status == "reset_to_remote" + assert _wedged_branches(local) == [] + assert _head(local) == _head(local, "origin/main") + + def test_diverged_history_takes_the_remote_as_the_new_base( + self, git_diverged_clone, + ): + local = git_diverged_clone.local + + result = reconcile_with_remote(local, "origin", recovery=RESET_LOCAL) + + assert result.status == "reset_to_remote" + assert _head(local) == _head(local, "origin/main") + + def test_unpushed_local_commits_are_left_for_the_caller_to_push( + self, git_healthy_clone, git_commit, + ): + """Last cycle's projection waiting on a push is not a divergence. + Resetting it away would drop work the caller is about to deliver.""" + local = git_healthy_clone.local + sha = git_commit(local, "index.md", "generated\n", "brain: project") + + result = reconcile_with_remote(local, "origin", recovery=RESET_LOCAL) + + assert result.status == "ahead" + assert _head(local) == sha + + +class TestThePoliciesDiffer: + + def test_source_preserves_the_history_a_projection_may_discard( + self, git_unrelated_history_clone, tmp_path, + ): + """Same wedge, two owners, two answers. If these ever converge, + either memory started losing commits or brain started hoarding + branches it can regenerate.""" + source = git_unrelated_history_clone.local + projection = tmp_path / "projection" + shutil.copytree(source, projection) + + preserved = reconcile_with_remote(source, "origin", recovery=PRESERVE_LOCAL) + reset = reconcile_with_remote(projection, "origin", recovery=RESET_LOCAL) + + assert preserved.status == "preserved_and_reset" + assert reset.status == "reset_to_remote" + assert len(_wedged_branches(source)) == 1 + assert _wedged_branches(projection) == [] + # Both end up at the remote — only the cost of getting there differs. + assert _head(source) == _head(projection) + + +# ── A remote that stopped answering ────────────────────────────────────── + +class TestRemoteFailures: + + def test_unreachable_remote_leaves_the_working_copy_alone( + self, git_unreachable_remote_clone, + ): + local = git_unreachable_remote_clone.local + before = _head(local) + + result = reconcile_with_remote(local, "origin", recovery=PRESERVE_LOCAL) + + assert result.status == "unreachable" + assert _head(local) == before + + def test_rejected_credentials_read_differently_from_a_dead_host(self): + """An expired token and an unplugged network both fail the fetch, + and only one of them is fixed by re-deriving the URL.""" + assert is_auth_failure( + "remote: Forgejo: Credentials are incorrect or have expired" + ) + assert is_auth_failure("fatal: Authentication failed for 'http://host/x.git'") + assert is_auth_failure( + "fatal: unable to access 'http://host/x.git': " + "The requested URL returned error: 403" + ) + assert not is_auth_failure("fatal: unable to access: Could not resolve host") + assert not is_auth_failure("") + + +# ── The curator's own wiring ───────────────────────────────────────────── + +@pytest.fixture +def auth_failing_transport(tmp_path, monkeypatch) -> str: + """A git transport that always answers "your credentials are wrong". + + A real `git-remote-` helper on PATH, so git produces the + failure itself: the code under test sees the same stderr Forgejo + would produce with an expired token, through the same code path. + Returns the URL to point a remote at. + """ + bindir = tmp_path / "fake-git-transports" + bindir.mkdir() + helper = bindir / "git-remote-authfail" + helper.write_text( + "#!/bin/sh\n" + "echo \"fatal: Authentication failed for 'authfail://memory.git'\" >&2\n" + "exit 1\n", + encoding="utf-8", + ) + helper.chmod(0o755) + monkeypatch.setenv("PATH", f"{bindir}{os.pathsep}{os.environ['PATH']}") + return "authfail://memory.git" + + +@pytest.fixture +def warnings_logged(): + """Collect anything the curator says at WARNING or above. + + Loguru sinks are the one collaborator these tests stub, because the + assertion *is* about the logging: an outage that only shows up at + DEBUG is how this went unnoticed for weeks. + """ + from loguru import logger + + lines: list[str] = [] + sink_id = logger.add(lines.append, level="WARNING") + yield lines + logger.remove(sink_id) + + +class TestVaultSync: + """The curator's vault sync: source policy, one auth retry, loud.""" + + async def test_recovers_a_wedged_vault_and_says_so( + self, git_unrelated_history_clone, warnings_logged, + ): + from curator import CURATOR_REMOTE, Vault + + local = git_unrelated_history_clone.local + _git(local, "remote", "add", CURATOR_REMOTE, + str(git_unrelated_history_clone.remote)) + + result = await Vault(local).sync() + + assert result.status == "preserved_and_reset" + assert len(_wedged_branches(local)) == 1 + assert any("unrelated history" in line for line in warnings_logged) + + async def test_a_stale_credential_is_re_derived_and_retried_once( + self, git_healthy_clone, auth_failing_transport, monkeypatch, tmp_path, + ): + """The token baked into the remote URL expires on its own + schedule. Re-reading the environment is the whole fix, and it + only counts if the sync then actually completes.""" + from curator import CURATOR_REMOTE, Vault + + # Where `vault_remote_url` will look: /family/memory.git + forgejo = tmp_path / "forgejo" + (forgejo / "family").mkdir(parents=True) + (forgejo / "family" / "memory.git").symlink_to(git_healthy_clone.remote) + monkeypatch.setenv("CODE_URL", str(forgejo)) + monkeypatch.setenv("MATRIX_ADMIN_USER", "stackadmin") + monkeypatch.setenv("MATRIX_ADMIN_PASSWORD", "hunter2") + + local = git_healthy_clone.local + _git(local, "remote", "add", CURATOR_REMOTE, auth_failing_transport) + + result = await Vault(local).sync() + + assert result.status == "up_to_date" + + async def test_a_credential_that_cannot_be_re_derived_is_an_error( + self, git_healthy_clone, auth_failing_transport, monkeypatch, + warnings_logged, + ): + from curator import CURATOR_REMOTE, Vault + + for key in ("CODE_URL", "MATRIX_ADMIN_USER", "MATRIX_ADMIN_PASSWORD"): + monkeypatch.delenv(key, raising=False) + local = git_healthy_clone.local + _git(local, "remote", "add", CURATOR_REMOTE, auth_failing_transport) + + result = await Vault(local).sync() + + assert result.status == "auth_failed" + assert any("rejected the credentials" in line for line in warnings_logged) + + +class TestBrainSync: + """Brain realigns instead of overwriting, and a push that never + lands is reported rather than forced.""" + + async def test_a_recreated_remote_realigns_the_projection( + self, git_unrelated_history_clone, tmp_path, + ): + from curator import CURATOR_REMOTE, Brain + + local = git_unrelated_history_clone.local + _git(local, "remote", "add", CURATOR_REMOTE, + str(git_unrelated_history_clone.remote)) + + result = await Brain(local, tmp_path / "source").sync() + + assert result.status == "reset_to_remote" + assert _wedged_branches(local) == [] + + async def test_a_push_that_cannot_land_is_reported_not_forced( + self, git_unreachable_remote_clone, tmp_path, warnings_logged, + ): + """The incident's brain symptom: `family/brain` did not exist, so + every push was refused. The old fallback answered by force-pushing + and logging "remote diverged", which was both useless and untrue.""" + from curator import CURATOR_REMOTE, Brain + + local = git_unreachable_remote_clone.local + _git(local, "remote", "add", CURATOR_REMOTE, + str(git_unreachable_remote_clone.remote)) + (local / "index.md").write_text("generated\n", encoding="utf-8") + + pushed = await Brain(local, tmp_path / "source").commit_push("brain: project") + + assert pushed is False + assert any("not reaching Forgejo" in line for line in warnings_logged) + # The commit is still here for the next cycle to deliver. + assert "brain: project" in _log_subjects(local) diff --git a/tests/stacklets/test_memory_vault.py b/tests/stacklets/test_memory_vault.py index b3646321..48445a7f 100644 --- a/tests/stacklets/test_memory_vault.py +++ b/tests/stacklets/test_memory_vault.py @@ -21,8 +21,10 @@ authenticated_remote, ensure_vault_cloned, get_ontology, + host_code_url, load_ontology_from_vault, load_seed_ontology, + point_remote_at, pull_vault, refresh_vault_if_stale, vault_local_head, @@ -112,6 +114,71 @@ def test_authenticated_remote_leaves_pathless_string_alone(self): assert authenticated_remote("not a url", "u", "t") == "not a url" +class TestHostSideRemoteUrl: + """A remote URL the host keeps has to survive the host changing address. + + In port mode `{code_url}` renders the LAN IP, because it is also + the URL a phone on the couch clicks. Written into a git remote on + the machine itself, it lasts exactly as long as the DHCP lease. + """ + + def test_port_mode_remote_carries_no_lan_address(self): + remote = vault_remote_url(host_code_url("http://192.168.188.42:42040")) + + assert "192.168.188.42" not in remote + assert remote == "http://127.0.0.1:42040/family/memory.git" + + def test_the_published_port_is_kept(self): + # Loopback only reaches Forgejo on the port the stack publishes. + assert host_code_url("http://10.0.0.7:42040").endswith(":42040") + + def test_domain_mode_hostname_is_left_alone(self): + # A DNS name does not move with the lease; rewriting it would + # break the one hosting mode that was never at risk. + assert host_code_url("https://code.simpsons.family") == \ + "https://code.simpsons.family" + + def test_container_service_name_is_left_alone(self): + # The curator's plane. `stack-code` is what makes it immune to + # the failure this function exists to fix. + assert host_code_url("http://stack-code:3000") == "http://stack-code:3000" + + def test_operator_configured_hostname_is_left_alone(self): + # `[core].host = my-mac.local` is a deliberate, machine-following + # name, not an address handed out by a router. + assert host_code_url("http://my-mac.local:42040") == \ + "http://my-mac.local:42040" + + +class TestPointRemoteAt: + """Both halves of a remote URL rot: the host part with the next DHCP + lease, the embedded token when Forgejo expires it. Re-deriving the + whole URL on every start is what makes a restart the fix.""" + + def test_repoints_an_existing_remote(self, tmp_path, seeded_upstream): + vault = tmp_path / "vault" + ensure_vault_cloned(vault, str(seeded_upstream)) + + assert point_remote_at(vault, "http://memory-bot:fresh@127.0.0.1:42040/family/memory.git") + + url = _run("git", "-C", str(vault), "remote", "get-url", "origin").stdout.strip() + assert url == "http://memory-bot:fresh@127.0.0.1:42040/family/memory.git" + + def test_adds_the_remote_when_the_clone_has_none(self, tmp_path): + vault = tmp_path / "no-remote-vault" + _run("git", "init", "--initial-branch=main", str(vault)) + + assert point_remote_at(vault, "http://127.0.0.1:42040/family/memory.git") + + url = _run("git", "-C", str(vault), "remote", "get-url", "origin").stdout.strip() + assert url == "http://127.0.0.1:42040/family/memory.git" + + def test_a_directory_that_is_not_a_clone_is_not_touched(self, tmp_path): + plain = tmp_path / "plain" + plain.mkdir() + assert point_remote_at(plain, "http://127.0.0.1:42040/x.git") is False + + # ─── ensure_vault_cloned ───────────────────────────────────────────────── class TestEnsureVaultCloned: