Skip to content

fix(cloakserve): evict poll()-alive but CDP-dead Chrome instead of 502ing (ADR-0163 stage 0) #1

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
DivMode merged 1 commit into from
Aug 8, 2026
Merged

fix(cloakserve): evict poll()-alive but CDP-dead Chrome instead of 502ing (ADR-0163 stage 0) #1

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
154 changes: 114 additions & 40 deletions bin/cloakserve
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -319,7 +319,14 @@ class ChromePool:
# Check if already running (including default fast-path)
if seed_key in self._processes:
proc = self._processes[seed_key]
if proc.process.poll() is None:
# poll() alone is not enough: during an identity-lease rebuild a
# client closes Chrome and immediately relaunches on the same
# seed, and Chrome tears down its DevTools/CDP socket seconds
# before the OS process actually exits. So a poll()-alive process
# can already be refusing CDP connections — returning it hands the
# client a corpse. Verify the CDP port is still accepting
# connections (cheap, sub-second, local) before reusing it.
if proc.process.poll() is None and await self._cdp_port_alive(proc.cdp_port):
if seed_key in self._idle_tasks:
self._schedule_idle_cleanup(seed_key)
if any([extra_args, timezone, locale, proxy, geoip]):
Expand All @@ -330,7 +337,14 @@ class ChromePool:
proc.timezone, proc.locale, proc.proxy,
)
return proc
# Dead — clean up
# Dead: the process exited, or it is poll()-alive but its CDP
# port is no longer accepting connections (a dying corpse). Evict
# and fall through to a fresh launch.
logger.info(
"Seed %s pooled process is no longer serving CDP (port %d) — "
"evicting and relaunching",
seed_key, proc.cdp_port,
)
await self._cleanup_process(seed_key)

# Resolve geoip if requested
Expand Down Expand Up @@ -409,6 +423,19 @@ class ChromePool:
logger.info("Chrome ready (seed=%s, port=%d, pid=%d)", actual_seed, port, process.pid)
return cp

def _pooled_key(self, cp: ChromeProcess) -> str | None:
"""Return the pool key currently mapping to cp.

Returns None when cp is no longer the pooled process for its seed —
i.e. it was already evicted, or a concurrent request relaunched a fresh
process in its place. Callers evict by this key so a stale corpse is
removed without ever nuking a healthy relaunched process.
"""
for key, proc in self._processes.items():
if proc is cp:
return key
return None

async def _cleanup_process(self, key: str) -> None:
"""Terminate a Chrome process and clean up."""
self._cancel_idle_cleanup(key)
Expand Down Expand Up @@ -464,6 +491,31 @@ class ChromePool:
finally:
await session.close()

@staticmethod
async def _cdp_port_alive(port: int, timeout: float = 0.5) -> bool:
"""Cheap liveness probe: is the CDP port still accepting TCP connections?

Chrome closes its DevTools listening socket seconds before the OS
process exits, so process.poll() alone reports a dying Chrome as alive.
A single sub-second local TCP connect distinguishes a genuinely-serving
CDP endpoint from a poll()-alive corpse, keeping the warm-hit happy path
fast (a live listener completes the handshake in well under a
millisecond; a dead port refuses immediately).
"""
try:
_reader, writer = await asyncio.wait_for(
asyncio.open_connection("127.0.0.1", port),
timeout=timeout,
)
except (OSError, asyncio.TimeoutError):
return False
writer.close()
try:
await writer.wait_closed()
except Exception:
pass
return True


# ---------------------------------------------------------------------------
# Query param parsing
Expand Down Expand Up @@ -556,25 +608,37 @@ async def handle_json_version(request: web.Request) -> web.Response:
pool: ChromePool = request.app["pool"]
params = parse_connection_params(request.query_string)

cp = await pool.get_or_launch(
seed=params["seed"],
extra_args=params["extra_args"] or None,
timezone=params["timezone"],
locale=params["locale"],
proxy=params["proxy"],
geoip=params["geoip"],
)

try:
async with aiohttp.ClientSession() as session:
async with session.get(
f"http://127.0.0.1:{cp.cdp_port}/json/version",
timeout=aiohttp.ClientTimeout(total=5),
) as resp:
data = await resp.json()
except Exception as exc:
logger.error("Failed to reach Chrome CDP (port %d): %s", cp.cdp_port, exc)
return web.json_response({"error": "CDP endpoint unreachable"}, status=502)
# Defense-in-depth: get_or_launch already probes CDP liveness before reuse,
# but a process can still die in the window between that probe and this
# fetch. If the fetch fails with a connection error, evict the stale pool
# entry and relaunch exactly once so a corpse becomes a single cold
# relaunch instead of a persistent 502.
data = None
for attempt in range(2):
cp = await pool.get_or_launch(
seed=params["seed"],
extra_args=params["extra_args"] or None,
timezone=params["timezone"],
locale=params["locale"],
proxy=params["proxy"],
geoip=params["geoip"],
)
try:
async with aiohttp.ClientSession() as session:
async with session.get(
f"http://127.0.0.1:{cp.cdp_port}/json/version",
timeout=aiohttp.ClientTimeout(total=5),
) as resp:
data = await resp.json()
break
except Exception as exc:
logger.error("Failed to reach Chrome CDP (port %d): %s", cp.cdp_port, exc)
if attempt == 0:
stale_key = pool._pooled_key(cp)
if stale_key is not None:
await pool._cleanup_process(stale_key)
continue
return web.json_response({"error": "CDP endpoint unreachable"}, status=502)

# Rewrite webSocketDebuggerUrl to route through our multiplexer
host = _external_host(request)
Expand All @@ -598,25 +662,35 @@ async def handle_json_list(request: web.Request) -> web.Response:
pool: ChromePool = request.app["pool"]
params = parse_connection_params(request.query_string)

cp = await pool.get_or_launch(
seed=params["seed"],
extra_args=params["extra_args"] or None,
timezone=params["timezone"],
locale=params["locale"],
proxy=params["proxy"],
geoip=params["geoip"],
)

try:
async with aiohttp.ClientSession() as session:
async with session.get(
f"http://127.0.0.1:{cp.cdp_port}/json/list",
timeout=aiohttp.ClientTimeout(total=5),
) as resp:
data = await resp.json()
except Exception as exc:
logger.error("Failed to reach Chrome CDP (port %d): %s", cp.cdp_port, exc)
return web.json_response({"error": "CDP endpoint unreachable"}, status=502)
# Defense-in-depth: evict + relaunch once if a pooled process dies in the
# window between get_or_launch's liveness probe and this fetch (see
# handle_json_version for the rationale).
data = None
for attempt in range(2):
cp = await pool.get_or_launch(
seed=params["seed"],
extra_args=params["extra_args"] or None,
timezone=params["timezone"],
locale=params["locale"],
proxy=params["proxy"],
geoip=params["geoip"],
)
try:
async with aiohttp.ClientSession() as session:
async with session.get(
f"http://127.0.0.1:{cp.cdp_port}/json/list",
timeout=aiohttp.ClientTimeout(total=5),
) as resp:
data = await resp.json()
break
except Exception as exc:
logger.error("Failed to reach Chrome CDP (port %d): %s", cp.cdp_port, exc)
if attempt == 0:
stale_key = pool._pooled_key(cp)
if stale_key is not None:
await pool._cleanup_process(stale_key)
continue
return web.json_response({"error": "CDP endpoint unreachable"}, status=502)

host = _external_host(request)
scheme = _ws_scheme(request)
Expand Down
Loading
Loading