Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ on:
workflow_dispatch:

jobs:
frontend:
frontend-nextjs:
runs-on: ubuntu-latest
name: Frontend (Next.js)
defaults:
Expand All @@ -28,7 +28,7 @@ jobs:
- name: Setup Node
uses: actions/setup-node@v4
with:
node-version: "20"
node-version: "22"
cache: pnpm
cache-dependency-path: frontend/pnpm-lock.yaml

Expand Down Expand Up @@ -99,8 +99,8 @@ jobs:
with:
python-version: "3.13"

- name: Install workflow contract dependency
run: python -m pip install "pydantic>=2.10.0"
- name: Install workflow compiler dependencies
run: python -m pip install -e ..

- name: Install dependencies
run: |
Expand Down
4 changes: 3 additions & 1 deletion Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -45,12 +45,14 @@ RUN npm install -g @jackwener/opencli@${OPENCLI_VERSION} \
&& rm -rf /root/.npm

ARG OHMYOPENCLI_REPO=https://github.com/2233admin/OhMyOpenCLI.git
ARG OHMYOPENCLI_COMMIT=73cc60c83586ef2c95469b3b70d6cfc80fa5bc53
ARG OHMYOPENCLI_COMMIT=b0fdd513f64899b068103ddd7ff0de957d778b5c
ARG OFFICIAL_SITE_CAPABILITY_COMMIT=73cc60c83586ef2c95469b3b70d6cfc80fa5bc53
ARG DOUBAO_CAPABILITY_COMMIT=b0fdd513f64899b068103ddd7ff0de957d778b5c
RUN git clone ${OHMYOPENCLI_REPO} /opt/ohmyopencli \
&& cd /opt/ohmyopencli \
&& git checkout --detach ${OHMYOPENCLI_COMMIT} \
&& git merge-base --is-ancestor ${OFFICIAL_SITE_CAPABILITY_COMMIT} HEAD \
&& git merge-base --is-ancestor ${DOUBAO_CAPABILITY_COMMIT} HEAD \
&& npm ci \
&& test "$(git rev-parse HEAD)" = "${OHMYOPENCLI_COMMIT}"

Expand Down
4 changes: 3 additions & 1 deletion agent/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -36,12 +36,14 @@ RUN npm install -g @jackwener/opencli@${OPENCLI_VERSION} \
# identities separate: the latter is the behavior change, while the former is
# the exact checkout certified by this image.
ARG OHMYOPENCLI_REPO=https://github.com/2233admin/OhMyOpenCLI.git
ARG OHMYOPENCLI_COMMIT=73cc60c83586ef2c95469b3b70d6cfc80fa5bc53
ARG OHMYOPENCLI_COMMIT=b0fdd513f64899b068103ddd7ff0de957d778b5c
ARG OFFICIAL_SITE_CAPABILITY_COMMIT=73cc60c83586ef2c95469b3b70d6cfc80fa5bc53
ARG DOUBAO_CAPABILITY_COMMIT=b0fdd513f64899b068103ddd7ff0de957d778b5c
RUN git clone ${OHMYOPENCLI_REPO} /opt/ohmyopencli \
&& cd /opt/ohmyopencli \
&& git checkout --detach ${OHMYOPENCLI_COMMIT} \
&& git merge-base --is-ancestor ${OFFICIAL_SITE_CAPABILITY_COMMIT} HEAD \
&& git merge-base --is-ancestor ${DOUBAO_CAPABILITY_COMMIT} HEAD \
&& npm ci \
&& test "$(git rev-parse HEAD)" = "${OHMYOPENCLI_COMMIT}"

Expand Down
148 changes: 138 additions & 10 deletions backend/acquisition/capabilities.py
Original file line number Diff line number Diff line change
@@ -1,8 +1,11 @@
"""Runtime-probed capability catalog for managed GEO acquisition."""

import asyncio
import json
import os
import re
from typing import Any
from urllib.parse import urlparse

from backend.acquisition.registry import (
OHMYOPENCLI_COMMIT,
Expand All @@ -16,6 +19,18 @@
COMMAND_TIMEOUT_SECONDS = 15.0


def _opencli_environment(*, cdp_endpoint: str | None = None) -> dict[str, str]:
"""Build one unambiguous OpenCLI browser-routing environment."""
env = os.environ.copy()
env.pop("OPENCLI_DAEMON_HOST", None)
env.pop("OPENCLI_DAEMON_PORT", None)
if cdp_endpoint is None:
env.pop("OPENCLI_CDP_ENDPOINT", None)
else:
env["OPENCLI_CDP_ENDPOINT"] = cdp_endpoint
return env


async def _command(*args: str, env: dict[str, str] | None = None) -> tuple[int, str]:
try:
process = await asyncio.create_subprocess_exec(
Expand Down Expand Up @@ -73,7 +88,11 @@ async def _runtime_is_installed() -> bool:
return False

opencli_bin = resolve_opencli_bin()
version_rc, version_output = await _command(opencli_bin, "--version")
version_rc, version_output = await _command(
opencli_bin,
"--version",
env=_opencli_environment(),
)
versions = re.findall(r"\d+\.\d+\.\d+", version_output)
if version_rc != 0 or OPENCLI_VERSION not in versions:
return False
Expand All @@ -86,13 +105,14 @@ async def _registration_is_available(
) -> bool:
opencli_bin = resolve_opencli_bin()
command_rc, command_output = await _command(
opencli_bin, *registration.probe_args
opencli_bin,
*registration.probe_args,
env=_opencli_environment(),
)
if command_rc != 0 or registration.help_marker not in command_output:
return False

patch_env = os.environ.copy()
patch_env["OPENCLI_CDP_ENDPOINT"] = "http://127.0.0.1:9"
patch_env = _opencli_environment(cdp_endpoint="http://127.0.0.1:9")
patch_rc, patch_output = await _command(
opencli_bin,
*registration.route_probe_args,
Expand All @@ -101,15 +121,104 @@ async def _registration_is_available(
return patch_rc != 0 and registration.route_probe_error in patch_output


def _anonymous_profile_available() -> bool:
def _profile_unavailable_reason(profile_kind: str) -> str:
return (
"no_clean_profile"
if profile_kind == "anonymous"
else f"no_{profile_kind}_profile"
)


def _profile_endpoints(profile_kind: str) -> tuple[object | None, list[str]]:
from backend.browser_pool import get_pool

try:
pool = get_pool()
except RuntimeError:
return False
return any(
pool.get_profile_kind(endpoint) == "anonymous" for endpoint in pool.endpoints
return None, []
return pool, [
endpoint
for endpoint in pool.endpoints
if pool.get_profile_kind(endpoint) == profile_kind
]


def _browser_environment(pool: Any, endpoint: str) -> dict[str, str]:
env = os.environ.copy()
if pool.get_mode(endpoint) == "bridge":
env.pop("OPENCLI_CDP_ENDPOINT", None)
env["OPENCLI_DAEMON_HOST"] = urlparse(endpoint).hostname or "agent-1"
env["OPENCLI_DAEMON_PORT"] = "19825"
else:
env.pop("OPENCLI_DAEMON_HOST", None)
env.pop("OPENCLI_DAEMON_PORT", None)
env["OPENCLI_CDP_ENDPOINT"] = endpoint
return env


def _json_payload(output: str) -> dict | None:
start = next((index for index, char in enumerate(output) if char in "[{"), None)
if start is None:
return None
try:
parsed, _ = json.JSONDecoder().raw_decode(output[start:])
except json.JSONDecodeError:
return None
if isinstance(parsed, list):
parsed = parsed[0] if parsed else None
return parsed if isinstance(parsed, dict) else None


async def _session_is_ready(
registration: CapabilityRegistration,
pool: Any,
endpoint: str,
) -> bool:
if not registration.session_probe_args:
return True
from backend.config import get_settings

if get_settings().collection_mode == "agent":
from backend.channels.opencli_channel import (
_collect_via_agent,
_collect_via_ws_agent,
)

site, command = registration.session_probe_args[:2]
mode = pool.get_mode(endpoint)
get_protocol = getattr(pool, "get_agent_protocol", None)
get_agent_url = getattr(pool, "get_agent_url", None)
protocol = get_protocol(endpoint) if get_protocol else "http"
agent_url = (get_agent_url(endpoint) if get_agent_url else None) or endpoint
if protocol == "ws":
result = await _collect_via_ws_agent(
agent_url, site, command, {}, [], "json", mode, None
)
else:
result = await _collect_via_agent(
agent_url, site, command, {}, [], "json", mode, None
)
payload = result.items[0] if result.success and result.items else None
rc = 0 if payload is not None else 1
else:
opencli_bin = resolve_opencli_bin()
rc, output = await _command(
opencli_bin,
*registration.session_probe_args,
env=_browser_environment(pool, endpoint),
)
payload = _json_payload(output)
return bool(
rc == 0
and payload
and payload.get("unattendedReady") is True
and payload.get("loginDetected") is False
and payload.get("promptInputDetected") is True
and (
registration.session_expected_host is None
or urlparse(str(payload.get("url", ""))).hostname
== registration.session_expected_host
)
)


Expand All @@ -118,19 +227,38 @@ async def probe_capabilities() -> list[CapabilityDescriptor]:
if not await _runtime_is_installed():
return []

ready = _anonymous_profile_available()
descriptors = []
for registration in list_capability_registrations():
if not await _registration_is_available(registration):
continue
pool, endpoints = _profile_endpoints(registration.required_profile_kind)
ready = bool(endpoints)
unavailable_reason = (
None
if ready
else _profile_unavailable_reason(registration.required_profile_kind)
)
if ready and registration.session_probe_args:
ready = any(
[
await _session_is_ready(registration, pool, endpoint)
for endpoint in endpoints
]
)
if not ready:
unavailable_reason = (
registration.session_unavailable_reason
or "browser_session_not_ready"
)
Comment on lines +241 to +252

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🚀 Performance & Scalability | 🟠 Major | ⚡ Quick win

any([... for endpoint in endpoints]) probes every endpoint even after one succeeds.

ready = any(
    [
        await _session_is_ready(registration, pool, endpoint)
        for endpoint in endpoints
    ]
)

The list comprehension eagerly awaits _session_is_ready for every endpoint before any() is applied — there's no short-circuiting once a ready endpoint is found. probe_capabilities() runs on the hot submission path (_validate_capability in backend/api/v1/geo_acquisition.py calls it per request), so with multiple authenticated endpoints this needlessly serializes extra subprocess/HTTP round-trips (each probe can take up to COMMAND_TIMEOUT_SECONDS) on every capability check.

⚡ Proposed fix: short-circuit with an explicit loop
         if ready and registration.session_probe_args:
-            ready = any(
-                [
-                    await _session_is_ready(registration, pool, endpoint)
-                    for endpoint in endpoints
-                ]
-            )
+            ready = False
+            for endpoint in endpoints:
+                if await _session_is_ready(registration, pool, endpoint):
+                    ready = True
+                    break
🤖 Prompt for AI Agents
Verify each finding against current code. Fix only still-valid issues, skip the
rest with a brief reason, keep changes minimal, and validate.

In `@backend/acquisition/capabilities.py` around lines 241 - 252, Update the
endpoint readiness logic in probe_capabilities to avoid eagerly evaluating every
_session_is_ready call: iterate through endpoints, await each probe
sequentially, and stop immediately when one returns true. Preserve ready=false
when no endpoint succeeds and keep the existing unavailable_reason handling
unchanged.

descriptors.append(
CapabilityDescriptor(
capability_id=registration.capability_id,
capability_version=registration.capability_version,
output_schema_version=registration.output_schema_version,
target=registration.target,
ready=ready,
runtime=registration.runtime_identity(),
unavailable_reason=None if ready else "no_clean_profile",
unavailable_reason=None if ready else unavailable_reason,
)
)
return descriptors
48 changes: 45 additions & 3 deletions backend/acquisition/registry.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,9 @@

from dataclasses import dataclass

OHMYOPENCLI_COMMIT = "73cc60c83586ef2c95469b3b70d6cfc80fa5bc53"
OHMYOPENCLI_COMMIT = "b0fdd513f64899b068103ddd7ff0de957d778b5c"
OFFICIAL_SITE_CAPABILITY_COMMIT = "73cc60c83586ef2c95469b3b70d6cfc80fa5bc53"
DOUBAO_CAPABILITY_COMMIT = "b0fdd513f64899b068103ddd7ff0de957d778b5c"
OPENCLI_VERSION = "1.8.5"


Expand All @@ -19,13 +20,19 @@ class CapabilityRegistration:
route_probe_args: tuple[str, ...]
route_probe_error: str
required_profile_kind: str = "anonymous"
url_input_field: str | None = "url"
target: str | None = None
session_probe_args: tuple[str, ...] = ()
session_unavailable_reason: str | None = None
session_expected_host: str | None = None

@property
def identity(self) -> tuple[str, str, str]:
def identity(self) -> tuple[str, str, str, str | None]:
return (
self.capability_id,
self.capability_version,
self.output_schema_version,
self.target,
)

def runtime_identity(self) -> dict[str, str]:
Expand Down Expand Up @@ -59,6 +66,40 @@ def runtime_identity(self) -> dict[str, str]:
),
route_probe_error="CDP not reachable at http://127.0.0.1:9",
),
CapabilityRegistration(
capability_id="chat-ai.capture",
capability_version="1.0.0",
output_schema_version="1",
source_commit=DOUBAO_CAPABILITY_COMMIT,
invocation={
"site": "doubao",
"command": "capture",
"format": "json",
},
probe_args=("doubao", "capture", "--help"),
help_marker="doubao capture",
route_probe_args=(
"doubao",
"capture",
"runtime-route-probe",
"-f",
"json",
),
route_probe_error="CDP not reachable at http://127.0.0.1:9",
required_profile_kind="authenticated",
url_input_field=None,
target="doubao",
session_probe_args=(
"doubao",
"session-probe",
"--strict",
"true",
"-f",
"json",
),
session_unavailable_reason="doubao_session_not_ready",
session_expected_host="www.doubao.com",
),
)


Expand All @@ -71,8 +112,9 @@ def get_capability_registration(
capability_id: str,
capability_version: str,
output_schema_version: str,
target: str | None = None,
) -> CapabilityRegistration | None:
identity = (capability_id, capability_version, output_schema_version)
identity = (capability_id, capability_version, output_schema_version, target)
return next(
(registration for registration in _REGISTRATIONS if registration.identity == identity),
None,
Expand Down
Loading