From be05d877e080d2e3ba3dd0d0457dd28493a4abed Mon Sep 17 00:00:00 2001 From: 1012839419a-alt <1012839419a-alt@users.noreply.github.com> Date: Fri, 7 Aug 2026 07:30:36 +0800 Subject: [PATCH] feat(doubao): capture conversation_url and classify captcha blocks - after a successful ask, best-effort `doubao status` to grab the active conversation URL (https://www.doubao.com/chat/); status failure never fails the collect - classify captcha/verification-wall errors as error_type "captcha_challenge" so runners can apply cooldown/retry policy - keep default site_session=ephemeral (fresh conversation per ask); add capture_conversation_url config flag (default true) - unit tests: conversation_url parsing (id / root / garbage), status captured + tolerated, captcha classified vs generic --- backend/channels/doubao_research_channel.py | 60 +++++++++++- .../channels/test_doubao_research_channel.py | 97 ++++++++++++++++++- 2 files changed, 154 insertions(+), 3 deletions(-) diff --git a/backend/channels/doubao_research_channel.py b/backend/channels/doubao_research_channel.py index 44766ef..6a9f430 100644 --- a/backend/channels/doubao_research_channel.py +++ b/backend/channels/doubao_research_channel.py @@ -7,8 +7,18 @@ from backend.channels.base import AbstractChannel, Capabilities, ChannelResult from backend.channels.registry import register_channel -_URL_RE = re.compile(r"https?://[^\s<>\]\[\](){}\"']+", re.IGNORECASE) +_URL_RE = re.compile(r"https?://[^\s<>\[\](){}'\"]+", re.IGNORECASE) _TRAILING_URL_PUNCTUATION = ".,;:!?\uff0c\u3002\uff1b\uff1a\uff01\uff1f" +#: OpenCLI adapter reports a captcha wall this way (verified on opencli 1.8.6). +_CAPTCHA_MARKERS = ( + "verification challenge", + "captcha", + "blocked the request", + "人机验证", + "验证码", +) + + def _citations(text: str) -> list[dict[str, str]]: """Extract and de-duplicate URLs while preserving the answer's order.""" seen: set[str] = set() @@ -37,6 +47,25 @@ def _answer(rows: list[dict[str, Any]]) -> str: ).strip() +def _conversation_url(stdout: str) -> str: + """Extract https://www.doubao.com/chat/ from `doubao status -f json` output.""" + try: + rows = _parse_opencli_rows(stdout) + except Exception: + return "" + for row in rows: + url = str(row.get("Url", row.get("url", "")) or "").strip() + if "/chat/" in url: + return url + return "" + + +def _is_captcha_block(stderr: str, stdout: str) -> bool: + """True when the adapter reports a captcha/verification wall.""" + text = f"{stderr} {stdout}".lower() + return any(marker in text for marker in _CAPTCHA_MARKERS) + + async def _run_doubao_command(command: list[str]) -> tuple[int, str, str]: """Late import avoids the channel registry's legacy OpenCLI import cycle.""" from backend.channels.opencli_channel import _run_opencli @@ -102,8 +131,12 @@ async def collect(self, config: dict[str, Any], parameters: dict[str, Any]) -> C ) if returncode: + # Classify captcha walls so the runner can apply a human-in-the-loop + # or cooldown-retry policy instead of treating it as a permanent failure. + error_type = "captcha_challenge" if _is_captcha_block(stderr, stdout) else None return ChannelResult.fail( - f"opencli doubao ask exited with code {returncode}: {stderr[:500]}" + f"opencli doubao ask exited with code {returncode}: {stderr[:500]}", + error_type=error_type, ) try: answer = _answer(_parse_opencli_rows(stdout)) @@ -114,6 +147,28 @@ async def collect(self, config: dict[str, Any], parameters: dict[str, Any]) -> C if not answer: return ChannelResult.fail("Doubao returned no assistant text") + # Best-effort conversation URL: `doubao status -f json` exposes the + # active chat id (https://www.doubao.com/chat/). This is a + # read-only query against the same browser session; a failure here + # must not fail the collect — the answer is already in hand. + conversation_url = "" + if config.get("capture_conversation_url", True): + status_command = [ + _opencli_binary(), + "doubao", + "status", + "-f", + "json", + "--site-session", + str(config.get("site_session", "ephemeral")), + ] + try: + rc, so, se = await _run_doubao_command(status_command) + if rc == 0: + conversation_url = _conversation_url(so) + except Exception: + conversation_url = "" + citations = _citations(answer) if extract_citations else [] return ChannelResult.ok( [ @@ -122,6 +177,7 @@ async def collect(self, config: dict[str, Any], parameters: dict[str, Any]) -> C "content": answer, "author": "doubao", "question": question, + "conversation_url": conversation_url, "citations": citations, "citation_count": len(citations), "citation_capture": ( diff --git a/tests/unit/channels/test_doubao_research_channel.py b/tests/unit/channels/test_doubao_research_channel.py index 3d76410..d2475c9 100644 --- a/tests/unit/channels/test_doubao_research_channel.py +++ b/tests/unit/channels/test_doubao_research_channel.py @@ -1,6 +1,10 @@ import pytest -from backend.channels.doubao_research_channel import DoubaoResearchChannel, _citations +from backend.channels.doubao_research_channel import ( + DoubaoResearchChannel, + _citations, + _conversation_url, +) from backend.schemas.source import DataSourceCreate @@ -13,6 +17,29 @@ def test_citations_preserve_order_and_strip_punctuation(): ] +def test_conversation_url_extracts_chat_id(): + status = ( + '[{"Status": "Connected", "Url": ' + '"https://www.doubao.com/chat/38436240748612354", "Title": "x"}]' + ) + assert ( + _conversation_url(status) + == "https://www.doubao.com/chat/38436240748612354" + ) + + +def test_conversation_url_ignores_root_chat(): + # A freshly opened /chat page has no conversation id yet — must not be picked up. + status = ( + '[{"Status": "Connected", "Url": "https://www.doubao.com/chat", "Title": "x"}]' + ) + assert _conversation_url(status) == "" + + +def test_conversation_url_tolerates_garbage(): + assert _conversation_url("not json at all") == "" + + @pytest.mark.asyncio async def test_collect_stores_answer_and_citations(monkeypatch): async def fake_run(command): @@ -48,8 +75,76 @@ async def fake_run(command): assert result.items[0]["citations"] == [{"url": "https://example.com/"}] +@pytest.mark.asyncio +async def test_collect_captures_conversation_url(monkeypatch): + calls = [] + + async def fake_run(command): + calls.append(command) + if command[2] == "ask": + return 0, '[{"Role":"assistant","Text":"回答"}]', "" + if command[2] == "status": + return 0, ( + '[{"Status": "Connected", "Url": ' + '"https://www.doubao.com/chat/12345", "Title": "t"}]' + ), "" + return 0, "", "" + + monkeypatch.setattr("backend.channels.doubao_research_channel._run_doubao_command", fake_run) + result = await DoubaoResearchChannel().collect({"question": "测试"}, {}) + + assert result.success + assert result.items[0]["conversation_url"] == "https://www.doubao.com/chat/12345" + # ask + status both hit the adapter + assert [c[2] for c in calls] == ["ask", "status"] + + +@pytest.mark.asyncio +async def test_collect_tolerates_status_failure(monkeypatch): + async def fake_run(command): + if command[2] == "ask": + return 0, '[{"Role":"assistant","Text":"回答"}]', "" + return 1, "", "status exploded" + + monkeypatch.setattr("backend.channels.doubao_research_channel._run_doubao_command", fake_run) + result = await DoubaoResearchChannel().collect({"question": "测试"}, {}) + + # A failed status must NOT fail the collect — answer is already in hand. + assert result.success + assert result.items[0]["conversation_url"] == "" + + +@pytest.mark.asyncio +async def test_collect_classifies_captcha_block(monkeypatch): + async def fake_run(command): + return 1, "", ( + "ok: false\nerror:\n code: COMMAND_EXEC\n" + " message: Doubao blocked the request with a verification challenge\n" + " help: 'Detected challenge signal: iframe[src*=\"captcha\"]'" + ) + + monkeypatch.setattr("backend.channels.doubao_research_channel._run_doubao_command", fake_run) + result = await DoubaoResearchChannel().collect({"question": "测试"}, {}) + + assert not result.success + assert result.error_type == "captcha_challenge" + + +@pytest.mark.asyncio +async def test_collect_does_not_classify_generic_error(monkeypatch): + async def fake_run(command): + return 1, "", "some unrelated error" + + monkeypatch.setattr("backend.channels.doubao_research_channel._run_doubao_command", fake_run) + result = await DoubaoResearchChannel().collect({"question": "测试"}, {}) + + assert not result.success + assert result.error_type is None + + def test_source_schema_accepts_doubao_research_channel(): source = DataSourceCreate( name="Doubao research", channel_type="doubao_research", channel_config={"question": "test"} ) assert source.channel_type == "doubao_research" +