Files
2026-11-02 22:54:14 +08:00

3208 lines
135 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""
app/core/network_monitor.py - 网络响应拦截监听器
职责:
- 拦截网络请求响应
- 解析增量数据并流式输出
- 支持超时和取消机制
- 失败时触发回退到 DOM 模式
"""
import time
import logging
import json
import re
from collections import deque
from typing import Generator, Optional, Dict, Callable, Any
from pathlib import Path
from app.core.config import (
logger,
SSEFormatter,
BrowserConstants,
sanitize_sensitive_data,
DEFAULT_LOG_DIR,
)
from app.core.background_image_downloader import (
background_image_downloader,
build_image_download_request_context,
get_image_download_partition,
normalize_remote_image_url,
)
from app.core.parsers import ParserRegistry, ResponseParser
from app.core.driver import driver_for_tab
def _network_debug_capture_retention_seconds() -> float:
"""S12:调试快照保留期(小时,默认 24;<=0 表示只按容量清理)。"""
try:
hours = float(BrowserConstants.get("NETWORK_DEBUG_CAPTURE_RETENTION_HOURS"))
except Exception:
hours = 24.0
if hours != hours: # NaN
hours = 24.0
return max(0.0, hours) * 3600.0
def trim_network_parser_debug_dir(
target_dir: Optional[Path] = None,
max_total_bytes: Optional[int] = None,
exclude_paths: Optional[Any] = None,
max_age_seconds: Optional[float] = None,
) -> int:
"""确保 logs/network_parser_debug 目录的总占用不超过 max_total_bytes(默认 50MB)。
超过阈值时按时间先后顺序(st_mtime 升序)自动删除最早的文件。
S12:另外删除超过保留期(NETWORK_DEBUG_CAPTURE_RETENTION_HOURS,默认 24h)的快照,
避免调试期间写下的响应正文长期留在磁盘上。
Returns:
删除的文件数量。
"""
if target_dir is None:
target_dir = (DEFAULT_LOG_DIR / "network_parser_debug").resolve()
else:
target_dir = Path(target_dir).resolve()
if not target_dir.exists() or not target_dir.is_dir():
return 0
if max_total_bytes is None:
try:
mb = int(BrowserConstants.get("NETWORK_DEBUG_CAPTURE_MAX_TOTAL_MB"))
max_total_bytes = max(1, mb) * 1024 * 1024
except Exception:
max_total_bytes = 50 * 1024 * 1024
if max_age_seconds is None:
max_age_seconds = _network_debug_capture_retention_seconds()
exclude_set = set()
if exclude_paths:
for p in exclude_paths:
try:
exclude_set.add(Path(p).resolve())
except Exception:
pass
try:
file_entries = []
total_size = 0
for entry in target_dir.iterdir():
if not entry.is_file():
continue
try:
st = entry.stat()
file_entries.append((st.st_mtime, entry.name, st.st_size, entry))
total_size += st.st_size
except OSError:
continue
deleted_count = 0
freed_bytes = 0
if max_age_seconds and max_age_seconds > 0:
cutoff = time.time() - float(max_age_seconds)
kept_entries = []
for entry_tuple in file_entries:
mtime, name, size, file_path = entry_tuple
try:
resolved = file_path.resolve()
except Exception:
resolved = file_path
if mtime < cutoff and resolved not in exclude_set:
try:
file_path.unlink(missing_ok=True)
total_size -= size
freed_bytes += size
deleted_count += 1
continue
except OSError as unlink_exc:
logger.debug(f"[NetworkMonitor] 删除过期调试文件失败 ({name}): {unlink_exc}")
kept_entries.append(entry_tuple)
file_entries = kept_entries
if total_size <= max_total_bytes:
if deleted_count > 0:
logger.info(
f"[NetworkMonitor] 已删除 {deleted_count} 个超过保留期的网络解析调试快照"
)
return deleted_count
# 按修改时间升序排序(最早的文件排在最前面),次关键字为文件名
file_entries.sort(key=lambda x: (x[0], x[1]))
for mtime, name, size, file_path in file_entries:
if total_size <= max_total_bytes:
break
try:
resolved = file_path.resolve()
except Exception:
resolved = file_path
if resolved in exclude_set:
continue
try:
file_path.unlink(missing_ok=True)
total_size -= size
freed_bytes += size
deleted_count += 1
except OSError as unlink_exc:
logger.debug(f"[NetworkMonitor] 删除旧调试文件失败 ({name}): {unlink_exc}")
if deleted_count > 0:
logger.info(
f"[NetworkMonitor] network_parser_debug 目录超过容量限制 "
f"({max_total_bytes / (1024 * 1024):.1f}MB),已自动删除 {deleted_count} 个最早的历史文件,"
f"释放 {freed_bytes / (1024 * 1024):.2f}MB,当前总大小 {total_size / (1024 * 1024):.2f}MB"
)
return deleted_count
except Exception as exc:
logger.debug(f"[NetworkMonitor] 清理 network_parser_debug 失败: {exc}")
return 0
_NETWORK_DEBUG_CAPTURE_WARNED = False
def _warn_network_debug_capture_enabled_once() -> None:
global _NETWORK_DEBUG_CAPTURE_WARNED
if _NETWORK_DEBUG_CAPTURE_WARNED:
return
_NETWORK_DEBUG_CAPTURE_WARNED = True
logger.warning(
"⚠️ 网络响应调试抓取已开启(NETWORK_DEBUG_CAPTURE_ENABLED):命中解析器的响应正文片段"
"(可能含聊天内容)会写入 logs/network_parser_debug,调试完成后请关闭并清理该目录"
)
def _debug_preview(value: Any, limit: int = 240) -> str:
text = repr(value)
if len(text) <= limit:
return text
return text[: limit - 3] + "..."
# ================= 自定义异常 =================
class NetworkMonitorTimeout(Exception):
"""网络监听超时异常(触发回退到 DOM 模式)"""
pass
class NetworkMonitorError(Exception):
"""网络监听错误异常"""
pass
class NetworkInterceptionTriggered(NetworkMonitorError):
"""网络拦截命中后主动中断当前监听。"""
pass
class NetworkMonitorTerminalError(NetworkMonitorError):
"""解析器确认当前目标流已失败,应立即终止工作流而非回退 DOM。"""
pass
class NetworkMonitorRecoverableError(NetworkMonitorError):
"""可恢复的流请求错误(如 Arena 429/403/503 或 post-to-evaluation 404),应保持监听等待页面重发。"""
def __init__(self, event: Dict[str, Any], status_code: int, error_text: str, raw_body: str = ""):
super().__init__(error_text)
self.event = event
self.status_code = status_code
self.error_text = error_text
self.raw_body = raw_body
class _EventOnlyParser(ResponseParser):
"""仅用于消费网络事件,不输出任何流式内容。"""
@classmethod
def get_id(cls) -> str:
return "event_only"
def reset(self):
return None
def parse_chunk(self, raw_response: str) -> Dict[str, Any]:
return {
"content": "",
"images": [],
"done": False,
"error": None,
}
# ================= 网络监听器 =================
class NetworkMonitor:
"""
网络响应拦截监听器
核心流程:
1. 启动网络监听(page.listen.start)
2. 循环等待响应(page.listen.wait)
3. 解析响应增量(parser.parse_chunk)
4. 流式输出(yield SSE chunk)
5. 检测结束条件(超时/done/取消)
回退机制:
- 首次响应超时(5s)→ 抛出 NetworkMonitorTimeout
- executor 捕获后切换到 StreamMonitor
"""
# 默认超时配置
DEFAULT_FIRST_RESPONSE_TIMEOUT = 300.0 # 首次响应超时(触发回退)
DEFAULT_HARD_TIMEOUT = 300 # 全局硬超时
DEFAULT_RESPONSE_INTERVAL = 0.5 # 响应轮询间隔
DEFAULT_SILENCE_THRESHOLD = 4.0 # 静默超时(无新数据)
DEFAULT_FIRST_CONTENT_TIMEOUT = 15.0 # 命中目标流后,等待首个有效正文的宽限
DEFAULT_IMAGE_FIRST_CONTENT_TIMEOUT = 60.0 # 生图首字(媒体)等待宽限
DEFAULT_POST_CONTENT_SILENCE_THRESHOLD = 15.0 # 已有输出后的自然停顿宽限
DEFAULT_INITIAL_TARGET_BODY_WAIT = 4.0 # 首个目标响应空 body 时的补等宽限
MAX_LISTEN_RESTARTS = 3 # 监听状态异常后的最大重建次数
MAX_PREFETCHED_RESPONSES = 64 # 发送后预取响应缓存上限,避免噪声网络无限堆积
DEFAULT_LMARENA_STREAM_EXCLUDE_PATTERNS = (
"/nextjs-api/stream/stop/",
)
MAX_STREAM_CHUNK_MERGE_CACHE = 8 # 活跃流 chunks 合并缓存上限,避免长流重复全量 join
MAX_RAW_BODY_SIGNATURE_EDGE = 512 # 媒体状态缓存只保留首尾签名片段,避免挂住完整 body
CANCEL_CHECK_SLICE = 1.0 # 长等待期间的取消检查切片(秒)
ACTIVE_STREAM_RESPONSE_POLL_TIMEOUT = 0.01 # 锁定 SSE 后仅快速扫队列,不阻塞吐出
LISTEN_RESTART_BACKOFF = 0.1 # 异常重建后的最小退避,避免忙循环
PRESERVED_LISTENER_TTL = 120.0 # 修复#3b:可恢复错误后保留监听的最长存活时间(秒)
def __init__(self, tab, formatter: SSEFormatter,
parser: ResponseParser,
stop_checker: Optional[Callable[[], bool]] = None,
stream_config: Optional[Dict] = None,
event_handler: Optional[Callable[[Dict[str, Any]], bool]] = None,
result_handler: Optional[Callable[[Dict[str, Any]], bool]] = None,
image_config: Optional[Dict] = None):
"""
初始化网络监听器
Args:
tab: DrissionPage 标签页对象
formatter: SSE 格式化器
parser: 响应解析器
stop_checker: 取消检查函数
stream_config: 流式配置
"""
self.tab = tab
self.formatter = formatter
self.parser = parser
self._should_stop = stop_checker or (lambda: False)
self._event_handler = event_handler
self._result_handler = result_handler
# 从配置中加载参数
self._stream_config = stream_config or {}
self._image_config = image_config or {}
network_config = self._stream_config.get("network", {})
top_level_hard_timeout = self._stream_config.get(
"hard_timeout",
self.DEFAULT_HARD_TIMEOUT
)
self._listen_pattern = network_config.get("listen_pattern", "")
self._stream_match_pattern = network_config.get(
"stream_match_pattern",
self._listen_pattern,
)
self._stream_match_mode = str(
network_config.get("stream_match_mode", "keyword") or "keyword"
).strip().lower()
try:
self._parser_id = str(parser.get_id() or "").strip().lower()
except Exception:
self._parser_id = parser.__class__.__name__.strip().lower()
configured_excludes = network_config.get("stream_exclude_patterns")
if configured_excludes is None:
configured_excludes = (
self.DEFAULT_LMARENA_STREAM_EXCLUDE_PATTERNS
if self._parser_id.startswith("lmarena")
else ()
)
elif isinstance(configured_excludes, str):
configured_excludes = (configured_excludes,)
self._stream_exclude_patterns = tuple(
str(pattern or "").strip().lower()
for pattern in (configured_excludes or ())
if str(pattern or "").strip()
)
self._hard_timeout = network_config.get(
"hard_timeout",
top_level_hard_timeout
)
self._first_response_timeout = network_config.get(
"first_response_timeout",
self._hard_timeout
)
self._response_interval = network_config.get(
"response_interval",
self.DEFAULT_RESPONSE_INTERVAL
)
self._silence_threshold = network_config.get(
"silence_threshold",
self.DEFAULT_SILENCE_THRESHOLD
)
first_content_timeout = network_config.get(
"first_content_timeout",
max(
(
self.DEFAULT_IMAGE_FIRST_CONTENT_TIMEOUT
if bool(
(self._image_config.get("_arena_image_generation_active"))
or self._image_config.get("arena_image_generation")
)
else self.DEFAULT_FIRST_CONTENT_TIMEOUT
),
float(self._silence_threshold or self.DEFAULT_SILENCE_THRESHOLD) * 4.0,
),
)
try:
first_content_timeout = float(first_content_timeout)
except (TypeError, ValueError, OverflowError):
first_content_timeout = max(
(
self.DEFAULT_IMAGE_FIRST_CONTENT_TIMEOUT
if bool(
(self._image_config.get("_arena_image_generation_active"))
or self._image_config.get("arena_image_generation")
)
else self.DEFAULT_FIRST_CONTENT_TIMEOUT
),
float(self._silence_threshold or self.DEFAULT_SILENCE_THRESHOLD) * 4.0,
)
self._first_content_timeout = min(
max(first_content_timeout, float(self._silence_threshold or self.DEFAULT_SILENCE_THRESHOLD)),
max(float(self._hard_timeout or self.DEFAULT_HARD_TIMEOUT), 1.0),
)
post_content_silence_threshold = network_config.get(
"post_content_silence_threshold",
max(
self.DEFAULT_POST_CONTENT_SILENCE_THRESHOLD,
float(self._silence_threshold or self.DEFAULT_SILENCE_THRESHOLD) * 2.0,
),
)
try:
post_content_silence_threshold = float(post_content_silence_threshold)
except (TypeError, ValueError, OverflowError):
post_content_silence_threshold = max(
self.DEFAULT_POST_CONTENT_SILENCE_THRESHOLD,
float(self._silence_threshold or self.DEFAULT_SILENCE_THRESHOLD) * 2.0,
)
self._post_content_silence_threshold = min(
max(
post_content_silence_threshold,
float(self._silence_threshold or self.DEFAULT_SILENCE_THRESHOLD),
),
max(float(self._hard_timeout or self.DEFAULT_HARD_TIMEOUT), 1.0),
)
initial_target_body_wait = network_config.get(
"initial_target_body_wait",
max(
self.DEFAULT_INITIAL_TARGET_BODY_WAIT,
float(self._silence_threshold or self.DEFAULT_SILENCE_THRESHOLD)
+ float(self._response_interval or self.DEFAULT_RESPONSE_INTERVAL),
),
)
try:
initial_target_body_wait = float(initial_target_body_wait)
except (TypeError, ValueError, OverflowError):
initial_target_body_wait = max(
self.DEFAULT_INITIAL_TARGET_BODY_WAIT,
float(self._silence_threshold or self.DEFAULT_SILENCE_THRESHOLD)
+ float(self._response_interval or self.DEFAULT_RESPONSE_INTERVAL),
)
self._initial_target_body_wait = min(
max(initial_target_body_wait, max(float(self._response_interval or 0.5), 0.2)),
max(float(self._hard_timeout or self.DEFAULT_HARD_TIMEOUT), 1.0),
)
# 监听预启动标记(用于提前启动监听)
self._pre_started = False
# 状态追踪
self._is_listening = False
self._cdp_session_listening = False
self._total_chunks = 0
self._total_content_chars = 0
self._reset_prefetched_responses()
self._reset_stream_chunk_merge_cache()
self._debug_capture_counter = 0
self._debug_capture_session_key = f"{int(time.time() * 1000)}_{id(self):x}"
self._debug_capture_written_stages = set()
self._debug_capture_has_content_snapshot = False
self._debug_trace_counter = 0
self._debug_trace_path: Optional[Path] = None
self._last_stream_event: Dict[str, Any] = {}
self._last_stream_raw_body: str = ""
self._last_stream_raw_body_len: int = 0
self._last_stream_raw_body_signature = None
self._last_stream_parse_result: Dict[str, Any] = {}
self._reset_media_generation_state_cache()
self._last_stream_media_items: list[Dict[str, Any]] = []
self._prefetched_image_urls: set[str] = set()
self._send_attempt_baseline_targets = 0
self._send_attempt_baseline_requests = 0
self._send_attempt_marked_at = 0.0
# 修复#3b:保留监听的过期时间戳(0 表示当前无保留状态)
self._preserve_deadline = 0.0
logger.debug(
f"[NetworkMonitor] 初始化完成 "
f"(pattern={self._listen_pattern!r}, "
f"parser={parser.get_id()}, "
f"first_response_timeout={self._first_response_timeout}, "
f"first_content_timeout={self._first_content_timeout})"
)
def _handle_parse_result(self, parse_result: Dict[str, Any]) -> Dict[str, Any]:
if not isinstance(parse_result, dict):
return {
"content": "",
"images": [],
"done": False,
"error": None,
}
error_text = str(parse_result.get("error") or "").strip()
if not error_text:
return parse_result
clean_error_text = error_text
try:
from app.services.error_metadata import resolve_error_metadata
meta = resolve_error_metadata(error_text)
if meta and meta.message:
clean_error_text = f"[{meta.code}] {meta.message}" if (meta.code and meta.code not in {"error", "workflow_failed", "upstream_error"}) else meta.message
except Exception:
pass
logger.warning(f"[NetworkMonitor] 解析失败: {clean_error_text}")
should_abort = False
try:
should_abort = bool(self.parser.should_abort_on_error())
except Exception:
should_abort = False
if should_abort:
raise NetworkMonitorTerminalError(error_text)
return {
**parse_result,
"content": "",
"done": False,
}
def _should_fallback_to_dom_on_empty_stream(self) -> bool:
try:
return bool(self.parser.should_fallback_to_dom_when_no_visible_content())
except Exception:
return False
def _should_require_explicit_done(self) -> bool:
checker = getattr(self.parser, "should_require_explicit_done", None)
if not callable(checker):
return False
try:
return bool(checker())
except Exception:
return False
def _should_wait_for_replacement_stream_on_incomplete_capture(self) -> bool:
checker = getattr(
self.parser,
"should_wait_for_replacement_stream_on_incomplete_capture",
None,
)
if not callable(checker):
return False
try:
return bool(checker())
except Exception:
return False
def _should_wait_for_followup_stream(self) -> bool:
checker = getattr(self.parser, "should_wait_for_followup_stream", None)
if not callable(checker):
return False
try:
return bool(checker())
except Exception:
return False
def _prepare_parser_for_followup_stream(self) -> None:
prepare = getattr(self.parser, "prepare_for_followup_stream", None)
if callable(prepare):
try:
prepare()
return
except Exception as e:
logger.debug(f"[NetworkMonitor] 准备后续流解析器失败,回退普通重置: {e}")
self._reset_parser_state()
@staticmethod
def _parse_result_has_unclosed_render_output(parse_result: Dict[str, Any]) -> bool:
return bool(
isinstance(parse_result, dict)
and parse_result.get("unclosed_render_output")
)
@staticmethod
def _parse_result_has_media(parse_result: Dict[str, Any]) -> bool:
if not isinstance(parse_result, dict):
return False
images = parse_result.get("images")
return isinstance(images, list) and bool(images)
def _should_fallback_on_empty_done(self, parse_result: Dict[str, Any]) -> bool:
return bool(
self._total_chunks == 0
and not self._parse_result_has_media(parse_result)
and not self._last_stream_media_items
and self._should_fallback_to_dom_on_empty_stream()
)
def _should_fallback_on_incomplete_empty_capture(self) -> bool:
return bool(
self._total_chunks == 0
and not self._last_stream_media_items
and self._should_fallback_to_dom_on_empty_stream()
)
def _has_stream_output(self, parse_result: Optional[Dict[str, Any]] = None) -> bool:
"""Return whether the target stream produced text, reasoning, or media."""
return bool(
self._total_chunks > 0
or self._last_stream_media_items
or self._parse_result_has_media(parse_result or self._last_stream_parse_result)
)
def _effective_silence_threshold(
self,
*,
active_stream_response: Any = None,
completed_by_done: bool = False,
waiting_for_followup_stream: bool = False,
) -> float:
"""Choose a silence budget that distinguishes startup from post-output pauses."""
threshold = float(self._silence_threshold)
if self._has_stream_output():
threshold = max(threshold, float(self._post_content_silence_threshold))
if (
active_stream_response is not None
and self._should_require_explicit_done()
and not completed_by_done
):
threshold = max(threshold, float(self._first_content_timeout))
if waiting_for_followup_stream:
threshold = max(threshold, 30.0, float(self._first_content_timeout))
if (
active_stream_response is not None
and not self._has_stream_output()
and not self._stream_capture_complete(active_stream_response)
):
threshold = max(threshold, float(self._first_content_timeout))
return threshold
@staticmethod
def _extract_http_status(event: Dict[str, Any]) -> int:
try:
return int(event.get("status") or 0)
except Exception:
return 0
@staticmethod
def _extract_http_error_detail(raw_body: str) -> str:
text = str(raw_body or "").strip()
if not text:
return ""
try:
data = json.loads(text)
except (TypeError, ValueError, RecursionError):
data = None
if isinstance(data, dict):
for key in ("message", "error", "detail", "title", "reason"):
value = data.get(key)
if isinstance(value, str) and value.strip():
return " ".join(value.split())[:240]
if isinstance(value, dict):
nested = value.get("message") or value.get("detail")
if isinstance(nested, str) and nested.strip():
return " ".join(nested.split())[:240]
return " ".join(text.split())[:240]
@classmethod
def _build_http_status_error_text(cls, event: Dict[str, Any], raw_body: str = "") -> str:
status_code = cls._extract_http_status(event)
if status_code <= 0:
return ""
reason_map = {
400: "Bad Request",
401: "Unauthorized",
403: "Forbidden",
404: "Not Found",
408: "Request Timeout",
409: "Conflict",
422: "Unprocessable Entity",
429: "Too Many Requests",
500: "Internal Server Error",
502: "Bad Gateway",
503: "Service Unavailable",
504: "Gateway Timeout",
}
reason = reason_map.get(status_code, "HTTP Error")
detail = cls._extract_http_error_detail(raw_body)
if detail:
return f"HTTP {status_code} {reason}: {detail}"
return f"HTTP {status_code} {reason}"
@staticmethod
def _extract_status_code_from_error_detail(error: Any) -> int:
"""修复#3a:从错误详情中解析 HTTP 状态码数值,解析不出返回 0。
本文件的终态 HTTP 错误统一经 _build_http_status_error_text 生成
"HTTP <code> <reason>" 格式;另兼容 "status=<code>" 类写法。
"""
detail = str(error or "")
match = re.search(r"\bHTTP[ _:-]?\s*(\d{3})\b", detail, flags=re.IGNORECASE)
if not match:
match = re.search(
r"\bstatus(?:[ _-]?code)?\s*[=:]\s*(\d{3})\b",
detail,
flags=re.IGNORECASE,
)
if not match:
return 0
try:
return int(match.group(1))
except Exception:
return 0
@staticmethod
def _should_preserve_listener_after_terminal_error(error: Any) -> bool:
"""Keep Network listeners alive for challenge flows that auto-replay the request.
修复#3a:不再用 "429"/"403" 子串匹配(URL、token 数等无关数字会误触发),
改为解析出 HTTP 状态码数值后与可自动重放的状态码集合比较;
解析不出数值则不保留监听。
"""
status_code = NetworkMonitor._extract_status_code_from_error_detail(error)
return status_code in (403, 429, 503)
def _listener(self):
"""R2-1:经驱动取得本标签页的监听器包装;没有标签页或标签页不支持监听时返回 None。"""
if self.tab is None:
return None
try:
return driver_for_tab(self.tab).listener_or_none
except Exception:
return None
def _listen_is_active(self) -> bool:
listener = self._listener()
return bool(listener is not None and listener.is_active())
def _force_reset_listen_state(self):
listener = self._listener()
if listener is not None:
listener.force_reset()
def _safe_stop_listen(self):
listener = self._listener()
if listener is not None:
listener.safe_stop()
@staticmethod
def _is_restartable_listen_error(err_text: str) -> bool:
err_text = str(err_text or "")
return (
"监听未启动或已停止" in err_text
or ("NoneType" in err_text and "is_running" in err_text)
)
def _sleep_after_listen_restart(self, attempts: int) -> None:
delay = min(0.5, self.LISTEN_RESTART_BACKOFF * max(1, int(attempts or 1)))
time.sleep(delay)
def _reset_prefetched_responses(self) -> None:
self._prefetched_responses = deque(maxlen=self.MAX_PREFETCHED_RESPONSES)
def _reset_stream_chunk_merge_cache(self) -> None:
self._stream_chunk_merge_cache: Dict[int, Dict[str, Any]] = {}
def _reset_media_generation_state_cache(self) -> None:
self._last_media_generation_state: Dict[str, Any] = {}
self._last_media_generation_state_body_signature = None
self._last_media_generation_state_raw_body = ""
self._last_media_generation_state_parse_result: Dict[str, Any] = {}
self._last_media_generation_state_cached: bool = False
def _reset_parser_state(self) -> None:
reset = getattr(self.parser, "reset", None)
if not callable(reset):
return
try:
reset()
except Exception as e:
logger.debug(f"[NetworkMonitor] 重置解析器状态失败(忽略): {e}")
def _raw_body_cache_signature(self, raw_body: Any):
text = str(raw_body or "")
length = len(text)
edge = max(1, int(self.MAX_RAW_BODY_SIGNATURE_EDGE))
if length <= edge * 2:
return (length, text)
return (length, hash(text), text[:edge], text[-edge:])
def _get_media_generation_state_cached(
self,
raw_body: Any,
parse_result: Dict[str, Any],
) -> Dict[str, Any]:
body_signature = self._raw_body_cache_signature(raw_body)
if (
self._last_media_generation_state_cached
and body_signature == getattr(self, "_last_media_generation_state_body_signature", None)
and parse_result == self._last_media_generation_state_parse_result
):
return dict(self._last_media_generation_state)
media_state = self.parser.get_media_generation_state(
raw_response=raw_body,
parse_result=parse_result,
)
normalized = dict(media_state) if isinstance(media_state, dict) else {}
self._last_media_generation_state = normalized
self._last_media_generation_state_body_signature = body_signature
self._last_media_generation_state_raw_body = ""
self._last_media_generation_state_parse_result = dict(parse_result or {})
self._last_media_generation_state_cached = True
return dict(normalized)
def _raw_body_preview(self, raw_body: Any) -> str:
text = str(raw_body or "")
edge = max(1, int(self.MAX_RAW_BODY_SIGNATURE_EDGE))
if len(text) <= edge * 2:
return text
return f"{text[:edge]}...{text[-edge:]}"
def _remember_last_stream_result(
self,
event: Dict[str, Any],
raw_body: Any,
parse_result: Dict[str, Any],
) -> None:
self._last_stream_event = dict(event or {})
text = str(raw_body or "")
self._last_stream_raw_body = self._raw_body_preview(text)
self._last_stream_raw_body_len = len(text)
self._last_stream_raw_body_signature = self._raw_body_cache_signature(text)
self._last_stream_parse_result = dict(parse_result or {})
def _remember_prefetched_response(self, response: Any) -> None:
if response is None or response is False:
return
self._prefetched_responses.append(response)
def _is_recoverable_stream_error(self, event: Dict[str, Any], status_code: int = 0) -> bool:
"""判断是否为 Arena 场景下的可恢复流错误。
- 403/429/503:适用于 Arena evaluation stream。
- 404:仅适用于 /nextjs-api/stream/post-to-evaluation/{id}。
- 400/401/422/500 等保持立即失败。
"""
if status_code <= 0:
status_code = self._extract_http_status(event)
url = str(event.get("url", "") or "")
if status_code == 404 and "post-to-evaluation" in url:
return True
if status_code in (403, 429, 503):
if "/nextjs-api/stream/" in url or "arena.ai" in url:
return True
return False
def _raise_for_http_error_status(
self,
event: Dict[str, Any],
raw_body: str,
raw_body_source: str = "",
is_event_stream: bool = False,
) -> None:
status_code = self._extract_http_status(event)
if status_code < 400:
return
error_text = self._build_http_status_error_text(event, raw_body)
is_recoverable = self._is_recoverable_stream_error(event, status_code)
self._write_parser_debug_dump(
raw_body,
event,
{
"content": "",
"images": [],
"done": False,
"error": error_text or f"HTTP {status_code}",
},
raw_body_source or "http_error",
is_event_stream,
force_stage="http_error",
extra_payload={
"http_status": status_code,
"is_recoverable": is_recoverable,
},
)
if is_recoverable:
logger.warning(
f"[NetworkMonitor] 捕获可恢复状态码 (status={status_code}, url={event.get('url', '')[:120]}, body_len={len(raw_body or '')}), 准备保持监听等待重试"
)
raise NetworkMonitorRecoverableError(event, status_code, error_text or f"HTTP {status_code}", raw_body)
logger.warning(
"[NetworkMonitor] 目标流返回异常状态码,终止工作流 "
f"(status={status_code}, url={event.get('url', '')[:120]}, "
f"body_len={len(raw_body or '')})"
)
raise NetworkMonitorTerminalError(error_text or f"HTTP {status_code}")
def _pop_prefetched_response(self) -> Optional[Any]:
if not self._prefetched_responses:
return None
return self._prefetched_responses.popleft()
def _start_listen(self):
if not self._listen_pattern:
raise NetworkMonitorError("listen_pattern 未配置")
self._reset_prefetched_responses()
self._reset_stream_chunk_merge_cache()
driver_for_tab(self.tab).listener.reuse_driver = True
try:
# tab.listen 与全局监听共享:显式 res_type=True,避免继承全局监听的资源类型过滤
driver_for_tab(self.tab).listener.start(self._listen_pattern, res_type=True)
except TypeError:
driver_for_tab(self.tab).listener.start(self._listen_pattern)
if not self._listen_is_active():
raise NetworkMonitorError("监听启动后未进入活动状态")
self._pre_started = True
self._is_listening = True
def _read_listen_counters(self) -> Dict[str, int]:
listener = self._listener()
if listener is None:
return {
"running_targets": 0,
"running_requests": 0,
"queued_packets": 0,
}
return listener.counters()
def mark_send_attempt(self):
"""Record the listener baseline immediately before a submit action."""
if not self._listen_pattern:
return
try:
if not self._listen_is_active():
self._ensure_listening("mark_send_attempt")
except Exception as e:
logger.debug(f"[NetworkMonitor] 记录发送基线失败: {e}")
return
snapshot = self._read_listen_counters()
self._send_attempt_baseline_targets = snapshot["running_targets"]
self._send_attempt_baseline_requests = snapshot["running_requests"]
self._send_attempt_marked_at = time.time()
logger.debug(
"[NetworkMonitor] 已记录发送基线 "
f"(targets={snapshot['running_targets']}, "
f"requests={snapshot['running_requests']}, "
f"queued={snapshot['queued_packets']})"
)
def rebuild_after_external_interruption(self, reason: str = "external_interrupt") -> None:
"""Restart listening after a command/captcha interruption may have replayed the request."""
if not self._listen_pattern:
return
self._send_attempt_baseline_targets = 0
self._send_attempt_baseline_requests = 0
self._send_attempt_marked_at = 0.0
if self._listen_is_active():
self._is_listening = True
self._pre_started = True
counters = self._read_listen_counters()
logger.debug(
f"[NetworkMonitor] 外部中断后保留现有监听 ({reason or 'external_interrupt'}, "
f"targets={counters['running_targets']}, requests={counters['running_requests']}, "
f"queued={counters['queued_packets']})"
)
return
self._is_listening = False
self._pre_started = False
self._safe_stop_listen()
self._force_reset_listen_state()
self._start_listen()
logger.debug(f"[NetworkMonitor] 已重建监听 ({reason or 'external_interrupt'})")
self.mark_send_attempt()
def poll_send_activity(self, timeout: float = 0.25) -> Dict[str, Any]:
"""
发送后短窗口里轻量探测一次网络活动。
如果拿到了响应对象,会先缓存起来,避免后续 monitor() 丢掉首个事件。
"""
if not self._listen_pattern:
return {"seen": False, "matched": False}
try:
if not self._listen_is_active():
self._ensure_listening("poll_send_activity")
except Exception as e:
err_text = str(e)
if self._is_restartable_listen_error(err_text):
try:
self._ensure_listening("poll_send_activity_restart")
except Exception:
return {"seen": False, "matched": False, "error": err_text}
self._sleep_after_listen_restart(1)
return {"seen": False, "matched": False, "error": err_text}
return {"seen": False, "matched": False, "error": err_text}
baseline_targets = int(getattr(self, "_send_attempt_baseline_targets", 0) or 0)
deadline = time.time() + max(0.01, float(timeout or 0.01))
saw_any_response = False
last_event: Dict[str, Any] = {}
listen_restart_attempts = 0
while True:
counters = self._read_listen_counters()
if counters["running_targets"] > baseline_targets:
return {
"seen": True,
"matched": True,
"source": "request_started",
**counters,
}
remaining = deadline - time.time()
if remaining <= 0:
return {
"seen": saw_any_response,
"matched": False,
"source": "timeout",
"event": last_event,
**counters,
}
wait_timeout = min(0.05, max(0.01, remaining))
try:
response = driver_for_tab(self.tab).listener.wait(timeout=wait_timeout)
except Exception as e:
err_text = str(e)
if self._is_restartable_listen_error(err_text):
listen_restart_attempts += 1
if listen_restart_attempts > self.MAX_LISTEN_RESTARTS:
return {
"seen": saw_any_response,
"matched": False,
"error": (
f"监听状态恢复失败(已重试 {self.MAX_LISTEN_RESTARTS} 次): {err_text}"
),
**counters,
}
try:
self._ensure_listening("poll_send_activity_wait_restart")
except Exception:
return {"seen": saw_any_response, "matched": False, "error": err_text, **counters}
self._sleep_after_listen_restart(listen_restart_attempts)
continue
return {"seen": saw_any_response, "matched": False, "error": err_text, **counters}
if response is None or response is False:
continue
saw_any_response = True
self._remember_prefetched_response(response)
event = self._extract_event(response)
last_event = event
matched = False
try:
matched = self.parser.get_id() != "event_only" and self._matches_stream_target(event)
except Exception:
matched = False
counters = self._read_listen_counters()
if matched:
return {
"seen": True,
"matched": True,
"source": "response_packet",
"event": event,
**counters,
}
def _ensure_listening(self, reason: str):
if self._is_listening and self._listen_is_active():
return
self._is_listening = False
self._pre_started = False
self._safe_stop_listen()
self._force_reset_listen_state()
try:
self._start_listen()
logger.debug(f"[NetworkMonitor] 已重建监听 ({reason})")
except Exception as e:
logger.error(f"[NetworkMonitor] 启动监听失败 ({reason}): {e}")
raise NetworkMonitorError(f"启动监听失败: {e}")
def _wait_for_response(self, timeout: float) -> Any:
wait_budget = max(0.01, float(timeout or 0.01))
remaining = wait_budget
while remaining > 0:
if self._should_stop():
return False
try:
if hasattr(self.tab, "states") and not self.tab.states.is_alive:
logger.warning("[NetworkMonitor] 检测到标签页已被关闭,强行退出网络监听")
return False
except Exception:
pass
step_timeout = min(remaining, self.CANCEL_CHECK_SLICE)
if not self._listen_is_active():
self._ensure_listening("wait_inactive")
response = driver_for_tab(self.tab).listener.wait(timeout=step_timeout)
if response not in (None, False):
return response
remaining -= step_timeout
return False
def _extract_event(self, response: Any) -> Dict[str, Any]:
req = getattr(response, "request", None)
resp = getattr(response, "response", None)
url = (
getattr(req, "url", None)
or getattr(resp, "url", None)
or getattr(response, "url", None)
or ""
)
method = (
getattr(req, "method", None)
or getattr(response, "method", None)
or ""
)
status = (
getattr(resp, "status", None)
or getattr(resp, "status_code", None)
or getattr(response, "status", None)
or 0
)
try:
status = int(status)
except (TypeError, ValueError, OverflowError):
status = 0
return {
"url": str(url or ""),
"method": str(method or "").upper(),
"status": status,
"timestamp": time.time(),
}
def _dispatch_event(self, event: Dict[str, Any]) -> bool:
if not self._event_handler:
return False
try:
return bool(self._event_handler(event))
except Exception as e:
logger.debug(f"[NetworkMonitor] 事件回调异常(忽略): {e}")
return False
def _dispatch_result(
self,
event: Dict[str, Any],
raw_body: Any,
parse_result: Dict[str, Any],
raw_body_source: str,
) -> bool:
if not self._result_handler:
return False
try:
parser_id = ""
try:
parser_id = str(self.parser.get_id() or "")
except Exception:
parser_id = self.parser.__class__.__name__
return bool(
self._result_handler(
{
"event": dict(event or {}),
"raw_body": raw_body,
"raw_body_source": str(raw_body_source or ""),
"parse_result": dict(parse_result or {}),
"parser_id": parser_id,
}
)
)
except Exception as e:
logger.debug(f"[NetworkMonitor] 结果回调异常(忽略): {e}")
return False
def _matches_stream_target(self, event: Dict[str, Any]) -> bool:
if self._is_excluded_stream_target(event):
return False
pattern = str(self._stream_match_pattern or "").strip()
if not pattern:
return True
url = str(event.get("url", "") or "")
if self._stream_match_mode == "regex":
try:
return bool(re.search(pattern, url, flags=re.IGNORECASE))
except re.error:
logger.debug(
f"[NetworkMonitor] 无效 stream_match_pattern 正则,回退关键字匹配: {pattern}"
)
return pattern.lower() in url.lower()
def _is_excluded_stream_target(self, event: Dict[str, Any]) -> bool:
"""Reject control responses that share the site's broad stream path."""
patterns = getattr(self, "_stream_exclude_patterns", ())
if not patterns:
return False
url = str(event.get("url", "") or "").strip().lower()
if not url:
return False
matched = next(
(pattern for pattern in patterns if pattern in url),
None,
)
if matched:
logger.debug(
"[NetworkMonitor] 忽略流控制接口响应 "
f"(pattern={matched!r}, url={url[:160]})"
)
return True
return False
@staticmethod
def _nested_get(container: Any, *path: str) -> Any:
current = container
for key in path:
if current is None:
return None
if isinstance(current, dict):
current = current.get(key)
else:
current = getattr(current, key, None)
return current
def _extract_raw_body(self, response: Any) -> tuple[Any, str]:
resp = getattr(response, "response", None)
if resp is None:
return None, "missing_response"
for source_name, source_value in (
("response._stream.fullText", self._nested_get(resp, "_stream", "fullText")),
("response.stream.fullText", self._nested_get(resp, "stream", "fullText")),
("response._stream.chunks", self._nested_get(resp, "_stream", "chunks")),
("response.stream.chunks", self._nested_get(resp, "stream", "chunks")),
("event._stream.fullText", self._nested_get(response, "_stream", "fullText")),
("event.stream.fullText", self._nested_get(response, "stream", "fullText")),
("event._stream.chunks", self._nested_get(response, "_stream", "chunks")),
("event.stream.chunks", self._nested_get(response, "stream", "chunks")),
):
if source_value in (None, "", [], ()):
continue
if source_name.endswith(".chunks"):
merged = self._merge_stream_chunks_cached(source_value)
if merged:
return merged, source_name
continue
return source_value, source_name
direct_body = getattr(resp, "body", None)
if direct_body not in (None, "", b"", bytearray()):
direct_body_container = self._coerce_json_container(direct_body)
if isinstance(direct_body_container, dict):
for source_name, source_value in (
("body._stream.fullText", self._nested_get(direct_body_container, "_stream", "fullText")),
("body.stream.fullText", self._nested_get(direct_body_container, "stream", "fullText")),
("body._stream.chunks", self._nested_get(direct_body_container, "_stream", "chunks")),
("body.stream.chunks", self._nested_get(direct_body_container, "stream", "chunks")),
):
if source_value in (None, "", [], ()):
continue
if source_name.endswith(".chunks"):
merged = self._merge_stream_chunks_cached(source_value)
if merged:
return merged, source_name
continue
return source_value, source_name
logger.debug(
"[NetworkMonitor][DirectBody] no stream field found in body container: "
f"{self._describe_json_container(direct_body)}"
)
else:
direct_body_text = self._normalize_raw_body(direct_body)
if not self._looks_like_sse_payload(direct_body_text):
logger.debug(
"[NetworkMonitor][DirectBody] body is not a JSON container: "
f"{self._describe_json_container(direct_body)}"
)
return direct_body, "body"
return None, "empty"
@classmethod
def _normalize_stream_chunk_data(cls, chunk: Any) -> str:
data = cls._nested_get(chunk, "data")
if data in (None, ""):
return ""
if isinstance(data, (bytes, bytearray)):
# 修复#5:补丁层(patch_drissionpage.py 的 _listener_append_stream_v1)
# 已用增量 UTF-8 解码器保证 chunks[].data 恒为 str,此分支仅为防御性兜底;
# 若真出现 bytes,用 replace 暴露问题而非 ignore 静默丢字。
return data.decode("utf-8", errors="replace")
if not isinstance(data, str):
return str(data)
return data
@staticmethod
def _merge_stream_chunks(chunks: Any) -> str:
if not isinstance(chunks, list):
return ""
parts = []
for chunk in chunks:
data = NetworkMonitor._normalize_stream_chunk_data(chunk)
if not data:
continue
parts.append(data)
return "".join(parts)
def _merge_stream_chunks_cached(self, chunks: Any) -> str:
if not isinstance(chunks, list):
return ""
if not chunks:
return ""
cache = getattr(self, "_stream_chunk_merge_cache", None)
if not isinstance(cache, dict):
self._reset_stream_chunk_merge_cache()
cache = self._stream_chunk_merge_cache
chunk_key = id(chunks)
chunk_len = len(chunks)
entry = cache.get(chunk_key)
if (
entry
and entry.get("chunks") is chunks
and entry.get("chunk_len", 0) <= chunk_len
and self._cached_stream_chunks_prefix_is_valid(entry, chunks)
):
cached_len = int(entry.get("chunk_len", 0) or 0)
if cached_len == chunk_len:
return str(entry.get("body", "") or "")
parts = [str(entry.get("body", "") or "")]
for chunk in chunks[cached_len:]:
parts.append(self._normalize_stream_chunk_data(chunk))
body = "".join(parts)
else:
body = self._merge_stream_chunks(chunks)
cache[chunk_key] = {
"chunks": chunks,
"chunk_len": chunk_len,
"tail_marker": self._stream_chunk_tail_marker(chunks),
"body": body,
}
self._trim_stream_chunk_merge_cache(cache)
return body
@classmethod
def _stream_chunk_tail_marker(cls, chunks: list[Any]) -> tuple[int, int, int, str]:
if not chunks:
return (0, 0, 0, "")
tail = chunks[-1]
return cls._stream_chunk_marker(tail)
@classmethod
def _stream_chunk_marker(cls, chunk: Any) -> tuple[int, int, int, str]:
data = cls._nested_get(chunk, "data")
try:
data_len = len(data) if data is not None else 0
except Exception:
data_len = 0
return (id(chunk), id(data), data_len, type(data).__name__)
def _cached_stream_chunks_prefix_is_valid(
self,
entry: Dict[str, Any],
chunks: list[Any],
) -> bool:
cached_len = int(entry.get("chunk_len", 0) or 0)
if cached_len <= 0:
return True
if len(chunks) < cached_len:
return False
return entry.get("tail_marker") == self._stream_chunk_marker(chunks[cached_len - 1])
def _trim_stream_chunk_merge_cache(self, cache: Dict[int, Dict[str, Any]]) -> None:
max_entries = max(1, int(self.MAX_STREAM_CHUNK_MERGE_CACHE or 1))
overflow = len(cache) - max_entries
if overflow <= 0:
return
for key in list(cache.keys())[:overflow]:
cache.pop(key, None)
@staticmethod
def _normalize_raw_body(raw_body: Any) -> str:
if raw_body is None:
return ""
if isinstance(raw_body, str):
return raw_body
if isinstance(raw_body, (bytes, bytearray)):
# 修复#5:流式路径的 body 由补丁层拼接为 str,bytes 仅出现在非流式兜底;
# 统一用 replace 暴露编码问题而非 ignore 静默丢字。
try:
return bytes(raw_body).decode("utf-8", errors="replace")
except Exception:
return bytes(raw_body).decode("utf-8", "replace")
if isinstance(raw_body, (dict, list)):
return json.dumps(raw_body, ensure_ascii=False)
return str(raw_body)
@staticmethod
def _coerce_json_container(value: Any) -> Any:
if isinstance(value, dict):
return value
if isinstance(value, str):
stripped = value.strip()
if stripped.startswith("{") and stripped.endswith("}"):
try:
parsed = json.loads(stripped)
except (TypeError, ValueError, RecursionError):
return None
return parsed if isinstance(parsed, dict) else None
return None
@staticmethod
def _describe_json_container(value: Any) -> str:
container = NetworkMonitor._coerce_json_container(value)
if not isinstance(container, dict):
return f"type={type(value).__name__}, preview={_debug_preview(value, 320)}"
def _keys_of(obj: Any) -> list[str]:
if isinstance(obj, dict):
return [str(k) for k in list(obj.keys())[:8]]
return []
message = container.get("message")
body = container.get("body")
message_content = message.get("content") if isinstance(message, dict) else None
message_content_len = len(message_content) if isinstance(message_content, str) else 0
return (
f"keys={_keys_of(container)}, "
f"stream_keys={_keys_of(container.get('stream'))}, "
f"_stream_keys={_keys_of(container.get('_stream'))}, "
f"body_keys={_keys_of(body)}, "
f"message_keys={_keys_of(message)}, "
f"message_content_len={message_content_len}, "
f"preview={_debug_preview(container, 320)}"
)
@staticmethod
def _looks_like_sse_payload(value: Any) -> bool:
if not isinstance(value, str):
return False
stripped = value.lstrip("\ufeff\r\n\t ")
if not stripped:
return False
return (
stripped.startswith("id:")
or stripped.startswith("event:")
or stripped.startswith("data:")
or "\nevent:" in stripped
or "\ndata:" in stripped
)
def _extract_content_type(self, response: Any) -> str:
resp = getattr(response, "response", None)
resp_payload = getattr(resp, "_response", None)
resp_headers = resp_payload.get("headers") if isinstance(resp_payload, dict) else None
candidates = (
self._nested_get(resp_headers, "content-type"),
self._nested_get(resp_headers, "Content-Type"),
self._nested_get(resp_headers, "contentType"),
getattr(resp, "content_type", None),
getattr(resp, "contentType", None),
self._nested_get(response, "headers", "content-type"),
self._nested_get(response, "headers", "Content-Type"),
self._nested_get(response, "headers", "contentType"),
getattr(response, "content_type", None),
getattr(response, "contentType", None),
)
for value in candidates:
if value:
return str(value).strip().lower()
return ""
def _is_event_stream_response(self, response: Any) -> bool:
content_type = self._extract_content_type(response)
if "text/event-stream" in content_type:
return True
direct_body = self._coerce_json_container(
self._nested_get(getattr(response, "response", None), "body")
)
for source_name, source_value in (
("response._stream", self._nested_get(getattr(response, "response", None), "_stream")),
("response.stream", self._nested_get(getattr(response, "response", None), "stream")),
("event._stream", self._nested_get(response, "_stream")),
("event.stream", self._nested_get(response, "stream")),
("body._stream", self._nested_get(direct_body, "_stream") if isinstance(direct_body, dict) else None),
("body.stream", self._nested_get(direct_body, "stream") if isinstance(direct_body, dict) else None),
):
if source_value not in (None, "", [], ()):
logger.debug(f"[NetworkMonitor] 检测到流响应结构: {source_name}")
return True
return False
def _stream_capture_complete(self, response: Any) -> bool:
direct_body = self._coerce_json_container(
self._nested_get(getattr(response, "response", None), "body")
)
for value in (
self._nested_get(getattr(response, "response", None), "_stream", "complete"),
self._nested_get(getattr(response, "response", None), "stream", "complete"),
self._nested_get(response, "_stream", "complete"),
self._nested_get(response, "stream", "complete"),
self._nested_get(direct_body, "_stream", "complete") if isinstance(direct_body, dict) else None,
self._nested_get(direct_body, "stream", "complete") if isinstance(direct_body, dict) else None,
):
if value is not None:
return bool(value)
return False
def _wait_for_stream_body(
self,
response: Any,
initial_body: str,
initial_source: str,
wait_budget: Optional[float] = None,
) -> tuple[str, str]:
body = initial_body
source = initial_source
if body:
return body, source
if wait_budget is None:
wait_budget = min(max(float(self._response_interval or 0.5), 0.2), 1.5)
else:
try:
wait_budget = max(float(wait_budget), 0.2)
except Exception:
wait_budget = min(max(float(self._response_interval or 0.5), 0.2), 1.5)
deadline = time.time() + wait_budget
while time.time() < deadline:
if self._should_stop():
break
raw_body, raw_body_source = self._extract_raw_body(response)
body = self._normalize_raw_body(raw_body)
if body:
logger.debug(
f"[NetworkMonitor] 流响应正文已就绪 "
f"(source={raw_body_source}, size={len(body)} chars)"
)
return body, raw_body_source
if self._stream_capture_complete(response):
break
time.sleep(0.05)
return body, source
def _wait_for_stream_progress(self, response: Any, current_body: str, current_source: str) -> tuple[str, str]:
body = current_body or ""
source = current_source
wait_budget = min(max(float(self._response_interval or 0.5), 0.2), 1.5)
deadline = time.time() + wait_budget
previous_len = len(body)
while time.time() < deadline:
if self._should_stop():
break
raw_body, raw_body_source = self._extract_raw_body(response)
next_body = self._normalize_raw_body(raw_body)
if self._is_stream_body_replacement(body, next_body):
return next_body, raw_body_source
if len(next_body) > previous_len:
return next_body, raw_body_source
if self._stream_capture_complete(response):
return body, source
time.sleep(0.05)
return body, source
@staticmethod
def _is_stream_body_replacement(previous_body: str, next_body: str) -> bool:
previous_body = previous_body or ""
next_body = next_body or ""
if not previous_body or not next_body or next_body == previous_body:
return False
return not next_body.startswith(previous_body)
def _handle_stream_body_replacement(
self,
previous_body: str,
next_body: str,
source: str,
) -> bool:
if not self._is_stream_body_replacement(previous_body, next_body):
return False
if self._total_chunks > 0:
logger.warning(
"[NetworkMonitor] 流响应原始快照发生非前缀替换,回退到 DOM 补齐 "
f"(old_len={len(previous_body or '')}, new_len={len(next_body or '')}, "
f"source={source}, chunks={self._total_chunks})"
)
raise NetworkMonitorTimeout("目标流响应快照发生非前缀替换")
logger.warning(
"[NetworkMonitor] 流响应原始快照发生非前缀替换,重置解析器后按新快照解析 "
f"(old_len={len(previous_body or '')}, new_len={len(next_body or '')}, "
f"source={source})"
)
try:
self.parser.reset()
except Exception as e:
logger.debug(f"[NetworkMonitor] 重置解析器失败(忽略): {e}")
return True
def _drain_completed_stream_tail(
self,
response: Any,
current_body: str,
current_source: str,
event: Dict[str, Any],
completion_id: str,
) -> Generator[str, None, tuple[str, str, bool]]:
"""修复#2:capture_complete 收尾竞态的尾部补读。
补丁层是"先追加尾块、后置 complete",而读取侧是"先取 body 快照、再查
complete";若尾块追加+置位发生在两步之间,外层看到 complete 直接 break
会丢失尾部。判定 capture_complete 成立后调用本方法强制再做一次 body
提取 + parse_chunk 增量下发(解析器有增量游标,重复调用幂等、代价极小)。
Returns:
(最新 body, body 来源, 补读解析是否产出 done 标志)
"""
try:
raw_body, raw_body_source = self._extract_raw_body(response)
next_body = self._normalize_raw_body(raw_body)
except Exception as e:
logger.debug(f"[NetworkMonitor] 收尾补读 body 失败(忽略): {e}")
return current_body, current_source, False
if not next_body or next_body == (current_body or ""):
return current_body, current_source, False
# 非前缀替换交由既有逻辑处理(可能重置解析器或触发 DOM 回退),保持行为一致
self._handle_stream_body_replacement(current_body, next_body, raw_body_source)
try:
parse_result = self.parser.parse_chunk(next_body)
except Exception as e:
logger.warning(f"[NetworkMonitor] 收尾补读解析异常: {e}")
return current_body, current_source, False
self._append_network_debug_trace(
"capture_complete_tail_drain",
response,
next_body,
raw_body_source,
parse_result,
True,
extra_payload={
"previous_body_len": len(current_body or ""),
"next_body_len": len(next_body),
},
)
parse_result = self._handle_parse_result(parse_result)
if parse_result.get("error"):
return next_body, raw_body_source, False
self._dispatch_result(event, next_body, parse_result, raw_body_source)
self._remember_last_stream_result(event, next_body, parse_result)
try:
self._record_parse_result_media(parse_result)
except Exception as media_exc:
logger.debug(f"[NetworkMonitor] 媒体结果记录失败(忽略): {media_exc}")
reasoning_content = parse_result.get("reasoning_content", "")
content = parse_result.get("content", "")
if reasoning_content:
self._total_chunks += 1
yield self.formatter.pack_chunk(
content="",
reasoning_content=reasoning_content,
completion_id=completion_id,
)
if content:
self._total_chunks += 1
self._total_content_chars += len(content)
logger.debug(
f"[NetworkMonitor] capture_complete 收尾补发尾部增量({len(content)} 字符)"
)
yield self.formatter.pack_chunk(content, completion_id=completion_id)
return next_body, raw_body_source, bool(parse_result.get("done", False))
def _write_parser_debug_dump(
self,
raw_body: str,
event: Dict[str, Any],
parse_result: Dict[str, Any],
raw_body_source: str,
is_event_stream: bool,
*,
force_stage: str = "",
extra_payload: Optional[Dict[str, Any]] = None,
) -> None:
if not self._is_network_debug_capture_enabled():
return
try:
parser_id = str(self.parser.get_id() or "").strip().lower()
parser_filter = self._get_network_debug_capture_parser_filter()
if parser_filter and parser_id != parser_filter:
return
capture_stage = str(force_stage or "").strip() or self._select_network_debug_capture_stage(parse_result)
if not capture_stage:
return
if force_stage:
if capture_stage in self._debug_capture_written_stages:
return
max_files = self._get_network_debug_capture_max_files_per_request()
if self._debug_capture_counter >= max_files:
return
self._debug_capture_counter += 1
max_chars = self._get_network_debug_capture_max_body_chars()
raw_body_text = str(raw_body or "")
truncated = len(raw_body_text) > max_chars
stored_body = raw_body_text[:max_chars] if truncated else raw_body_text
stored_body = str(sanitize_sensitive_data(stored_body))
has_content = bool(str(parse_result.get("content", "") or ""))
has_reasoning = bool(str(parse_result.get("reasoning_content", "") or ""))
parser_debug = None
if hasattr(self.parser, "export_debug_data"):
try:
parser_debug = sanitize_sensitive_data(
self.parser.export_debug_data(raw_body_text)
)
except Exception as parser_exc:
parser_debug = {"error": str(parser_exc)}
content_preview = str(parse_result.get("content", "") or "")[:800]
reasoning_text = str(parse_result.get("reasoning_content", "") or "")
context_data = {
"domain": getattr(self.tab, "current_domain", None),
"preset_name": getattr(self.tab, "current_preset_name", None),
"task_id": getattr(self.tab, "current_task_id", None),
"tab_id": getattr(self.tab, "id", None),
"stream_mode": self._stream_config.get("mode"),
"listen_pattern": self._listen_pattern,
"current_command": getattr(self.tab, "current_command_name", None),
}
context_data = {k: v for k, v in context_data.items() if v is not None}
payload = {
"captured_at": int(time.time()),
"parser": parser_id,
"capture_session": self._debug_capture_session_key,
"capture_index": self._debug_capture_counter,
"capture_stage": capture_stage,
"context": context_data,
"event": {
"url": sanitize_sensitive_data(str(event.get("url", "") or "")),
"method": str(event.get("method", "") or ""),
"status": int(event.get("status", 0) or 0),
"timestamp": float(event.get("timestamp", 0) or 0),
},
"source": str(raw_body_source or ""),
"is_event_stream": bool(is_event_stream),
"raw_body_len": len(raw_body_text),
"raw_body_truncated": truncated,
"raw_body": stored_body,
"parse_result": {
"content_len": len(str(parse_result.get("content", "") or "")),
"content_preview": sanitize_sensitive_data(content_preview),
"reasoning_len": len(reasoning_text),
"reasoning_preview": sanitize_sensitive_data(reasoning_text[:800]),
"done": bool(parse_result.get("done", False)),
"error": sanitize_sensitive_data(str(parse_result.get("error") or "")),
"image_count": len(parse_result.get("images", []) or []),
},
"parser_debug": parser_debug,
}
if extra_payload:
payload["diagnostics"] = sanitize_sensitive_data(dict(extra_payload))
dump_dir = (DEFAULT_LOG_DIR / "network_parser_debug").resolve()
dump_dir.mkdir(parents=True, exist_ok=True)
filename = (
f"{self._debug_capture_session_key}_{self._debug_capture_counter:03d}_"
f"{capture_stage}_{parser_id or 'unknown'}.json"
)
dump_path = dump_dir / filename
dump_path.write_text(
json.dumps(payload, ensure_ascii=False, indent=2) + "\n",
encoding="utf-8",
newline="\n",
)
self._debug_capture_written_stages.add(capture_stage)
if has_content or has_reasoning:
self._debug_capture_has_content_snapshot = True
logger.debug_throttled(
f"network.parser_debug_dump.{id(self)}",
"[NetworkMonitor] 网络解析调试快照已写入 "
f"({dump_path}, stage={capture_stage}, parser={parser_id}, "
f"body_len={len(raw_body_text)}, truncated={truncated})",
interval_sec=3.0,
)
active_excludes = {dump_path}
if self._debug_trace_path is not None:
active_excludes.add(self._debug_trace_path)
trim_network_parser_debug_dir(
target_dir=dump_dir,
max_total_bytes=self._get_network_debug_capture_max_total_bytes(),
exclude_paths=active_excludes,
)
except Exception as exc:
logger.debug(f"[NetworkMonitor] 写入网络解析调试快照失败: {exc}")
def _append_network_debug_trace(
self,
phase: str,
response: Any = None,
raw_body: Any = "",
raw_body_source: str = "",
parse_result: Optional[Dict[str, Any]] = None,
is_event_stream: bool = False,
extra_payload: Optional[Dict[str, Any]] = None,
) -> None:
"""Append low-level stream diagnostics without consuming a stage slot."""
if not self._is_network_debug_capture_enabled():
return
try:
parser_id = str(self.parser.get_id() or "").strip().lower()
parser_filter = self._get_network_debug_capture_parser_filter()
if parser_filter and parser_id != parser_filter:
return
max_events = 2000
if self._debug_trace_counter >= max_events:
return
body_text = self._normalize_raw_body(raw_body)
normalized_body = body_text.replace("\r\n", "\n").replace("\r", "\n")
event_names = []
for line in normalized_body.split("\n"):
line = line.strip()
if line.startswith("event:"):
name = line[6:].strip()
if name and (not event_names or event_names[-1] != name):
event_names.append(name)
response_obj = getattr(response, "response", None)
stream_records = []
for stream_path, stream_obj in (
("response._stream", self._nested_get(response_obj, "_stream")),
("response.stream", self._nested_get(response_obj, "stream")),
("event._stream", self._nested_get(response, "_stream")),
("event.stream", self._nested_get(response, "stream")),
):
if stream_obj in (None, "", [], ()):
continue
full_text = self._nested_get(stream_obj, "fullText")
chunks = self._nested_get(stream_obj, "chunks")
stream_records.append(
{
"path": stream_path,
"type": type(stream_obj).__name__,
"complete": self._nested_get(stream_obj, "complete"),
"full_text_len": len(self._normalize_raw_body(full_text)) if full_text not in (None, "") else 0,
"chunks_len": len(chunks) if isinstance(chunks, list) else None,
}
)
parser_state = {
"type": type(self.parser).__name__,
"pending_len": len(str(getattr(self.parser, "_pending", "") or "")),
"last_raw_length": int(getattr(self.parser, "_last_raw_length", 0) or 0),
"last_raw_response_len": len(str(getattr(self.parser, "_last_raw_response", "") or "")),
}
result = parse_result if isinstance(parse_result, dict) else {}
payload = {
"trace_index": self._debug_trace_counter + 1,
"captured_at": time.time(),
"phase": str(phase or "unknown"),
"parser": parser_id,
"capture_session": self._debug_capture_session_key,
"event": {
"url": sanitize_sensitive_data(str((self._extract_event(response) if response is not None else {}).get("url", "") or "")),
"method": str((self._extract_event(response) if response is not None else {}).get("method", "") or ""),
"status": int((self._extract_event(response) if response is not None else {}).get("status", 0) or 0),
},
"source": str(raw_body_source or ""),
"is_event_stream": bool(is_event_stream),
"body": {
"length": len(body_text),
"prefix": sanitize_sensitive_data(body_text[:600]),
"suffix": sanitize_sensitive_data(body_text[-1200:]) if body_text else "",
"event_names": event_names[-40:],
"has_message_stop": "message_stop" in event_names,
},
"response": {
"type": type(response).__name__ if response is not None else None,
"response_type": type(response_obj).__name__ if response_obj is not None else None,
"content_type": self._extract_content_type(response) if response is not None else "",
"streams": stream_records,
},
"parser_state": parser_state,
"parse_result": {
"content_len": len(str(result.get("content", "") or "")),
"reasoning_len": len(str(result.get("reasoning_content", "") or "")),
"done": bool(result.get("done", False)),
"error": sanitize_sensitive_data(str(result.get("error") or "")),
},
}
if extra_payload:
payload["extra"] = sanitize_sensitive_data(dict(extra_payload))
dump_dir = (DEFAULT_LOG_DIR / "network_parser_debug").resolve()
dump_dir.mkdir(parents=True, exist_ok=True)
if self._debug_trace_path is None:
self._debug_trace_path = dump_dir / f"{self._debug_capture_session_key}_trace_{parser_id or 'unknown'}.jsonl"
logger.debug(
f"[NetworkMonitor] 网络解析 trace 已启用 ({self._debug_trace_path})"
)
trim_network_parser_debug_dir(
target_dir=dump_dir,
max_total_bytes=self._get_network_debug_capture_max_total_bytes(),
exclude_paths={self._debug_trace_path},
)
with self._debug_trace_path.open("a", encoding="utf-8", newline="\n") as handle:
handle.write(json.dumps(payload, ensure_ascii=False) + "\n")
self._debug_trace_counter += 1
except Exception as exc:
logger.debug(f"[NetworkMonitor] 写入网络解析 trace 失败: {exc}")
def _select_network_debug_capture_stage(self, parse_result: Dict[str, Any]) -> str:
error_text = str(parse_result.get("error") or "").strip()
has_content = bool(str(parse_result.get("content", "") or ""))
has_reasoning = bool(str(parse_result.get("reasoning_content", "") or ""))
done = bool(parse_result.get("done", False))
if error_text:
stage = "error"
elif done:
stage = "done"
elif self._debug_capture_counter == 0:
stage = "initial"
elif (has_content or has_reasoning) and not self._debug_capture_has_content_snapshot:
stage = "first_content"
else:
return ""
if stage in self._debug_capture_written_stages:
return ""
max_files = self._get_network_debug_capture_max_files_per_request()
remaining_slots = max_files - self._debug_capture_counter
if remaining_slots <= 0:
return ""
# 预留最后一个槽位给结束态,避免中途流式增量把限额写满。
if stage == "first_content" and remaining_slots <= 1:
return ""
return stage
@staticmethod
def _is_network_debug_capture_enabled() -> bool:
# S12:严格布尔解析。旧实现 bool("false") == True,字符串 "false" 反而开启抓取。
try:
value = BrowserConstants.get("NETWORK_DEBUG_CAPTURE_ENABLED")
except Exception:
return False
if isinstance(value, bool):
enabled = value
elif isinstance(value, (int, float)):
enabled = value == 1
else:
enabled = str(value or "").strip().lower() in {"1", "true", "yes", "on"}
if enabled:
_warn_network_debug_capture_enabled_once()
return enabled
@staticmethod
def _get_network_debug_capture_max_body_chars() -> int:
try:
value = int(BrowserConstants.get("NETWORK_DEBUG_CAPTURE_MAX_BODY_CHARS"))
return max(2000, value)
except Exception:
return 50000
@staticmethod
def _get_network_debug_capture_max_files_per_request() -> int:
try:
value = int(BrowserConstants.get("NETWORK_DEBUG_CAPTURE_MAX_FILES_PER_REQUEST"))
return max(2, value)
except Exception:
return 3
@staticmethod
def _get_network_debug_capture_parser_filter() -> str:
try:
return str(BrowserConstants.get("NETWORK_DEBUG_CAPTURE_PARSER_FILTER") or "").strip().lower()
except Exception:
return ""
@staticmethod
def _get_network_debug_capture_max_total_bytes() -> int:
try:
mb = int(BrowserConstants.get("NETWORK_DEBUG_CAPTURE_MAX_TOTAL_MB"))
return max(1, mb) * 1024 * 1024
except Exception:
return 50 * 1024 * 1024
def _expire_stale_preserved_listener(self) -> None:
"""修复#3b:处理上一轮因可恢复错误(429/403/503)保留下来的监听。
保留监听后若上层不再重建,监听回调与 DrissionPage 无上限抓包队列会
随 tab 存活期一直泄漏。此处在下一次进入本实例时检查:
- 保留状态已超过 PRESERVED_LISTENER_TTL 仍未被复用 → 先执行 _cleanup();
- 未过期 → 视为本次进入即复用,仅清除截止标记。
"""
deadline = float(getattr(self, "_preserve_deadline", 0.0) or 0.0)
if deadline <= 0:
return
self._preserve_deadline = 0.0
if time.time() >= deadline:
logger.debug("[NetworkMonitor] 保留监听已超过存活上限,先执行清理")
self._cleanup()
def pre_start(self):
"""
在发送动作之前启动网络监听
v5.11 改进:
- 恢复实际启动(延迟启动会错过 requestWillBeSent 事件)
- 但调用时机从 FILL_INPUT 延后到 CLICK send_btn / KEY_PRESS Enter 之前
- 暴露窗口:从"发送前一刻"到"回复结束",而非"输入开始"到"回复结束"
调用时机:仅在 CLICK send_btn 或 KEY_PRESS Enter 之前
"""
# 修复#3b:先处理过期的保留监听,避免复用已泄漏的监听状态
self._expire_stale_preserved_listener()
if self._pre_started and self._listen_is_active():
return
if not self._listen_pattern:
logger.warning("[NetworkMonitor] listen_pattern 未配置")
return
try:
# 启用复用模式:使用 tab 主连接,不创建额外 CDP session
self._ensure_listening("pre_start")
logger.debug(f"[NetworkMonitor] 发送前启动监听 - 复用模式 (pattern={self._listen_pattern!r})")
except Exception as e:
logger.error(f"[NetworkMonitor] 预启动失败: {e}")
def monitor(self, selector: str = None, user_input: str = "",
completion_id: Optional[str] = None) -> Generator[str, None, None]:
"""
监听网络响应并流式输出
v5.11:
- pre_start 已在发送前启动监听
- 此处仅做兜底检查(正常不应走到未启动的情况)
- 响应结束后立即 stop(),最小化暴露窗口
Args:
selector: 选择器(兼容参数,实际不使用)
user_input: 用户输入(用于日志)
completion_id: 完成 ID
Yields:
SSE 格式的数据块
Raises:
NetworkMonitorTimeout: 首次响应超时(触发回退)
NetworkMonitorError: 其他网络监听错误
"""
if not self._listen_pattern:
raise NetworkMonitorError("listen_pattern 未配置")
# 修复#3b:进入 monitor 前先检查上一轮保留监听是否已过期,过期则先清理
self._expire_stale_preserved_listener()
if completion_id is None:
completion_id = SSEFormatter._generate_id()
# 重置解析器状态
self._reset_parser_state()
self._total_chunks = 0
self._total_content_chars = 0
self._last_stream_event = {}
self._last_stream_raw_body = ""
self._last_stream_raw_body_len = 0
self._last_stream_raw_body_signature = None
self._last_stream_parse_result = {}
self._reset_media_generation_state_cache()
self._last_stream_media_items = []
self._prefetched_image_urls = set()
# 兜底:如果 pre_start 未被调用,在此启动(可能错过首包)
if not self._is_listening or not self._listen_is_active():
logger.warning(
"[NetworkMonitor] 监听未预启动,在此启动"
"(可能错过 requestWillBeSent)"
)
self._ensure_listening("monitor_start")
try:
yield from self._stream_output_phase(completion_id)
finally:
# 退出时清理 listener、CDP session 和临时状态,避免监听泄漏
self._cleanup()
self._pre_started = False
self._preserve_deadline = 0.0
def _stream_output_phase(self, completion_id: str) -> Generator[str, None, None]:
"""
流式输出阶段
"""
phase_start = time.time()
try:
hard_timeout = max(0.01, float(self._hard_timeout or self.DEFAULT_HARD_TIMEOUT))
except Exception:
hard_timeout = float(self.DEFAULT_HARD_TIMEOUT)
hard_timeout_message = f"网络监听超过最大时间({hard_timeout:.1f}s)"
try:
first_target_timeout = max(0.01, float(self._first_response_timeout))
except Exception:
first_target_timeout = hard_timeout
first_target_deadline = phase_start + min(first_target_timeout, hard_timeout)
has_received_response = False
has_seen_stream_target = False
last_activity_time = time.time()
listen_restart_attempts = 0
total_responses = 0
non_target_skips = 0
empty_body_skips = 0
stream_target_hits = 0
active_stream_response = None
active_stream_event: Dict[str, Any] = {}
active_stream_body = ""
active_stream_body_source = ""
completed_by_done = False
waiting_for_followup_stream = False
followup_stream_deadline = 0.0
recorded_recoverable_error = None
recorded_recoverable_status = 0
recorded_recoverable_url = ""
completion_reason = "unknown"
while True:
# 检查全局超时
now = time.time()
elapsed = now - phase_start
remaining_hard_timeout = hard_timeout - elapsed
if remaining_hard_timeout <= 0:
logger.error(f"[NetworkMonitor] 超过最大监听时间 {hard_timeout:.1f}s,触发回退")
raise NetworkMonitorTimeout(hard_timeout_message)
if not has_seen_stream_target and now >= first_target_deadline:
logger.warning(
"[NetworkMonitor] 目标流首次响应超时 "
f"({elapsed:.1f}s, limit={first_target_timeout:.1f}s)"
)
raise NetworkMonitorTimeout(f"目标流响应超时({elapsed:.1f}s)")
# 检查取消信号
if self._should_stop():
logger.debug("[NetworkMonitor] 监听被取消")
completion_reason = "cancelled"
self._append_network_debug_trace(
"cancelled",
active_stream_response,
active_stream_body,
active_stream_body_source,
is_event_stream=active_stream_response is not None,
extra_payload={"elapsed": time.time() - phase_start},
)
break
# 设置超时时间
if active_stream_response is not None:
timeout = self.ACTIVE_STREAM_RESPONSE_POLL_TIMEOUT
elif waiting_for_followup_stream and followup_stream_deadline > 0:
timeout = min(self._response_interval, max(0.01, followup_stream_deadline - now))
else:
timeout = (
min(
first_target_timeout,
max(0.01, first_target_deadline - now),
)
if not has_seen_stream_target
else self._response_interval
)
try:
timeout = float(timeout)
except (TypeError, ValueError, OverflowError):
timeout = self.DEFAULT_RESPONSE_INTERVAL
timeout = min(max(0.01, timeout), max(0.01, remaining_hard_timeout))
# 等待响应
try:
if self._prefetched_responses:
response = self._pop_prefetched_response()
else:
response = self._wait_for_response(timeout)
except Exception as e:
err_text = str(e)
if self._is_restartable_listen_error(err_text):
listen_restart_attempts += 1
if listen_restart_attempts > self.MAX_LISTEN_RESTARTS:
raise NetworkMonitorError(
f"监听状态恢复失败(已重试 {self.MAX_LISTEN_RESTARTS} 次): {err_text}"
) from e
logger.warning(
"[NetworkMonitor] wait 期间监听状态失效,尝试重建后重试 "
f"({listen_restart_attempts}/{self.MAX_LISTEN_RESTARTS})"
)
self._ensure_listening("wait_restart")
self._sleep_after_listen_restart(listen_restart_attempts)
continue
raise NetworkMonitorError(err_text) from e
# 检查是否为无效响应
if response is None or response is False:
# 修复#4:_wait_for_response 在收到取消信号时会提前返回 False,
# 先识别取消并走既有取消收尾路径(静默 break),
# 避免被误报为 NetworkMonitorTimeout 触发 DOM 回退
if self._should_stop():
logger.debug("[NetworkMonitor] 等待响应期间监听被取消")
completion_reason = "cancelled"
self._append_network_debug_trace(
"cancelled",
active_stream_response,
active_stream_body,
active_stream_body_source,
is_event_stream=active_stream_response is not None,
extra_payload={"elapsed": time.time() - phase_start},
)
break
elapsed = time.time() - phase_start
if elapsed >= hard_timeout:
logger.error(f"[NetworkMonitor] 超过最大监听时间 {hard_timeout:.1f}s,触发回退")
raise NetworkMonitorTimeout(hard_timeout_message)
if not has_seen_stream_target:
logger.warning(f"[NetworkMonitor] 目标流响应超时 ({elapsed:.1f}s),触发回退")
raise NetworkMonitorTimeout(f"目标流响应超时({elapsed:.1f}s)")
if active_stream_response is not None:
next_body, next_source = self._wait_for_stream_progress(
active_stream_response,
active_stream_body,
active_stream_body_source,
)
self._append_network_debug_trace(
"active_wait_result",
active_stream_response,
next_body or active_stream_body,
next_source or active_stream_body_source,
is_event_stream=True,
extra_payload={
"previous_body_len": len(active_stream_body or ""),
"next_body_len": len(next_body or ""),
"changed": bool(next_body and next_body != active_stream_body),
"capture_complete": self._stream_capture_complete(active_stream_response),
},
)
if next_body and next_body != active_stream_body:
self._handle_stream_body_replacement(
active_stream_body,
next_body,
next_source,
)
active_stream_body = next_body
active_stream_body_source = next_source
last_activity_time = time.time()
logger.debug_throttled(
f"network.active_stream_growth.{id(self)}",
"[NetworkMonitor] 流响应增长中 "
f"(当前大小={len(next_body)} 字节)",
interval_sec=10.0,
)
try:
parse_result = self.parser.parse_chunk(active_stream_body)
except Exception as e:
logger.warning(f"[NetworkMonitor] 活跃流二次解析异常: {e}")
self._append_network_debug_trace(
"active_growth_parse_error",
active_stream_response,
active_stream_body,
active_stream_body_source,
is_event_stream=True,
extra_payload={"error": str(e)},
)
continue
self._append_network_debug_trace(
"active_growth_parse",
active_stream_response,
active_stream_body,
active_stream_body_source,
parse_result,
True,
)
self._write_parser_debug_dump(
active_stream_body,
active_stream_event,
parse_result,
active_stream_body_source,
True,
)
parse_result = self._handle_parse_result(parse_result)
if parse_result.get("error"):
continue
self._dispatch_result(
active_stream_event,
active_stream_body,
parse_result,
active_stream_body_source,
)
self._remember_last_stream_result(
active_stream_event,
active_stream_body,
parse_result,
)
try:
self._record_parse_result_media(parse_result)
except Exception as media_exc:
logger.debug(f"[NetworkMonitor] 媒体结果记录失败(忽略): {media_exc}")
try:
media_state = self._get_media_generation_state_cached(
active_stream_body,
parse_result,
)
except Exception as parser_exc:
logger.debug(f"[NetworkMonitor] 媒体状态提取失败(忽略): {parser_exc}")
content = parse_result.get("content", "")
reasoning_content = parse_result.get("reasoning_content", "")
done = parse_result.get("done", False)
if reasoning_content:
last_activity_time = time.time()
self._total_chunks += 1
yield self.formatter.pack_chunk(
content="",
reasoning_content=reasoning_content,
completion_id=completion_id,
)
if content:
self._total_chunks += 1
self._total_content_chars += len(content)
yield self.formatter.pack_chunk(content, completion_id=completion_id)
if done:
if self._should_wait_for_followup_stream():
waiting_for_followup_stream = True
active_stream_response = None
active_stream_event = {}
active_stream_body = ""
active_stream_body_source = ""
self._prepare_parser_for_followup_stream()
last_activity_time = time.time()
logger.info("[NetworkMonitor] GLM 中间流已结束,继续等待自动确认后的后续流")
continue
if self._should_fallback_on_empty_done(parse_result):
logger.warning(
"[NetworkMonitor] 目标流返回完成标志但未产出有效正文,回退到 DOM 监听 "
f"(source={active_stream_body_source}, body_len={len(active_stream_body or '')})"
)
raise NetworkMonitorTimeout("目标流完成但未产出有效正文")
self._append_network_debug_trace(
"protocol_done",
active_stream_response,
active_stream_body,
active_stream_body_source,
parse_result,
True,
extra_payload={"total_chunks": self._total_chunks},
)
completed_by_done = True
completion_reason = "done"
logger._logger.log(logging.DEBUG - 5, "[NetworkMonitor] 检测到结束标志,完成监听")
break
continue
# 修复#2:capture_complete 判定成立后,先强制补读一次 body 并做
# parse_chunk 增量下发,再进入两处 break/回退判定;
# 避免"取快照→尾块追加+complete 置位→查 complete"竞态丢失尾部。
capture_complete_now = self._stream_capture_complete(active_stream_response)
if capture_complete_now:
(
active_stream_body,
active_stream_body_source,
tail_done,
) = yield from self._drain_completed_stream_tail(
active_stream_response,
active_stream_body,
active_stream_body_source,
active_stream_event,
completion_id,
)
if tail_done:
# 尾部补读解析出结束标志:按协议完成处理,
# 使下方 require_explicit_done 分支不再误判缺失结束事件
completed_by_done = True
if (
capture_complete_now
and (self._has_stream_output() or completed_by_done)
and (completed_by_done or not self._should_require_explicit_done())
):
if self._should_wait_for_followup_stream():
waiting_for_followup_stream = True
active_stream_response = None
active_stream_event = {}
active_stream_body = ""
active_stream_body_source = ""
self._prepare_parser_for_followup_stream()
last_activity_time = time.time()
logger.info("[NetworkMonitor] GLM 中间流已完成,继续等待自动确认后的后续流")
continue
# A completed image-only stream has no text chunks by design. Reuse
# the media-aware empty-result check so captured images are treated
# as a valid network result instead of triggering a DOM fallback.
if self._should_fallback_on_empty_done(self._last_stream_parse_result):
logger.warning(
"[NetworkMonitor] 流响应已结束但仍无有效正文,回退到 DOM 监听 "
f"(source={active_stream_body_source}, body_len={len(active_stream_body or '')})"
)
raise NetworkMonitorTimeout("目标流未产出有效正文")
if (
self._has_stream_output()
and not completed_by_done
and self._should_fallback_to_dom_on_empty_stream()
):
logger.warning(
"[NetworkMonitor] 流响应已结束但未收到完成标志,回退到 DOM 补齐 "
f"(source={active_stream_body_source}, chunks={self._total_chunks}, "
f"body_len={len(active_stream_body or '')})"
)
raise NetworkMonitorTimeout("目标流未收到完成标志")
completion_reason = "capture_complete"
logger.debug("[NetworkMonitor] 活跃流响应已完成,结束监听")
break
if (
active_stream_response is not None
# 修复#2:复用上方已补读过尾部的 capture_complete 判定结果
and capture_complete_now
and self._should_require_explicit_done()
and not completed_by_done
):
if self._should_wait_for_replacement_stream_on_incomplete_capture():
logger.warning(
"[NetworkMonitor] Claude 当前流已结束但缺少 message_stop,继续等待后续流"
)
self._append_network_debug_trace(
"await_replacement_stream",
active_stream_response,
active_stream_body,
active_stream_body_source,
is_event_stream=True,
extra_payload={
"capture_complete": True,
"total_chunks": self._total_chunks,
"last_activity_age": time.time() - last_activity_time,
},
)
waiting_for_followup_stream = True
active_stream_response = None
active_stream_event = {}
active_stream_body = ""
active_stream_body_source = ""
self._prepare_parser_for_followup_stream()
last_activity_time = time.time()
continue
logger.warning(
"[NetworkMonitor] 流响应已捕获完成但缺少协议结束事件,回退到 DOM 监听"
)
self._append_network_debug_trace(
"terminate_missing_message_stop",
active_stream_response,
active_stream_body,
active_stream_body_source,
is_event_stream=True,
extra_payload={
"capture_complete": True,
"total_chunks": self._total_chunks,
"last_activity_age": time.time() - last_activity_time,
},
)
raise NetworkMonitorTimeout("目标流缺少明确结束事件")
if (
active_stream_response is not None
and capture_complete_now
and not completed_by_done
and self._should_fallback_on_incomplete_empty_capture()
):
logger.warning(
"[NetworkMonitor] 目标流连接已关闭但没有完成事件、正文或媒体,"
"判定为断流并回退到 DOM 接管"
)
self._append_network_debug_trace(
"fallback_incomplete_empty_capture",
active_stream_response,
active_stream_body,
active_stream_body_source,
is_event_stream=True,
extra_payload={
"capture_complete": True,
"total_chunks": self._total_chunks,
"media_count": len(self._last_stream_media_items or []),
},
)
raise NetworkMonitorTimeout("目标流提前关闭且未产出有效结果")
silence_duration = time.time() - last_activity_time
effective_silence_threshold = self._effective_silence_threshold(
active_stream_response=active_stream_response,
completed_by_done=completed_by_done,
waiting_for_followup_stream=waiting_for_followup_stream,
)
if (
active_stream_response is not None
and not self._has_stream_output()
and not self._stream_capture_complete(active_stream_response)
and silence_duration >= 3.0
):
logger.debug_throttled(
f"network.wait_first_content.{id(self)}",
"[NetworkMonitor] 等待流式文本解析产出 "
f"(已捕获流长度={len(active_stream_body or '')}, "
f"已静默等待={silence_duration:.1f}s/上限={effective_silence_threshold:.1f}s)",
interval_sec=10.0,
)
if (
silence_duration > effective_silence_threshold
or (waiting_for_followup_stream and followup_stream_deadline > 0 and time.time() >= followup_stream_deadline)
):
if waiting_for_followup_stream or recorded_recoverable_status > 0:
if recorded_recoverable_status in (403, 429, 503):
logger.warning(
"[NetworkMonitor] 可恢复重试窗口已耗尽(30s),抛出终态限流/验证错误 "
f"(idle={silence_duration:.1f}s, status={recorded_recoverable_status})"
)
raise NetworkMonitorTerminalError(
recorded_recoverable_error or f"HTTP {recorded_recoverable_status}"
)
logger.warning(
"[NetworkMonitor] 可恢复流等待重试超时,回退到 DOM 监听 "
f"(idle={silence_duration:.1f}s, limit={effective_silence_threshold:.1f}s, status={recorded_recoverable_status})"
)
raise NetworkMonitorTimeout(
f"后续流未在宽限期内到达({silence_duration:.1f}s)"
)
if (
self._has_stream_output()
and self._parse_result_has_unclosed_render_output(
self._last_stream_parse_result
)
and self._should_fallback_to_dom_on_empty_stream()
):
body_len = len(active_stream_body or "") or int(
self._last_stream_raw_body_len or 0
)
logger.warning(
"[NetworkMonitor] 流式响应静默但前端 render 输出包装未闭合,回退到 DOM 补齐 "
f"(idle={silence_duration:.1f}s, chunks={self._total_chunks}, "
f"body_len={body_len})"
)
raise NetworkMonitorTimeout(
f"目标流 render 输出包装未完整结束({silence_duration:.1f}s)"
)
if (
active_stream_response is not None
and not self._has_stream_output()
and self._should_fallback_to_dom_on_empty_stream()
):
logger.warning(
"[NetworkMonitor] 首段等待超时且仍无有效正文,回退到 DOM 监听 "
f"(idle={silence_duration:.1f}s, limit={effective_silence_threshold:.1f}s, "
f"body_len={len(active_stream_body or '')})"
)
raise NetworkMonitorTimeout(
f"目标流未产出有效正文({silence_duration:.1f}s)"
)
if (
active_stream_response is not None
and self._has_stream_output()
and self._should_fallback_to_dom_on_empty_stream()
):
if active_stream_response is not None and not completed_by_done:
logger.warning(
"[NetworkMonitor] 流式响应静默但未收到完成标志,回退到 DOM 补齐 "
f"(idle={silence_duration:.1f}s, chunks={self._total_chunks}, "
f"body_len={len(active_stream_body or '')})"
)
raise NetworkMonitorTimeout(
f"目标流未完整结束({silence_duration:.1f}s)"
)
if (
active_stream_response is not None
and self._should_require_explicit_done()
and not completed_by_done
):
logger.warning(
"[NetworkMonitor] Claude 流静默且未收到 message_stop,回退到 DOM 监听"
)
self._append_network_debug_trace(
"terminate_silence_without_message_stop",
active_stream_response,
active_stream_body,
active_stream_body_source,
is_event_stream=True,
extra_payload={
"silence_duration": silence_duration,
"effective_silence_threshold": effective_silence_threshold,
"total_chunks": self._total_chunks,
},
)
raise NetworkMonitorTimeout("目标流静默且缺少明确结束事件")
completion_reason = "silence"
logger.debug(f"[NetworkMonitor] 静默超时 ({silence_duration:.1f}s),结束监听")
break
continue
# 标记已收到响应(在读取 body 之前!)
if not has_received_response:
has_received_response = True
logger._logger.log(logging.DEBUG - 5, "[NetworkMonitor] 已捕获到首次响应")
total_responses += 1
# 修复#1:此处不再重置 last_activity_time——任意命中 listen_pattern 的
# 响应(如周期性心跳/埋点)都会把静默计时清零,导致 silence 结束条件
# 永远达不到、请求挂满硬超时。仅当 _matches_stream_target 通过后
# (下方目标命中处)才视为目标流活动并重置计时;event_only 模式在其
# 自身分支内仍按"任意事件即活动"处理。
listen_restart_attempts = 0
event = self._extract_event(response)
if self._dispatch_event(event):
logger.warning(
"[NetworkMonitor] 命中网络异常拦截,主动中断监听 "
f"(status={event.get('status')}, url={event.get('url', '')[:100]})"
)
raise NetworkInterceptionTriggered("network_intercepted")
if self.parser.get_id() == "event_only":
if total_responses == 1:
logger._logger.log(logging.DEBUG - 5, "[NetworkMonitor] event-only 已捕获到首个网络事件")
last_activity_time = time.time()
continue
if not self._matches_stream_target(event):
non_target_skips += 1
logger.debug_throttled(
"network.non_target_response",
f"[NetworkMonitor] 非流式目标响应,跳过解析 "
f"(count={non_target_skips}, url={event.get('url', '')[:100]})",
interval_sec=5.0,
)
continue
if not has_seen_stream_target:
has_seen_stream_target = True
logger._logger.log(logging.DEBUG - 5, "[NetworkMonitor] 已捕获到首个流目标响应")
stream_target_hits += 1
logger._logger.log(
logging.DEBUG - 5,
"[NetworkMonitor] 命中流目标 "
f"(status={event.get('status')}, method={event.get('method')}, "
f"url={event.get('url', '')[:120]}, count={stream_target_hits})"
)
if stream_target_hits == 1:
logger._logger.log(logging.DEBUG - 5, "[NetworkMonitor] 已捕获到首个有效流响应")
last_activity_time = time.time()
active_stream_response = None
active_stream_event = dict(event or {})
active_stream_body = ""
active_stream_body_source = ""
# 检查响应对象结构
response_obj = getattr(response, "response", None)
if response_obj is None:
logger.debug(f"[NetworkMonitor] 响应对象结构异常: {type(response).__name__}")
continue
try:
# 读取响应体,流式协议优先使用 _stream.fullText
raw_body, raw_body_source = self._extract_raw_body(response)
raw_body = self._normalize_raw_body(raw_body)
self._raise_for_http_error_status(event, raw_body, raw_body_source)
if getattr(response_obj, "_response", None) is None and not raw_body:
logger.warning(
"[NetworkMonitor] 目标流响应缺少响应元数据和响应体,回退到 DOM 监听 "
f"(status={event.get('status')}, url={event.get('url', '')[:120]})"
)
raise NetworkMonitorError("incomplete_target_response")
if self.parser.get_id() == "doubao" and raw_body_source == "body":
if self._looks_like_sse_payload(raw_body):
logger.debug(
"[NetworkMonitor][DoubaoDebug] body source contains raw SSE payload, "
"continue parsing in network mode"
)
else:
logger.debug(
"[NetworkMonitor][DoubaoDebug] body-only response summary: "
f"{self._describe_json_container(raw_body)}"
)
logger.warning(
"[NetworkMonitor] 豆包网络响应仅返回 body 包装结果,回退到 DOM 监听"
)
raise NetworkMonitorError("doubao_body_only_response")
is_event_stream = self._is_event_stream_response(response)
should_probe_initial_target_body = (
(stream_target_hits == 1 or waiting_for_followup_stream or recorded_recoverable_status > 0)
and self._total_chunks == 0
)
if not raw_body and (is_event_stream or should_probe_initial_target_body):
wait_budget = (
self._initial_target_body_wait
if should_probe_initial_target_body
else None
)
raw_body, raw_body_source = self._wait_for_stream_body(
response,
raw_body,
raw_body_source,
wait_budget=wait_budget,
)
if raw_body and not is_event_stream and self._looks_like_sse_payload(raw_body):
is_event_stream = True
self._raise_for_http_error_status(
event,
raw_body,
raw_body_source,
is_event_stream,
)
except NetworkMonitorRecoverableError as rec_err:
recorded_recoverable_error = rec_err.error_text
recorded_recoverable_status = rec_err.status_code
recorded_recoverable_url = str(rec_err.event.get("url", "") or "")
waiting_for_followup_stream = True
if followup_stream_deadline <= 0:
followup_stream_deadline = min(
time.time() + 30.0,
phase_start + hard_timeout,
)
active_stream_response = None
active_stream_event = {}
active_stream_body = ""
active_stream_body_source = ""
self._prepare_parser_for_followup_stream()
last_activity_time = time.time()
rem_sec = max(0.0, followup_stream_deadline - time.time())
logger.warning(
f"[NetworkMonitor] 捕获可恢复流错误 (status={rec_err.status_code}, url={recorded_recoverable_url[:120]}), "
f"保持监听等待重发 (剩余重试窗口={rem_sec:.1f}s)..."
)
continue
if is_event_stream and response is not None:
active_stream_response = response
active_stream_event = dict(event or {})
if not raw_body:
empty_body_skips += 1
if should_probe_initial_target_body:
body_wait_elapsed = max(
0.0,
time.time() - float(event.get("timestamp", 0.0) or 0.0),
)
logger.warning(
"[NetworkMonitor] 首个流目标响应正文未在宽限期内就绪,回退到 DOM 监听 "
f"(wait={body_wait_elapsed:.1f}s, status={event.get('status')}, "
f"url={event.get('url', '')[:120]}, source={raw_body_source})"
)
raise NetworkMonitorTimeout(
f"目标流响应正文未就绪({body_wait_elapsed:.1f}s)"
)
logger.debug_throttled(
"network.empty_body",
"[NetworkMonitor] 响应体为空,跳过 "
f"(count={empty_body_skips}, stream={is_event_stream}, source={raw_body_source})",
interval_sec=5.0,
)
continue
if stream_target_hits == 1:
logger.info(
"[NetworkMonitor] 成功锁定流目标响应 "
f"(status={event.get('status')}, method={event.get('method')}, "
f"url={event.get('url', '')[:120]}, 初始长度={len(raw_body)} 字符)"
)
else:
logger.debug_throttled(
"network.body_captured",
f"[NetworkMonitor] 持续捕获流响应分块 "
f"(targets={stream_target_hits}, source={raw_body_source}, 长度={len(raw_body)} 字符)",
interval_sec=5.0,
)
if waiting_for_followup_stream or recorded_recoverable_status > 0:
waiting_for_followup_stream = False
followup_stream_deadline = 0.0
recorded_recoverable_error = None
recorded_recoverable_status = 0
logger.info(
"[NetworkMonitor] 已成功捕获后续流正文 "
f"(target={stream_target_hits}, body_len={len(raw_body)})"
)
# 解析响应
try:
parse_result = self.parser.parse_chunk(raw_body)
except Exception as e:
logger.warning(f"[NetworkMonitor] 解析异常: {e}")
self._append_network_debug_trace(
"initial_parse_error",
response,
raw_body,
raw_body_source,
is_event_stream=is_event_stream,
extra_payload={"error": str(e)},
)
continue
self._append_network_debug_trace(
"target_response_parse",
response,
raw_body,
raw_body_source,
parse_result,
is_event_stream,
extra_payload={
"stream_target_hits": stream_target_hits,
"total_responses": total_responses,
"capture_complete": self._stream_capture_complete(response),
},
)
if (
is_event_stream
and not parse_result.get("content")
and not parse_result.get("done", False)
and not parse_result.get("error")
):
next_body, next_source = self._wait_for_stream_progress(
response,
raw_body,
raw_body_source,
)
if next_body and next_body != raw_body:
self._handle_stream_body_replacement(
raw_body,
next_body,
next_source,
)
raw_body = next_body
raw_body_source = next_source
last_activity_time = time.time()
try:
parse_result = self.parser.parse_chunk(raw_body)
except Exception as e:
logger.warning(f"[NetworkMonitor] 二次解析异常: {e}")
self._append_network_debug_trace(
"initial_growth_parse_error",
response,
raw_body,
raw_body_source,
is_event_stream=is_event_stream,
extra_payload={"error": str(e)},
)
continue
self._append_network_debug_trace(
"initial_growth_parse",
response,
raw_body,
raw_body_source,
parse_result,
is_event_stream,
extra_payload={"capture_complete": self._stream_capture_complete(response)},
)
self._write_parser_debug_dump(
raw_body,
event,
parse_result,
raw_body_source,
is_event_stream,
)
parse_result = self._handle_parse_result(parse_result)
if parse_result.get("error"):
continue
self._dispatch_result(event, raw_body, parse_result, raw_body_source)
self._remember_last_stream_result(event, raw_body, parse_result)
try:
self._record_parse_result_media(parse_result)
except Exception as media_exc:
logger.debug(f"[NetworkMonitor] 媒体结果记录失败(忽略): {media_exc}")
try:
media_state = self._get_media_generation_state_cached(
raw_body,
parse_result,
)
except Exception as parser_exc:
logger.debug(f"[NetworkMonitor] 媒体状态提取失败(忽略): {parser_exc}")
if is_event_stream:
active_stream_response = response
active_stream_event = dict(event or {})
active_stream_body = str(raw_body or "")
active_stream_body_source = raw_body_source
# 提取内容
content = parse_result.get("content", "")
reasoning_content = parse_result.get("reasoning_content", "")
done = parse_result.get("done", False)
if reasoning_content:
last_activity_time = time.time()
self._total_chunks += 1
yield self.formatter.pack_chunk(
content="",
reasoning_content=reasoning_content,
completion_id=completion_id,
)
if content:
last_activity_time = time.time()
self._total_chunks += 1
self._total_content_chars += len(content)
yield self.formatter.pack_chunk(content, completion_id=completion_id)
if done:
if self._should_wait_for_followup_stream():
waiting_for_followup_stream = True
active_stream_response = None
active_stream_event = {}
active_stream_body = ""
active_stream_body_source = ""
self._prepare_parser_for_followup_stream()
last_activity_time = time.time()
logger.info("[NetworkMonitor] GLM 中间流已结束,继续等待自动确认后的后续流")
continue
if self._should_fallback_on_empty_done(parse_result):
logger.warning(
"[NetworkMonitor] 目标流返回完成标志但未产出有效正文,回退到 DOM 监听 "
f"(source={raw_body_source}, body_len={len(raw_body or '')})"
)
raise NetworkMonitorTimeout("目标流完成但未产出有效正文")
self._append_network_debug_trace(
"protocol_done",
response,
raw_body,
raw_body_source,
parse_result,
is_event_stream,
extra_payload={"total_chunks": self._total_chunks},
)
completed_by_done = True
completion_reason = "done"
logger._logger.log(logging.DEBUG - 5, "[NetworkMonitor] 检测到结束标志,完成监听")
break
reason_text = {
"done": "检测到结束标志",
"capture_complete": "流响应捕获完成",
"silence": "静默超时",
"cancelled": "监听被取消",
"unknown": "循环结束",
}.get(completion_reason, completion_reason)
logger.info(
"[NetworkMonitor] 网络流监听完成 "
f"(reason={reason_text}, 历时={time.time() - phase_start:.1f}s, "
f"捕获响应={total_responses}, 产出文本块={self._total_chunks}, "
f"提取字符数={self._total_content_chars})"
)
def get_media_generation_state(self) -> Dict[str, Any]:
return dict(self._last_media_generation_state or {})
def get_stream_media_items(self) -> list[Dict[str, Any]]:
return [dict(item) for item in (self._last_stream_media_items or []) if isinstance(item, dict)]
def get_total_content_chars(self) -> int:
try:
return max(0, int(self._total_content_chars or 0))
except Exception:
return 0
def _prefetch_image_url(self, url: str) -> bool:
normalized = normalize_remote_image_url(url)
if not normalized or normalized in self._prefetched_image_urls:
return False
cookies_dict, headers = build_image_download_request_context(self.tab)
result = background_image_downloader.start_download(
normalized,
cookies=cookies_dict,
headers=headers,
partition_key=get_image_download_partition(normalized, cookies_dict, headers),
max_bytes=max(
1,
int(self._image_config.get("max_size_mb") or 10),
) * 1024 * 1024,
)
if result:
self._prefetched_image_urls.add(normalized)
return True
return False
def _record_parse_result_media(self, parse_result: Dict[str, Any]) -> None:
raw_items = parse_result.get("images")
if not isinstance(raw_items, list) or not raw_items:
return
seen = {
(
str(item.get("media_type") or "image").strip().lower(),
str(item.get("url") or item.get("data_uri") or "").strip(),
)
for item in (self._last_stream_media_items or [])
if isinstance(item, dict)
}
for raw_item in raw_items:
if isinstance(raw_item, str):
normalized = {
"media_type": "image",
"kind": "url",
"url": raw_item,
"data_uri": None,
"mime": None,
"byte_size": None,
"source": f"{self.parser.get_id()}_stream",
}
elif isinstance(raw_item, dict):
normalized = dict(raw_item)
else:
continue
media_type = str(normalized.get("media_type") or "image").strip().lower() or "image"
if normalized.get("data_uri"):
normalized["kind"] = "data_uri"
normalized["url"] = None
ref = str(normalized.get("data_uri") or "").strip()
else:
normalized["kind"] = "url"
normalized["data_uri"] = None
ref = str(normalized.get("url") or normalized.get("src") or "").strip()
normalized["url"] = ref or None
if not ref:
continue
key = (media_type, ref)
if key in seen:
continue
seen.add(key)
normalized["media_type"] = media_type
normalized.pop("src", None)
normalized.setdefault("mime", None)
normalized.setdefault("byte_size", None)
normalized.setdefault("source", f"{self.parser.get_id()}_stream")
self._last_stream_media_items.append(normalized)
if media_type == "image" and normalized.get("kind") == "url":
self._prefetch_image_url(normalized.get("url"))
def _clear_cached_results(self, *, include_media: bool = False) -> None:
self._reset_prefetched_responses()
self._last_stream_event = {}
self._last_stream_raw_body = ""
self._last_stream_raw_body_len = 0
self._last_stream_raw_body_signature = None
self._last_stream_parse_result = {}
self._prefetched_image_urls = set()
if include_media:
self._reset_media_generation_state_cache()
self._last_stream_media_items = []
def cleanup(self) -> None:
self._cleanup(include_media=True)
def _cleanup(self, *, include_media: bool = False):
"""
清理:停止网络监听并释放额外的 CDP session
tab.listen.stop() 内部会:
1. 移除所有 Network.* 事件回调
2. 关闭独立的 Driver 连接(释放额外的 CDP session)
这会关闭 Target.attachToTarget 创建的额外 session,
消除 Network.enable 的全局副作用。
"""
if self._is_listening:
try:
self._safe_stop_listen()
logger._logger.log(logging.DEBUG - 5, "[NetworkMonitor] 已停止监听(listen 已释放)")
except Exception as e:
logger.debug(f"[NetworkMonitor] 停止监听失败: {e}")
finally:
self._is_listening = False
self._pre_started = False
try:
if getattr(self, "_cdp_session_listening", False):
network = getattr(self.tab, "network", None)
stop_interception = getattr(network, "stop_interception", None)
if callable(stop_interception):
stop_interception()
self._cdp_session_listening = False
logger._logger.log(logging.DEBUG - 5, "[NetworkMonitor] 已停止 CDP interception")
except Exception as e:
logger.debug(f"[NetworkMonitor] 停止 CDP interception 失败: {e}")
# 即使 _is_listening 已经是 False,也尝试确保 listen 已停止
# (防止异常路径导致状态不一致)
elif self._listen_is_active():
try:
self._safe_stop_listen()
logger.debug("[NetworkMonitor] 补充停止残留监听")
except Exception:
pass
self._clear_cached_results(include_media=include_media)
self._reset_parser_state()
# ================= 工厂函数 =================
def create_network_monitor(tab, formatter: SSEFormatter,
stream_config: Dict,
stop_checker: Optional[Callable[[], bool]] = None,
event_handler: Optional[Callable[[Dict[str, Any]], bool]] = None,
result_handler: Optional[Callable[[Dict[str, Any]], bool]] = None,
image_config: Optional[Dict] = None) -> NetworkMonitor:
"""
创建网络监听器(工厂函数)
Args:
tab: DrissionPage 标签页
formatter: SSE 格式化器
stream_config: 流式配置(必须包含 network.parser)
stop_checker: 取消检查函数
Returns:
NetworkMonitor 实例
Raises:
ValueError: 配置缺失或解析器不存在
"""
network_config = stream_config.get("network", {})
# 获取解析器 ID
parser_id = network_config.get("parser")
event_only = bool(network_config.get("event_only", False))
if not parser_id:
if event_only and event_handler is not None:
parser = _EventOnlyParser()
else:
raise ValueError("network.parser 未配置")
else:
# 获取解析器实例
try:
parser = ParserRegistry.get(parser_id)
except ValueError as e:
raise ValueError(f"解析器不存在: {e}")
return NetworkMonitor(
tab=tab,
formatter=formatter,
parser=parser,
stop_checker=stop_checker,
stream_config=stream_config,
image_config=image_config,
event_handler=event_handler,
result_handler=result_handler,
)
__all__ = [
'NetworkMonitor',
'NetworkMonitorTimeout',
'NetworkMonitorError',
'NetworkMonitorTerminalError',
'NetworkInterceptionTriggered',
'create_network_monitor',
]