""" lmarena_parser.py - Arena.ai 响应解析器 响应格式:Vercel AI SDK Data Stream Protocol - 每行格式: {prefix}:{json_data} - a0: 文本增量(JSON 编码的字符串) - a2: 心跳/元数据(JSON 数组) - ad: 结束元数据(含 finishReason) - ae: 错误信息 编码问题: DrissionPage 通过 CDP 获取响应体时,UTF-8 字节被混合解码为 cp1252 映射字符(如 0x92→' U+2019)和 latin-1 直通字符 (如 0x90→U+0090),导致双重编码(mojibake)。 需要两步修复:translate 统一到 latin-1 范围 → encode('latin-1') 还原字节 调用方式: NetworkMonitor 每次调用 parse_chunk 传入一个完整的 HTTP 响应体 (DrissionPage listen.wait 是非流式的,一次性拿到整个 SSE body) """ from __future__ import annotations import json from datetime import datetime, timezone from typing import Any, Dict, List from app.core.config import logger from .base import ResponseParser # ───────────────────────────────────────────────────── # cp1252 → latin-1 逆映射表(模块级常量,只构建一次) # ───────────────────────────────────────────────────── # cp1252 在 0x80-0x9F 区间将部分字节映射到 U+0100 以上的 Unicode 字符, # 而 latin-1 将相同字节直通映射到 U+0080-U+009F。 # 此表将 cp1252 的特殊映射还原为 latin-1 范围的等价字符, # 使后续 encode('latin-1') 能正确还原原始字节。 # # 5 个未定义位置 (0x81, 0x8D, 0x8F, 0x90, 0x9D) 已经是 # latin-1 直通映射 (U+0081 等),无需处理。 _CP1252_TO_LATIN1 = str.maketrans({ '\u20ac': '\x80', # € '\u201a': '\x82', # ‚ '\u0192': '\x83', # ƒ '\u201e': '\x84', # „ '\u2026': '\x85', # … '\u2020': '\x86', # † '\u2021': '\x87', # ‡ '\u02c6': '\x88', # ˆ '\u2030': '\x89', # ‰ '\u0160': '\x8a', # Š '\u2039': '\x8b', # ‹ '\u0152': '\x8c', # Œ '\u017d': '\x8e', # Ž '\u2018': '\x91', # ' '\u2019': '\x92', # ' '\u201c': '\x93', # " '\u201d': '\x94', # " '\u2022': '\x95', # • '\u2013': '\x96', # – '\u2014': '\x97', # — '\u02dc': '\x98', # ˜ '\u2122': '\x99', # ™ '\u0161': '\x9a', # š '\u203a': '\x9b', # › '\u0153': '\x9c', # œ '\u017e': '\x9e', # ž '\u0178': '\x9f', # Ÿ }) def _looks_like_image_ref(value: str) -> bool: text = str(value or "").strip() if not text: return False lowered = text.lower() if lowered.startswith(("http://", "https://", "data:image/", "blob:")): return True return any(token in lowered for token in (".png", ".jpg", ".jpeg", ".webp", ".gif", ".bmp", ".svg", ".avif")) def _extract_arena_image_items( payload: str, seen_refs: set[str], *, source: str, ) -> List[Dict[str, Any]]: items: List[Dict[str, Any]] = [] try: data = json.loads(payload) except (json.JSONDecodeError, ValueError): return items entries = data if isinstance(data, list) else [data] for entry in entries: if not isinstance(entry, dict): continue entry_type = str(entry.get("type") or "").strip().lower() if entry_type and entry_type != "image": continue image_ref = "" for key in ("image", "url", "src"): candidate = str(entry.get(key) or "").strip() if _looks_like_image_ref(candidate): image_ref = candidate break if not image_ref: continue dedupe_key = f"image:{image_ref}" if dedupe_key in seen_refs: continue seen_refs.add(dedupe_key) mime = str(entry.get("mimeType") or entry.get("mime") or "").strip() or None kind = "data_uri" if image_ref.startswith("data:image/") else "url" items.append( { "media_type": "image", "kind": kind, "url": image_ref if kind == "url" else None, "data_uri": image_ref if kind == "data_uri" else None, "mime": mime, "byte_size": None, "alt": str(entry.get("alt") or "").strip(), "width": entry.get("width"), "height": entry.get("height"), "detected_at": datetime.now(timezone.utc).isoformat().replace("+00:00", "Z"), "source": source, } ) return items class _LmarenaScanState: """LmarenaParser 增量扫描的累计状态(只覆盖已提交的完整行 + 本次半行)。""" __slots__ = ( "content", "reasoning", "done", "error", "line_count", "prefix_counts", "unknown_prefix_count", "non_protocol_line_count", "parse_errors", "committed_len", "committed_raw", ) def __init__(self) -> None: self.content = "" self.reasoning = "" self.done = False self.error = None self.line_count = 0 self.prefix_counts: Dict[str, int] = {} self.unknown_prefix_count = 0 self.non_protocol_line_count = 0 self.parse_errors: list[Dict[str, Any]] = [] self.committed_len = 0 self.committed_raw = "" def copy(self) -> "_LmarenaScanState": other = _LmarenaScanState() for name in self.__slots__: setattr(other, name, getattr(self, name)) other.prefix_counts = dict(self.prefix_counts) other.parse_errors = list(self.parse_errors) return other def add_parse_error(self, item: Dict[str, Any]) -> None: self.parse_errors.append(item) if len(self.parse_errors) > 16: del self.parse_errors[:-8] class LmarenaParser(ResponseParser): """ Arena.ai 响应解析器 URL 特征: /nextjs-api/stream/create-evaluation 响应格式: Vercel AI SDK Data Stream Protocol (行分隔) """ _PROTOCOL_PREFIXES = {"a0", "a2", "ad", "ae", "a3", "ag", "bg"} def __init__(self) -> None: self._accumulated = "" self._accumulated_reasoning = "" self._seen_image_refs: set[str] = set() self._last_debug_summary: Dict[str, Any] = {} self._last_debug_raw_signature = "" self._inc_state: "_LmarenaScanState | None" = None @staticmethod def _content_delta( accumulated: str, candidate: str, *, append_disjoint: bool = False, ) -> tuple[str, str]: """Return unseen text and the new accumulated value for repeated SSE bodies.""" if not candidate: return "", accumulated if not accumulated: return candidate, candidate if candidate == accumulated or accumulated.startswith(candidate): return "", accumulated if candidate.startswith(accumulated): return candidate[len(accumulated):], candidate max_overlap = min(len(accumulated), len(candidate)) for overlap in range(max_overlap, 0, -1): if accumulated.endswith(candidate[:overlap]): delta = candidate[overlap:] return delta, accumulated + delta if append_disjoint: return candidate, accumulated + candidate # A parser instance belongs to one response stream. A body that neither # extends nor overlaps the emitted text is a stale/non-monotonic replay; # emitting it wholesale would duplicate the response already sent. return "", accumulated # ============ 增量扫描(P0-5) ============ # 置为 False 时每次整段解析(与旧实现逐字节一致),用于回归对照 _INCREMENTAL = True def _scan_response(self, raw_response: str) -> tuple["_LmarenaScanState", list]: """返回 (扫描状态, 本次新发现的图片)。""" images: list[Dict[str, Any]] = [] if not self._INCREMENTAL: # 旧行为:整段修复 mojibake 后逐行解析 scan = _LmarenaScanState() self._scan_lines(self._fix_mojibake(raw_response), scan, images) return scan, images cached = getattr(self, "_inc_state", None) if cached is not None and cached.committed_len <= len(raw_response) and raw_response.startswith(cached.committed_raw): scan = cached.copy() offset = cached.committed_len else: scan = _LmarenaScanState() offset = 0 last_newline = raw_response.rfind("\n", offset) if last_newline >= offset: committed_end = last_newline + 1 # 完整行一定以 ASCII 换行结尾,不会截断多字节序列,可以按段修复 mojibake self._scan_lines(self._fix_mojibake(raw_response[offset:committed_end]), scan, images) scan.committed_len = committed_end scan.committed_raw = raw_response[:committed_end] self._inc_state = scan.copy() offset = committed_end elif cached is None or scan.committed_len == 0: self._inc_state = None tail = raw_response[offset:] if tail: self._scan_lines(self._fix_mojibake(tail), scan, images) return scan, images def _scan_lines(self, text: str, scan: "_LmarenaScanState", images: list) -> None: content_parts: list[str] = [] reasoning_parts: list[str] = [] for line in text.split("\n"): line = line.strip() if not line: continue scan.line_count += 1 colon_idx = line.find(":") if colon_idx < 1: scan.non_protocol_line_count += 1 continue prefix = line[:colon_idx] payload = line[colon_idx + 1:] if prefix in self._PROTOCOL_PREFIXES: scan.prefix_counts[prefix] = scan.prefix_counts.get(prefix, 0) + 1 else: scan.unknown_prefix_count += 1 if prefix == "a0": text_chunk = self._parse_text_chunk(payload) if text_chunk is not None: content_parts.append(text_chunk) else: scan.add_parse_error({"prefix": prefix, "payload_preview": payload[:160]}) elif prefix in {"ag", "bg"}: text_chunk = self._parse_text_chunk(payload) if text_chunk is not None: reasoning_parts.append(text_chunk) else: scan.add_parse_error({"prefix": prefix, "payload_preview": payload[:160]}) elif prefix == "a2": images.extend( _extract_arena_image_items( payload, self._seen_image_refs, source="lmarena_stream", ) ) elif prefix == "ad": if self._is_finish_signal(payload): scan.done = True elif prefix in {"ae", "a3"}: error_msg = self._parse_error(payload) if error_msg: scan.error = error_msg scan.done = True if content_parts: scan.content += "".join(content_parts) if reasoning_parts: scan.reasoning += "".join(reasoning_parts) # ============ 对外接口 ============ def parse_chunk(self, raw_response: str) -> Dict[str, Any]: """ 解析完整的 HTTP 响应体(一次性包含所有 SSE 行) """ result: Dict[str, Any] = { "content": "", "reasoning_content": "", "images": [], "done": False, "error": None, } if isinstance(raw_response, (bytes, bytearray)): raw_response = raw_response.decode("utf-8", errors="ignore") if not raw_response or not isinstance(raw_response, str): return result debug_raw_signature = self._debug_body_signature(raw_response) try: # P0-5:增量解析。网络监听每次传入的是“不断增长的完整响应体”,旧实现 # 每次都整段 split + json.loads,一个 160KB 的回答流累计要花数秒 CPU。 # 这里把“已以换行结尾的完整行”的解析结果缓存下来,只解析新增部分; # 末尾未结束的半行每次重新解析、不入缓存。响应体不是上次的前缀扩展 # (换了一条流)时回退为整段解析。 scan, images = self._scan_response(raw_response) done = scan.done if scan.error: result["error"] = scan.error new_content = scan.content new_reasoning = scan.reasoning if new_content: delta, next_accumulated = self._content_delta(self._accumulated, new_content) if delta: result["content"] = delta self._accumulated = next_accumulated if new_reasoning: delta_r, next_acc_r = self._content_delta(self._accumulated_reasoning, new_reasoning) if delta_r: result["reasoning_content"] = delta_r self._accumulated_reasoning = next_acc_r if images: result["images"] = images result["done"] = done self._last_debug_summary = { "raw_body_len": len(raw_response), "line_count": scan.line_count, "prefix_counts": dict(scan.prefix_counts), "unknown_prefix_count": scan.unknown_prefix_count, "non_protocol_line_count": scan.non_protocol_line_count, "content_candidate_len": len(new_content), "emitted_content_len": len(str(result.get("content") or "")), "accumulated_len": len(self._accumulated), "reasoning_candidate_len": len(new_reasoning), "emitted_reasoning_len": len(str(result.get("reasoning_content") or "")), "accumulated_reasoning_len": len(self._accumulated_reasoning), "done": bool(done), "error": str(result.get("error") or ""), "image_count": len(images), "parse_errors": scan.parse_errors[-8:], "incremental_offset": scan.committed_len if self._INCREMENTAL else 0, } self._last_debug_raw_signature = debug_raw_signature except Exception as e: logger.debug(f"[LmarenaParser] 解析异常: {e}") result["error"] = str(e) self._inc_state = None self._last_debug_summary = { "raw_body_len": len(raw_response), "exception": str(e), } self._last_debug_raw_signature = debug_raw_signature return result def reset(self) -> None: """重置状态""" self._accumulated = "" self._accumulated_reasoning = "" self._seen_image_refs.clear() self._last_debug_summary = {} self._last_debug_raw_signature = "" self._inc_state = None def should_abort_on_error(self) -> bool: """当 Arena 返回 ae: 或 a3: 等错误协议行时,立即终止工作流。""" return True def should_fallback_to_dom_when_no_visible_content(self) -> bool: """Arena 的心跳流可能因网络波动提前关闭,空结果必须交给 DOM 接管。""" return True def export_debug_data(self, raw_response: str = "") -> Dict[str, Any]: """Return a compact parser-side protocol summary for network_parser_debug.""" summary = dict(self._last_debug_summary or {}) if raw_response: text = self._normalize_debug_text(raw_response) raw_summary = self._summarize_protocol_lines(text) if self._last_debug_raw_signature == self._debug_body_signature(text): summary.update(raw_summary) else: summary = raw_summary summary.update({ "accumulated_len": len(self._accumulated), "seen_image_refs": len(self._seen_image_refs), }) return summary # ============ 内部方法 ============ @staticmethod def _normalize_debug_text(raw_response: Any) -> str: if isinstance(raw_response, (bytes, bytearray)): return raw_response.decode("utf-8", errors="ignore") return str(raw_response or "") @classmethod def _debug_body_signature(cls, raw_response: Any) -> str: text = cls._normalize_debug_text(raw_response) return f"{len(text)}:{text[:80]}:{text[-80:]}" @classmethod def _summarize_protocol_lines(cls, raw_response: Any) -> Dict[str, Any]: text = cls._normalize_debug_text(raw_response) prefix_counts: Dict[str, int] = {} line_count = 0 unknown_prefix_count = 0 non_protocol_line_count = 0 for raw_line in text.split("\n"): line = raw_line.strip() if not line: continue line_count += 1 colon_idx = line.find(":") if colon_idx < 1: non_protocol_line_count += 1 continue prefix = line[:colon_idx] if prefix in cls._PROTOCOL_PREFIXES: prefix_counts[prefix] = prefix_counts.get(prefix, 0) + 1 else: unknown_prefix_count += 1 return { "raw_body_len": len(text), "line_count": line_count, "prefix_counts": prefix_counts, "unknown_prefix_count": unknown_prefix_count, "non_protocol_line_count": non_protocol_line_count, } @staticmethod def _fix_mojibake(text: str) -> str: """ 修复双重 UTF-8 编码(mojibake) DrissionPage 通过 CDP 获取的响应体存在编码错误: UTF-8 字节被混合解码为 cp1252 + latin-1,导致: 原始 UTF-8: f0 9f 92 9e (💞) 被 cp1252 解码: ð(F0) Ÿ(9F→U+0178) '(92→U+2019) ž(9E→U+017E) 但 0x90 → U+0090(latin-1 直通,因 cp1252 未定义此位置) cp1252 有 5 个未定义位置 (0x81,0x8D,0x8F,0x90,0x9D) 走 latin-1 直通映射,导致 encode('cp1252') 和 encode('latin-1') 都无法单独处理全部字符。 修复方案(两步): 1. translate: 将 cp1252 特有字符 (U+2019等) 映射回 latin-1 范围 (U+0092等) 2. encode('latin-1'): 所有字符现在都在 U+0000-U+00FF,可一一还原原始字节 3. decode('utf-8'): 原始字节 → 正确文本 """ try: mapped = text.translate(_CP1252_TO_LATIN1) return mapped.encode("latin-1").decode("utf-8") except (UnicodeDecodeError, UnicodeEncodeError): # 不是 mojibake(或只有部分是),返回原样 return text @staticmethod def _parse_text_chunk(payload: str) -> str | None: """解析 a0 行的 payload""" try: value = json.loads(payload) if isinstance(value, str): return value except (json.JSONDecodeError, ValueError): pass return None @staticmethod def _is_finish_signal(payload: str) -> bool: """检查 ad 行是否表示结束""" try: data = json.loads(payload) if isinstance(data, dict) and data.get("finishReason"): return True except (json.JSONDecodeError, ValueError): pass return False @staticmethod def _parse_error(payload: str) -> str | None: """解析 ae 行的错误信息""" try: data = json.loads(payload) if isinstance(data, dict): return data.get("message", str(data)) return str(data) except (json.JSONDecodeError, ValueError): return payload.strip() if payload.strip() else None # ============ 元数据 ============ @classmethod def get_id(cls) -> str: return "lmarena" @classmethod def get_name(cls) -> str: return "Arena.ai" @classmethod def get_description(cls) -> str: return "解析 Arena.ai 的流式响应 (Vercel AI SDK Data Stream Protocol)" @classmethod def get_supported_patterns(cls) -> List[str]: return ["nextjs-api/stream/create-evaluation"] __all__ = [ "LmarenaParser", "_CP1252_TO_LATIN1", "_extract_arena_image_items", ]