Files
universal-web-api/app/core/parsers/lmarena_parser.py
T
2026-11-02 22:54:14 +08:00

565 lines
21 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""
lmarena_parser.py - Arena.ai 响应解析器
响应格式:Vercel AI SDK Data Stream Protocol
- 每行格式: {prefix}:{json_data}
- a0: 文本增量(JSON 编码的字符串)
- a2: 心跳/元数据(JSON 数组)
- ad: 结束元数据(含 finishReason)
- ae: 错误信息
编码问题:
DrissionPage 通过 CDP 获取响应体时,UTF-8 字节被混合解码为
cp1252 映射字符(如 0x92→' U+2019)和 latin-1 直通字符
(如 0x90→U+0090),导致双重编码(mojibake)。
需要两步修复:translate 统一到 latin-1 范围 → encode('latin-1') 还原字节
调用方式:
NetworkMonitor 每次调用 parse_chunk 传入一个完整的 HTTP 响应体
(DrissionPage listen.wait 是非流式的,一次性拿到整个 SSE body)
"""
from __future__ import annotations
import json
from datetime import datetime, timezone
from typing import Any, Dict, List
from app.core.config import logger
from .base import ResponseParser
# ─────────────────────────────────────────────────────
# cp1252 → latin-1 逆映射表(模块级常量,只构建一次)
# ─────────────────────────────────────────────────────
# cp1252 在 0x80-0x9F 区间将部分字节映射到 U+0100 以上的 Unicode 字符,
# 而 latin-1 将相同字节直通映射到 U+0080-U+009F。
# 此表将 cp1252 的特殊映射还原为 latin-1 范围的等价字符,
# 使后续 encode('latin-1') 能正确还原原始字节。
#
# 5 个未定义位置 (0x81, 0x8D, 0x8F, 0x90, 0x9D) 已经是
# latin-1 直通映射 (U+0081 等),无需处理。
_CP1252_TO_LATIN1 = str.maketrans({
'\u20ac': '\x80', # €
'\u201a': '\x82', # ‚
'\u0192': '\x83', # ƒ
'\u201e': '\x84', # „
'\u2026': '\x85', # …
'\u2020': '\x86', # †
'\u2021': '\x87', # ‡
'\u02c6': '\x88', # ˆ
'\u2030': '\x89', # ‰
'\u0160': '\x8a', # Š
'\u2039': '\x8b', # ‹
'\u0152': '\x8c', # Œ
'\u017d': '\x8e', # Ž
'\u2018': '\x91', # '
'\u2019': '\x92', # '
'\u201c': '\x93', # "
'\u201d': '\x94', # "
'\u2022': '\x95', # •
'\u2013': '\x96', # –
'\u2014': '\x97', # —
'\u02dc': '\x98', # ˜
'\u2122': '\x99', # ™
'\u0161': '\x9a', # š
'\u203a': '\x9b', # ›
'\u0153': '\x9c', # œ
'\u017e': '\x9e', # ž
'\u0178': '\x9f', # Ÿ
})
def _looks_like_image_ref(value: str) -> bool:
text = str(value or "").strip()
if not text:
return False
lowered = text.lower()
if lowered.startswith(("http://", "https://", "data:image/", "blob:")):
return True
return any(token in lowered for token in (".png", ".jpg", ".jpeg", ".webp", ".gif", ".bmp", ".svg", ".avif"))
def _extract_arena_image_items(
payload: str,
seen_refs: set[str],
*,
source: str,
) -> List[Dict[str, Any]]:
items: List[Dict[str, Any]] = []
try:
data = json.loads(payload)
except (json.JSONDecodeError, ValueError):
return items
entries = data if isinstance(data, list) else [data]
for entry in entries:
if not isinstance(entry, dict):
continue
entry_type = str(entry.get("type") or "").strip().lower()
if entry_type and entry_type != "image":
continue
image_ref = ""
for key in ("image", "url", "src"):
candidate = str(entry.get(key) or "").strip()
if _looks_like_image_ref(candidate):
image_ref = candidate
break
if not image_ref:
continue
dedupe_key = f"image:{image_ref}"
if dedupe_key in seen_refs:
continue
seen_refs.add(dedupe_key)
mime = str(entry.get("mimeType") or entry.get("mime") or "").strip() or None
kind = "data_uri" if image_ref.startswith("data:image/") else "url"
items.append(
{
"media_type": "image",
"kind": kind,
"url": image_ref if kind == "url" else None,
"data_uri": image_ref if kind == "data_uri" else None,
"mime": mime,
"byte_size": None,
"alt": str(entry.get("alt") or "").strip(),
"width": entry.get("width"),
"height": entry.get("height"),
"detected_at": datetime.now(timezone.utc).isoformat().replace("+00:00", "Z"),
"source": source,
}
)
return items
class _LmarenaScanState:
"""LmarenaParser 增量扫描的累计状态(只覆盖已提交的完整行 + 本次半行)。"""
__slots__ = (
"content", "reasoning", "done", "error", "line_count", "prefix_counts",
"unknown_prefix_count", "non_protocol_line_count", "parse_errors",
"committed_len", "committed_raw",
)
def __init__(self) -> None:
self.content = ""
self.reasoning = ""
self.done = False
self.error = None
self.line_count = 0
self.prefix_counts: Dict[str, int] = {}
self.unknown_prefix_count = 0
self.non_protocol_line_count = 0
self.parse_errors: list[Dict[str, Any]] = []
self.committed_len = 0
self.committed_raw = ""
def copy(self) -> "_LmarenaScanState":
other = _LmarenaScanState()
for name in self.__slots__:
setattr(other, name, getattr(self, name))
other.prefix_counts = dict(self.prefix_counts)
other.parse_errors = list(self.parse_errors)
return other
def add_parse_error(self, item: Dict[str, Any]) -> None:
self.parse_errors.append(item)
if len(self.parse_errors) > 16:
del self.parse_errors[:-8]
class LmarenaParser(ResponseParser):
"""
Arena.ai 响应解析器
URL 特征: /nextjs-api/stream/create-evaluation
响应格式: Vercel AI SDK Data Stream Protocol (行分隔)
"""
_PROTOCOL_PREFIXES = {"a0", "a2", "ad", "ae", "a3", "ag", "bg"}
def __init__(self) -> None:
self._accumulated = ""
self._accumulated_reasoning = ""
self._seen_image_refs: set[str] = set()
self._last_debug_summary: Dict[str, Any] = {}
self._last_debug_raw_signature = ""
self._inc_state: "_LmarenaScanState | None" = None
@staticmethod
def _content_delta(
accumulated: str,
candidate: str,
*,
append_disjoint: bool = False,
) -> tuple[str, str]:
"""Return unseen text and the new accumulated value for repeated SSE bodies."""
if not candidate:
return "", accumulated
if not accumulated:
return candidate, candidate
if candidate == accumulated or accumulated.startswith(candidate):
return "", accumulated
if candidate.startswith(accumulated):
return candidate[len(accumulated):], candidate
max_overlap = min(len(accumulated), len(candidate))
for overlap in range(max_overlap, 0, -1):
if accumulated.endswith(candidate[:overlap]):
delta = candidate[overlap:]
return delta, accumulated + delta
if append_disjoint:
return candidate, accumulated + candidate
# A parser instance belongs to one response stream. A body that neither
# extends nor overlaps the emitted text is a stale/non-monotonic replay;
# emitting it wholesale would duplicate the response already sent.
return "", accumulated
# ============ 增量扫描(P0-5) ============
# 置为 False 时每次整段解析(与旧实现逐字节一致),用于回归对照
_INCREMENTAL = True
def _scan_response(self, raw_response: str) -> tuple["_LmarenaScanState", list]:
"""返回 (扫描状态, 本次新发现的图片)。"""
images: list[Dict[str, Any]] = []
if not self._INCREMENTAL:
# 旧行为:整段修复 mojibake 后逐行解析
scan = _LmarenaScanState()
self._scan_lines(self._fix_mojibake(raw_response), scan, images)
return scan, images
cached = getattr(self, "_inc_state", None)
if cached is not None and cached.committed_len <= len(raw_response) and raw_response.startswith(cached.committed_raw):
scan = cached.copy()
offset = cached.committed_len
else:
scan = _LmarenaScanState()
offset = 0
last_newline = raw_response.rfind("\n", offset)
if last_newline >= offset:
committed_end = last_newline + 1
# 完整行一定以 ASCII 换行结尾,不会截断多字节序列,可以按段修复 mojibake
self._scan_lines(self._fix_mojibake(raw_response[offset:committed_end]), scan, images)
scan.committed_len = committed_end
scan.committed_raw = raw_response[:committed_end]
self._inc_state = scan.copy()
offset = committed_end
elif cached is None or scan.committed_len == 0:
self._inc_state = None
tail = raw_response[offset:]
if tail:
self._scan_lines(self._fix_mojibake(tail), scan, images)
return scan, images
def _scan_lines(self, text: str, scan: "_LmarenaScanState", images: list) -> None:
content_parts: list[str] = []
reasoning_parts: list[str] = []
for line in text.split("\n"):
line = line.strip()
if not line:
continue
scan.line_count += 1
colon_idx = line.find(":")
if colon_idx < 1:
scan.non_protocol_line_count += 1
continue
prefix = line[:colon_idx]
payload = line[colon_idx + 1:]
if prefix in self._PROTOCOL_PREFIXES:
scan.prefix_counts[prefix] = scan.prefix_counts.get(prefix, 0) + 1
else:
scan.unknown_prefix_count += 1
if prefix == "a0":
text_chunk = self._parse_text_chunk(payload)
if text_chunk is not None:
content_parts.append(text_chunk)
else:
scan.add_parse_error({"prefix": prefix, "payload_preview": payload[:160]})
elif prefix in {"ag", "bg"}:
text_chunk = self._parse_text_chunk(payload)
if text_chunk is not None:
reasoning_parts.append(text_chunk)
else:
scan.add_parse_error({"prefix": prefix, "payload_preview": payload[:160]})
elif prefix == "a2":
images.extend(
_extract_arena_image_items(
payload,
self._seen_image_refs,
source="lmarena_stream",
)
)
elif prefix == "ad":
if self._is_finish_signal(payload):
scan.done = True
elif prefix in {"ae", "a3"}:
error_msg = self._parse_error(payload)
if error_msg:
scan.error = error_msg
scan.done = True
if content_parts:
scan.content += "".join(content_parts)
if reasoning_parts:
scan.reasoning += "".join(reasoning_parts)
# ============ 对外接口 ============
def parse_chunk(self, raw_response: str) -> Dict[str, Any]:
"""
解析完整的 HTTP 响应体(一次性包含所有 SSE 行)
"""
result: Dict[str, Any] = {
"content": "",
"reasoning_content": "",
"images": [],
"done": False,
"error": None,
}
if isinstance(raw_response, (bytes, bytearray)):
raw_response = raw_response.decode("utf-8", errors="ignore")
if not raw_response or not isinstance(raw_response, str):
return result
debug_raw_signature = self._debug_body_signature(raw_response)
try:
# P0-5:增量解析。网络监听每次传入的是“不断增长的完整响应体”,旧实现
# 每次都整段 split + json.loads,一个 160KB 的回答流累计要花数秒 CPU。
# 这里把“已以换行结尾的完整行”的解析结果缓存下来,只解析新增部分;
# 末尾未结束的半行每次重新解析、不入缓存。响应体不是上次的前缀扩展
# (换了一条流)时回退为整段解析。
scan, images = self._scan_response(raw_response)
done = scan.done
if scan.error:
result["error"] = scan.error
new_content = scan.content
new_reasoning = scan.reasoning
if new_content:
delta, next_accumulated = self._content_delta(self._accumulated, new_content)
if delta:
result["content"] = delta
self._accumulated = next_accumulated
if new_reasoning:
delta_r, next_acc_r = self._content_delta(self._accumulated_reasoning, new_reasoning)
if delta_r:
result["reasoning_content"] = delta_r
self._accumulated_reasoning = next_acc_r
if images:
result["images"] = images
result["done"] = done
self._last_debug_summary = {
"raw_body_len": len(raw_response),
"line_count": scan.line_count,
"prefix_counts": dict(scan.prefix_counts),
"unknown_prefix_count": scan.unknown_prefix_count,
"non_protocol_line_count": scan.non_protocol_line_count,
"content_candidate_len": len(new_content),
"emitted_content_len": len(str(result.get("content") or "")),
"accumulated_len": len(self._accumulated),
"reasoning_candidate_len": len(new_reasoning),
"emitted_reasoning_len": len(str(result.get("reasoning_content") or "")),
"accumulated_reasoning_len": len(self._accumulated_reasoning),
"done": bool(done),
"error": str(result.get("error") or ""),
"image_count": len(images),
"parse_errors": scan.parse_errors[-8:],
"incremental_offset": scan.committed_len if self._INCREMENTAL else 0,
}
self._last_debug_raw_signature = debug_raw_signature
except Exception as e:
logger.debug(f"[LmarenaParser] 解析异常: {e}")
result["error"] = str(e)
self._inc_state = None
self._last_debug_summary = {
"raw_body_len": len(raw_response),
"exception": str(e),
}
self._last_debug_raw_signature = debug_raw_signature
return result
def reset(self) -> None:
"""重置状态"""
self._accumulated = ""
self._accumulated_reasoning = ""
self._seen_image_refs.clear()
self._last_debug_summary = {}
self._last_debug_raw_signature = ""
self._inc_state = None
def should_abort_on_error(self) -> bool:
"""当 Arena 返回 ae: 或 a3: 等错误协议行时,立即终止工作流。"""
return True
def should_fallback_to_dom_when_no_visible_content(self) -> bool:
"""Arena 的心跳流可能因网络波动提前关闭,空结果必须交给 DOM 接管。"""
return True
def export_debug_data(self, raw_response: str = "") -> Dict[str, Any]:
"""Return a compact parser-side protocol summary for network_parser_debug."""
summary = dict(self._last_debug_summary or {})
if raw_response:
text = self._normalize_debug_text(raw_response)
raw_summary = self._summarize_protocol_lines(text)
if self._last_debug_raw_signature == self._debug_body_signature(text):
summary.update(raw_summary)
else:
summary = raw_summary
summary.update({
"accumulated_len": len(self._accumulated),
"seen_image_refs": len(self._seen_image_refs),
})
return summary
# ============ 内部方法 ============
@staticmethod
def _normalize_debug_text(raw_response: Any) -> str:
if isinstance(raw_response, (bytes, bytearray)):
return raw_response.decode("utf-8", errors="ignore")
return str(raw_response or "")
@classmethod
def _debug_body_signature(cls, raw_response: Any) -> str:
text = cls._normalize_debug_text(raw_response)
return f"{len(text)}:{text[:80]}:{text[-80:]}"
@classmethod
def _summarize_protocol_lines(cls, raw_response: Any) -> Dict[str, Any]:
text = cls._normalize_debug_text(raw_response)
prefix_counts: Dict[str, int] = {}
line_count = 0
unknown_prefix_count = 0
non_protocol_line_count = 0
for raw_line in text.split("\n"):
line = raw_line.strip()
if not line:
continue
line_count += 1
colon_idx = line.find(":")
if colon_idx < 1:
non_protocol_line_count += 1
continue
prefix = line[:colon_idx]
if prefix in cls._PROTOCOL_PREFIXES:
prefix_counts[prefix] = prefix_counts.get(prefix, 0) + 1
else:
unknown_prefix_count += 1
return {
"raw_body_len": len(text),
"line_count": line_count,
"prefix_counts": prefix_counts,
"unknown_prefix_count": unknown_prefix_count,
"non_protocol_line_count": non_protocol_line_count,
}
@staticmethod
def _fix_mojibake(text: str) -> str:
"""
修复双重 UTF-8 编码(mojibake)
DrissionPage 通过 CDP 获取的响应体存在编码错误:
UTF-8 字节被混合解码为 cp1252 + latin-1,导致:
原始 UTF-8: f0 9f 92 9e (💞)
被 cp1252 解码: ð(F0) Ÿ(9F→U+0178) '(92→U+2019) ž(9E→U+017E)
但 0x90 → U+0090(latin-1 直通,因 cp1252 未定义此位置)
cp1252 有 5 个未定义位置 (0x81,0x8D,0x8F,0x90,0x9D)
走 latin-1 直通映射,导致 encode('cp1252') 和 encode('latin-1')
都无法单独处理全部字符。
修复方案(两步):
1. translate: 将 cp1252 特有字符 (U+2019等) 映射回 latin-1 范围 (U+0092等)
2. encode('latin-1'): 所有字符现在都在 U+0000-U+00FF,可一一还原原始字节
3. decode('utf-8'): 原始字节 → 正确文本
"""
try:
mapped = text.translate(_CP1252_TO_LATIN1)
return mapped.encode("latin-1").decode("utf-8")
except (UnicodeDecodeError, UnicodeEncodeError):
# 不是 mojibake(或只有部分是),返回原样
return text
@staticmethod
def _parse_text_chunk(payload: str) -> str | None:
"""解析 a0 行的 payload"""
try:
value = json.loads(payload)
if isinstance(value, str):
return value
except (json.JSONDecodeError, ValueError):
pass
return None
@staticmethod
def _is_finish_signal(payload: str) -> bool:
"""检查 ad 行是否表示结束"""
try:
data = json.loads(payload)
if isinstance(data, dict) and data.get("finishReason"):
return True
except (json.JSONDecodeError, ValueError):
pass
return False
@staticmethod
def _parse_error(payload: str) -> str | None:
"""解析 ae 行的错误信息"""
try:
data = json.loads(payload)
if isinstance(data, dict):
return data.get("message", str(data))
return str(data)
except (json.JSONDecodeError, ValueError):
return payload.strip() if payload.strip() else None
# ============ 元数据 ============
@classmethod
def get_id(cls) -> str:
return "lmarena"
@classmethod
def get_name(cls) -> str:
return "Arena.ai"
@classmethod
def get_description(cls) -> str:
return "解析 Arena.ai 的流式响应 (Vercel AI SDK Data Stream Protocol)"
@classmethod
def get_supported_patterns(cls) -> List[str]:
return ["nextjs-api/stream/create-evaluation"]
__all__ = [
"LmarenaParser",
"_CP1252_TO_LATIN1",
"_extract_arena_image_items",
]