565 lines
21 KiB
Python
565 lines
21 KiB
Python
"""
|
||
lmarena_parser.py - Arena.ai 响应解析器
|
||
|
||
响应格式:Vercel AI SDK Data Stream Protocol
|
||
- 每行格式: {prefix}:{json_data}
|
||
- a0: 文本增量(JSON 编码的字符串)
|
||
- a2: 心跳/元数据(JSON 数组)
|
||
- ad: 结束元数据(含 finishReason)
|
||
- ae: 错误信息
|
||
|
||
编码问题:
|
||
DrissionPage 通过 CDP 获取响应体时,UTF-8 字节被混合解码为
|
||
cp1252 映射字符(如 0x92→' U+2019)和 latin-1 直通字符
|
||
(如 0x90→U+0090),导致双重编码(mojibake)。
|
||
需要两步修复:translate 统一到 latin-1 范围 → encode('latin-1') 还原字节
|
||
|
||
调用方式:
|
||
NetworkMonitor 每次调用 parse_chunk 传入一个完整的 HTTP 响应体
|
||
(DrissionPage listen.wait 是非流式的,一次性拿到整个 SSE body)
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import json
|
||
from datetime import datetime, timezone
|
||
from typing import Any, Dict, List
|
||
|
||
from app.core.config import logger
|
||
from .base import ResponseParser
|
||
|
||
|
||
# ─────────────────────────────────────────────────────
|
||
# cp1252 → latin-1 逆映射表(模块级常量,只构建一次)
|
||
# ─────────────────────────────────────────────────────
|
||
# cp1252 在 0x80-0x9F 区间将部分字节映射到 U+0100 以上的 Unicode 字符,
|
||
# 而 latin-1 将相同字节直通映射到 U+0080-U+009F。
|
||
# 此表将 cp1252 的特殊映射还原为 latin-1 范围的等价字符,
|
||
# 使后续 encode('latin-1') 能正确还原原始字节。
|
||
#
|
||
# 5 个未定义位置 (0x81, 0x8D, 0x8F, 0x90, 0x9D) 已经是
|
||
# latin-1 直通映射 (U+0081 等),无需处理。
|
||
_CP1252_TO_LATIN1 = str.maketrans({
|
||
'\u20ac': '\x80', # €
|
||
'\u201a': '\x82', # ‚
|
||
'\u0192': '\x83', # ƒ
|
||
'\u201e': '\x84', # „
|
||
'\u2026': '\x85', # …
|
||
'\u2020': '\x86', # †
|
||
'\u2021': '\x87', # ‡
|
||
'\u02c6': '\x88', # ˆ
|
||
'\u2030': '\x89', # ‰
|
||
'\u0160': '\x8a', # Š
|
||
'\u2039': '\x8b', # ‹
|
||
'\u0152': '\x8c', # Œ
|
||
'\u017d': '\x8e', # Ž
|
||
'\u2018': '\x91', # '
|
||
'\u2019': '\x92', # '
|
||
'\u201c': '\x93', # "
|
||
'\u201d': '\x94', # "
|
||
'\u2022': '\x95', # •
|
||
'\u2013': '\x96', # –
|
||
'\u2014': '\x97', # —
|
||
'\u02dc': '\x98', # ˜
|
||
'\u2122': '\x99', # ™
|
||
'\u0161': '\x9a', # š
|
||
'\u203a': '\x9b', # ›
|
||
'\u0153': '\x9c', # œ
|
||
'\u017e': '\x9e', # ž
|
||
'\u0178': '\x9f', # Ÿ
|
||
})
|
||
|
||
|
||
def _looks_like_image_ref(value: str) -> bool:
|
||
text = str(value or "").strip()
|
||
if not text:
|
||
return False
|
||
lowered = text.lower()
|
||
if lowered.startswith(("http://", "https://", "data:image/", "blob:")):
|
||
return True
|
||
return any(token in lowered for token in (".png", ".jpg", ".jpeg", ".webp", ".gif", ".bmp", ".svg", ".avif"))
|
||
|
||
|
||
def _extract_arena_image_items(
|
||
payload: str,
|
||
seen_refs: set[str],
|
||
*,
|
||
source: str,
|
||
) -> List[Dict[str, Any]]:
|
||
items: List[Dict[str, Any]] = []
|
||
|
||
try:
|
||
data = json.loads(payload)
|
||
except (json.JSONDecodeError, ValueError):
|
||
return items
|
||
|
||
entries = data if isinstance(data, list) else [data]
|
||
for entry in entries:
|
||
if not isinstance(entry, dict):
|
||
continue
|
||
|
||
entry_type = str(entry.get("type") or "").strip().lower()
|
||
if entry_type and entry_type != "image":
|
||
continue
|
||
|
||
image_ref = ""
|
||
for key in ("image", "url", "src"):
|
||
candidate = str(entry.get(key) or "").strip()
|
||
if _looks_like_image_ref(candidate):
|
||
image_ref = candidate
|
||
break
|
||
|
||
if not image_ref:
|
||
continue
|
||
|
||
dedupe_key = f"image:{image_ref}"
|
||
if dedupe_key in seen_refs:
|
||
continue
|
||
seen_refs.add(dedupe_key)
|
||
|
||
mime = str(entry.get("mimeType") or entry.get("mime") or "").strip() or None
|
||
kind = "data_uri" if image_ref.startswith("data:image/") else "url"
|
||
items.append(
|
||
{
|
||
"media_type": "image",
|
||
"kind": kind,
|
||
"url": image_ref if kind == "url" else None,
|
||
"data_uri": image_ref if kind == "data_uri" else None,
|
||
"mime": mime,
|
||
"byte_size": None,
|
||
"alt": str(entry.get("alt") or "").strip(),
|
||
"width": entry.get("width"),
|
||
"height": entry.get("height"),
|
||
"detected_at": datetime.now(timezone.utc).isoformat().replace("+00:00", "Z"),
|
||
"source": source,
|
||
}
|
||
)
|
||
|
||
return items
|
||
|
||
|
||
class _LmarenaScanState:
|
||
"""LmarenaParser 增量扫描的累计状态(只覆盖已提交的完整行 + 本次半行)。"""
|
||
|
||
__slots__ = (
|
||
"content", "reasoning", "done", "error", "line_count", "prefix_counts",
|
||
"unknown_prefix_count", "non_protocol_line_count", "parse_errors",
|
||
"committed_len", "committed_raw",
|
||
)
|
||
|
||
def __init__(self) -> None:
|
||
self.content = ""
|
||
self.reasoning = ""
|
||
self.done = False
|
||
self.error = None
|
||
self.line_count = 0
|
||
self.prefix_counts: Dict[str, int] = {}
|
||
self.unknown_prefix_count = 0
|
||
self.non_protocol_line_count = 0
|
||
self.parse_errors: list[Dict[str, Any]] = []
|
||
self.committed_len = 0
|
||
self.committed_raw = ""
|
||
|
||
def copy(self) -> "_LmarenaScanState":
|
||
other = _LmarenaScanState()
|
||
for name in self.__slots__:
|
||
setattr(other, name, getattr(self, name))
|
||
other.prefix_counts = dict(self.prefix_counts)
|
||
other.parse_errors = list(self.parse_errors)
|
||
return other
|
||
|
||
def add_parse_error(self, item: Dict[str, Any]) -> None:
|
||
self.parse_errors.append(item)
|
||
if len(self.parse_errors) > 16:
|
||
del self.parse_errors[:-8]
|
||
|
||
|
||
class LmarenaParser(ResponseParser):
|
||
"""
|
||
Arena.ai 响应解析器
|
||
|
||
URL 特征: /nextjs-api/stream/create-evaluation
|
||
响应格式: Vercel AI SDK Data Stream Protocol (行分隔)
|
||
"""
|
||
|
||
_PROTOCOL_PREFIXES = {"a0", "a2", "ad", "ae", "a3", "ag", "bg"}
|
||
|
||
def __init__(self) -> None:
|
||
self._accumulated = ""
|
||
self._accumulated_reasoning = ""
|
||
self._seen_image_refs: set[str] = set()
|
||
self._last_debug_summary: Dict[str, Any] = {}
|
||
self._last_debug_raw_signature = ""
|
||
self._inc_state: "_LmarenaScanState | None" = None
|
||
|
||
@staticmethod
|
||
def _content_delta(
|
||
accumulated: str,
|
||
candidate: str,
|
||
*,
|
||
append_disjoint: bool = False,
|
||
) -> tuple[str, str]:
|
||
"""Return unseen text and the new accumulated value for repeated SSE bodies."""
|
||
if not candidate:
|
||
return "", accumulated
|
||
if not accumulated:
|
||
return candidate, candidate
|
||
if candidate == accumulated or accumulated.startswith(candidate):
|
||
return "", accumulated
|
||
if candidate.startswith(accumulated):
|
||
return candidate[len(accumulated):], candidate
|
||
|
||
max_overlap = min(len(accumulated), len(candidate))
|
||
for overlap in range(max_overlap, 0, -1):
|
||
if accumulated.endswith(candidate[:overlap]):
|
||
delta = candidate[overlap:]
|
||
return delta, accumulated + delta
|
||
|
||
if append_disjoint:
|
||
return candidate, accumulated + candidate
|
||
|
||
# A parser instance belongs to one response stream. A body that neither
|
||
# extends nor overlaps the emitted text is a stale/non-monotonic replay;
|
||
# emitting it wholesale would duplicate the response already sent.
|
||
return "", accumulated
|
||
|
||
# ============ 增量扫描(P0-5) ============
|
||
|
||
# 置为 False 时每次整段解析(与旧实现逐字节一致),用于回归对照
|
||
_INCREMENTAL = True
|
||
|
||
def _scan_response(self, raw_response: str) -> tuple["_LmarenaScanState", list]:
|
||
"""返回 (扫描状态, 本次新发现的图片)。"""
|
||
images: list[Dict[str, Any]] = []
|
||
if not self._INCREMENTAL:
|
||
# 旧行为:整段修复 mojibake 后逐行解析
|
||
scan = _LmarenaScanState()
|
||
self._scan_lines(self._fix_mojibake(raw_response), scan, images)
|
||
return scan, images
|
||
|
||
cached = getattr(self, "_inc_state", None)
|
||
if cached is not None and cached.committed_len <= len(raw_response) and raw_response.startswith(cached.committed_raw):
|
||
scan = cached.copy()
|
||
offset = cached.committed_len
|
||
else:
|
||
scan = _LmarenaScanState()
|
||
offset = 0
|
||
|
||
last_newline = raw_response.rfind("\n", offset)
|
||
if last_newline >= offset:
|
||
committed_end = last_newline + 1
|
||
# 完整行一定以 ASCII 换行结尾,不会截断多字节序列,可以按段修复 mojibake
|
||
self._scan_lines(self._fix_mojibake(raw_response[offset:committed_end]), scan, images)
|
||
scan.committed_len = committed_end
|
||
scan.committed_raw = raw_response[:committed_end]
|
||
self._inc_state = scan.copy()
|
||
offset = committed_end
|
||
elif cached is None or scan.committed_len == 0:
|
||
self._inc_state = None
|
||
|
||
tail = raw_response[offset:]
|
||
if tail:
|
||
self._scan_lines(self._fix_mojibake(tail), scan, images)
|
||
return scan, images
|
||
|
||
def _scan_lines(self, text: str, scan: "_LmarenaScanState", images: list) -> None:
|
||
content_parts: list[str] = []
|
||
reasoning_parts: list[str] = []
|
||
for line in text.split("\n"):
|
||
line = line.strip()
|
||
if not line:
|
||
continue
|
||
scan.line_count += 1
|
||
|
||
colon_idx = line.find(":")
|
||
if colon_idx < 1:
|
||
scan.non_protocol_line_count += 1
|
||
continue
|
||
|
||
prefix = line[:colon_idx]
|
||
payload = line[colon_idx + 1:]
|
||
if prefix in self._PROTOCOL_PREFIXES:
|
||
scan.prefix_counts[prefix] = scan.prefix_counts.get(prefix, 0) + 1
|
||
else:
|
||
scan.unknown_prefix_count += 1
|
||
|
||
if prefix == "a0":
|
||
text_chunk = self._parse_text_chunk(payload)
|
||
if text_chunk is not None:
|
||
content_parts.append(text_chunk)
|
||
else:
|
||
scan.add_parse_error({"prefix": prefix, "payload_preview": payload[:160]})
|
||
|
||
elif prefix in {"ag", "bg"}:
|
||
text_chunk = self._parse_text_chunk(payload)
|
||
if text_chunk is not None:
|
||
reasoning_parts.append(text_chunk)
|
||
else:
|
||
scan.add_parse_error({"prefix": prefix, "payload_preview": payload[:160]})
|
||
|
||
elif prefix == "a2":
|
||
images.extend(
|
||
_extract_arena_image_items(
|
||
payload,
|
||
self._seen_image_refs,
|
||
source="lmarena_stream",
|
||
)
|
||
)
|
||
|
||
elif prefix == "ad":
|
||
if self._is_finish_signal(payload):
|
||
scan.done = True
|
||
|
||
elif prefix in {"ae", "a3"}:
|
||
error_msg = self._parse_error(payload)
|
||
if error_msg:
|
||
scan.error = error_msg
|
||
scan.done = True
|
||
if content_parts:
|
||
scan.content += "".join(content_parts)
|
||
if reasoning_parts:
|
||
scan.reasoning += "".join(reasoning_parts)
|
||
|
||
# ============ 对外接口 ============
|
||
|
||
def parse_chunk(self, raw_response: str) -> Dict[str, Any]:
|
||
"""
|
||
解析完整的 HTTP 响应体(一次性包含所有 SSE 行)
|
||
"""
|
||
result: Dict[str, Any] = {
|
||
"content": "",
|
||
"reasoning_content": "",
|
||
"images": [],
|
||
"done": False,
|
||
"error": None,
|
||
}
|
||
|
||
if isinstance(raw_response, (bytes, bytearray)):
|
||
raw_response = raw_response.decode("utf-8", errors="ignore")
|
||
|
||
if not raw_response or not isinstance(raw_response, str):
|
||
return result
|
||
|
||
debug_raw_signature = self._debug_body_signature(raw_response)
|
||
|
||
try:
|
||
# P0-5:增量解析。网络监听每次传入的是“不断增长的完整响应体”,旧实现
|
||
# 每次都整段 split + json.loads,一个 160KB 的回答流累计要花数秒 CPU。
|
||
# 这里把“已以换行结尾的完整行”的解析结果缓存下来,只解析新增部分;
|
||
# 末尾未结束的半行每次重新解析、不入缓存。响应体不是上次的前缀扩展
|
||
# (换了一条流)时回退为整段解析。
|
||
scan, images = self._scan_response(raw_response)
|
||
done = scan.done
|
||
if scan.error:
|
||
result["error"] = scan.error
|
||
new_content = scan.content
|
||
new_reasoning = scan.reasoning
|
||
|
||
if new_content:
|
||
delta, next_accumulated = self._content_delta(self._accumulated, new_content)
|
||
if delta:
|
||
result["content"] = delta
|
||
self._accumulated = next_accumulated
|
||
|
||
if new_reasoning:
|
||
delta_r, next_acc_r = self._content_delta(self._accumulated_reasoning, new_reasoning)
|
||
if delta_r:
|
||
result["reasoning_content"] = delta_r
|
||
self._accumulated_reasoning = next_acc_r
|
||
|
||
if images:
|
||
result["images"] = images
|
||
result["done"] = done
|
||
self._last_debug_summary = {
|
||
"raw_body_len": len(raw_response),
|
||
"line_count": scan.line_count,
|
||
"prefix_counts": dict(scan.prefix_counts),
|
||
"unknown_prefix_count": scan.unknown_prefix_count,
|
||
"non_protocol_line_count": scan.non_protocol_line_count,
|
||
"content_candidate_len": len(new_content),
|
||
"emitted_content_len": len(str(result.get("content") or "")),
|
||
"accumulated_len": len(self._accumulated),
|
||
"reasoning_candidate_len": len(new_reasoning),
|
||
"emitted_reasoning_len": len(str(result.get("reasoning_content") or "")),
|
||
"accumulated_reasoning_len": len(self._accumulated_reasoning),
|
||
"done": bool(done),
|
||
"error": str(result.get("error") or ""),
|
||
"image_count": len(images),
|
||
"parse_errors": scan.parse_errors[-8:],
|
||
"incremental_offset": scan.committed_len if self._INCREMENTAL else 0,
|
||
}
|
||
self._last_debug_raw_signature = debug_raw_signature
|
||
|
||
except Exception as e:
|
||
logger.debug(f"[LmarenaParser] 解析异常: {e}")
|
||
result["error"] = str(e)
|
||
self._inc_state = None
|
||
self._last_debug_summary = {
|
||
"raw_body_len": len(raw_response),
|
||
"exception": str(e),
|
||
}
|
||
self._last_debug_raw_signature = debug_raw_signature
|
||
|
||
return result
|
||
|
||
def reset(self) -> None:
|
||
"""重置状态"""
|
||
self._accumulated = ""
|
||
self._accumulated_reasoning = ""
|
||
self._seen_image_refs.clear()
|
||
self._last_debug_summary = {}
|
||
self._last_debug_raw_signature = ""
|
||
self._inc_state = None
|
||
|
||
def should_abort_on_error(self) -> bool:
|
||
"""当 Arena 返回 ae: 或 a3: 等错误协议行时,立即终止工作流。"""
|
||
return True
|
||
|
||
def should_fallback_to_dom_when_no_visible_content(self) -> bool:
|
||
"""Arena 的心跳流可能因网络波动提前关闭,空结果必须交给 DOM 接管。"""
|
||
return True
|
||
|
||
def export_debug_data(self, raw_response: str = "") -> Dict[str, Any]:
|
||
"""Return a compact parser-side protocol summary for network_parser_debug."""
|
||
summary = dict(self._last_debug_summary or {})
|
||
if raw_response:
|
||
text = self._normalize_debug_text(raw_response)
|
||
raw_summary = self._summarize_protocol_lines(text)
|
||
if self._last_debug_raw_signature == self._debug_body_signature(text):
|
||
summary.update(raw_summary)
|
||
else:
|
||
summary = raw_summary
|
||
summary.update({
|
||
"accumulated_len": len(self._accumulated),
|
||
"seen_image_refs": len(self._seen_image_refs),
|
||
})
|
||
return summary
|
||
|
||
# ============ 内部方法 ============
|
||
|
||
@staticmethod
|
||
def _normalize_debug_text(raw_response: Any) -> str:
|
||
if isinstance(raw_response, (bytes, bytearray)):
|
||
return raw_response.decode("utf-8", errors="ignore")
|
||
return str(raw_response or "")
|
||
|
||
@classmethod
|
||
def _debug_body_signature(cls, raw_response: Any) -> str:
|
||
text = cls._normalize_debug_text(raw_response)
|
||
return f"{len(text)}:{text[:80]}:{text[-80:]}"
|
||
|
||
@classmethod
|
||
def _summarize_protocol_lines(cls, raw_response: Any) -> Dict[str, Any]:
|
||
text = cls._normalize_debug_text(raw_response)
|
||
prefix_counts: Dict[str, int] = {}
|
||
line_count = 0
|
||
unknown_prefix_count = 0
|
||
non_protocol_line_count = 0
|
||
for raw_line in text.split("\n"):
|
||
line = raw_line.strip()
|
||
if not line:
|
||
continue
|
||
line_count += 1
|
||
colon_idx = line.find(":")
|
||
if colon_idx < 1:
|
||
non_protocol_line_count += 1
|
||
continue
|
||
prefix = line[:colon_idx]
|
||
if prefix in cls._PROTOCOL_PREFIXES:
|
||
prefix_counts[prefix] = prefix_counts.get(prefix, 0) + 1
|
||
else:
|
||
unknown_prefix_count += 1
|
||
return {
|
||
"raw_body_len": len(text),
|
||
"line_count": line_count,
|
||
"prefix_counts": prefix_counts,
|
||
"unknown_prefix_count": unknown_prefix_count,
|
||
"non_protocol_line_count": non_protocol_line_count,
|
||
}
|
||
|
||
@staticmethod
|
||
def _fix_mojibake(text: str) -> str:
|
||
"""
|
||
修复双重 UTF-8 编码(mojibake)
|
||
|
||
DrissionPage 通过 CDP 获取的响应体存在编码错误:
|
||
UTF-8 字节被混合解码为 cp1252 + latin-1,导致:
|
||
|
||
原始 UTF-8: f0 9f 92 9e (💞)
|
||
被 cp1252 解码: ð(F0) Ÿ(9F→U+0178) '(92→U+2019) ž(9E→U+017E)
|
||
但 0x90 → U+0090(latin-1 直通,因 cp1252 未定义此位置)
|
||
|
||
cp1252 有 5 个未定义位置 (0x81,0x8D,0x8F,0x90,0x9D)
|
||
走 latin-1 直通映射,导致 encode('cp1252') 和 encode('latin-1')
|
||
都无法单独处理全部字符。
|
||
|
||
修复方案(两步):
|
||
1. translate: 将 cp1252 特有字符 (U+2019等) 映射回 latin-1 范围 (U+0092等)
|
||
2. encode('latin-1'): 所有字符现在都在 U+0000-U+00FF,可一一还原原始字节
|
||
3. decode('utf-8'): 原始字节 → 正确文本
|
||
"""
|
||
try:
|
||
mapped = text.translate(_CP1252_TO_LATIN1)
|
||
return mapped.encode("latin-1").decode("utf-8")
|
||
except (UnicodeDecodeError, UnicodeEncodeError):
|
||
# 不是 mojibake(或只有部分是),返回原样
|
||
return text
|
||
|
||
@staticmethod
|
||
def _parse_text_chunk(payload: str) -> str | None:
|
||
"""解析 a0 行的 payload"""
|
||
try:
|
||
value = json.loads(payload)
|
||
if isinstance(value, str):
|
||
return value
|
||
except (json.JSONDecodeError, ValueError):
|
||
pass
|
||
return None
|
||
|
||
@staticmethod
|
||
def _is_finish_signal(payload: str) -> bool:
|
||
"""检查 ad 行是否表示结束"""
|
||
try:
|
||
data = json.loads(payload)
|
||
if isinstance(data, dict) and data.get("finishReason"):
|
||
return True
|
||
except (json.JSONDecodeError, ValueError):
|
||
pass
|
||
return False
|
||
|
||
@staticmethod
|
||
def _parse_error(payload: str) -> str | None:
|
||
"""解析 ae 行的错误信息"""
|
||
try:
|
||
data = json.loads(payload)
|
||
if isinstance(data, dict):
|
||
return data.get("message", str(data))
|
||
return str(data)
|
||
except (json.JSONDecodeError, ValueError):
|
||
return payload.strip() if payload.strip() else None
|
||
|
||
# ============ 元数据 ============
|
||
|
||
@classmethod
|
||
def get_id(cls) -> str:
|
||
return "lmarena"
|
||
|
||
@classmethod
|
||
def get_name(cls) -> str:
|
||
return "Arena.ai"
|
||
|
||
@classmethod
|
||
def get_description(cls) -> str:
|
||
return "解析 Arena.ai 的流式响应 (Vercel AI SDK Data Stream Protocol)"
|
||
|
||
@classmethod
|
||
def get_supported_patterns(cls) -> List[str]:
|
||
return ["nextjs-api/stream/create-evaluation"]
|
||
|
||
|
||
__all__ = [
|
||
"LmarenaParser",
|
||
"_CP1252_TO_LATIN1",
|
||
"_extract_arena_image_items",
|
||
]
|