Initial commit

This commit is contained in:
Alex committed 2026-11-02 22:54:14 +08:00
commit 57fc3525eb
7311 files changed
+1291398

No files matched your search

+564
View File
@@ -0,0 +1,564 @@
"""
lmarena_parser.py - Arena.ai 响应解析器
响应格式:Vercel AI SDK Data Stream Protocol
- 每行格式: {prefix}:{json_data}
- a0: 文本增量(JSON 编码的字符串)
- a2: 心跳/元数据(JSON 数组)
- ad: 结束元数据(含 finishReason)
- ae: 错误信息
编码问题:
DrissionPage 通过 CDP 获取响应体时,UTF-8 字节被混合解码为
cp1252 映射字符(如 0x92→' U+2019)和 latin-1 直通字符
(如 0x90→U+0090),导致双重编码(mojibake)。
需要两步修复:translate 统一到 latin-1 范围 → encode('latin-1') 还原字节
调用方式:
NetworkMonitor 每次调用 parse_chunk 传入一个完整的 HTTP 响应体
(DrissionPage listen.wait 是非流式的,一次性拿到整个 SSE body)
"""
from __future__ import annotations
import json
from datetime import datetime, timezone
from typing import Any, Dict, List
from app.core.config import logger
from .base import ResponseParser
# ─────────────────────────────────────────────────────
# cp1252 → latin-1 逆映射表(模块级常量,只构建一次)
# ─────────────────────────────────────────────────────
# cp1252 在 0x80-0x9F 区间将部分字节映射到 U+0100 以上的 Unicode 字符,
# 而 latin-1 将相同字节直通映射到 U+0080-U+009F。
# 此表将 cp1252 的特殊映射还原为 latin-1 范围的等价字符,
# 使后续 encode('latin-1') 能正确还原原始字节。
#
# 5 个未定义位置 (0x81, 0x8D, 0x8F, 0x90, 0x9D) 已经是
# latin-1 直通映射 (U+0081 等),无需处理。
_CP1252_TO_LATIN1 = str.maketrans({
'\u20ac': '\x80', # €
'\u201a': '\x82', # ‚
'\u0192': '\x83', # ƒ
'\u201e': '\x84', # „
'\u2026': '\x85', # …
'\u2020': '\x86', # †
'\u2021': '\x87', # ‡
'\u02c6': '\x88', # ˆ
'\u2030': '\x89', # ‰
'\u0160': '\x8a', # Š
'\u2039': '\x8b', # ‹
'\u0152': '\x8c', # Œ
'\u017d': '\x8e', # Ž
'\u2018': '\x91', # '
'\u2019': '\x92', # '
'\u201c': '\x93', # "
'\u201d': '\x94', # "
'\u2022': '\x95', # •
'\u2013': '\x96', # –
'\u2014': '\x97', # —
'\u02dc': '\x98', # ˜
'\u2122': '\x99', # ™
'\u0161': '\x9a', # š
'\u203a': '\x9b', # ›
'\u0153': '\x9c', # œ
'\u017e': '\x9e', # ž
'\u0178': '\x9f', # Ÿ
})
def _looks_like_image_ref(value: str) -> bool:
text = str(value or "").strip()
if not text:
return False
lowered = text.lower()
if lowered.startswith(("http://", "https://", "data:image/", "blob:")):
return True
return any(token in lowered for token in (".png", ".jpg", ".jpeg", ".webp", ".gif", ".bmp", ".svg", ".avif"))
def _extract_arena_image_items(
payload: str,
seen_refs: set[str],
*,
source: str,
) -> List[Dict[str, Any]]:
items: List[Dict[str, Any]] = []
try:
data = json.loads(payload)
except (json.JSONDecodeError, ValueError):
return items
entries = data if isinstance(data, list) else [data]
for entry in entries:
if not isinstance(entry, dict):
continue
entry_type = str(entry.get("type") or "").strip().lower()
if entry_type and entry_type != "image":
continue
image_ref = ""
for key in ("image", "url", "src"):
candidate = str(entry.get(key) or "").strip()
if _looks_like_image_ref(candidate):
image_ref = candidate
break
if not image_ref:
continue
dedupe_key = f"image:{image_ref}"
if dedupe_key in seen_refs:
continue
seen_refs.add(dedupe_key)
mime = str(entry.get("mimeType") or entry.get("mime") or "").strip() or None
kind = "data_uri" if image_ref.startswith("data:image/") else "url"
items.append(
{
"media_type": "image",
"kind": kind,
"url": image_ref if kind == "url" else None,
"data_uri": image_ref if kind == "data_uri" else None,
"mime": mime,
"byte_size": None,
"alt": str(entry.get("alt") or "").strip(),
"width": entry.get("width"),
"height": entry.get("height"),
"detected_at": datetime.now(timezone.utc).isoformat().replace("+00:00", "Z"),
"source": source,
}
)
return items
class _LmarenaScanState:
"""LmarenaParser 增量扫描的累计状态(只覆盖已提交的完整行 + 本次半行)。"""
__slots__ = (
"content", "reasoning", "done", "error", "line_count", "prefix_counts",
"unknown_prefix_count", "non_protocol_line_count", "parse_errors",
"committed_len", "committed_raw",
)
def __init__(self) -> None:
self.content = ""
self.reasoning = ""
self.done = False
self.error = None
self.line_count = 0
self.prefix_counts: Dict[str, int] = {}
self.unknown_prefix_count = 0
self.non_protocol_line_count = 0
self.parse_errors: list[Dict[str, Any]] = []
self.committed_len = 0
self.committed_raw = ""
def copy(self) -> "_LmarenaScanState":
other = _LmarenaScanState()
for name in self.__slots__:
setattr(other, name, getattr(self, name))
other.prefix_counts = dict(self.prefix_counts)
other.parse_errors = list(self.parse_errors)
return other
def add_parse_error(self, item: Dict[str, Any]) -> None:
self.parse_errors.append(item)
if len(self.parse_errors) > 16:
del self.parse_errors[:-8]
class LmarenaParser(ResponseParser):
"""
Arena.ai 响应解析器
URL 特征: /nextjs-api/stream/create-evaluation
响应格式: Vercel AI SDK Data Stream Protocol (行分隔)
"""
_PROTOCOL_PREFIXES = {"a0", "a2", "ad", "ae", "a3", "ag", "bg"}
def __init__(self) -> None:
self._accumulated = ""
self._accumulated_reasoning = ""
self._seen_image_refs: set[str] = set()
self._last_debug_summary: Dict[str, Any] = {}
self._last_debug_raw_signature = ""
self._inc_state: "_LmarenaScanState | None" = None
@staticmethod
def _content_delta(
accumulated: str,
candidate: str,
*,
append_disjoint: bool = False,
) -> tuple[str, str]:
"""Return unseen text and the new accumulated value for repeated SSE bodies."""
if not candidate:
return "", accumulated
if not accumulated:
return candidate, candidate
if candidate == accumulated or accumulated.startswith(candidate):
return "", accumulated
if candidate.startswith(accumulated):
return candidate[len(accumulated):], candidate
max_overlap = min(len(accumulated), len(candidate))
for overlap in range(max_overlap, 0, -1):
if accumulated.endswith(candidate[:overlap]):
delta = candidate[overlap:]
return delta, accumulated + delta
if append_disjoint:
return candidate, accumulated + candidate
# A parser instance belongs to one response stream. A body that neither
# extends nor overlaps the emitted text is a stale/non-monotonic replay;
# emitting it wholesale would duplicate the response already sent.
return "", accumulated
# ============ 增量扫描(P0-5) ============
# 置为 False 时每次整段解析(与旧实现逐字节一致),用于回归对照
_INCREMENTAL = True
def _scan_response(self, raw_response: str) -> tuple["_LmarenaScanState", list]:
"""返回 (扫描状态, 本次新发现的图片)。"""
images: list[Dict[str, Any]] = []
if not self._INCREMENTAL:
# 旧行为:整段修复 mojibake 后逐行解析
scan = _LmarenaScanState()
self._scan_lines(self._fix_mojibake(raw_response), scan, images)
return scan, images
cached = getattr(self, "_inc_state", None)
if cached is not None and cached.committed_len <= len(raw_response) and raw_response.startswith(cached.committed_raw):
scan = cached.copy()
offset = cached.committed_len
else:
scan = _LmarenaScanState()
offset = 0
last_newline = raw_response.rfind("\n", offset)
if last_newline >= offset:
committed_end = last_newline + 1
# 完整行一定以 ASCII 换行结尾,不会截断多字节序列,可以按段修复 mojibake
self._scan_lines(self._fix_mojibake(raw_response[offset:committed_end]), scan, images)
scan.committed_len = committed_end
scan.committed_raw = raw_response[:committed_end]
self._inc_state = scan.copy()
offset = committed_end
elif cached is None or scan.committed_len == 0:
self._inc_state = None
tail = raw_response[offset:]
if tail:
self._scan_lines(self._fix_mojibake(tail), scan, images)
return scan, images
def _scan_lines(self, text: str, scan: "_LmarenaScanState", images: list) -> None:
content_parts: list[str] = []
reasoning_parts: list[str] = []
for line in text.split("\n"):
line = line.strip()
if not line:
continue
scan.line_count += 1
colon_idx = line.find(":")
if colon_idx < 1:
scan.non_protocol_line_count += 1
continue
prefix = line[:colon_idx]
payload = line[colon_idx + 1:]
if prefix in self._PROTOCOL_PREFIXES:
scan.prefix_counts[prefix] = scan.prefix_counts.get(prefix, 0) + 1
else:
scan.unknown_prefix_count += 1
if prefix == "a0":
text_chunk = self._parse_text_chunk(payload)
if text_chunk is not None:
content_parts.append(text_chunk)
else:
scan.add_parse_error({"prefix": prefix, "payload_preview": payload[:160]})
elif prefix in {"ag", "bg"}:
text_chunk = self._parse_text_chunk(payload)
if text_chunk is not None:
reasoning_parts.append(text_chunk)
else:
scan.add_parse_error({"prefix": prefix, "payload_preview": payload[:160]})
elif prefix == "a2":
images.extend(
_extract_arena_image_items(
payload,
self._seen_image_refs,
source="lmarena_stream",
)
)
elif prefix == "ad":
if self._is_finish_signal(payload):
scan.done = True
elif prefix in {"ae", "a3"}:
error_msg = self._parse_error(payload)
if error_msg:
scan.error = error_msg
scan.done = True
if content_parts:
scan.content += "".join(content_parts)
if reasoning_parts:
scan.reasoning += "".join(reasoning_parts)
# ============ 对外接口 ============
def parse_chunk(self, raw_response: str) -> Dict[str, Any]:
"""
解析完整的 HTTP 响应体(一次性包含所有 SSE 行)
"""
result: Dict[str, Any] = {
"content": "",
"reasoning_content": "",
"images": [],
"done": False,
"error": None,
}
if isinstance(raw_response, (bytes, bytearray)):
raw_response = raw_response.decode("utf-8", errors="ignore")
if not raw_response or not isinstance(raw_response, str):
return result
debug_raw_signature = self._debug_body_signature(raw_response)
try:
# P0-5:增量解析。网络监听每次传入的是“不断增长的完整响应体”,旧实现
# 每次都整段 split + json.loads,一个 160KB 的回答流累计要花数秒 CPU。
# 这里把“已以换行结尾的完整行”的解析结果缓存下来,只解析新增部分;
# 末尾未结束的半行每次重新解析、不入缓存。响应体不是上次的前缀扩展
# (换了一条流)时回退为整段解析。
scan, images = self._scan_response(raw_response)
done = scan.done
if scan.error:
result["error"] = scan.error
new_content = scan.content
new_reasoning = scan.reasoning
if new_content:
delta, next_accumulated = self._content_delta(self._accumulated, new_content)
if delta:
result["content"] = delta
self._accumulated = next_accumulated
if new_reasoning:
delta_r, next_acc_r = self._content_delta(self._accumulated_reasoning, new_reasoning)
if delta_r:
result["reasoning_content"] = delta_r
self._accumulated_reasoning = next_acc_r
if images:
result["images"] = images
result["done"] = done
self._last_debug_summary = {
"raw_body_len": len(raw_response),
"line_count": scan.line_count,
"prefix_counts": dict(scan.prefix_counts),
"unknown_prefix_count": scan.unknown_prefix_count,
"non_protocol_line_count": scan.non_protocol_line_count,
"content_candidate_len": len(new_content),
"emitted_content_len": len(str(result.get("content") or "")),
"accumulated_len": len(self._accumulated),
"reasoning_candidate_len": len(new_reasoning),
"emitted_reasoning_len": len(str(result.get("reasoning_content") or "")),
"accumulated_reasoning_len": len(self._accumulated_reasoning),
"done": bool(done),
"error": str(result.get("error") or ""),
"image_count": len(images),
"parse_errors": scan.parse_errors[-8:],
"incremental_offset": scan.committed_len if self._INCREMENTAL else 0,
}
self._last_debug_raw_signature = debug_raw_signature
except Exception as e:
logger.debug(f"[LmarenaParser] 解析异常: {e}")
result["error"] = str(e)
self._inc_state = None
self._last_debug_summary = {
"raw_body_len": len(raw_response),
"exception": str(e),
}
self._last_debug_raw_signature = debug_raw_signature
return result
def reset(self) -> None:
"""重置状态"""
self._accumulated = ""
self._accumulated_reasoning = ""
self._seen_image_refs.clear()
self._last_debug_summary = {}
self._last_debug_raw_signature = ""
self._inc_state = None
def should_abort_on_error(self) -> bool:
"""当 Arena 返回 ae: 或 a3: 等错误协议行时,立即终止工作流。"""
return True
def should_fallback_to_dom_when_no_visible_content(self) -> bool:
"""Arena 的心跳流可能因网络波动提前关闭,空结果必须交给 DOM 接管。"""
return True
def export_debug_data(self, raw_response: str = "") -> Dict[str, Any]:
"""Return a compact parser-side protocol summary for network_parser_debug."""
summary = dict(self._last_debug_summary or {})
if raw_response:
text = self._normalize_debug_text(raw_response)
raw_summary = self._summarize_protocol_lines(text)
if self._last_debug_raw_signature == self._debug_body_signature(text):
summary.update(raw_summary)
else:
summary = raw_summary
summary.update({
"accumulated_len": len(self._accumulated),
"seen_image_refs": len(self._seen_image_refs),
})
return summary
# ============ 内部方法 ============
@staticmethod
def _normalize_debug_text(raw_response: Any) -> str:
if isinstance(raw_response, (bytes, bytearray)):
return raw_response.decode("utf-8", errors="ignore")
return str(raw_response or "")
@classmethod
def _debug_body_signature(cls, raw_response: Any) -> str:
text = cls._normalize_debug_text(raw_response)
return f"{len(text)}:{text[:80]}:{text[-80:]}"
@classmethod
def _summarize_protocol_lines(cls, raw_response: Any) -> Dict[str, Any]:
text = cls._normalize_debug_text(raw_response)
prefix_counts: Dict[str, int] = {}
line_count = 0
unknown_prefix_count = 0
non_protocol_line_count = 0
for raw_line in text.split("\n"):
line = raw_line.strip()
if not line:
continue
line_count += 1
colon_idx = line.find(":")
if colon_idx < 1:
non_protocol_line_count += 1
continue
prefix = line[:colon_idx]
if prefix in cls._PROTOCOL_PREFIXES:
prefix_counts[prefix] = prefix_counts.get(prefix, 0) + 1
else:
unknown_prefix_count += 1
return {
"raw_body_len": len(text),
"line_count": line_count,
"prefix_counts": prefix_counts,
"unknown_prefix_count": unknown_prefix_count,
"non_protocol_line_count": non_protocol_line_count,
}
@staticmethod
def _fix_mojibake(text: str) -> str:
"""
修复双重 UTF-8 编码(mojibake)
DrissionPage 通过 CDP 获取的响应体存在编码错误:
UTF-8 字节被混合解码为 cp1252 + latin-1,导致:
原始 UTF-8: f0 9f 92 9e (💞)
被 cp1252 解码: ð(F0) Ÿ(9F→U+0178) '(92→U+2019) ž(9E→U+017E)
但 0x90 → U+0090(latin-1 直通,因 cp1252 未定义此位置)
cp1252 有 5 个未定义位置 (0x81,0x8D,0x8F,0x90,0x9D)
走 latin-1 直通映射,导致 encode('cp1252') 和 encode('latin-1')
都无法单独处理全部字符。
修复方案(两步):
1. translate: 将 cp1252 特有字符 (U+2019等) 映射回 latin-1 范围 (U+0092等)
2. encode('latin-1'): 所有字符现在都在 U+0000-U+00FF,可一一还原原始字节
3. decode('utf-8'): 原始字节 → 正确文本
"""
try:
mapped = text.translate(_CP1252_TO_LATIN1)
return mapped.encode("latin-1").decode("utf-8")
except (UnicodeDecodeError, UnicodeEncodeError):
# 不是 mojibake(或只有部分是),返回原样
return text
@staticmethod
def _parse_text_chunk(payload: str) -> str | None:
"""解析 a0 行的 payload"""
try:
value = json.loads(payload)
if isinstance(value, str):
return value
except (json.JSONDecodeError, ValueError):
pass
return None
@staticmethod
def _is_finish_signal(payload: str) -> bool:
"""检查 ad 行是否表示结束"""
try:
data = json.loads(payload)
if isinstance(data, dict) and data.get("finishReason"):
return True
except (json.JSONDecodeError, ValueError):
pass
return False
@staticmethod
def _parse_error(payload: str) -> str | None:
"""解析 ae 行的错误信息"""
try:
data = json.loads(payload)
if isinstance(data, dict):
return data.get("message", str(data))
return str(data)
except (json.JSONDecodeError, ValueError):
return payload.strip() if payload.strip() else None
# ============ 元数据 ============
@classmethod
def get_id(cls) -> str:
return "lmarena"
@classmethod
def get_name(cls) -> str:
return "Arena.ai"
@classmethod
def get_description(cls) -> str:
return "解析 Arena.ai 的流式响应 (Vercel AI SDK Data Stream Protocol)"
@classmethod
def get_supported_patterns(cls) -> List[str]:
return ["nextjs-api/stream/create-evaluation"]
__all__ = [
"LmarenaParser",
"_CP1252_TO_LATIN1",
"_extract_arena_image_items",
]