Initial commit
This commit is contained in:
commit
57fc3525eb
7311 files changed
+1291398
No files matched your search
@@ -0,0 +1,74 @@
|
||||
"""
|
||||
app/core/parsers - 网络响应解析器
|
||||
|
||||
职责:
|
||||
- 定义响应解析的标准接口
|
||||
- 提供注册机制支持多站点适配
|
||||
- 解析增量响应数据
|
||||
"""
|
||||
|
||||
from .base import ResponseParser
|
||||
from .registry import ParserRegistry
|
||||
from .gemini_parser import GeminiParser
|
||||
from .chatgpt_parser import ChatGPTParser
|
||||
from .deepseek_parser import DeepSeekParser
|
||||
from .aistudio_parser import AIStudioParser
|
||||
from .doubao_parser import DoubaoParser
|
||||
from .claude_parser import ClaudeParser
|
||||
from .kimi_parser import KimiParser
|
||||
from .glm_parser import GLMParser
|
||||
from .qwen_parser import QwenParser
|
||||
from .mimo_parser import MimoParser
|
||||
from .lmarena_parser import LmarenaParser
|
||||
from .lmarena_side_left_parser import LmarenaSideLeftParser
|
||||
from .lmarena_battle_side_parser import (
|
||||
LmarenaBattleSideLeftParser,
|
||||
LmarenaBattleSideRightParser,
|
||||
LmarenaBattleWinnerParser,
|
||||
)
|
||||
from .lmarena_image_side_left_parser import LmarenaImageSideLeftParser
|
||||
from .lmarena_image_side_right_parser import LmarenaImageSideRightParser
|
||||
from .grok_parser import GrokParser
|
||||
|
||||
# 自动注册内置解析器
|
||||
ParserRegistry.register_class(GeminiParser)
|
||||
ParserRegistry.register_class(ChatGPTParser)
|
||||
ParserRegistry.register_class(DeepSeekParser)
|
||||
ParserRegistry.register_class(AIStudioParser)
|
||||
ParserRegistry.register_class(DoubaoParser)
|
||||
ParserRegistry.register_class(ClaudeParser)
|
||||
ParserRegistry.register_class(KimiParser)
|
||||
ParserRegistry.register_class(GLMParser)
|
||||
ParserRegistry.register_class(QwenParser)
|
||||
ParserRegistry.register_class(MimoParser)
|
||||
ParserRegistry.register_class(LmarenaParser)
|
||||
ParserRegistry.register_class(LmarenaSideLeftParser)
|
||||
ParserRegistry.register_class(LmarenaBattleWinnerParser)
|
||||
ParserRegistry.register_class(LmarenaBattleSideLeftParser)
|
||||
ParserRegistry.register_class(LmarenaBattleSideRightParser)
|
||||
ParserRegistry.register_class(LmarenaImageSideLeftParser)
|
||||
ParserRegistry.register_class(LmarenaImageSideRightParser)
|
||||
ParserRegistry.register_class(GrokParser)
|
||||
|
||||
__all__ = [
|
||||
'ResponseParser',
|
||||
'ParserRegistry',
|
||||
'GeminiParser',
|
||||
'ChatGPTParser',
|
||||
'DeepSeekParser',
|
||||
'AIStudioParser',
|
||||
'DoubaoParser',
|
||||
'ClaudeParser',
|
||||
'KimiParser',
|
||||
'GLMParser',
|
||||
'QwenParser',
|
||||
'MimoParser',
|
||||
'LmarenaParser',
|
||||
'LmarenaSideLeftParser',
|
||||
'LmarenaBattleWinnerParser',
|
||||
'LmarenaBattleSideLeftParser',
|
||||
'LmarenaBattleSideRightParser',
|
||||
'LmarenaImageSideLeftParser',
|
||||
'LmarenaImageSideRightParser',
|
||||
'GrokParser',
|
||||
]
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -0,0 +1,514 @@
|
||||
"""
|
||||
aistudio_parser.py - Google AI Studio 响应解析器
|
||||
|
||||
响应格式特征:
|
||||
- Content-Type: application/json+protobuf
|
||||
- 嵌套数组结构
|
||||
- 增量文本模式
|
||||
- 包含 thinking 内容(需过滤)
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
from typing import Any, Dict, List, Tuple
|
||||
|
||||
from app.core.config import logger
|
||||
from .base import ResponseParser
|
||||
|
||||
|
||||
class AIStudioParser(ResponseParser):
|
||||
"""
|
||||
Google AI Studio (MakerSuite) 响应解析器
|
||||
|
||||
URL 特征: MakerSuiteService/GenerateContent
|
||||
响应格式: JSON+Protobuf (嵌套数组)
|
||||
"""
|
||||
|
||||
_VISIBLE_TEXT_BLOCK_RE = re.compile(
|
||||
r'null,"((?:\\.|[^"\\])*)"\]\],"model"'
|
||||
)
|
||||
_VISIBLE_TEXT_BLOCK_PREFIX = 'null,"'
|
||||
_OBFUSCATED_PAYLOAD_RE = re.compile(r"^[A-Za-z0-9+/=_-]+$")
|
||||
_CJK_RE = re.compile(r"[\u3400-\u9fff]")
|
||||
|
||||
def __init__(self):
|
||||
self._accumulated_content = ""
|
||||
self._accumulated_reasoning = ""
|
||||
self._is_done = False
|
||||
self._last_raw_length = 0
|
||||
self._pending_raw = ""
|
||||
self._visible_text_scan_pending = ""
|
||||
self._visible_text_fallback_content = ""
|
||||
|
||||
def parse_chunk(self, raw_response: str) -> Dict[str, Any]:
|
||||
"""
|
||||
解析响应(返回完整内容的增量)
|
||||
"""
|
||||
result = {
|
||||
"content": "",
|
||||
"reasoning_content": "",
|
||||
"images": [],
|
||||
"done": False,
|
||||
"error": None
|
||||
}
|
||||
|
||||
try:
|
||||
if isinstance(raw_response, bytes):
|
||||
raw_response = raw_response.decode("utf-8", errors="ignore")
|
||||
elif not isinstance(raw_response, str):
|
||||
raw_response = str(raw_response)
|
||||
|
||||
new_data = self._prepare_incremental_raw_response(raw_response)
|
||||
if not new_data:
|
||||
return result
|
||||
|
||||
payloads = self._decode_payloads(new_data, incremental=True)
|
||||
content = ""
|
||||
is_done = False
|
||||
|
||||
reasoning = ""
|
||||
for data in payloads:
|
||||
current_content, current_reasoning, current_done = self._extract_content(data)
|
||||
if current_content:
|
||||
content = current_content
|
||||
if current_reasoning:
|
||||
reasoning = current_reasoning
|
||||
if current_done:
|
||||
is_done = True
|
||||
|
||||
if not content:
|
||||
content = self._extract_visible_text_from_increment(new_data)
|
||||
|
||||
if content:
|
||||
if content.startswith(self._accumulated_content):
|
||||
delta = content[len(self._accumulated_content):]
|
||||
self._accumulated_content = content
|
||||
else:
|
||||
delta = content
|
||||
self._accumulated_content += content
|
||||
if delta:
|
||||
result["content"] = delta
|
||||
|
||||
if reasoning:
|
||||
if reasoning.startswith(self._accumulated_reasoning):
|
||||
delta_r = reasoning[len(self._accumulated_reasoning):]
|
||||
self._accumulated_reasoning = reasoning
|
||||
else:
|
||||
delta_r = reasoning
|
||||
self._accumulated_reasoning += reasoning
|
||||
if delta_r:
|
||||
result["reasoning_content"] = delta_r
|
||||
|
||||
result["done"] = is_done
|
||||
|
||||
except json.JSONDecodeError as e:
|
||||
logger.debug(f"[AIStudioParser] JSON 解析失败: {e}")
|
||||
result["error"] = str(e)
|
||||
except Exception as e:
|
||||
logger.debug(f"[AIStudioParser] 解析异常: {e}")
|
||||
result["error"] = str(e)
|
||||
|
||||
return result
|
||||
|
||||
def reset(self):
|
||||
"""重置状态"""
|
||||
self._accumulated_content = ""
|
||||
self._accumulated_reasoning = ""
|
||||
self._is_done = False
|
||||
self._last_raw_length = 0
|
||||
self._last_raw_response = ""
|
||||
self._pending_raw = ""
|
||||
self._visible_text_scan_pending = ""
|
||||
self._visible_text_fallback_content = ""
|
||||
|
||||
@staticmethod
|
||||
def _normalize_raw_text(raw_response: Any) -> str:
|
||||
if isinstance(raw_response, bytes):
|
||||
raw_response = raw_response.decode("utf-8", errors="ignore")
|
||||
elif not isinstance(raw_response, str):
|
||||
raw_response = str(raw_response)
|
||||
|
||||
text = raw_response.lstrip("\ufeff")
|
||||
stripped = text.lstrip()
|
||||
if stripped.startswith(")]}'"):
|
||||
stripped = stripped[4:].lstrip("\r\n")
|
||||
return stripped
|
||||
|
||||
def _decode_payloads(self, raw_response: Any, incremental: bool = False) -> List[Any]:
|
||||
if isinstance(raw_response, (list, dict)):
|
||||
return [raw_response]
|
||||
|
||||
normalized = self._normalize_raw_text(raw_response)
|
||||
if not normalized:
|
||||
return []
|
||||
|
||||
if incremental:
|
||||
normalized = f"{self._pending_raw}{normalized}"
|
||||
|
||||
try:
|
||||
payload = json.loads(normalized)
|
||||
if incremental:
|
||||
self._pending_raw = ""
|
||||
return [payload]
|
||||
except json.JSONDecodeError as exc:
|
||||
payloads, remainder = self._decode_json_prefix_values(normalized)
|
||||
if incremental:
|
||||
self._pending_raw = remainder
|
||||
if payloads:
|
||||
return payloads
|
||||
|
||||
repaired = self._try_repair_unclosed_json(normalized)
|
||||
if repaired is not None:
|
||||
return [repaired]
|
||||
|
||||
logger.debug_throttled(
|
||||
f"aistudio.incomplete_json.{id(self)}",
|
||||
"[AIStudioParser] JSON 未完整,继续等待流增长 "
|
||||
f"(body_len={len(normalized)}, error={exc})",
|
||||
interval_sec=3.0,
|
||||
)
|
||||
return []
|
||||
|
||||
@staticmethod
|
||||
def _try_repair_unclosed_json(text: str) -> Any:
|
||||
stripped = text.strip()
|
||||
if not stripped:
|
||||
return None
|
||||
for pad in range(0, 30):
|
||||
try:
|
||||
attempt = stripped + (']' * pad)
|
||||
return json.loads(attempt)
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
attempt = stripped + '"]' + (']' * pad)
|
||||
return json.loads(attempt)
|
||||
except Exception:
|
||||
pass
|
||||
return None
|
||||
|
||||
@classmethod
|
||||
def _extract_visible_text_from_raw_body(cls, raw_response: Any) -> str:
|
||||
raw_text = cls._normalize_raw_text(raw_response)
|
||||
if not raw_text:
|
||||
return ""
|
||||
|
||||
pieces, _last_match_end = cls._scan_visible_text_matches(raw_text)
|
||||
return "".join(pieces)
|
||||
|
||||
@classmethod
|
||||
def _scan_visible_text_matches(cls, raw_text: str) -> Tuple[List[str], int]:
|
||||
pieces: List[str] = []
|
||||
last_match_end = 0
|
||||
for match in cls._VISIBLE_TEXT_BLOCK_RE.finditer(raw_text):
|
||||
last_match_end = match.end()
|
||||
piece = cls._decode_json_fragment(match.group(1))
|
||||
if piece and cls._looks_like_visible_text(piece):
|
||||
pieces.append(piece)
|
||||
|
||||
return pieces, last_match_end
|
||||
|
||||
@classmethod
|
||||
def _build_visible_text_scan_pending(cls, scan_text: str, last_match_end: int) -> str:
|
||||
remainder = scan_text[last_match_end:] if last_match_end > 0 else scan_text
|
||||
marker_index = remainder.rfind(cls._VISIBLE_TEXT_BLOCK_PREFIX)
|
||||
if marker_index >= 0:
|
||||
return remainder[marker_index:]
|
||||
|
||||
keep_chars = max(0, len(cls._VISIBLE_TEXT_BLOCK_PREFIX) - 1)
|
||||
return remainder[-keep_chars:] if keep_chars else ""
|
||||
|
||||
def _extract_visible_text_from_increment(self, new_data: str) -> str:
|
||||
if not new_data and not self._visible_text_scan_pending:
|
||||
return self._visible_text_fallback_content
|
||||
|
||||
normalized_new_data = self._normalize_raw_text(new_data)
|
||||
scan_text = f"{self._visible_text_scan_pending}{normalized_new_data}"
|
||||
if not scan_text:
|
||||
return self._visible_text_fallback_content
|
||||
|
||||
pieces, last_match_end = self._scan_visible_text_matches(scan_text)
|
||||
if pieces:
|
||||
self._visible_text_fallback_content += "".join(pieces)
|
||||
self._visible_text_scan_pending = self._build_visible_text_scan_pending(
|
||||
scan_text,
|
||||
last_match_end,
|
||||
)
|
||||
return self._visible_text_fallback_content
|
||||
|
||||
@classmethod
|
||||
def _looks_like_obfuscated_payload(cls, value: Any) -> bool:
|
||||
text = str(value or "").strip()
|
||||
if len(text) < 96:
|
||||
return False
|
||||
if any(ch.isspace() for ch in text):
|
||||
return False
|
||||
if cls._CJK_RE.search(text):
|
||||
return False
|
||||
if not cls._OBFUSCATED_PAYLOAD_RE.fullmatch(text):
|
||||
return False
|
||||
if not any(ch in "+/=_-" for ch in text):
|
||||
return False
|
||||
|
||||
distinct_chars = len(set(text))
|
||||
if distinct_chars < 20:
|
||||
return False
|
||||
|
||||
alpha_num_ratio = sum(ch.isalnum() for ch in text) / len(text)
|
||||
return alpha_num_ratio >= 0.85
|
||||
|
||||
@classmethod
|
||||
def _looks_like_visible_text(cls, value: Any) -> bool:
|
||||
text = str(value or "")
|
||||
stripped = text.strip()
|
||||
if not stripped:
|
||||
return False
|
||||
if cls._looks_like_obfuscated_payload(stripped):
|
||||
return False
|
||||
if cls._CJK_RE.search(stripped):
|
||||
return True
|
||||
if any(ch.isspace() for ch in stripped):
|
||||
return True
|
||||
if stripped.startswith(("{", "[", "<", "```", "http://", "https://")):
|
||||
return True
|
||||
|
||||
punctuation_count = sum(
|
||||
ch in '.,!?;:()[]{}<>"\'`~@#$%^&*\\|'
|
||||
for ch in stripped
|
||||
)
|
||||
if punctuation_count > 0:
|
||||
return True
|
||||
|
||||
return len(stripped) <= 80
|
||||
|
||||
@classmethod
|
||||
def _pick_visible_text_candidate(cls, values: List[Any]) -> str:
|
||||
for item in values:
|
||||
if not isinstance(item, str):
|
||||
continue
|
||||
if cls._looks_like_visible_text(item):
|
||||
return item
|
||||
return ""
|
||||
|
||||
@staticmethod
|
||||
def _decode_json_fragment(value: str) -> str:
|
||||
text = str(value or "")
|
||||
if not text:
|
||||
return ""
|
||||
|
||||
try:
|
||||
decoded = json.loads(f'"{text}"')
|
||||
return decoded if isinstance(decoded, str) else ""
|
||||
except json.JSONDecodeError:
|
||||
return (
|
||||
text.replace("\\n", "\n")
|
||||
.replace("\\r", "\r")
|
||||
.replace("\\t", "\t")
|
||||
.replace('\\"', '"')
|
||||
.replace("\\\\", "\\")
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _decode_json_prefix_values(raw_text: str) -> Tuple[List[Any], str]:
|
||||
decoder = json.JSONDecoder()
|
||||
values: List[Any] = []
|
||||
index = 0
|
||||
length = len(raw_text)
|
||||
|
||||
while index < length:
|
||||
while index < length and raw_text[index].isspace():
|
||||
index += 1
|
||||
if index >= length:
|
||||
break
|
||||
|
||||
try:
|
||||
value, next_index = decoder.raw_decode(raw_text, index)
|
||||
except json.JSONDecodeError:
|
||||
return values, raw_text[index:]
|
||||
|
||||
values.append(value)
|
||||
index = next_index
|
||||
|
||||
return values, ""
|
||||
|
||||
@staticmethod
|
||||
def _normalize_outer_blocks(data: Any) -> List[Any]:
|
||||
if not isinstance(data, list) or not data:
|
||||
return []
|
||||
|
||||
if any(isinstance(item, list) for item in data):
|
||||
if (
|
||||
len(data) == 1
|
||||
and isinstance(data[0], list)
|
||||
and any(isinstance(item, list) for item in data[0])
|
||||
):
|
||||
return data[0]
|
||||
return data
|
||||
|
||||
return []
|
||||
|
||||
def _extract_content(self, data: Any) -> Tuple[str, str, bool]:
|
||||
"""
|
||||
从响应数据中提取文本内容与思考过程
|
||||
|
||||
Returns:
|
||||
(accumulated_text, accumulated_reasoning, is_done)
|
||||
"""
|
||||
try:
|
||||
outer = self._normalize_outer_blocks(data)
|
||||
if not outer:
|
||||
return "", "", False
|
||||
|
||||
accumulated = ""
|
||||
accumulated_reasoning = ""
|
||||
is_done = False
|
||||
|
||||
for block in outer:
|
||||
if not isinstance(block, list):
|
||||
continue
|
||||
|
||||
# 检查是否是统计块(结束标志之一)
|
||||
if self._is_stats_block(block):
|
||||
is_done = True
|
||||
continue
|
||||
|
||||
# 提取文本
|
||||
text, block_done, is_thinking = self._extract_block_content(block)
|
||||
|
||||
if is_thinking:
|
||||
if text:
|
||||
accumulated_reasoning += text
|
||||
else:
|
||||
if text:
|
||||
accumulated += text
|
||||
|
||||
if block_done:
|
||||
is_done = True
|
||||
|
||||
return accumulated, accumulated_reasoning, is_done
|
||||
|
||||
except Exception as e:
|
||||
logger.debug(f"[AIStudioParser] _extract_content 异常: {e}")
|
||||
return "", "", False
|
||||
|
||||
def _is_stats_block(self, block: list) -> bool:
|
||||
"""检查是否是统计块(响应结束)"""
|
||||
try:
|
||||
# 格式: [null, null, null, [timestamp, ...]]
|
||||
if len(block) >= 4:
|
||||
if block[0] is None and block[1] is None and block[2] is None:
|
||||
if isinstance(block[3], list):
|
||||
return True
|
||||
return False
|
||||
except:
|
||||
return False
|
||||
|
||||
def _extract_block_content(self, block: list) -> tuple:
|
||||
"""
|
||||
从单个块提取内容
|
||||
|
||||
Returns:
|
||||
(text, is_done, is_thinking)
|
||||
"""
|
||||
try:
|
||||
# 路径: block[0][0][0][0][0]
|
||||
# thinking: [13 items, None, "**Thinking...", ..., 1]
|
||||
# 正常内容: [2 items, None, "文本内容"]
|
||||
|
||||
if not isinstance(block, list) or len(block) == 0:
|
||||
return "", False, False
|
||||
|
||||
level1 = block[0]
|
||||
if not isinstance(level1, list) or len(level1) == 0:
|
||||
return "", False, False
|
||||
|
||||
level2 = level1[0]
|
||||
if not isinstance(level2, list) or len(level2) == 0:
|
||||
return "", False, False
|
||||
|
||||
level3 = level2[0]
|
||||
if not isinstance(level3, list) or len(level3) == 0:
|
||||
return "", False, False
|
||||
|
||||
level4 = level3[0]
|
||||
if not isinstance(level4, list) or len(level4) == 0:
|
||||
return "", False, False
|
||||
|
||||
# 这里就是 content_arr
|
||||
content_arr = level4[0]
|
||||
if not isinstance(content_arr, list):
|
||||
return "", False, False
|
||||
|
||||
# 提取文本:只接受像“可见正文”的字符串,避免把内部高熵 payload 误当回复。
|
||||
text = ""
|
||||
primary_text = content_arr[1] if len(content_arr) > 1 else ""
|
||||
if self._looks_like_visible_text(primary_text):
|
||||
text = primary_text
|
||||
else:
|
||||
text = self._pick_visible_text_candidate(content_arr[2:])
|
||||
if not text:
|
||||
suppressed_payload = next(
|
||||
(
|
||||
item for item in content_arr[2:]
|
||||
if isinstance(item, str)
|
||||
and self._looks_like_obfuscated_payload(item)
|
||||
),
|
||||
"",
|
||||
)
|
||||
if suppressed_payload:
|
||||
logger.debug_throttled(
|
||||
f"aistudio.hidden_payload.{id(self)}",
|
||||
"[AIStudioParser] 忽略疑似内部高熵 payload "
|
||||
f"(len={len(str(suppressed_payload).strip())})",
|
||||
interval_sec=3.0,
|
||||
)
|
||||
|
||||
# 判断是否是 thinking
|
||||
# thinking 块特征:len >= 13 且索引 12 == 1
|
||||
is_thinking = False
|
||||
if len(content_arr) >= 13:
|
||||
if len(content_arr) > 12 and content_arr[12] == 1:
|
||||
is_thinking = True
|
||||
|
||||
# 检查是否结束
|
||||
# 1. level2 的第二个元素是 1
|
||||
is_done = False
|
||||
if len(level2) > 1 and level2[1] == 1:
|
||||
is_done = True
|
||||
|
||||
# 2. 有长 token 字符串(索引 13 或 14)
|
||||
if len(content_arr) > 13 and isinstance(content_arr[13], str):
|
||||
if len(content_arr[13]) > 50:
|
||||
is_done = True
|
||||
|
||||
return text, is_done, is_thinking
|
||||
|
||||
except Exception as e:
|
||||
logger.debug(f"[AIStudioParser] _extract_block_content 异常: {e}")
|
||||
return "", False, False
|
||||
|
||||
# ============ 元数据 ============
|
||||
|
||||
@classmethod
|
||||
def get_id(cls) -> str:
|
||||
return "aistudio"
|
||||
|
||||
@classmethod
|
||||
def get_name(cls) -> str:
|
||||
return "Google AI Studio"
|
||||
|
||||
@classmethod
|
||||
def get_description(cls) -> str:
|
||||
return "解析 Google AI Studio (MakerSuite) 的 GenerateContent 响应"
|
||||
|
||||
@classmethod
|
||||
def get_supported_patterns(cls) -> List[str]:
|
||||
return ["MakerSuiteService/GenerateContent", "GenerateContent"]
|
||||
|
||||
def should_fallback_to_dom_when_no_visible_content(self) -> bool:
|
||||
return True
|
||||
|
||||
|
||||
__all__ = ['AIStudioParser']
|
||||
@@ -0,0 +1,180 @@
|
||||
"""
|
||||
app/core/parsers/base.py - 响应解析器基类
|
||||
|
||||
定义网络响应解析的标准接口(支持增量响应)
|
||||
"""
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import Dict, List, Optional, Any
|
||||
|
||||
|
||||
class ResponseParser(ABC):
|
||||
"""
|
||||
响应解析器抽象基类
|
||||
|
||||
职责:
|
||||
- 解析网络拦截到的原始响应体
|
||||
- 支持增量响应(流式传输)
|
||||
- 提取文本内容和图片
|
||||
"""
|
||||
|
||||
# ============ 抽象方法(子类必须实现)============
|
||||
|
||||
@abstractmethod
|
||||
def parse_chunk(self, raw_response: str) -> Dict[str, Any]:
|
||||
"""
|
||||
解析单次响应数据(增量)
|
||||
|
||||
Args:
|
||||
raw_response: 原始响应体(字符串)
|
||||
|
||||
Returns:
|
||||
{
|
||||
"content": str, # 本次增量的文本内容
|
||||
"images": List[Dict], # 本次增量的图片(可选)
|
||||
"done": bool, # 是否为最后一块数据
|
||||
"error": Optional[str] # 解析错误信息
|
||||
}
|
||||
|
||||
注意:
|
||||
- content 为空字符串表示本次无新增文本
|
||||
- images 为空列表表示本次无新增图片
|
||||
- done=True 表示流式传输结束
|
||||
- error 不为 None 表示解析失败
|
||||
"""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def reset(self):
|
||||
"""
|
||||
重置解析器状态
|
||||
|
||||
用于新一轮对话开始时清空累积状态
|
||||
"""
|
||||
pass
|
||||
|
||||
def _prepare_incremental_raw_response(self, raw_response: str) -> str:
|
||||
"""
|
||||
Return the append-only delta for a full raw response snapshot.
|
||||
|
||||
Network listeners usually pass a growing full body to parsers. When the
|
||||
browser replaces that snapshot with another response of the same or
|
||||
greater length, slicing by length alone loses content. This helper
|
||||
preserves the fast append path and resets parser state when the new
|
||||
snapshot is not a prefix extension of the previous one.
|
||||
"""
|
||||
if not isinstance(raw_response, str):
|
||||
raw_response = str(raw_response)
|
||||
|
||||
previous = getattr(self, "_last_raw_response", "")
|
||||
try:
|
||||
previous_len = int(getattr(self, "_last_raw_length", 0) or 0)
|
||||
except Exception:
|
||||
previous_len = 0
|
||||
|
||||
if previous_len > 0 and previous:
|
||||
if raw_response == previous:
|
||||
return ""
|
||||
if raw_response.startswith(previous):
|
||||
new_data = raw_response[len(previous):]
|
||||
else:
|
||||
self.reset()
|
||||
new_data = raw_response
|
||||
else:
|
||||
new_data = raw_response
|
||||
|
||||
self._last_raw_length = len(raw_response)
|
||||
self._last_raw_response = raw_response
|
||||
return new_data
|
||||
|
||||
# ============ 可选覆盖方法 ============
|
||||
|
||||
def validate_response(self, raw_response: str) -> bool:
|
||||
"""
|
||||
验证响应格式是否匹配此解析器
|
||||
|
||||
Args:
|
||||
raw_response: 原始响应体
|
||||
|
||||
Returns:
|
||||
True 表示格式匹配,False 表示不匹配
|
||||
"""
|
||||
try:
|
||||
result = self.parse_chunk(raw_response)
|
||||
return result.get("error") is None
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
def should_abort_on_error(self) -> bool:
|
||||
"""
|
||||
当 parse_chunk 返回 error 时,是否应立即终止当前工作流。
|
||||
|
||||
默认仅记录并继续,交给具体解析器按需升级为硬失败。
|
||||
"""
|
||||
return False
|
||||
|
||||
def should_fallback_to_dom_when_no_visible_content(self) -> bool:
|
||||
"""
|
||||
网络流存在但长期拿不到可见正文时,是否优先回退 DOM 监听。
|
||||
|
||||
适用于网络协议里混入推理/中间态,而页面真实正文由前端二次拼装的站点。
|
||||
"""
|
||||
return False
|
||||
|
||||
def should_require_explicit_done(self) -> bool:
|
||||
"""Whether a stream must expose its protocol-level completion event."""
|
||||
return False
|
||||
|
||||
def should_wait_for_replacement_stream_on_incomplete_capture(self) -> bool:
|
||||
"""Whether an incomplete captured response may be followed by another stream."""
|
||||
return False
|
||||
|
||||
def get_media_generation_state(
|
||||
self,
|
||||
raw_response: str = "",
|
||||
parse_result: Optional[Dict[str, Any]] = None,
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
提取通用的媒体生成状态。
|
||||
|
||||
返回约定:
|
||||
{
|
||||
"pending": bool,
|
||||
"media_type": "image" | "audio" | "video" | "",
|
||||
"hint_text": str,
|
||||
"wait_timeout_seconds": float | int | None,
|
||||
}
|
||||
|
||||
通用工作流层只消费这些字段,不关心站点私有协议细节。
|
||||
"""
|
||||
return {}
|
||||
|
||||
# ============ 元数据接口 ============
|
||||
|
||||
@classmethod
|
||||
def get_id(cls) -> str:
|
||||
"""解析器唯一标识符"""
|
||||
return cls.__name__.lower().replace('parser', '')
|
||||
|
||||
@classmethod
|
||||
def get_name(cls) -> str:
|
||||
"""解析器显示名称"""
|
||||
return cls.__name__
|
||||
|
||||
@classmethod
|
||||
def get_description(cls) -> str:
|
||||
"""解析器描述"""
|
||||
return "未提供描述"
|
||||
|
||||
@classmethod
|
||||
def get_supported_patterns(cls) -> List[str]:
|
||||
"""
|
||||
返回此解析器支持的 URL 匹配模式
|
||||
|
||||
Returns:
|
||||
URL 子串列表(用于 page.listen.start())
|
||||
"""
|
||||
return []
|
||||
|
||||
|
||||
__all__ = ['ResponseParser']
|
||||
@@ -0,0 +1,645 @@
|
||||
"""
|
||||
chatgpt_parser.py - ChatGPT 响应解析器
|
||||
|
||||
响应格式特征:
|
||||
- SSE (Server-Sent Events) 流式响应
|
||||
- 使用 v1 delta 编码
|
||||
- 增量文本通过 event: delta 事件传递
|
||||
- 结束标志: data: [DONE]
|
||||
"""
|
||||
|
||||
import json
|
||||
import re
|
||||
from typing import Dict, Any, List, Optional
|
||||
|
||||
from app.core.config import logger
|
||||
from .base import ResponseParser
|
||||
|
||||
|
||||
class ChatGPTParser(ResponseParser):
|
||||
"""
|
||||
ChatGPT API 响应解析器
|
||||
|
||||
URL 特征: /backend-api/f/conversation
|
||||
响应格式: SSE (text/event-stream) with v1 delta encoding
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
self._accumulated_content = ""
|
||||
self._last_raw_length = 0
|
||||
self._last_raw_response = ""
|
||||
self._rendered_content = ""
|
||||
self._message_parts: Dict[int, str] = {}
|
||||
self._message_parts_joined = ""
|
||||
self._message_parts_dirty = False
|
||||
self._content_references: List[Dict[str, Any]] = []
|
||||
self._content_references_sorted: List[Dict[str, Any]] = []
|
||||
self._content_references_dirty = False
|
||||
self._active_part_index: Optional[int] = None
|
||||
self._pending_sse_chunk = ""
|
||||
self._ref_start = "\ue200"
|
||||
self._ref_end = "\ue201"
|
||||
self._ref_sep = "\ue202"
|
||||
self._media_generation_state: Dict[str, Any] = {}
|
||||
self._inline_ref_pattern = re.compile(
|
||||
rf"{re.escape(self._ref_start)}([^{re.escape(self._ref_sep)}{re.escape(self._ref_end)}]+)"
|
||||
rf"{re.escape(self._ref_sep)}(.*?){re.escape(self._ref_end)}",
|
||||
re.DOTALL,
|
||||
)
|
||||
self._dangling_ref_pattern = re.compile(
|
||||
rf"{re.escape(self._ref_start)}[^{re.escape(self._ref_end)}]*$",
|
||||
re.DOTALL,
|
||||
)
|
||||
|
||||
def parse_chunk(self, raw_response: str) -> Dict[str, Any]:
|
||||
"""
|
||||
解析SSE响应流(返回增量)
|
||||
"""
|
||||
result = {
|
||||
"content": "",
|
||||
"images": [],
|
||||
"done": False,
|
||||
"error": None
|
||||
}
|
||||
|
||||
try:
|
||||
# 确保是字符串
|
||||
if isinstance(raw_response, bytes):
|
||||
raw_response = raw_response.decode('utf-8', errors='ignore')
|
||||
|
||||
if not isinstance(raw_response, str):
|
||||
raw_response = str(raw_response)
|
||||
|
||||
# 只处理新增部分;非前缀新快照会在 helper 内重置解析状态。
|
||||
new_data = self._prepare_incremental_raw_response(raw_response)
|
||||
if not new_data:
|
||||
return result
|
||||
|
||||
# 解析新增的SSE事件
|
||||
delta_content, done = self._parse_sse_chunk(new_data)
|
||||
|
||||
if delta_content:
|
||||
result["content"] = delta_content
|
||||
self._accumulated_content += delta_content
|
||||
|
||||
# 检测结束标志
|
||||
if done:
|
||||
result["done"] = True
|
||||
|
||||
except Exception as e:
|
||||
logger.debug(f"[ChatGPTParser] 解析异常: {e}")
|
||||
result["error"] = str(e)
|
||||
|
||||
return result
|
||||
|
||||
def _parse_sse_chunk(self, chunk: str) -> tuple[str, bool]:
|
||||
"""解析SSE数据块,提取文本增量"""
|
||||
chunk = self._extract_complete_sse_chunk(chunk)
|
||||
if not chunk:
|
||||
return "", False
|
||||
|
||||
previous_content = self._rendered_content
|
||||
done = False
|
||||
|
||||
for current_event, data in self._iter_sse_json_events(chunk):
|
||||
if not isinstance(data, dict):
|
||||
continue
|
||||
self._consume_event(current_event, data)
|
||||
|
||||
for raw_line in str(chunk or "").split('\n'):
|
||||
line = raw_line.strip()
|
||||
if line.startswith("data:") and line[5:].strip() == "[DONE]":
|
||||
done = True
|
||||
break
|
||||
|
||||
current_content = self._render_visible_text()
|
||||
self._rendered_content = current_content
|
||||
return self._compute_delta(previous_content, current_content), done
|
||||
|
||||
def _extract_complete_sse_chunk(self, chunk: str) -> str:
|
||||
"""Buffer partial SSE frames until a blank line terminates the event."""
|
||||
combined = f"{self._pending_sse_chunk}{str(chunk or '')}"
|
||||
if not combined:
|
||||
return ""
|
||||
|
||||
normalized = combined.replace("\r\n", "\n").replace("\r", "\n")
|
||||
parts = normalized.split("\n\n")
|
||||
if normalized.endswith("\n\n"):
|
||||
self._pending_sse_chunk = ""
|
||||
complete_parts = parts[:-1]
|
||||
elif normalized.rstrip("\n").endswith("data: [DONE]"):
|
||||
self._pending_sse_chunk = ""
|
||||
complete_parts = [normalized.rstrip("\n")]
|
||||
else:
|
||||
self._pending_sse_chunk = parts.pop() if parts else normalized
|
||||
complete_parts = parts
|
||||
|
||||
if not complete_parts:
|
||||
return ""
|
||||
return "\n\n".join(complete_parts) + "\n\n"
|
||||
|
||||
def _iter_sse_json_events(self, chunk: str):
|
||||
"""遍历 SSE 中可解析为 JSON 的事件。"""
|
||||
current_event = None
|
||||
|
||||
for raw_line in str(chunk or "").split('\n'):
|
||||
line = raw_line.strip()
|
||||
|
||||
if not line:
|
||||
current_event = None
|
||||
continue
|
||||
|
||||
if line.startswith('event:'):
|
||||
current_event = line[6:].strip()
|
||||
continue
|
||||
|
||||
if not line.startswith('data:'):
|
||||
continue
|
||||
|
||||
data_str = line[5:].strip()
|
||||
if not data_str or data_str == "[DONE]":
|
||||
continue
|
||||
|
||||
try:
|
||||
payload = json.loads(data_str)
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
|
||||
yield current_event, payload
|
||||
|
||||
def _consume_event(self, event_name: Optional[str], data: Dict[str, Any]) -> None:
|
||||
self._update_media_generation_state(event_name, data)
|
||||
if event_name != "delta":
|
||||
return
|
||||
|
||||
operations: List[Dict[str, Any]] = []
|
||||
if data.get("o") == "patch" and isinstance(data.get("v"), list):
|
||||
operations = [item for item in data.get("v", []) if isinstance(item, dict)]
|
||||
elif isinstance(data.get("v"), list):
|
||||
operations = [item for item in data.get("v", []) if isinstance(item, dict)]
|
||||
elif isinstance(data.get("p"), str) and data.get("o") is not None:
|
||||
operations = [data]
|
||||
|
||||
if operations:
|
||||
for operation in operations:
|
||||
self._apply_operation(operation)
|
||||
return
|
||||
|
||||
value = data.get("v")
|
||||
if isinstance(value, str) and self._active_part_index is not None:
|
||||
self._append_message_part(self._active_part_index, value)
|
||||
return
|
||||
|
||||
self._sync_message_snapshot_from_payload(data)
|
||||
|
||||
def _sync_message_snapshot_from_payload(self, payload: Dict[str, Any]) -> None:
|
||||
message = None
|
||||
value = payload.get("v")
|
||||
if isinstance(value, dict) and isinstance(value.get("message"), dict):
|
||||
message = value.get("message") or {}
|
||||
elif isinstance(payload.get("message"), dict):
|
||||
message = payload.get("message") or {}
|
||||
|
||||
if not isinstance(message, dict):
|
||||
return
|
||||
|
||||
author = message.get("author") or {}
|
||||
if str(author.get("role") or "").strip().lower() != "assistant":
|
||||
return
|
||||
|
||||
content = message.get("content") or {}
|
||||
if not isinstance(content, dict):
|
||||
return
|
||||
|
||||
if str(content.get("content_type") or "").strip().lower() != "text":
|
||||
return
|
||||
|
||||
parts = content.get("parts")
|
||||
if isinstance(parts, list):
|
||||
self._set_message_parts(parts)
|
||||
self._active_part_index = max(self._message_parts.keys(), default=0)
|
||||
|
||||
metadata = message.get("metadata") or {}
|
||||
refs = metadata.get("content_references")
|
||||
if isinstance(refs, list):
|
||||
self._content_references = [
|
||||
dict(item) for item in refs if isinstance(item, dict)
|
||||
]
|
||||
self._mark_content_references_dirty()
|
||||
|
||||
def _apply_operation(self, operation: Dict[str, Any]) -> None:
|
||||
path = str(operation.get("p") or "")
|
||||
action = str(operation.get("o") or "").strip().lower()
|
||||
value = operation.get("v")
|
||||
|
||||
if self._apply_content_part_operation(path, action, value):
|
||||
return
|
||||
|
||||
if self._apply_content_reference_operation(path, action, value):
|
||||
return
|
||||
|
||||
def _apply_content_part_operation(self, path: str, action: str, value: Any) -> bool:
|
||||
if not path.startswith("/message/content/parts"):
|
||||
return False
|
||||
|
||||
if path == "/message/content/parts":
|
||||
if action in {"add", "replace"} and isinstance(value, list):
|
||||
self._set_message_parts(value)
|
||||
self._active_part_index = max(self._message_parts.keys(), default=0)
|
||||
elif action == "append" and isinstance(value, list):
|
||||
start_index = max(self._message_parts.keys(), default=-1) + 1
|
||||
for offset, part in enumerate(value):
|
||||
if isinstance(part, str):
|
||||
self._set_message_part(start_index + offset, part)
|
||||
self._active_part_index = max(self._message_parts.keys(), default=0)
|
||||
elif action == "remove":
|
||||
self._clear_message_parts()
|
||||
self._active_part_index = None
|
||||
return True
|
||||
|
||||
match = re.match(r"^/message/content/parts/(\d+)$", path)
|
||||
if not match:
|
||||
return True
|
||||
|
||||
index = int(match.group(1))
|
||||
if action == "append" and isinstance(value, str):
|
||||
self._append_message_part(index, value)
|
||||
self._active_part_index = index
|
||||
elif action in {"add", "replace"}:
|
||||
if isinstance(value, str):
|
||||
self._set_message_part(index, value)
|
||||
self._active_part_index = index
|
||||
elif value is None:
|
||||
self._set_message_part(index, "")
|
||||
self._active_part_index = index
|
||||
elif action == "remove":
|
||||
self._remove_message_part(index)
|
||||
if self._active_part_index == index:
|
||||
self._active_part_index = max(self._message_parts.keys(), default=None)
|
||||
return True
|
||||
|
||||
def _apply_content_reference_operation(self, path: str, action: str, value: Any) -> bool:
|
||||
if not path.startswith("/message/metadata/content_references"):
|
||||
return False
|
||||
|
||||
match = re.match(r"^/message/metadata/content_references(?:/(\d+)(?:/(.+))?)?$", path)
|
||||
if not match:
|
||||
return True
|
||||
|
||||
index_text, subpath = match.groups()
|
||||
if index_text is None:
|
||||
if action in {"add", "replace"} and isinstance(value, list):
|
||||
self._content_references = [
|
||||
dict(item) for item in value if isinstance(item, dict)
|
||||
]
|
||||
self._mark_content_references_dirty()
|
||||
elif action == "append":
|
||||
changed = False
|
||||
items = value if isinstance(value, list) else [value]
|
||||
for item in items:
|
||||
if isinstance(item, dict):
|
||||
self._content_references.append(dict(item))
|
||||
changed = True
|
||||
if changed:
|
||||
self._mark_content_references_dirty()
|
||||
elif action == "remove":
|
||||
self._clear_content_references()
|
||||
return True
|
||||
|
||||
index = int(index_text)
|
||||
if self._ensure_reference_index(index):
|
||||
self._mark_content_references_dirty()
|
||||
|
||||
if not subpath:
|
||||
if action == "remove":
|
||||
if 0 <= index < len(self._content_references):
|
||||
self._content_references.pop(index)
|
||||
self._mark_content_references_dirty()
|
||||
elif action in {"add", "replace"} and isinstance(value, dict):
|
||||
self._content_references[index] = dict(value)
|
||||
self._mark_content_references_dirty()
|
||||
elif action == "append" and isinstance(value, dict):
|
||||
self._content_references[index].update(value)
|
||||
self._mark_content_references_dirty()
|
||||
return True
|
||||
|
||||
target = self._content_references[index]
|
||||
self._apply_nested_patch(target, subpath.split("/"), action, value)
|
||||
self._mark_content_references_dirty()
|
||||
return True
|
||||
|
||||
def _ensure_reference_index(self, index: int) -> bool:
|
||||
changed = False
|
||||
while len(self._content_references) <= index:
|
||||
self._content_references.append({})
|
||||
changed = True
|
||||
return changed
|
||||
|
||||
def _apply_nested_patch(
|
||||
self,
|
||||
container: Dict[str, Any],
|
||||
path_segments: List[str],
|
||||
action: str,
|
||||
value: Any,
|
||||
) -> None:
|
||||
current = container
|
||||
for segment in path_segments[:-1]:
|
||||
next_value = current.get(segment)
|
||||
if not isinstance(next_value, dict):
|
||||
next_value = {}
|
||||
current[segment] = next_value
|
||||
current = next_value
|
||||
|
||||
key = path_segments[-1]
|
||||
existing = current.get(key)
|
||||
if action == "append":
|
||||
if isinstance(existing, str) and isinstance(value, str):
|
||||
current[key] = existing + value
|
||||
elif isinstance(existing, list) and isinstance(value, list):
|
||||
current[key] = existing + value
|
||||
elif isinstance(existing, dict) and isinstance(value, dict):
|
||||
merged = dict(existing)
|
||||
merged.update(value)
|
||||
current[key] = merged
|
||||
elif existing is None:
|
||||
current[key] = value
|
||||
else:
|
||||
current[key] = value
|
||||
elif action in {"add", "replace"}:
|
||||
current[key] = value
|
||||
elif action == "remove":
|
||||
current.pop(key, None)
|
||||
|
||||
def _render_visible_text(self) -> str:
|
||||
if not self._message_parts:
|
||||
return self._rendered_content
|
||||
|
||||
text = self._render_message_parts()
|
||||
text = self._replace_content_references(text)
|
||||
text = self._strip_private_markup(text)
|
||||
return text
|
||||
|
||||
def _set_message_parts(self, parts: List[Any]) -> None:
|
||||
self._message_parts = {
|
||||
index: str(part)
|
||||
for index, part in enumerate(parts)
|
||||
if isinstance(part, str)
|
||||
}
|
||||
self._mark_message_parts_dirty()
|
||||
|
||||
def _set_message_part(self, index: int, value: str) -> None:
|
||||
self._message_parts[index] = str(value)
|
||||
self._mark_message_parts_dirty()
|
||||
|
||||
def _append_message_part(self, index: int, value: str) -> None:
|
||||
if not value:
|
||||
return
|
||||
self._message_parts[index] = self._message_parts.get(index, "") + str(value)
|
||||
self._mark_message_parts_dirty()
|
||||
|
||||
def _remove_message_part(self, index: int) -> None:
|
||||
if index in self._message_parts:
|
||||
self._message_parts.pop(index, None)
|
||||
self._mark_message_parts_dirty()
|
||||
|
||||
def _clear_message_parts(self) -> None:
|
||||
if self._message_parts or self._message_parts_joined:
|
||||
self._message_parts = {}
|
||||
self._message_parts_joined = ""
|
||||
self._message_parts_dirty = False
|
||||
|
||||
def _mark_message_parts_dirty(self) -> None:
|
||||
self._message_parts_dirty = True
|
||||
|
||||
def _render_message_parts(self) -> str:
|
||||
if self._message_parts_dirty:
|
||||
self._message_parts_joined = "".join(
|
||||
self._message_parts[index] for index in sorted(self._message_parts)
|
||||
)
|
||||
self._message_parts_dirty = False
|
||||
return self._message_parts_joined
|
||||
|
||||
def _mark_content_references_dirty(self) -> None:
|
||||
self._content_references_dirty = True
|
||||
|
||||
def _clear_content_references(self) -> None:
|
||||
if self._content_references or self._content_references_sorted:
|
||||
self._content_references = []
|
||||
self._content_references_sorted = []
|
||||
self._content_references_dirty = False
|
||||
|
||||
def _sorted_content_references(self) -> List[Dict[str, Any]]:
|
||||
if self._content_references_dirty:
|
||||
self._content_references_sorted = sorted(
|
||||
(
|
||||
ref for ref in self._content_references
|
||||
if isinstance(ref, dict) and str(ref.get("matched_text") or "")
|
||||
),
|
||||
key=lambda ref: len(str(ref.get("matched_text") or "")),
|
||||
reverse=True,
|
||||
)
|
||||
self._content_references_dirty = False
|
||||
return self._content_references_sorted
|
||||
|
||||
def _replace_content_references(self, text: str) -> str:
|
||||
rendered = text
|
||||
|
||||
for ref in self._sorted_content_references():
|
||||
matched_text = str(ref.get("matched_text") or "")
|
||||
if not matched_text or matched_text not in rendered:
|
||||
continue
|
||||
rendered = rendered.replace(
|
||||
matched_text,
|
||||
self._build_reference_replacement(ref),
|
||||
)
|
||||
|
||||
rendered = self._inline_ref_pattern.sub(
|
||||
lambda match: self._decode_inline_reference(match.group(1), match.group(2)),
|
||||
rendered,
|
||||
)
|
||||
return rendered
|
||||
|
||||
def _build_reference_replacement(self, ref: Dict[str, Any]) -> str:
|
||||
ref_type = str(ref.get("type") or "").strip().lower()
|
||||
if ref_type == "hidden":
|
||||
return ""
|
||||
|
||||
label = ""
|
||||
for key in ("alt", "prompt_text", "name"):
|
||||
value = str(ref.get(key) or "").strip()
|
||||
if value:
|
||||
label = value
|
||||
break
|
||||
|
||||
if not label:
|
||||
matched_text = str(ref.get("matched_text") or "")
|
||||
if matched_text:
|
||||
label = self._decode_inline_reference_from_text(matched_text)
|
||||
|
||||
safe_urls = ref.get("safe_urls")
|
||||
first_url = ""
|
||||
if isinstance(safe_urls, list):
|
||||
for item in safe_urls:
|
||||
candidate = str(item or "").strip()
|
||||
if candidate:
|
||||
first_url = candidate
|
||||
break
|
||||
|
||||
if first_url and label:
|
||||
if first_url in label or "](" in label or label.startswith("http://") or label.startswith("https://"):
|
||||
return label
|
||||
return f"{label} ({first_url})"
|
||||
if label:
|
||||
return label
|
||||
if first_url:
|
||||
return first_url
|
||||
return ""
|
||||
|
||||
def _decode_inline_reference_from_text(self, text: str) -> str:
|
||||
match = self._inline_ref_pattern.search(str(text or ""))
|
||||
if not match:
|
||||
return ""
|
||||
return self._decode_inline_reference(match.group(1), match.group(2))
|
||||
|
||||
def _decode_inline_reference(self, ref_kind: str, payload: str) -> str:
|
||||
kind = str(ref_kind or "").strip().lower()
|
||||
payload_text = str(payload or "")
|
||||
|
||||
if kind == "entity":
|
||||
try:
|
||||
decoded = json.loads(payload_text)
|
||||
if isinstance(decoded, list):
|
||||
for index in (1, 0, 2):
|
||||
if index < len(decoded):
|
||||
value = str(decoded[index] or "").strip()
|
||||
if value:
|
||||
return value
|
||||
except Exception:
|
||||
return ""
|
||||
|
||||
return ""
|
||||
|
||||
def _strip_private_markup(self, text: str) -> str:
|
||||
cleaned = self._dangling_ref_pattern.sub("", text)
|
||||
cleaned = cleaned.replace(self._ref_start, "")
|
||||
cleaned = cleaned.replace(self._ref_sep, "")
|
||||
cleaned = cleaned.replace(self._ref_end, "")
|
||||
return cleaned
|
||||
|
||||
def _update_media_generation_state(self, event_name: Optional[str], payload: Dict[str, Any]) -> None:
|
||||
pending = False
|
||||
media_type = str(self._media_generation_state.get("media_type") or "")
|
||||
hint_parts = [
|
||||
item for item in str(self._media_generation_state.get("hint_text") or "").split("\n\n")
|
||||
if item
|
||||
]
|
||||
wait_timeout_seconds = self._media_generation_state.get("wait_timeout_seconds")
|
||||
|
||||
payload_type = str(payload.get("type") or "").strip().lower()
|
||||
if payload_type == "server_ste_metadata":
|
||||
metadata = payload.get("metadata") or {}
|
||||
if str(metadata.get("turn_use_case") or "").strip().lower() == "image gen":
|
||||
pending = True
|
||||
media_type = media_type or "image"
|
||||
wait_timeout_seconds = max(float(wait_timeout_seconds or 0), 90.0)
|
||||
|
||||
message = None
|
||||
if event_name == "delta":
|
||||
value = payload.get("v")
|
||||
if isinstance(value, dict) and isinstance(value.get("message"), dict):
|
||||
message = value.get("message") or {}
|
||||
elif isinstance(payload.get("message"), dict):
|
||||
message = payload.get("message") or {}
|
||||
|
||||
if isinstance(message, dict):
|
||||
metadata = message.get("metadata") or {}
|
||||
content = message.get("content") or {}
|
||||
|
||||
if str(metadata.get("image_gen_task_id") or "").strip():
|
||||
pending = True
|
||||
media_type = media_type or "image"
|
||||
wait_timeout_seconds = max(float(wait_timeout_seconds or 0), 90.0)
|
||||
|
||||
if bool(metadata.get("image_gen_multi_stream")):
|
||||
pending = True
|
||||
media_type = media_type or "image"
|
||||
wait_timeout_seconds = max(float(wait_timeout_seconds or 0), 90.0)
|
||||
|
||||
for field in ("ui_card_title", "ui_card_description"):
|
||||
value = str(metadata.get(field) or "").strip()
|
||||
if value and value not in hint_parts:
|
||||
hint_parts.append(value)
|
||||
|
||||
if isinstance(content, dict):
|
||||
parts = content.get("parts")
|
||||
if isinstance(parts, list):
|
||||
for item in parts:
|
||||
value = str(item or "").strip() if isinstance(item, str) else ""
|
||||
if value and value not in hint_parts:
|
||||
hint_parts.append(value)
|
||||
|
||||
if pending and media_type:
|
||||
self._media_generation_state = {
|
||||
"pending": True,
|
||||
"media_type": media_type,
|
||||
"hint_text": "\n\n".join(hint_parts[:3]),
|
||||
"wait_timeout_seconds": wait_timeout_seconds,
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _compute_delta(previous: str, current: str) -> str:
|
||||
if not current or current == previous:
|
||||
return ""
|
||||
if not previous:
|
||||
return current
|
||||
if current.startswith(previous):
|
||||
return current[len(previous):]
|
||||
|
||||
prefix_len = 0
|
||||
max_prefix = min(len(previous), len(current))
|
||||
while prefix_len < max_prefix and previous[prefix_len] == current[prefix_len]:
|
||||
prefix_len += 1
|
||||
|
||||
return current[prefix_len:]
|
||||
|
||||
def reset(self):
|
||||
"""重置状态"""
|
||||
self._accumulated_content = ""
|
||||
self._last_raw_length = 0
|
||||
self._last_raw_response = ""
|
||||
self._rendered_content = ""
|
||||
self._message_parts = {}
|
||||
self._message_parts_joined = ""
|
||||
self._message_parts_dirty = False
|
||||
self._content_references = []
|
||||
self._content_references_sorted = []
|
||||
self._content_references_dirty = False
|
||||
self._active_part_index = None
|
||||
self._pending_sse_chunk = ""
|
||||
self._media_generation_state = {}
|
||||
|
||||
def get_media_generation_state(
|
||||
self,
|
||||
raw_response: str = "",
|
||||
parse_result: Optional[Dict[str, Any]] = None,
|
||||
) -> Dict[str, Any]:
|
||||
return dict(self._media_generation_state or {})
|
||||
|
||||
@classmethod
|
||||
def get_id(cls) -> str:
|
||||
return "chatgpt"
|
||||
|
||||
@classmethod
|
||||
def get_name(cls) -> str:
|
||||
return "ChatGPT API"
|
||||
|
||||
@classmethod
|
||||
def get_description(cls) -> str:
|
||||
return "解析 ChatGPT 的 API 响应(SSE流式,v1 delta编码)"
|
||||
|
||||
@classmethod
|
||||
def get_supported_patterns(cls) -> List[str]:
|
||||
return ["**/backend-api/f/conversation**"]
|
||||
|
||||
|
||||
__all__ = ['ChatGPTParser']
|
||||
@@ -0,0 +1,179 @@
|
||||
"""
|
||||
claude_parser.py - Claude SSE response parser.
|
||||
|
||||
Observed stream traits:
|
||||
- event: content_block_delta with delta.type=text_delta carries answer text
|
||||
- event: content_block_delta with delta.type=thinking_delta carries Extended Thinking CoT text
|
||||
- event: message_stop marks stream completion
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from typing import Any, Dict, List, Tuple
|
||||
|
||||
from app.core.config import logger
|
||||
from .base import ResponseParser
|
||||
|
||||
|
||||
class ClaudeParser(ResponseParser):
|
||||
"""Parse Claude web SSE responses."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._last_raw_length = 0
|
||||
self._pending = ""
|
||||
|
||||
def parse_chunk(self, raw_response: str) -> Dict[str, Any]:
|
||||
result: Dict[str, Any] = {
|
||||
"content": "",
|
||||
"reasoning_content": "",
|
||||
"images": [],
|
||||
"done": False,
|
||||
"error": None,
|
||||
}
|
||||
|
||||
try:
|
||||
if isinstance(raw_response, (bytes, bytearray)):
|
||||
raw_response = raw_response.decode("utf-8", errors="ignore")
|
||||
elif not isinstance(raw_response, str):
|
||||
raw_response = str(raw_response)
|
||||
|
||||
new_data = self._prepare_incremental_raw_response(raw_response)
|
||||
if not new_data:
|
||||
return result
|
||||
|
||||
delta_content, delta_reasoning, done = self._consume_new_data(new_data)
|
||||
if delta_content:
|
||||
result["content"] = delta_content
|
||||
if delta_reasoning:
|
||||
result["reasoning_content"] = delta_reasoning
|
||||
result["done"] = done
|
||||
|
||||
except Exception as e:
|
||||
logger.debug(f"[ClaudeParser] parse exception: {e}")
|
||||
result["error"] = str(e)
|
||||
|
||||
return result
|
||||
|
||||
def reset(self) -> None:
|
||||
self._last_raw_length = 0
|
||||
self._last_raw_response = ""
|
||||
self._pending = ""
|
||||
|
||||
def _consume_new_data(self, new_data: str) -> Tuple[str, str, bool]:
|
||||
# Browser stream snapshots can use either CRLF or lone CR line endings.
|
||||
normalized = (self._pending + new_data).replace("\r\n", "\n").replace("\r", "\n")
|
||||
if not normalized:
|
||||
return "", "", False
|
||||
|
||||
blocks = normalized.split("\n\n")
|
||||
if normalized.endswith("\n\n"):
|
||||
self._pending = ""
|
||||
complete_blocks = [block for block in blocks if block.strip()]
|
||||
else:
|
||||
self._pending = blocks.pop() if blocks else normalized
|
||||
complete_blocks = [block for block in blocks if block.strip()]
|
||||
|
||||
content_parts: List[str] = []
|
||||
reasoning_parts: List[str] = []
|
||||
done = False
|
||||
|
||||
for block in complete_blocks:
|
||||
block_content, block_reasoning, block_done = self._parse_event_block(block)
|
||||
if block_content:
|
||||
content_parts.append(block_content)
|
||||
if block_reasoning:
|
||||
reasoning_parts.append(block_reasoning)
|
||||
if block_done:
|
||||
done = True
|
||||
|
||||
return "".join(content_parts), "".join(reasoning_parts), done
|
||||
|
||||
def _parse_event_block(self, block: str) -> Tuple[str, str, bool]:
|
||||
event_name = ""
|
||||
data_lines: List[str] = []
|
||||
|
||||
for raw_line in block.split("\n"):
|
||||
line = raw_line.strip()
|
||||
if not line:
|
||||
continue
|
||||
if line.startswith("event:"):
|
||||
event_name = line[6:].strip()
|
||||
elif line.startswith("data:"):
|
||||
data_lines.append(line[5:].strip())
|
||||
|
||||
if event_name == "message_stop":
|
||||
return "", "", True
|
||||
|
||||
payload = "\n".join(data_lines).strip()
|
||||
if not payload:
|
||||
return "", "", False
|
||||
|
||||
try:
|
||||
data = json.loads(payload)
|
||||
except json.JSONDecodeError:
|
||||
return "", "", False
|
||||
|
||||
# Some response wrappers omit the SSE event line but preserve the
|
||||
# protocol event type in the JSON payload.
|
||||
if not event_name:
|
||||
event_name = str(data.get("type") or "").strip()
|
||||
|
||||
if event_name == "message_stop":
|
||||
return "", "", True
|
||||
|
||||
if event_name == "content_block_start":
|
||||
cb = data.get("content_block", {})
|
||||
if isinstance(cb, dict):
|
||||
cb_type = cb.get("type")
|
||||
if cb_type == "text" and isinstance(cb.get("text"), str):
|
||||
return cb.get("text", ""), "", False
|
||||
elif cb_type == "thinking" and isinstance(cb.get("thinking"), str):
|
||||
return "", cb.get("thinking", ""), False
|
||||
|
||||
if event_name == "content_block_delta":
|
||||
delta = data.get("delta", {})
|
||||
if isinstance(delta, dict):
|
||||
delta_type = delta.get("type")
|
||||
if delta_type == "text_delta":
|
||||
text = delta.get("text", "")
|
||||
return (text, "", False) if isinstance(text, str) else ("", "", False)
|
||||
elif delta_type == "thinking_delta":
|
||||
thinking_text = delta.get("thinking", "")
|
||||
return ("", thinking_text, False) if isinstance(thinking_text, str) else ("", "", False)
|
||||
|
||||
elif delta_type == "thinking_summary_delta":
|
||||
summary_text = delta.get("summary", delta.get("thinking", ""))
|
||||
return ("", summary_text, False) if isinstance(summary_text, str) else ("", "", False)
|
||||
|
||||
return "", "", False
|
||||
|
||||
@classmethod
|
||||
def get_id(cls) -> str:
|
||||
return "claude"
|
||||
|
||||
@classmethod
|
||||
def get_name(cls) -> str:
|
||||
return "Claude"
|
||||
|
||||
@classmethod
|
||||
def get_description(cls) -> str:
|
||||
return "Parse Claude SSE streams including Extended Thinking blocks"
|
||||
|
||||
def should_require_explicit_done(self) -> bool:
|
||||
# Claude's stream object may report capture completion after an
|
||||
# intermediate snapshot; message_stop is the reliable protocol end.
|
||||
return True
|
||||
|
||||
def should_wait_for_replacement_stream_on_incomplete_capture(self) -> bool:
|
||||
return True
|
||||
|
||||
@classmethod
|
||||
def get_supported_patterns(cls) -> List[str]:
|
||||
# The conversation path is also used by /title and other JSON requests.
|
||||
# Match the actual SSE completion endpoint so auxiliary requests cannot
|
||||
# replace the active stream.
|
||||
return ["/completion"]
|
||||
|
||||
|
||||
__all__ = ["ClaudeParser"]
|
||||
@@ -0,0 +1,488 @@
|
||||
"""
|
||||
deepseek_parser.py - DeepSeek SSE response parser.
|
||||
|
||||
Observed stream traits:
|
||||
- data: {"v": {"response": {"fragments": [...]}}} initializes fragment state
|
||||
- thinking mode sends THINK fragments first and answer text later in RESPONSE fragments
|
||||
- path-less data: {"v": "..."} continues the current fragment's content
|
||||
- response/status or quasi_status=FINISHED ends the stream
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from typing import Any, Dict, List, Tuple
|
||||
|
||||
from app.core.config import logger
|
||||
from .base import ResponseParser
|
||||
|
||||
|
||||
class DeepSeekParser(ResponseParser):
|
||||
"""Parse DeepSeek web SSE streams and ignore THINK fragments."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._last_raw_length = 0
|
||||
self._pending = ""
|
||||
self._fragment_types: List[str] = []
|
||||
self._fragment_contents: List[str] = []
|
||||
self._current_fragment_type = ""
|
||||
self._current_fragment_index = -1
|
||||
self._debug_records: List[Dict[str, Any]] = []
|
||||
self._emit_buffer = ""
|
||||
self._has_seen_visible_response_text = False
|
||||
|
||||
def parse_chunk(self, raw_response: str) -> Dict[str, Any]:
|
||||
result: Dict[str, Any] = {
|
||||
"content": "",
|
||||
"images": [],
|
||||
"done": False,
|
||||
"error": None,
|
||||
}
|
||||
|
||||
try:
|
||||
if isinstance(raw_response, (bytes, bytearray)):
|
||||
raw_response = raw_response.decode("utf-8", errors="ignore")
|
||||
elif not isinstance(raw_response, str):
|
||||
raw_response = str(raw_response)
|
||||
|
||||
new_data = self._prepare_incremental_raw_response(raw_response)
|
||||
if not new_data:
|
||||
return result
|
||||
|
||||
delta_content, done = self._consume_new_data(new_data)
|
||||
safe_content = self._drain_safe_output(delta_content, force=done)
|
||||
if safe_content:
|
||||
result["content"] = safe_content
|
||||
result["done"] = done
|
||||
|
||||
except Exception as e:
|
||||
logger.debug(f"[DeepSeekParser] parse exception: {e}")
|
||||
result["error"] = str(e)
|
||||
|
||||
return result
|
||||
|
||||
def reset(self) -> None:
|
||||
self._last_raw_length = 0
|
||||
self._last_raw_response = ""
|
||||
self._pending = ""
|
||||
self._fragment_types = []
|
||||
self._fragment_contents = []
|
||||
self._current_fragment_type = ""
|
||||
self._current_fragment_index = -1
|
||||
self._debug_records = []
|
||||
self._emit_buffer = ""
|
||||
self._has_seen_visible_response_text = False
|
||||
|
||||
def _consume_new_data(self, new_data: str) -> Tuple[str, bool]:
|
||||
normalized = (self._pending + new_data).replace("\r\n", "\n")
|
||||
if not normalized:
|
||||
return "", False
|
||||
|
||||
blocks = normalized.split("\n\n")
|
||||
if normalized.endswith("\n\n"):
|
||||
self._pending = ""
|
||||
complete_blocks = [block for block in blocks if block.strip()]
|
||||
else:
|
||||
self._pending = blocks.pop() if blocks else normalized
|
||||
complete_blocks = [block for block in blocks if block.strip()]
|
||||
|
||||
content_parts: List[str] = []
|
||||
done = False
|
||||
|
||||
for block in complete_blocks:
|
||||
block_content, block_done = self._parse_event_block(block)
|
||||
if block_content:
|
||||
content_parts.append(block_content)
|
||||
if block_done:
|
||||
done = True
|
||||
|
||||
return "".join(content_parts), done
|
||||
|
||||
def _parse_event_block(self, block: str) -> Tuple[str, bool]:
|
||||
event_name = ""
|
||||
data_lines: List[str] = []
|
||||
|
||||
for raw_line in block.split("\n"):
|
||||
line = raw_line.strip()
|
||||
if not line:
|
||||
continue
|
||||
if line.startswith("event:"):
|
||||
event_name = line[6:].strip()
|
||||
elif line.startswith("data:"):
|
||||
data_lines.append(line[5:].strip())
|
||||
|
||||
if event_name in {"ready", "update_session", "title"}:
|
||||
return "", False
|
||||
if event_name in {"finish", "close"}:
|
||||
return "", self._can_accept_done_signal()
|
||||
|
||||
payload = "\n".join(data_lines).strip()
|
||||
if not payload or payload == "{}":
|
||||
return "", False
|
||||
|
||||
try:
|
||||
data = json.loads(payload)
|
||||
except json.JSONDecodeError:
|
||||
return "", False
|
||||
|
||||
content, done = self._extract_content(data)
|
||||
self._record_debug_event(event_name, data, content, done)
|
||||
return content, done
|
||||
|
||||
def _extract_content(self, data: Dict[str, Any]) -> Tuple[str, bool]:
|
||||
path = str(data.get("p", "") or "")
|
||||
operation = str(data.get("o", "") or "")
|
||||
value = data.get("v")
|
||||
|
||||
if operation == "BATCH" and isinstance(value, list):
|
||||
return self._extract_batch_content(value)
|
||||
|
||||
if isinstance(value, dict):
|
||||
response = value.get("response")
|
||||
if isinstance(response, dict):
|
||||
return self._extract_response_object(response)
|
||||
|
||||
if self._is_done_signal(path, value):
|
||||
return "", self._can_accept_done_signal()
|
||||
|
||||
if self._is_fragment_append(path, operation, value):
|
||||
return self._register_fragments(value), False
|
||||
|
||||
if self._is_fragment_content_patch(path, value):
|
||||
fragment_index = self._resolve_fragment_index(path)
|
||||
fragment_type = self._resolve_fragment_type(path)
|
||||
delta_text = self._apply_fragment_content_update(fragment_index, value)
|
||||
if self._should_emit_fragment_text(fragment_type):
|
||||
self._mark_visible_response_text(fragment_type, delta_text)
|
||||
return delta_text, False
|
||||
return "", False
|
||||
|
||||
if not path and not operation and isinstance(value, str):
|
||||
self._append_to_current_fragment(value)
|
||||
if self._should_emit_fragment_text(self._current_fragment_type):
|
||||
self._mark_visible_response_text(self._current_fragment_type, value)
|
||||
return value, False
|
||||
return "", False
|
||||
|
||||
return "", False
|
||||
|
||||
def _extract_batch_content(self, operations: List[Dict[str, Any]]) -> Tuple[str, bool]:
|
||||
content_parts: List[str] = []
|
||||
saw_done_signal = False
|
||||
|
||||
for op in operations:
|
||||
if not isinstance(op, dict):
|
||||
continue
|
||||
|
||||
path = str(op.get("p", "") or "")
|
||||
operation = str(op.get("o", "") or "")
|
||||
value = op.get("v")
|
||||
|
||||
if self._is_done_signal(path, value):
|
||||
saw_done_signal = True
|
||||
continue
|
||||
|
||||
if self._is_fragment_append(path, operation, value):
|
||||
fragment_content = self._register_fragments(value)
|
||||
if fragment_content:
|
||||
content_parts.append(fragment_content)
|
||||
continue
|
||||
|
||||
if self._is_fragment_content_patch(path, value):
|
||||
fragment_index = self._resolve_fragment_index(path)
|
||||
fragment_type = self._resolve_fragment_type(path)
|
||||
delta_text = self._apply_fragment_content_update(fragment_index, value)
|
||||
if self._should_emit_fragment_text(fragment_type):
|
||||
self._mark_visible_response_text(fragment_type, delta_text)
|
||||
content_parts.append(delta_text)
|
||||
|
||||
return "".join(content_parts), bool(saw_done_signal and self._can_accept_done_signal())
|
||||
|
||||
def _extract_response_object(self, response: Dict[str, Any]) -> Tuple[str, bool]:
|
||||
fragments = response.get("fragments")
|
||||
content = self._register_fragments(fragments) if isinstance(fragments, list) else ""
|
||||
|
||||
done = False
|
||||
if self._can_accept_done_signal():
|
||||
if self._is_done_signal("response/status", response.get("status")):
|
||||
done = True
|
||||
if self._is_done_signal("response/quasi_status", response.get("quasi_status")):
|
||||
done = True
|
||||
|
||||
return content, done
|
||||
|
||||
def _register_fragments(self, fragments: List[Dict[str, Any]]) -> str:
|
||||
content_parts: List[str] = []
|
||||
|
||||
for fragment in fragments:
|
||||
if not isinstance(fragment, dict):
|
||||
continue
|
||||
|
||||
fragment_type = str(fragment.get("type", "") or "").upper()
|
||||
self._fragment_types.append(fragment_type)
|
||||
self._fragment_contents.append("")
|
||||
self._current_fragment_index = len(self._fragment_types) - 1
|
||||
if fragment_type:
|
||||
self._current_fragment_type = fragment_type
|
||||
|
||||
text = fragment.get("content", "")
|
||||
if isinstance(text, str):
|
||||
self._fragment_contents[self._current_fragment_index] = text
|
||||
if text and self._should_emit_fragment_text(fragment_type):
|
||||
self._mark_visible_response_text(fragment_type, text)
|
||||
content_parts.append(text)
|
||||
|
||||
return "".join(content_parts)
|
||||
|
||||
def _resolve_fragment_index(self, path: str) -> int:
|
||||
marker = "response/fragments/"
|
||||
if marker not in path or not self._fragment_types:
|
||||
return self._current_fragment_index
|
||||
|
||||
index_part = path.split(marker, 1)[1].split("/", 1)[0]
|
||||
if index_part == "-1":
|
||||
return len(self._fragment_types) - 1
|
||||
|
||||
try:
|
||||
index = int(index_part)
|
||||
except ValueError:
|
||||
return self._current_fragment_index
|
||||
|
||||
if index < 0:
|
||||
index += len(self._fragment_types)
|
||||
if 0 <= index < len(self._fragment_types):
|
||||
return index
|
||||
|
||||
return self._current_fragment_index
|
||||
|
||||
def _resolve_fragment_type(self, path: str) -> str:
|
||||
fragment_index = self._resolve_fragment_index(path)
|
||||
if 0 <= fragment_index < len(self._fragment_types):
|
||||
fragment_type = self._fragment_types[fragment_index]
|
||||
self._current_fragment_index = fragment_index
|
||||
self._current_fragment_type = fragment_type
|
||||
return fragment_type
|
||||
return self._current_fragment_type
|
||||
|
||||
def _append_to_current_fragment(self, delta_text: str) -> None:
|
||||
if not isinstance(delta_text, str) or not delta_text:
|
||||
return
|
||||
if 0 <= self._current_fragment_index < len(self._fragment_contents):
|
||||
self._fragment_contents[self._current_fragment_index] += delta_text
|
||||
|
||||
def _apply_fragment_content_update(self, fragment_index: int, incoming_text: str) -> str:
|
||||
if not isinstance(incoming_text, str) or not incoming_text:
|
||||
return ""
|
||||
|
||||
if not (0 <= fragment_index < len(self._fragment_contents)):
|
||||
return incoming_text
|
||||
|
||||
previous_text = self._fragment_contents[fragment_index]
|
||||
new_text, delta_text = self._merge_fragment_text(previous_text, incoming_text)
|
||||
self._fragment_contents[fragment_index] = new_text
|
||||
self._current_fragment_index = fragment_index
|
||||
return delta_text
|
||||
|
||||
@staticmethod
|
||||
def _merge_fragment_text(previous_text: str, incoming_text: str) -> Tuple[str, str]:
|
||||
previous = previous_text if isinstance(previous_text, str) else ""
|
||||
incoming = incoming_text if isinstance(incoming_text, str) else ""
|
||||
|
||||
if not previous:
|
||||
return incoming, incoming
|
||||
|
||||
if not incoming:
|
||||
return previous, ""
|
||||
|
||||
if incoming.startswith(previous):
|
||||
return incoming, incoming[len(previous) :]
|
||||
|
||||
if previous.endswith(incoming):
|
||||
return previous, ""
|
||||
|
||||
common_prefix_len = 0
|
||||
max_prefix_len = min(len(previous), len(incoming))
|
||||
while common_prefix_len < max_prefix_len and previous[common_prefix_len] == incoming[common_prefix_len]:
|
||||
common_prefix_len += 1
|
||||
|
||||
if common_prefix_len > 0:
|
||||
return incoming, incoming[common_prefix_len:]
|
||||
|
||||
return previous + incoming, incoming
|
||||
|
||||
@staticmethod
|
||||
def _is_fragment_append(path: str, operation: str, value: Any) -> bool:
|
||||
if not isinstance(value, list):
|
||||
return False
|
||||
return operation == "APPEND" and path in {"response/fragments", "fragments"}
|
||||
|
||||
@staticmethod
|
||||
def _is_fragment_content_patch(path: str, value: Any) -> bool:
|
||||
return isinstance(value, str) and "fragments/" in path and path.endswith("/content")
|
||||
|
||||
@staticmethod
|
||||
def _is_done_signal(path: str, value: Any) -> bool:
|
||||
if not isinstance(value, str):
|
||||
return False
|
||||
if value.upper() != "FINISHED":
|
||||
return False
|
||||
return path.endswith("status") or path.endswith("quasi_status")
|
||||
|
||||
def _should_emit_fragment_text(self, fragment_type: str) -> bool:
|
||||
if not self._fragment_types:
|
||||
return True
|
||||
return fragment_type == "RESPONSE"
|
||||
|
||||
def _mark_visible_response_text(self, fragment_type: str, text: str) -> None:
|
||||
if not isinstance(text, str) or not text:
|
||||
return
|
||||
if self._should_emit_fragment_text(fragment_type):
|
||||
self._has_seen_visible_response_text = True
|
||||
|
||||
def _can_accept_done_signal(self) -> bool:
|
||||
return bool(self._has_seen_visible_response_text)
|
||||
|
||||
def _drain_safe_output(self, delta_text: str, force: bool = False) -> str:
|
||||
if delta_text:
|
||||
self._emit_buffer += delta_text
|
||||
|
||||
if not self._emit_buffer:
|
||||
return ""
|
||||
|
||||
if force:
|
||||
flushed = self._emit_buffer
|
||||
self._emit_buffer = ""
|
||||
return flushed
|
||||
|
||||
safe_index = self._find_last_safe_emit_index(self._emit_buffer)
|
||||
if safe_index <= 0:
|
||||
return ""
|
||||
|
||||
flushed = self._emit_buffer[:safe_index]
|
||||
self._emit_buffer = self._emit_buffer[safe_index:]
|
||||
return flushed
|
||||
|
||||
@staticmethod
|
||||
def _find_last_safe_emit_index(text: str) -> int:
|
||||
if not text:
|
||||
return 0
|
||||
|
||||
inside_tag = False
|
||||
quote_char = ""
|
||||
last_safe_index = 0
|
||||
|
||||
for index, char in enumerate(text):
|
||||
if inside_tag:
|
||||
if quote_char:
|
||||
if char == quote_char:
|
||||
quote_char = ""
|
||||
continue
|
||||
|
||||
if char in {'"', "'"}:
|
||||
quote_char = char
|
||||
continue
|
||||
|
||||
if char == ">":
|
||||
inside_tag = False
|
||||
last_safe_index = index + 1
|
||||
continue
|
||||
|
||||
if char == "<":
|
||||
inside_tag = True
|
||||
continue
|
||||
|
||||
last_safe_index = index + 1
|
||||
|
||||
if inside_tag:
|
||||
last_open = text.rfind("<")
|
||||
if last_open != -1:
|
||||
return last_open
|
||||
|
||||
return last_safe_index
|
||||
|
||||
def export_debug_data(self, raw_response: str = "") -> Dict[str, Any]:
|
||||
return {
|
||||
"parser": self.get_id(),
|
||||
"fragment_types": list(self._fragment_types),
|
||||
"fragment_lengths": [len(item) for item in self._fragment_contents],
|
||||
"current_fragment_type": self._current_fragment_type,
|
||||
"current_fragment_index": self._current_fragment_index,
|
||||
"has_seen_visible_response_text": bool(self._has_seen_visible_response_text),
|
||||
"debug_records": self._debug_records[-120:],
|
||||
"raw_response_preview": str(raw_response or "")[:12000],
|
||||
}
|
||||
|
||||
def export_debug_snapshot(self, raw_response: str = "") -> str:
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
payload = {
|
||||
**self.export_debug_data(raw_response),
|
||||
"exported_at": int(time.time()),
|
||||
}
|
||||
path = Path("logs") / "deepseek_parser_debug.json"
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(
|
||||
json.dumps(payload, ensure_ascii=False, indent=2) + "\n",
|
||||
encoding="utf-8",
|
||||
newline="\n",
|
||||
)
|
||||
return str(path)
|
||||
|
||||
def _record_debug_event(self, event_name: str, data: Dict[str, Any], content: str, done: bool) -> None:
|
||||
value = data.get("v")
|
||||
path = str(data.get("p", "") or "")
|
||||
operation = str(data.get("o", "") or "")
|
||||
|
||||
record: Dict[str, Any] = {
|
||||
"event": str(event_name or ""),
|
||||
"path": path,
|
||||
"op": operation,
|
||||
"done": bool(done),
|
||||
"content_len": len(content or ""),
|
||||
"content_preview": str(content or "")[:240],
|
||||
"value_type": type(value).__name__,
|
||||
"fragment_count": len(self._fragment_types),
|
||||
"current_fragment_type": self._current_fragment_type,
|
||||
"current_fragment_index": self._current_fragment_index,
|
||||
}
|
||||
|
||||
if isinstance(value, str):
|
||||
record["value_len"] = len(value)
|
||||
record["value_preview"] = value[:240]
|
||||
elif isinstance(value, list):
|
||||
record["value_len"] = len(value)
|
||||
elif isinstance(value, dict):
|
||||
response = value.get("response") if isinstance(value.get("response"), dict) else {}
|
||||
fragments = response.get("fragments") if isinstance(response.get("fragments"), list) else []
|
||||
record["response_keys"] = list(response.keys())[:12]
|
||||
record["response_fragment_count"] = len(fragments)
|
||||
if fragments:
|
||||
record["response_fragment_types"] = [
|
||||
str(fragment.get("type", "") or "").upper()
|
||||
for fragment in fragments[:8]
|
||||
if isinstance(fragment, dict)
|
||||
]
|
||||
|
||||
self._debug_records.append(record)
|
||||
if len(self._debug_records) > 200:
|
||||
self._debug_records = self._debug_records[-200:]
|
||||
|
||||
@classmethod
|
||||
def get_id(cls) -> str:
|
||||
return "deepseek"
|
||||
|
||||
@classmethod
|
||||
def get_name(cls) -> str:
|
||||
return "DeepSeek API"
|
||||
|
||||
@classmethod
|
||||
def get_description(cls) -> str:
|
||||
return "Parse DeepSeek SSE streams and ignore THINK fragments"
|
||||
|
||||
@classmethod
|
||||
def get_supported_patterns(cls) -> List[str]:
|
||||
return ["**/api/v0/chat/completion**"]
|
||||
|
||||
|
||||
__all__ = ["DeepSeekParser"]
|
||||
@@ -0,0 +1,693 @@
|
||||
"""
|
||||
doubao_parser.py - Doubao SSE response parser.
|
||||
|
||||
Observed stream traits:
|
||||
- event: STREAM_MSG_NOTIFY carries the first assistant block
|
||||
- event: STREAM_CHUNK may carry extra content_block patches
|
||||
- event: CHUNK_DELTA carries plain text deltas
|
||||
- event: SSE_REPLY_END marks the answer as finished
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
from typing import Any, Dict, List, Tuple
|
||||
|
||||
from app.core.config import logger
|
||||
from .base import ResponseParser
|
||||
|
||||
|
||||
class DoubaoParser(ResponseParser):
|
||||
"""Parse Doubao's SSE response stream."""
|
||||
|
||||
# 生成图片以 image_ori / image_thumb 等字段挂在 SSE 事件里,
|
||||
# 同一张图的各尺寸共享 tos 路径(差异仅在 ~tplv 模板与签名参数)。
|
||||
_IMAGE_FIELD_RE = re.compile(
|
||||
r'"(image_ori|image_ori_raw|image_preview_resize|image_preview|image_thumb)"\s*:\s*'
|
||||
r'\{[^{}]*?"url"\s*:\s*"(https?:\\?/\\?/[^"\s]+)"'
|
||||
)
|
||||
_IMAGE_KEY_PRIORITY = {
|
||||
"image_ori": 0,
|
||||
"image_ori_raw": 1,
|
||||
"image_preview": 2,
|
||||
"image_preview_resize": 3,
|
||||
"image_thumb": 4,
|
||||
}
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._last_raw = ""
|
||||
self._pending = ""
|
||||
self._block_texts: Dict[str, str] = {}
|
||||
self._last_full_message = ""
|
||||
self._assembled_text = ""
|
||||
self._has_seen_visible_text = False
|
||||
self._seen_image_keys: set = set()
|
||||
|
||||
def _collect_image_urls(self, new_data: str) -> List[str]:
|
||||
"""从新的 SSE 数据中提取生成图片 URL,按图片去重、尺寸优先级取最优。"""
|
||||
found: Dict[str, Tuple[int, str]] = {}
|
||||
for match in self._IMAGE_FIELD_RE.finditer(new_data):
|
||||
key, raw_url = match.group(1), match.group(2)
|
||||
url = raw_url.replace(r"\/", "/")
|
||||
# 去掉主机与签名参数/裁剪模板,同一张图的不同 CDN 副本视为一条
|
||||
base_key = re.sub(r"^https?://[^/]+/", "", url).split("?", 1)[0].split("~tplv", 1)[0]
|
||||
priority = self._IMAGE_KEY_PRIORITY.get(key, 99)
|
||||
current = found.get(base_key)
|
||||
if current is None or priority < current[0]:
|
||||
found[base_key] = (priority, url)
|
||||
fresh: List[str] = []
|
||||
for base_key, (_, url) in found.items():
|
||||
if base_key in self._seen_image_keys:
|
||||
continue
|
||||
self._seen_image_keys.add(base_key)
|
||||
fresh.append(url)
|
||||
return fresh
|
||||
|
||||
def parse_chunk(self, raw_response: str) -> Dict[str, Any]:
|
||||
result: Dict[str, Any] = {
|
||||
"content": "",
|
||||
"images": [],
|
||||
"done": False,
|
||||
"error": None,
|
||||
}
|
||||
|
||||
try:
|
||||
if isinstance(raw_response, (bytes, bytearray)):
|
||||
raw_response = raw_response.decode("utf-8", errors="ignore")
|
||||
elif not isinstance(raw_response, str):
|
||||
raw_response = str(raw_response)
|
||||
|
||||
if not raw_response:
|
||||
return result
|
||||
|
||||
if self._last_raw and raw_response == self._last_raw:
|
||||
return result
|
||||
|
||||
if self._last_raw and raw_response.startswith(self._last_raw):
|
||||
new_data = raw_response[len(self._last_raw):]
|
||||
else:
|
||||
new_data = raw_response
|
||||
|
||||
self._last_raw = raw_response
|
||||
|
||||
result["images"] = self._collect_image_urls(new_data)
|
||||
|
||||
direct_content, direct_done = self._parse_direct_payload(new_data)
|
||||
if direct_content or direct_done:
|
||||
repaired = self._repair_text(direct_content)
|
||||
result["content"] = repaired
|
||||
if repaired:
|
||||
self._assembled_text += repaired
|
||||
self._has_seen_visible_text = True
|
||||
result["done"] = bool(direct_done and self._can_accept_done_signal(repaired))
|
||||
return result
|
||||
|
||||
delta_content, done = self._consume_new_data(new_data)
|
||||
if delta_content:
|
||||
repaired = self._repair_text(delta_content)
|
||||
if repaired != delta_content and self._assembled_text.endswith(delta_content):
|
||||
self._assembled_text = self._assembled_text[:-len(delta_content)] + repaired
|
||||
result["content"] = repaired
|
||||
result["done"] = done
|
||||
|
||||
except Exception as e:
|
||||
logger.debug(f"[DoubaoParser] parse exception: {e}")
|
||||
result["error"] = str(e)
|
||||
|
||||
return result
|
||||
|
||||
def reset(self) -> None:
|
||||
self._last_raw = ""
|
||||
self._pending = ""
|
||||
self._block_texts.clear()
|
||||
self._last_full_message = ""
|
||||
self._assembled_text = ""
|
||||
self._has_seen_visible_text = False
|
||||
self._seen_image_keys.clear()
|
||||
|
||||
def _consume_new_data(self, new_data: str) -> Tuple[str, bool]:
|
||||
normalized = (self._pending + new_data).replace("\r\n", "\n")
|
||||
if not normalized:
|
||||
return "", False
|
||||
|
||||
blocks = normalized.split("\n\n")
|
||||
if normalized.endswith("\n\n"):
|
||||
self._pending = ""
|
||||
complete_blocks = [block for block in blocks if block.strip()]
|
||||
else:
|
||||
self._pending = blocks.pop() if blocks else normalized
|
||||
complete_blocks = [block for block in blocks if block.strip()]
|
||||
|
||||
content_parts: List[str] = []
|
||||
done = False
|
||||
|
||||
for block in complete_blocks:
|
||||
block_content, block_done = self._parse_event_block(block)
|
||||
if block_content:
|
||||
content_parts.append(block_content)
|
||||
self._assembled_text += block_content
|
||||
self._has_seen_visible_text = True
|
||||
if block_done:
|
||||
done = True
|
||||
|
||||
return "".join(content_parts), done
|
||||
|
||||
def _parse_direct_payload(self, payload: str) -> Tuple[str, bool]:
|
||||
stripped = (payload or "").strip()
|
||||
if not stripped or not stripped.startswith("{"):
|
||||
return "", False
|
||||
|
||||
try:
|
||||
data = json.loads(stripped)
|
||||
except json.JSONDecodeError:
|
||||
return "", False
|
||||
|
||||
logger.debug(f"[DoubaoParser][DirectJSON] {self._describe_direct_json(data)}")
|
||||
|
||||
embedded_stream = self._extract_embedded_stream_payload(data)
|
||||
if embedded_stream:
|
||||
logger.debug(
|
||||
"[DoubaoParser][DirectJSON] branch=embedded_stream "
|
||||
f"payload_len={len(embedded_stream)}"
|
||||
)
|
||||
return self._consume_new_data(embedded_stream)
|
||||
|
||||
logger.debug("[DoubaoParser][DirectJSON] branch=direct_content")
|
||||
return self._extract_direct_json_content(data)
|
||||
|
||||
@staticmethod
|
||||
def _extract_embedded_stream_payload(data: Any) -> str:
|
||||
if not isinstance(data, dict):
|
||||
return ""
|
||||
|
||||
containers: List[Dict[str, Any]] = [data]
|
||||
body = data.get("body")
|
||||
if isinstance(body, dict):
|
||||
containers.append(body)
|
||||
|
||||
for container in containers:
|
||||
for stream_key in ("_stream", "stream"):
|
||||
stream_data = container.get(stream_key)
|
||||
if not isinstance(stream_data, dict):
|
||||
continue
|
||||
|
||||
full_text = stream_data.get("fullText")
|
||||
if isinstance(full_text, str) and full_text:
|
||||
return full_text
|
||||
|
||||
chunks = stream_data.get("chunks")
|
||||
if not isinstance(chunks, list):
|
||||
continue
|
||||
|
||||
parts: List[str] = []
|
||||
for chunk in chunks:
|
||||
if not isinstance(chunk, dict):
|
||||
continue
|
||||
chunk_data = chunk.get("data")
|
||||
if isinstance(chunk_data, str) and chunk_data:
|
||||
parts.append(chunk_data)
|
||||
if parts:
|
||||
return "".join(parts)
|
||||
|
||||
return ""
|
||||
|
||||
def _parse_event_block(self, block: str) -> Tuple[str, bool]:
|
||||
event_name = ""
|
||||
data_lines: List[str] = []
|
||||
|
||||
for raw_line in block.split("\n"):
|
||||
line = raw_line.strip()
|
||||
if not line:
|
||||
continue
|
||||
if line.startswith("event:"):
|
||||
event_name = line[6:].strip()
|
||||
elif line.startswith("data:"):
|
||||
data_lines.append(line[5:].strip())
|
||||
|
||||
payload = "\n".join(data_lines).strip()
|
||||
if not event_name:
|
||||
return "", False
|
||||
|
||||
if not payload:
|
||||
return "", False
|
||||
|
||||
if event_name in {"STREAM_FINISH", "STREAM_END", "DONE", "COMPLETION"}:
|
||||
logger.debug(f"[DoubaoParser][SSE] event={event_name} done_only=1")
|
||||
return "", self._can_accept_done_signal()
|
||||
|
||||
if event_name == "CHUNK_DELTA":
|
||||
text = self._extract_chunk_delta(payload)
|
||||
text = self._repair_text(text)
|
||||
if text:
|
||||
logger.debug(
|
||||
f"[DoubaoParser][SSE] event=CHUNK_DELTA text_len={len(text)} "
|
||||
f"preview={text[:60]!r}"
|
||||
)
|
||||
return text, False
|
||||
|
||||
if event_name == "STREAM_MSG_NOTIFY":
|
||||
text = self._extract_stream_msg_notify(payload)
|
||||
text = self._repair_text(text)
|
||||
if text:
|
||||
logger.debug(
|
||||
f"[DoubaoParser][SSE] event=STREAM_MSG_NOTIFY text_len={len(text)} "
|
||||
f"preview={text[:60]!r}"
|
||||
)
|
||||
return text, False
|
||||
|
||||
if event_name == "STREAM_CHUNK":
|
||||
text, done = self._extract_stream_chunk(payload)
|
||||
text = self._repair_text(text)
|
||||
if text or done:
|
||||
logger.debug(
|
||||
f"[DoubaoParser][SSE] event=STREAM_CHUNK text_len={len(text)} "
|
||||
f"done={done} preview={text[:60]!r}"
|
||||
)
|
||||
return text, bool(done and self._can_accept_done_signal(text))
|
||||
|
||||
if event_name == "FULL_MSG_NOTIFY":
|
||||
text = self._extract_full_msg_notify(payload)
|
||||
text = self._repair_text(text)
|
||||
if text:
|
||||
logger.debug(
|
||||
f"[DoubaoParser][SSE] event=FULL_MSG_NOTIFY text_len={len(text)} "
|
||||
f"preview={text[:60]!r}"
|
||||
)
|
||||
return text, False
|
||||
|
||||
if event_name == "SSE_REPLY_END":
|
||||
text, done = self._extract_reply_end(payload)
|
||||
text = self._repair_text(text)
|
||||
logger.debug(
|
||||
f"[DoubaoParser][SSE] event=SSE_REPLY_END text_len={len(text)} "
|
||||
f"done={done} preview={text[:60]!r}"
|
||||
)
|
||||
return text, bool(done and self._can_accept_done_signal(text))
|
||||
|
||||
return "", False
|
||||
|
||||
def _extract_direct_json_content(self, data: Any) -> Tuple[str, bool]:
|
||||
if not isinstance(data, dict):
|
||||
return "", False
|
||||
|
||||
role = data.get("role")
|
||||
content = data.get("content")
|
||||
if role == "assistant" and isinstance(content, str) and content:
|
||||
logger.debug(
|
||||
"[DoubaoParser][DirectJSON] content_source=role.content "
|
||||
f"len={len(content)}"
|
||||
)
|
||||
return self._extract_full_message_delta(content), True
|
||||
|
||||
message = data.get("message")
|
||||
if isinstance(message, dict):
|
||||
role = message.get("role")
|
||||
content = message.get("content")
|
||||
if role == "assistant" and isinstance(content, str) and content:
|
||||
logger.debug(
|
||||
"[DoubaoParser][DirectJSON] content_source=message.content "
|
||||
f"len={len(content)}"
|
||||
)
|
||||
return self._extract_full_message_delta(content), True
|
||||
|
||||
choices = data.get("choices")
|
||||
if isinstance(choices, list):
|
||||
parts: List[str] = []
|
||||
for choice in choices:
|
||||
if not isinstance(choice, dict):
|
||||
continue
|
||||
delta = choice.get("delta")
|
||||
if isinstance(delta, dict):
|
||||
text = delta.get("content")
|
||||
if isinstance(text, str) and text:
|
||||
parts.append(text)
|
||||
message_obj = choice.get("message")
|
||||
if isinstance(message_obj, dict):
|
||||
text = message_obj.get("content")
|
||||
if isinstance(text, str) and text:
|
||||
parts.append(text)
|
||||
if parts:
|
||||
logger.debug(
|
||||
"[DoubaoParser][DirectJSON] content_source=choices "
|
||||
f"parts={len(parts)} total_len={len(''.join(parts))}"
|
||||
)
|
||||
return self._extract_full_message_delta("".join(parts)), bool(data.get("done", True))
|
||||
|
||||
return "", False
|
||||
|
||||
@staticmethod
|
||||
def _describe_direct_json(data: Any) -> str:
|
||||
if not isinstance(data, dict):
|
||||
return f"type={type(data).__name__}"
|
||||
|
||||
def _keys_of(obj: Any) -> list[str]:
|
||||
if isinstance(obj, dict):
|
||||
return [str(k) for k in list(obj.keys())[:8]]
|
||||
return []
|
||||
|
||||
message = data.get("message")
|
||||
body = data.get("body")
|
||||
message_content = message.get("content") if isinstance(message, dict) else None
|
||||
message_content_len = len(message_content) if isinstance(message_content, str) else 0
|
||||
choices = data.get("choices")
|
||||
choices_len = len(choices) if isinstance(choices, list) else 0
|
||||
|
||||
return (
|
||||
f"keys={_keys_of(data)}, "
|
||||
f"stream_keys={_keys_of(data.get('stream'))}, "
|
||||
f"_stream_keys={_keys_of(data.get('_stream'))}, "
|
||||
f"body_keys={_keys_of(body)}, "
|
||||
f"message_keys={_keys_of(message)}, "
|
||||
f"message_content_len={message_content_len}, "
|
||||
f"choices_len={choices_len}"
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _extract_chunk_delta(payload: str) -> str:
|
||||
try:
|
||||
data = json.loads(payload)
|
||||
except json.JSONDecodeError:
|
||||
return ""
|
||||
|
||||
text = data.get("text", "")
|
||||
return text if isinstance(text, str) else ""
|
||||
|
||||
def _extract_stream_msg_notify(self, payload: str) -> str:
|
||||
try:
|
||||
data = json.loads(payload)
|
||||
except json.JSONDecodeError:
|
||||
return ""
|
||||
|
||||
content = data.get("content", {})
|
||||
return self._extract_content_blocks(content.get("content_block"))
|
||||
|
||||
def _extract_full_msg_notify(self, payload: str) -> str:
|
||||
try:
|
||||
data = json.loads(payload)
|
||||
except json.JSONDecodeError:
|
||||
return ""
|
||||
|
||||
message = data.get("message")
|
||||
if not isinstance(message, dict):
|
||||
return ""
|
||||
|
||||
if message.get("user_type") != 2:
|
||||
return ""
|
||||
|
||||
content_block_text = self._extract_content_blocks(message.get("content_block"))
|
||||
if content_block_text:
|
||||
return content_block_text
|
||||
|
||||
raw_content = message.get("content")
|
||||
if not isinstance(raw_content, str) or not raw_content.strip():
|
||||
return ""
|
||||
|
||||
try:
|
||||
parsed_content = json.loads(raw_content)
|
||||
except json.JSONDecodeError:
|
||||
return ""
|
||||
|
||||
return self._extract_content_blocks(parsed_content)
|
||||
|
||||
def _extract_stream_chunk(self, payload: str) -> Tuple[str, bool]:
|
||||
try:
|
||||
data = json.loads(payload)
|
||||
except json.JSONDecodeError:
|
||||
return "", False
|
||||
|
||||
content_parts: List[str] = []
|
||||
done = False
|
||||
|
||||
for patch in data.get("patch_op", []) or []:
|
||||
if not isinstance(patch, dict):
|
||||
continue
|
||||
|
||||
patch_value = patch.get("patch_value")
|
||||
if not isinstance(patch_value, dict):
|
||||
continue
|
||||
|
||||
text = self._extract_content_blocks(patch_value.get("content_block"))
|
||||
if text:
|
||||
content_parts.append(text)
|
||||
|
||||
ext = patch_value.get("ext")
|
||||
if isinstance(ext, dict) and str(ext.get("is_finish", "")) == "1":
|
||||
done = True
|
||||
|
||||
return "".join(content_parts), done
|
||||
|
||||
def _extract_reply_end(self, payload: str) -> Tuple[str, bool]:
|
||||
try:
|
||||
data = json.loads(payload)
|
||||
except json.JSONDecodeError:
|
||||
return "", True
|
||||
|
||||
if self._assembled_text:
|
||||
return "", True
|
||||
|
||||
finish_attr = data.get("msg_finish_attr")
|
||||
if not isinstance(finish_attr, dict):
|
||||
return "", True
|
||||
|
||||
brief = finish_attr.get("brief")
|
||||
if not isinstance(brief, str) or not brief:
|
||||
return "", True
|
||||
|
||||
return self._extract_full_message_delta(brief), True
|
||||
|
||||
def _can_accept_done_signal(self, pending_text: str = "") -> bool:
|
||||
return bool(
|
||||
self._has_seen_visible_text
|
||||
or self._assembled_text
|
||||
or str(pending_text or "")
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _extract_text_from_block(block: Any) -> Tuple[str, str]:
|
||||
if not isinstance(block, dict):
|
||||
return "", ""
|
||||
|
||||
block_id = block.get("block_id") or block.get("id") or ""
|
||||
if not isinstance(block_id, str):
|
||||
block_id = str(block_id)
|
||||
|
||||
content = block.get("content")
|
||||
if not isinstance(content, dict):
|
||||
return block_id, ""
|
||||
|
||||
text_block = content.get("text_block")
|
||||
if not isinstance(text_block, dict):
|
||||
return block_id, ""
|
||||
|
||||
text = text_block.get("text", "")
|
||||
return block_id, text if isinstance(text, str) else ""
|
||||
|
||||
def _extract_content_blocks(self, blocks: Any) -> str:
|
||||
if not isinstance(blocks, list):
|
||||
return ""
|
||||
|
||||
parts: List[str] = []
|
||||
for block in blocks:
|
||||
block_id, text = self._extract_text_from_block(block)
|
||||
if not text:
|
||||
continue
|
||||
|
||||
previous = self._block_texts.get(block_id, "")
|
||||
if previous and text.startswith(previous):
|
||||
delta = text[len(previous):]
|
||||
elif previous == text:
|
||||
delta = ""
|
||||
else:
|
||||
delta = text
|
||||
|
||||
self._block_texts[block_id] = text
|
||||
|
||||
if delta:
|
||||
parts.append(delta)
|
||||
|
||||
return "".join(parts)
|
||||
|
||||
def _extract_full_message_delta(self, text: str) -> str:
|
||||
if not text:
|
||||
return ""
|
||||
|
||||
previous = self._last_full_message
|
||||
self._last_full_message = text
|
||||
|
||||
if self._assembled_text and text.startswith(self._assembled_text):
|
||||
return text[len(self._assembled_text):]
|
||||
if previous and text.startswith(previous):
|
||||
return text[len(previous):]
|
||||
if previous == text:
|
||||
return ""
|
||||
return text
|
||||
|
||||
@staticmethod
|
||||
def _repair_text(text: str) -> str:
|
||||
if not text:
|
||||
return text
|
||||
|
||||
repaired_whole = DoubaoParser._repair_text_candidate(text)
|
||||
if repaired_whole != text:
|
||||
return repaired_whole
|
||||
|
||||
repaired_segments: List[str] = []
|
||||
changed = False
|
||||
buffer = ""
|
||||
buffer_is_suspicious: bool | None = None
|
||||
|
||||
def flush() -> None:
|
||||
nonlocal buffer, buffer_is_suspicious, changed
|
||||
if not buffer:
|
||||
return
|
||||
if buffer_is_suspicious:
|
||||
repaired = DoubaoParser._repair_text_candidate(buffer)
|
||||
if repaired != buffer:
|
||||
changed = True
|
||||
repaired_segments.append(repaired)
|
||||
else:
|
||||
repaired_segments.append(buffer)
|
||||
buffer = ""
|
||||
buffer_is_suspicious = None
|
||||
|
||||
for ch in text:
|
||||
is_suspicious = DoubaoParser._is_suspicious_char(ch)
|
||||
if buffer_is_suspicious is None:
|
||||
buffer = ch
|
||||
buffer_is_suspicious = is_suspicious
|
||||
continue
|
||||
if is_suspicious == buffer_is_suspicious:
|
||||
buffer += ch
|
||||
continue
|
||||
flush()
|
||||
buffer = ch
|
||||
buffer_is_suspicious = is_suspicious
|
||||
|
||||
flush()
|
||||
if changed:
|
||||
repaired_text = "".join(repaired_segments)
|
||||
if repaired_text != text:
|
||||
logger.debug(
|
||||
"[DoubaoParser][Repair] repaired mixed mojibake "
|
||||
f"(before_len={len(text)}, after_len={len(repaired_text)})"
|
||||
)
|
||||
return repaired_text
|
||||
return text
|
||||
|
||||
@staticmethod
|
||||
def _repair_text_candidate(text: str) -> str:
|
||||
if not text or not DoubaoParser._should_try_repair_text(text):
|
||||
return text
|
||||
|
||||
for source_encoding in ("latin1", "cp1252"):
|
||||
repaired = DoubaoParser._try_redecode_text(text, source_encoding)
|
||||
if not repaired:
|
||||
continue
|
||||
|
||||
if DoubaoParser._looks_more_readable(repaired, text):
|
||||
logger.debug(
|
||||
"[DoubaoParser][Repair] repaired mojibake "
|
||||
f"(enc={source_encoding}, before_len={len(text)}, after_len={len(repaired)})"
|
||||
)
|
||||
return repaired
|
||||
return text
|
||||
|
||||
@staticmethod
|
||||
def _should_try_repair_text(text: str) -> bool:
|
||||
if not text:
|
||||
return False
|
||||
|
||||
if any(0x4E00 <= ord(ch) <= 0x9FFF or ord(ch) > 0xFFFF for ch in text):
|
||||
return False
|
||||
|
||||
cp1252_extra = {
|
||||
0x0152, 0x0153, 0x0160, 0x0161, 0x0178, 0x017D, 0x017E,
|
||||
0x02C6, 0x02DC, 0x2013, 0x2014, 0x2018, 0x2019, 0x201A,
|
||||
0x201C, 0x201D, 0x201E, 0x2020, 0x2021, 0x2022, 0x2026,
|
||||
0x2030, 0x2039, 0x203A, 0x20AC, 0x2122,
|
||||
}
|
||||
suspicious_count = 0
|
||||
for ch in text:
|
||||
if DoubaoParser._is_suspicious_char(ch, cp1252_extra):
|
||||
suspicious_count += 1
|
||||
|
||||
return suspicious_count >= max(2, len(text) // 4)
|
||||
|
||||
@staticmethod
|
||||
def _is_suspicious_char(ch: str, cp1252_extra: set[int] | None = None) -> bool:
|
||||
if not ch:
|
||||
return False
|
||||
if cp1252_extra is None:
|
||||
cp1252_extra = {
|
||||
0x0152, 0x0153, 0x0160, 0x0161, 0x0178, 0x017D, 0x017E,
|
||||
0x02C6, 0x02DC, 0x2013, 0x2014, 0x2018, 0x2019, 0x201A,
|
||||
0x201C, 0x201D, 0x201E, 0x2020, 0x2021, 0x2022, 0x2026,
|
||||
0x2030, 0x2039, 0x203A, 0x20AC, 0x2122,
|
||||
}
|
||||
code = ord(ch)
|
||||
return 0x80 <= code <= 0x00FF or code in cp1252_extra
|
||||
|
||||
@staticmethod
|
||||
def _try_redecode_text(text: str, source_encoding: str) -> str:
|
||||
data = bytearray()
|
||||
|
||||
for ch in text:
|
||||
code = ord(ch)
|
||||
if code <= 0xFF:
|
||||
data.append(code)
|
||||
continue
|
||||
|
||||
try:
|
||||
encoded = ch.encode(source_encoding)
|
||||
except Exception:
|
||||
return ""
|
||||
|
||||
if len(encoded) != 1:
|
||||
return ""
|
||||
|
||||
data.extend(encoded)
|
||||
|
||||
try:
|
||||
return bytes(data).decode("utf-8")
|
||||
except UnicodeDecodeError:
|
||||
return bytes(data).decode("utf-8", errors="ignore")
|
||||
|
||||
@staticmethod
|
||||
def _looks_more_readable(candidate: str, original: str) -> bool:
|
||||
readable_punct = set(".,!?;:'\"()[]{}-_/~`@#$%^&*+=<>,。!?:;()【】《》、~“”‘’…")
|
||||
|
||||
def score(value: str) -> int:
|
||||
total = 0
|
||||
for ch in value:
|
||||
code = ord(ch)
|
||||
if 0x4E00 <= code <= 0x9FFF:
|
||||
total += 2
|
||||
elif code > 0xFFFF:
|
||||
total += 2
|
||||
elif ch.isascii() and (ch.isalnum() or ch.isspace() or ch in readable_punct):
|
||||
total += 1
|
||||
elif ch in readable_punct or ch.isspace():
|
||||
total += 1
|
||||
return total
|
||||
|
||||
return score(candidate) > score(original)
|
||||
|
||||
@classmethod
|
||||
def get_id(cls) -> str:
|
||||
return "doubao"
|
||||
|
||||
@classmethod
|
||||
def get_name(cls) -> str:
|
||||
return "Doubao"
|
||||
|
||||
@classmethod
|
||||
def get_description(cls) -> str:
|
||||
return "Parse Doubao SSE streams with STREAM_MSG_NOTIFY / CHUNK_DELTA events"
|
||||
|
||||
@classmethod
|
||||
def get_supported_patterns(cls) -> List[str]:
|
||||
return ["**/chat/completion**"]
|
||||
|
||||
|
||||
__all__ = ["DoubaoParser"]
|
||||
@@ -0,0 +1,723 @@
|
||||
"""
|
||||
gemini_parser.py - Gemini StreamGenerate 响应解析器
|
||||
|
||||
响应格式特征
|
||||
1. 安全前缀: )]}'\n
|
||||
2. 多块格式: <len>\n[[JSON]]
|
||||
3. 文本路径: outer[0][2] ➜ json.loads ➜ inner[4][0][1][0]
|
||||
4. 文本是“累积”模式:每块都返回从开头到当前的完整文本
|
||||
5. 结束标志: [["di",...]] / [["e",...]] / [["af.httprm",...]]
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
|
||||
from app.core.config import logger
|
||||
from .base import ResponseParser
|
||||
|
||||
|
||||
# ----------------------------------------------------------
|
||||
# 私有工具
|
||||
# ----------------------------------------------------------
|
||||
_ESCAPE_FIXER = re.compile(r"\\([<>`#*_\[\]()])") # 常见转义符号:HTML/CSS/Markdown
|
||||
_EOL_FIXER = re.compile(r"\\\\n") # \\n → \n
|
||||
_STANDALONE_BS = re.compile(r"\\\n") # backslash + real LF
|
||||
_REASONING_HINTS = (
|
||||
"i'm now ",
|
||||
"i am now ",
|
||||
"i've ",
|
||||
"i have ",
|
||||
"i must ",
|
||||
"i clarified",
|
||||
"i've clarified",
|
||||
"i have clarified",
|
||||
"user's request",
|
||||
"assistant persona",
|
||||
"brainstorm",
|
||||
"analyzing",
|
||||
"analysis",
|
||||
"focusing on",
|
||||
"clarifying",
|
||||
"refining",
|
||||
"crafting",
|
||||
"delivering",
|
||||
"defining",
|
||||
"prioritizing",
|
||||
"interpr",
|
||||
)
|
||||
_REASONING_PROCESS_HINTS = (
|
||||
"define",
|
||||
"defined",
|
||||
"defining",
|
||||
"develop",
|
||||
"developing",
|
||||
"draft",
|
||||
"drafting",
|
||||
"craft",
|
||||
"crafted",
|
||||
"crafting",
|
||||
"focus",
|
||||
"focusing",
|
||||
"clarif",
|
||||
"refin",
|
||||
"break",
|
||||
"breaking",
|
||||
"apply",
|
||||
"applying",
|
||||
"confirm",
|
||||
"confirming",
|
||||
)
|
||||
_REASONING_META_HINTS = (
|
||||
"task",
|
||||
"request",
|
||||
"user",
|
||||
"intent",
|
||||
"persona",
|
||||
"story",
|
||||
"parameter",
|
||||
"format",
|
||||
"constraint",
|
||||
"component",
|
||||
"narrative",
|
||||
"title",
|
||||
"scene",
|
||||
"style",
|
||||
"protagonist",
|
||||
"plot",
|
||||
"character",
|
||||
"dialogue",
|
||||
"ending",
|
||||
"climax",
|
||||
"structure",
|
||||
)
|
||||
_GEMINI_GENERATED_MEDIA_URL_RE = re.compile(
|
||||
r"https?://(?:[\w.-]+\.)?(?:googleusercontent\.com|lh3\.googleusercontent\.com)/[^\s\"'<>\]\)\}\\]+",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
_GEMINI_PLACEHOLDER_MEDIA_RE = re.compile(
|
||||
r"https?://(?:[\w.-]+\.)?googleusercontent\.com/(?:image_generation_content|generated_music_content)/\d+",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
_RENDER_START_RE = re.compile(r"<\s*render\b", re.IGNORECASE)
|
||||
_RENDER_CLOSE_RE = re.compile(r"<\s*/\s*render\s*>", re.IGNORECASE)
|
||||
_MEDIA_SCAN_TAIL_CHARS = 4096
|
||||
_MEDIA_SCAN_DELIMITERS = (" ", "\n", "\r", "\t", '"', "'", "<", ">", "\\")
|
||||
|
||||
|
||||
def _clean_escaped(text: str) -> str:
|
||||
"""
|
||||
Gemini 返回的字符串仍可能残留多余的反斜杠:
|
||||
1. \\<ctx\\> → <ctx>
|
||||
2. \\\\n → \n → 换行
|
||||
3. \\`code\\` → `code`
|
||||
4. 反斜杠 + 真 \n → 真 \n
|
||||
"""
|
||||
if "\\" not in text:
|
||||
return text
|
||||
|
||||
# 1) 处理 \n / \\n
|
||||
text = _STANDALONE_BS.sub("\n", text) # \ + 真换行
|
||||
text = _EOL_FIXER.sub(r"\n", text) # \\n → \n
|
||||
|
||||
# 2) 去除转义符号前缀(只对 < > ` 三个常见 HTML 符号)
|
||||
text = _ESCAPE_FIXER.sub(r"\1", text)
|
||||
|
||||
return text
|
||||
|
||||
|
||||
# ----------------------------------------------------------
|
||||
# 解析器主体
|
||||
# ----------------------------------------------------------
|
||||
class GeminiParser(ResponseParser):
|
||||
"""
|
||||
Google Gemini StreamGenerate 响应解析器
|
||||
"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._last_len = 0 # 已发送给上层的字符数
|
||||
self._full_cache = "" # 最新完整文本
|
||||
self._last_reasoning_len = 0
|
||||
self._full_reasoning_cache = ""
|
||||
self._last_raw_length = 0
|
||||
self._pending_raw = ""
|
||||
self._seen_media_refs: set[str] = set()
|
||||
self._media_scan_tail = ""
|
||||
|
||||
# ---------- 对外接口 ---------- #
|
||||
def parse_chunk(self, raw: str | bytes) -> Dict[str, Any]:
|
||||
"""
|
||||
解析 *单个* HTTP chunk,返回增量内容
|
||||
"""
|
||||
if isinstance(raw, bytes):
|
||||
raw = raw.decode("utf-8", errors="ignore")
|
||||
elif not isinstance(raw, str):
|
||||
raw = str(raw)
|
||||
|
||||
try:
|
||||
previous_len = int(getattr(self, "_last_raw_length", 0) or 0)
|
||||
previous_raw = getattr(self, "_last_raw_response", "")
|
||||
will_reset = bool(previous_len and previous_raw and not raw.startswith(previous_raw))
|
||||
is_stream_start = (previous_len == 0 and not self._pending_raw) or will_reset
|
||||
new_data = self._prepare_incremental_raw_response(raw)
|
||||
if not new_data:
|
||||
return {
|
||||
"content": "",
|
||||
"reasoning_content": "",
|
||||
"images": [],
|
||||
"done": False,
|
||||
"error": None,
|
||||
"unclosed_render_output": self._has_unclosed_render_output(
|
||||
self._full_cache
|
||||
),
|
||||
}
|
||||
|
||||
full_txt, full_reasoning, done = self._parse_incremental(
|
||||
new_data,
|
||||
is_stream_start=is_stream_start,
|
||||
)
|
||||
images = self._extract_generated_images_from_increment(new_data)
|
||||
except Exception as exc: # pragma: no cover
|
||||
logger.debug(f"[GeminiParser] 解析异常: {exc}")
|
||||
return {"content": "", "reasoning_content": "", "images": [], "done": False, "error": str(exc)}
|
||||
|
||||
delta = ""
|
||||
if full_txt is not None and len(full_txt) > self._last_len:
|
||||
delta = full_txt[self._last_len :]
|
||||
self._last_len = len(full_txt)
|
||||
self._full_cache = full_txt
|
||||
|
||||
delta_reasoning = ""
|
||||
if full_reasoning is not None and len(full_reasoning) > self._last_reasoning_len:
|
||||
delta_reasoning = full_reasoning[self._last_reasoning_len :]
|
||||
self._last_reasoning_len = len(full_reasoning)
|
||||
self._full_reasoning_cache = full_reasoning
|
||||
|
||||
return {
|
||||
"content": delta,
|
||||
"reasoning_content": delta_reasoning,
|
||||
"images": images,
|
||||
"done": done,
|
||||
"error": None,
|
||||
"unclosed_render_output": self._has_unclosed_render_output(
|
||||
self._full_cache
|
||||
),
|
||||
}
|
||||
|
||||
def reset(self) -> None:
|
||||
self._last_len = 0
|
||||
self._full_cache = ""
|
||||
self._last_reasoning_len = 0
|
||||
self._full_reasoning_cache = ""
|
||||
self._last_raw_length = 0
|
||||
self._last_raw_response = ""
|
||||
self._pending_raw = ""
|
||||
self._seen_media_refs.clear()
|
||||
self._media_scan_tail = ""
|
||||
|
||||
@staticmethod
|
||||
def _list_get(value: Any, index: int) -> Any:
|
||||
if isinstance(value, list) and 0 <= index < len(value):
|
||||
return value[index]
|
||||
return None
|
||||
|
||||
@classmethod
|
||||
def _nested_list_get(cls, value: Any, *path: int) -> Any:
|
||||
current = value
|
||||
for index in path:
|
||||
current = cls._list_get(current, index)
|
||||
if current is None:
|
||||
return None
|
||||
return current
|
||||
|
||||
@staticmethod
|
||||
def _looks_like_content(value: Any) -> bool:
|
||||
if not isinstance(value, str):
|
||||
return False
|
||||
|
||||
text = value.strip()
|
||||
if not text:
|
||||
return False
|
||||
|
||||
lowered = text.lower()
|
||||
if lowered.startswith(("r_", "c_", "rc_")) and len(text) <= 80:
|
||||
return False
|
||||
|
||||
if text.startswith(("http://", "https://", "data:image", "blob:")):
|
||||
return True
|
||||
if "\n" in text or "\r" in text:
|
||||
return True
|
||||
if "<" in text and ">" in text:
|
||||
return True
|
||||
if any("\u4e00" <= ch <= "\u9fff" for ch in text):
|
||||
return True
|
||||
|
||||
alpha_count = sum(ch.isalpha() for ch in text)
|
||||
return len(text) >= 16 and alpha_count >= 6
|
||||
|
||||
@classmethod
|
||||
def _iter_content_candidates(cls, value: Any):
|
||||
if isinstance(value, str):
|
||||
if cls._looks_like_content(value):
|
||||
yield value
|
||||
return
|
||||
|
||||
if isinstance(value, list):
|
||||
for item in value:
|
||||
yield from cls._iter_content_candidates(item)
|
||||
return
|
||||
|
||||
if isinstance(value, dict):
|
||||
for item in value.values():
|
||||
yield from cls._iter_content_candidates(item)
|
||||
|
||||
@staticmethod
|
||||
def _decode_content_text(content: str) -> str:
|
||||
try:
|
||||
decoded = json.loads(f'"{content}"')
|
||||
if isinstance(decoded, str):
|
||||
content = decoded
|
||||
except json.JSONDecodeError:
|
||||
content = _clean_escaped(content)
|
||||
return _clean_escaped(content)
|
||||
|
||||
@staticmethod
|
||||
def _has_unclosed_render_output(text: str) -> bool:
|
||||
text = str(text or "")
|
||||
if not text:
|
||||
return False
|
||||
|
||||
stripped = text.lstrip()
|
||||
if not _RENDER_START_RE.match(stripped):
|
||||
return False
|
||||
|
||||
render_starts = list(_RENDER_START_RE.finditer(text))
|
||||
if not render_starts:
|
||||
return False
|
||||
|
||||
render_closes = list(_RENDER_CLOSE_RE.finditer(text))
|
||||
last_open_at = render_starts[-1].start()
|
||||
last_close_at = render_closes[-1].start() if render_closes else -1
|
||||
return last_open_at > last_close_at
|
||||
|
||||
@staticmethod
|
||||
def _looks_like_reasoning(value: str) -> bool:
|
||||
text = str(value or "").strip()
|
||||
if not text:
|
||||
return False
|
||||
|
||||
lowered = " ".join(text.lower().split())
|
||||
if not lowered:
|
||||
return False
|
||||
|
||||
looks_like_structured_answer = text.startswith(
|
||||
("{", "```json", "```", "<render", "<segment", "<kaitou", "<scene")
|
||||
) or any(token in text for token in ('"role"', '"content"', '"tool_calls"', '"decision"'))
|
||||
|
||||
if text.startswith("**") and any(hint in lowered for hint in _REASONING_HINTS):
|
||||
return True
|
||||
|
||||
if (
|
||||
not looks_like_structured_answer
|
||||
and lowered.startswith(("i'm ", "i am ", "i've ", "i have "))
|
||||
and any(hint in lowered for hint in _REASONING_PROCESS_HINTS)
|
||||
and any(hint in lowered for hint in _REASONING_META_HINTS)
|
||||
):
|
||||
return True
|
||||
|
||||
return any(
|
||||
marker in lowered
|
||||
for marker in (
|
||||
"i'm now carefully",
|
||||
"i am now carefully",
|
||||
"i'm now focused",
|
||||
"i am now focused",
|
||||
"i've established",
|
||||
"i have established",
|
||||
"the user's compositional request",
|
||||
"the user's intent",
|
||||
"helpful assistant persona",
|
||||
)
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def _score_candidate(cls, value: str) -> int:
|
||||
text = str(value or "").strip()
|
||||
if not text:
|
||||
return -10_000
|
||||
|
||||
score = 0
|
||||
lowered = text.lower()
|
||||
|
||||
if text.startswith(("{", "```json", "```", "<render", "<segment", "<kaitou", "<scene")):
|
||||
score += 120
|
||||
if any(token in text for token in ('"role"', '"content"', '"tool_calls"', '"decision"', '"name"', '"description"')):
|
||||
score += 80
|
||||
if any("\u4e00" <= ch <= "\u9fff" for ch in text):
|
||||
score += 60
|
||||
if text.startswith(("http://", "https://", "data:image", "blob:")):
|
||||
score += 40
|
||||
if "\n" in text or "\r" in text:
|
||||
score += 10
|
||||
if len(text) >= 80:
|
||||
score += 10
|
||||
elif len(text) >= 24:
|
||||
score += 5
|
||||
|
||||
if text.startswith("**"):
|
||||
score -= 30
|
||||
|
||||
if lowered.startswith(("r_", "c_", "rc_")) and len(text) <= 80:
|
||||
score -= 200
|
||||
|
||||
return score
|
||||
|
||||
@classmethod
|
||||
def _select_best_candidate(cls, value: Any) -> Optional[str]:
|
||||
seen: set[str] = set()
|
||||
best_text: Optional[str] = None
|
||||
best_score: Optional[int] = None
|
||||
|
||||
for candidate in cls._iter_content_candidates(value):
|
||||
decoded = cls._decode_content_text(candidate)
|
||||
normalized = decoded.strip()
|
||||
if not normalized or normalized in seen:
|
||||
continue
|
||||
seen.add(normalized)
|
||||
|
||||
score = cls._score_candidate(decoded)
|
||||
if (
|
||||
best_text is None
|
||||
or best_score is None
|
||||
or score > best_score
|
||||
or (score == best_score and len(decoded) > len(best_text))
|
||||
):
|
||||
best_text = decoded
|
||||
best_score = score
|
||||
|
||||
if best_text is None or best_score is None or best_score <= 0:
|
||||
return None
|
||||
|
||||
return best_text
|
||||
|
||||
@classmethod
|
||||
def _extract_direct_answer(cls, inner: Any) -> Optional[str]:
|
||||
content_block = cls._list_get(inner, 4)
|
||||
if not isinstance(content_block, list):
|
||||
return None
|
||||
|
||||
best_text: Optional[str] = None
|
||||
best_score: Optional[int] = None
|
||||
|
||||
for item in content_block:
|
||||
direct_payload = cls._list_get(item, 1)
|
||||
if direct_payload is None:
|
||||
continue
|
||||
|
||||
candidate = cls._select_best_candidate(direct_payload)
|
||||
if not candidate:
|
||||
continue
|
||||
|
||||
score = cls._score_candidate(candidate)
|
||||
if (
|
||||
best_text is None
|
||||
or best_score is None
|
||||
or score > best_score
|
||||
or (score == best_score and len(candidate) > len(best_text))
|
||||
):
|
||||
best_text = candidate
|
||||
best_score = score
|
||||
|
||||
return best_text
|
||||
|
||||
@classmethod
|
||||
def _extract_thinking(cls, inner: Any) -> Optional[str]:
|
||||
try:
|
||||
content_block = cls._list_get(inner, 4)
|
||||
if not isinstance(content_block, list) or not content_block:
|
||||
return None
|
||||
|
||||
for item in content_block:
|
||||
if not isinstance(item, list):
|
||||
continue
|
||||
|
||||
candidate = item
|
||||
if isinstance(item, list) and item and isinstance(item[0], list):
|
||||
candidate = item[0]
|
||||
|
||||
if not isinstance(candidate, list):
|
||||
continue
|
||||
|
||||
# 1. 优先读取标准 37 槽位(零开销命中)
|
||||
if len(candidate) > 37 and isinstance(candidate[37], list) and candidate[37]:
|
||||
thinking_data = candidate[37]
|
||||
if (
|
||||
isinstance(thinking_data, list)
|
||||
and thinking_data
|
||||
and isinstance(thinking_data[0], list)
|
||||
and thinking_data[0]
|
||||
and isinstance(thinking_data[0][0], str)
|
||||
):
|
||||
text = thinking_data[0][0].strip()
|
||||
if text:
|
||||
return text
|
||||
|
||||
# 2. 备用:结构化特征扫描(抵御未来索引偏移变动)
|
||||
for sub in candidate:
|
||||
if (
|
||||
isinstance(sub, list)
|
||||
and len(sub) >= 2
|
||||
and isinstance(sub[0], list)
|
||||
and sub[0]
|
||||
and isinstance(sub[0][0], str)
|
||||
):
|
||||
text = sub[0][0].strip()
|
||||
if text and (
|
||||
text.startswith(("**Initiating", "**Assessing", "**Thinking", "**Outlining", "**Dissecting", "**Refining"))
|
||||
or "I'm " in text or "I've " in text or "Initiating " in text
|
||||
):
|
||||
return text
|
||||
except Exception:
|
||||
pass
|
||||
return None
|
||||
|
||||
# ---------- 内部逻辑 ---------- #
|
||||
def _parse_incremental(
|
||||
self,
|
||||
raw_text: str,
|
||||
*,
|
||||
is_stream_start: bool = False,
|
||||
) -> Tuple[Optional[str], Optional[str], bool]:
|
||||
"""
|
||||
解析 Gemini 的 *增量* HTTP body 片段,返回 (完整文本, 完整思考过程, 是否结束)
|
||||
"""
|
||||
combined = f"{self._pending_raw}{str(raw_text or '')}"
|
||||
if not combined:
|
||||
return None, None, False
|
||||
|
||||
# 仅在全新流的首个片段上清理安全前缀。
|
||||
if is_stream_start:
|
||||
combined = combined.lstrip(")]}'\n")
|
||||
|
||||
lines = combined.splitlines(keepends=True)
|
||||
full_content: Optional[str] = None
|
||||
full_reasoning: Optional[str] = None
|
||||
done = False
|
||||
pending_start = len(combined)
|
||||
i = 0
|
||||
cursor = 0
|
||||
while i < len(lines):
|
||||
line = lines[i]
|
||||
line_start = cursor
|
||||
line_end = line_start + len(line)
|
||||
meta = line.strip()
|
||||
if not meta: # 空行
|
||||
cursor = line_end
|
||||
i += 1
|
||||
continue
|
||||
|
||||
if meta.isdigit(): # 长度行
|
||||
if i + 1 >= len(lines):
|
||||
pending_start = line_start
|
||||
break
|
||||
json_block = lines[i + 1].rstrip("\r\n")
|
||||
|
||||
try:
|
||||
outer = json.loads(json_block)
|
||||
except json.JSONDecodeError:
|
||||
pending_start = line_start
|
||||
break
|
||||
|
||||
if self._is_end_signal(outer):
|
||||
done = True
|
||||
else:
|
||||
content, reasoning = self._extract_content(outer)
|
||||
if content:
|
||||
full_content = content
|
||||
if reasoning:
|
||||
full_reasoning = reasoning
|
||||
cursor = line_end + len(lines[i + 1])
|
||||
i += 2
|
||||
else:
|
||||
cursor = line_end
|
||||
i += 1
|
||||
|
||||
self._pending_raw = combined[pending_start:] if pending_start < len(combined) else ""
|
||||
return full_content, full_reasoning, done
|
||||
|
||||
@staticmethod
|
||||
def _is_end_signal(data: list) -> bool:
|
||||
"""
|
||||
结束块格式:
|
||||
[["di", 123]] / [["e",...]] / [["af.httprm",...]]
|
||||
"""
|
||||
if (
|
||||
isinstance(data, list)
|
||||
and data
|
||||
and isinstance(data[0], list)
|
||||
and data[0]
|
||||
and data[0][0] in ("di", "e", "af.httprm")
|
||||
):
|
||||
return True
|
||||
return False
|
||||
|
||||
# -------- 核心:提取 & 转义修复 -------- #
|
||||
def _extract_content(self, outer: list) -> Tuple[Optional[str], Optional[str]]:
|
||||
"""
|
||||
outer → inner → (content, reasoning_content)
|
||||
"""
|
||||
try:
|
||||
first = outer[0]
|
||||
if not (isinstance(first, list) and len(first) >= 3 and first[0] == "wrb.fr"):
|
||||
return None, None
|
||||
|
||||
inner_raw: str = first[2]
|
||||
if not isinstance(inner_raw, str):
|
||||
return None, None
|
||||
inner = json.loads(inner_raw) # type: ignore[arg-type]
|
||||
|
||||
content = self._extract_direct_answer(inner)
|
||||
reasoning = self._extract_thinking(inner)
|
||||
return content, reasoning
|
||||
|
||||
except Exception as exc: # pragma: no cover
|
||||
logger.debug(f"[GeminiParser] 提取失败: {exc}")
|
||||
return None, None
|
||||
|
||||
def should_fallback_to_dom_when_no_visible_content(self) -> bool:
|
||||
return True
|
||||
|
||||
def get_media_generation_state(
|
||||
self,
|
||||
raw_response: str = "",
|
||||
parse_result: Optional[Dict[str, Any]] = None,
|
||||
) -> Dict[str, Any]:
|
||||
raw_text = str(raw_response or "")
|
||||
if not raw_text:
|
||||
return {}
|
||||
|
||||
parse_images = parse_result.get("images") if isinstance(parse_result, dict) else None
|
||||
if parse_images or self._seen_media_refs or self._has_generated_image(raw_text):
|
||||
return {}
|
||||
|
||||
hint_parts: List[str] = []
|
||||
lowered = raw_text.lower()
|
||||
wait_timeout_seconds = None
|
||||
|
||||
if "nano banana" in lowered:
|
||||
hint_parts.append("正在加载 Nano Banana 2...")
|
||||
wait_timeout_seconds = max(float(wait_timeout_seconds or 0), 90.0)
|
||||
|
||||
if "image_generation_content/" in lowered:
|
||||
hint_parts.append("image_generation_content")
|
||||
wait_timeout_seconds = max(float(wait_timeout_seconds or 0), 90.0)
|
||||
|
||||
if not hint_parts:
|
||||
return {}
|
||||
|
||||
deduped_hint_parts: List[str] = []
|
||||
seen = set()
|
||||
for item in hint_parts:
|
||||
if item in seen:
|
||||
continue
|
||||
seen.add(item)
|
||||
deduped_hint_parts.append(item)
|
||||
|
||||
return {
|
||||
"pending": True,
|
||||
"media_type": "image",
|
||||
"hint_text": "\n\n".join(deduped_hint_parts[:3]),
|
||||
"wait_timeout_seconds": wait_timeout_seconds,
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _build_media_scan_tail(raw_text: str) -> str:
|
||||
text = str(raw_text or "")
|
||||
if not text:
|
||||
return ""
|
||||
|
||||
last_delimiter = -1
|
||||
for delimiter in _MEDIA_SCAN_DELIMITERS:
|
||||
last_delimiter = max(last_delimiter, text.rfind(delimiter))
|
||||
|
||||
tail = text[last_delimiter + 1:] if last_delimiter >= 0 else text
|
||||
if len(tail) > _MEDIA_SCAN_TAIL_CHARS:
|
||||
tail = tail[-_MEDIA_SCAN_TAIL_CHARS:]
|
||||
return tail
|
||||
|
||||
@classmethod
|
||||
def _iter_generated_image_urls(cls, raw_response: str):
|
||||
raw_text = str(raw_response or "")
|
||||
if not raw_text:
|
||||
return
|
||||
|
||||
for match in _GEMINI_GENERATED_MEDIA_URL_RE.finditer(raw_text):
|
||||
url = str(match.group(0) or "").strip().rstrip(".,;:!?")
|
||||
if not url:
|
||||
continue
|
||||
if _GEMINI_PLACEHOLDER_MEDIA_RE.fullmatch(url):
|
||||
continue
|
||||
lowered = url.lower()
|
||||
if "/gg/" not in lowered and "/gg-dl/" not in lowered:
|
||||
continue
|
||||
if not any(ext in lowered for ext in (".png", ".jpg", ".jpeg", ".webp", ".gif", ".bmp", ".avif")):
|
||||
continue
|
||||
yield url
|
||||
|
||||
@classmethod
|
||||
def _has_generated_image(cls, raw_response: str) -> bool:
|
||||
return any(cls._iter_generated_image_urls(raw_response))
|
||||
|
||||
def _extract_generated_images_from_increment(self, new_data: str) -> List[Dict[str, Any]]:
|
||||
scan_text = f"{self._media_scan_tail}{str(new_data or '')}"
|
||||
images = self._extract_generated_images(scan_text)
|
||||
self._media_scan_tail = self._build_media_scan_tail(scan_text)
|
||||
return images
|
||||
|
||||
def _extract_generated_images(self, raw_response: str, *, mark_seen: bool = True) -> List[Dict[str, Any]]:
|
||||
local_seen: set[str] = set()
|
||||
|
||||
images: List[Dict[str, Any]] = []
|
||||
for url in self._iter_generated_image_urls(raw_response):
|
||||
if url in local_seen:
|
||||
continue
|
||||
local_seen.add(url)
|
||||
if mark_seen:
|
||||
if url in self._seen_media_refs:
|
||||
continue
|
||||
self._seen_media_refs.add(url)
|
||||
images.append(
|
||||
{
|
||||
"media_type": "image",
|
||||
"kind": "url",
|
||||
"url": url,
|
||||
"data_uri": None,
|
||||
"mime": None,
|
||||
"byte_size": None,
|
||||
"source": "gemini_stream",
|
||||
}
|
||||
)
|
||||
|
||||
return images
|
||||
|
||||
# ---------- 元数据 ---------- #
|
||||
@classmethod
|
||||
def get_id(cls) -> str:
|
||||
return "gemini"
|
||||
|
||||
@classmethod
|
||||
def get_name(cls) -> str:
|
||||
return "Gemini StreamGenerate"
|
||||
|
||||
@classmethod
|
||||
def get_description(cls) -> str:
|
||||
return "解析 Google Gemini 的 StreamGenerate 响应"
|
||||
|
||||
@classmethod
|
||||
def get_supported_patterns(cls) -> List[str]:
|
||||
return ["StreamGenerate"]
|
||||
|
||||
|
||||
__all__ = ["GeminiParser"]
|
||||
@@ -0,0 +1,384 @@
|
||||
"""
|
||||
glm_parser.py - ChatGLM SSE response parser.
|
||||
|
||||
Observed stream traits:
|
||||
- content-type: text/event-stream
|
||||
- each SSE block is carried in a data: line with a JSON payload
|
||||
- think/tool_calls/text/system_error are all emitted through parts[*].content[*]
|
||||
- text payloads are full rendered text snapshots, not pure deltas
|
||||
- stream ends when a text part status becomes finish or top-level status becomes finish
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
from typing import Any, Dict, List
|
||||
|
||||
from app.core.config import logger
|
||||
from .base import ResponseParser
|
||||
|
||||
|
||||
class GLMParser(ResponseParser):
|
||||
"""Parse ChatGLM SSE streams while ignoring think/tool-call noise."""
|
||||
|
||||
_TERMINAL_STATUSES = {"finish", "finished", "intervene", "intervened"}
|
||||
_PPT_FOLLOWUP_COMMANDS = {"change_mode_to_engine_ppt"}
|
||||
_PPT_TEMPLATE_MARKERS = ('"theme"', '"background"', '"palette"')
|
||||
_HIDDEN_REPAIR_TEXT_PATTERNS = (
|
||||
"xml-style tool call",
|
||||
"could not be parsed into a valid declared tool",
|
||||
"could not be parsed into valid tool_calls",
|
||||
"validation errors",
|
||||
"the rejected assistant reply was",
|
||||
)
|
||||
_THINK_REASONING_PREFIX_RE = re.compile(
|
||||
r"^\s*(let me\b|looking at\b|the user has\b|让我|我来|先分析|仔细分析)",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
_EMBEDDED_REPLY_LABEL_RE = re.compile(
|
||||
r"(?:the rejected assistant reply was|the rejected response was|被拒绝的响应是|被拒绝的助手回复是)\s*[::]?\s*```",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
_VISIBLE_MARKERS = ("<render", "<segment", "<context_summary", "<q>", "<act>", "<inner>", "<aside>", "<meme")
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._last_raw_length = 0
|
||||
self._pending = ""
|
||||
self._rendered_text = ""
|
||||
self._think_text = ""
|
||||
self._has_seen_visible_text = False
|
||||
self._saw_hidden_repair_flow = False
|
||||
self._saw_only_think_payloads = False
|
||||
self._awaiting_ppt_followup = False
|
||||
self._deferred_followup_text = ""
|
||||
self._ppt_followup_command = ""
|
||||
|
||||
def reset(self) -> None:
|
||||
self._last_raw_length = 0
|
||||
self._last_raw_response = ""
|
||||
self._pending = ""
|
||||
self._rendered_text = ""
|
||||
self._think_text = ""
|
||||
self._has_seen_visible_text = False
|
||||
self._saw_hidden_repair_flow = False
|
||||
self._saw_only_think_payloads = False
|
||||
self._awaiting_ppt_followup = False
|
||||
self._deferred_followup_text = ""
|
||||
self._ppt_followup_command = ""
|
||||
|
||||
def prepare_for_followup_stream(self) -> None:
|
||||
awaiting_ppt_followup = self._awaiting_ppt_followup
|
||||
ppt_followup_command = self._ppt_followup_command
|
||||
self.reset()
|
||||
self._awaiting_ppt_followup = awaiting_ppt_followup
|
||||
self._ppt_followup_command = ppt_followup_command
|
||||
|
||||
def parse_chunk(self, raw_response: str) -> Dict[str, Any]:
|
||||
result: Dict[str, Any] = {
|
||||
"content": "",
|
||||
"images": [],
|
||||
"done": False,
|
||||
"error": None,
|
||||
}
|
||||
|
||||
try:
|
||||
if isinstance(raw_response, (bytes, bytearray)):
|
||||
raw_response = raw_response.decode("utf-8", errors="ignore")
|
||||
elif not isinstance(raw_response, str):
|
||||
raw_response = str(raw_response)
|
||||
|
||||
new_data = self._prepare_incremental_raw_response(raw_response)
|
||||
if not new_data:
|
||||
return result
|
||||
|
||||
content, done = self._consume_new_data(new_data)
|
||||
if content:
|
||||
result["content"] = content
|
||||
result["done"] = done
|
||||
except Exception as e:
|
||||
logger.debug(f"[GLMParser] parse exception: {e}")
|
||||
result["error"] = str(e)
|
||||
|
||||
return result
|
||||
|
||||
def _consume_new_data(self, new_data: str) -> tuple[str, bool]:
|
||||
normalized = (self._pending + new_data).replace("\r\n", "\n")
|
||||
if not normalized:
|
||||
return "", False
|
||||
|
||||
blocks = normalized.split("\n\n")
|
||||
if normalized.endswith("\n\n"):
|
||||
self._pending = ""
|
||||
complete_blocks = [block for block in blocks if block.strip()]
|
||||
else:
|
||||
self._pending = blocks.pop() if blocks else normalized
|
||||
complete_blocks = [block for block in blocks if block.strip()]
|
||||
|
||||
content_parts: List[str] = []
|
||||
done = False
|
||||
|
||||
for block in complete_blocks:
|
||||
block_content, block_done = self._parse_event_block(block)
|
||||
if block_content:
|
||||
content_parts.append(block_content)
|
||||
if block_done:
|
||||
done = True
|
||||
|
||||
return "".join(content_parts), done
|
||||
|
||||
def _parse_event_block(self, block: str) -> tuple[str, bool]:
|
||||
data_lines: List[str] = []
|
||||
|
||||
for raw_line in block.split("\n"):
|
||||
line = raw_line.strip()
|
||||
if not line:
|
||||
continue
|
||||
if line.startswith("data:"):
|
||||
data_lines.append(line[5:].strip())
|
||||
|
||||
payload = "\n".join(data_lines).strip()
|
||||
if not payload:
|
||||
return "", False
|
||||
|
||||
try:
|
||||
data = json.loads(payload)
|
||||
except json.JSONDecodeError:
|
||||
return "", False
|
||||
|
||||
if not isinstance(data, dict):
|
||||
return "", False
|
||||
|
||||
return self._extract_payload(data)
|
||||
|
||||
def _extract_payload(self, data: Dict[str, Any]) -> tuple[str, bool]:
|
||||
top_status = str(data.get("status") or "").strip().lower()
|
||||
last_error = data.get("last_error")
|
||||
intervene_text = self._extract_intervene_text(last_error)
|
||||
if intervene_text and self._looks_like_hidden_repair_text(intervene_text):
|
||||
intervene_text = ""
|
||||
parts = data.get("parts")
|
||||
if not isinstance(parts, list) or not parts:
|
||||
if self._awaiting_ppt_followup and top_status in self._TERMINAL_STATUSES:
|
||||
snapshot = self._resolve_deferred_followup_text()
|
||||
if snapshot:
|
||||
delta = self._compute_delta(self._rendered_text, snapshot)
|
||||
self._rendered_text = snapshot
|
||||
self._has_seen_visible_text = True
|
||||
return delta, True
|
||||
return "", False
|
||||
if intervene_text and top_status in self._TERMINAL_STATUSES:
|
||||
delta = self._compute_delta(self._rendered_text, intervene_text)
|
||||
self._rendered_text = intervene_text
|
||||
if intervene_text:
|
||||
self._has_seen_visible_text = True
|
||||
return delta, True
|
||||
return "", bool(self._has_seen_visible_text and top_status in self._TERMINAL_STATUSES)
|
||||
|
||||
visible_snapshot = self._rendered_text
|
||||
saw_visible_text_in_payload = False
|
||||
saw_think_in_payload = False
|
||||
done = bool(self._has_seen_visible_text and top_status in self._TERMINAL_STATUSES)
|
||||
|
||||
for part in parts:
|
||||
if not isinstance(part, dict):
|
||||
continue
|
||||
|
||||
part_status = str(part.get("status") or "").strip().lower()
|
||||
|
||||
content_items = part.get("content")
|
||||
if not isinstance(content_items, list):
|
||||
continue
|
||||
|
||||
for item in content_items:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
item_type = str(item.get("type") or "").strip().lower()
|
||||
if item_type == "system_error":
|
||||
self._handle_system_error_item(item, part)
|
||||
continue
|
||||
if item_type == "think":
|
||||
think_text = str(item.get("think") or self._think_text)
|
||||
self._think_text = think_text
|
||||
saw_think_in_payload = True
|
||||
if self._awaiting_ppt_followup and think_text.strip():
|
||||
self._awaiting_ppt_followup = False
|
||||
self._deferred_followup_text = ""
|
||||
embedded_snapshot = self._extract_embedded_visible_snapshot(think_text)
|
||||
if embedded_snapshot:
|
||||
visible_snapshot = embedded_snapshot
|
||||
saw_visible_text_in_payload = True
|
||||
self._has_seen_visible_text = True
|
||||
if self._looks_like_hidden_repair_text(think_text):
|
||||
self._saw_hidden_repair_flow = True
|
||||
continue
|
||||
if item_type == "tool_calls":
|
||||
continue
|
||||
if item_type != "text":
|
||||
continue
|
||||
|
||||
snapshot = str(item.get("text") or "")
|
||||
if self._awaiting_ppt_followup:
|
||||
if snapshot:
|
||||
self._deferred_followup_text = snapshot
|
||||
if part_status in self._TERMINAL_STATUSES:
|
||||
snapshot = self._resolve_deferred_followup_text()
|
||||
if snapshot:
|
||||
visible_snapshot = snapshot
|
||||
saw_visible_text_in_payload = True
|
||||
self._has_seen_visible_text = True
|
||||
done = True
|
||||
continue
|
||||
if snapshot:
|
||||
visible_snapshot = snapshot
|
||||
saw_visible_text_in_payload = True
|
||||
self._has_seen_visible_text = True
|
||||
if part_status in self._TERMINAL_STATUSES and (snapshot or self._has_seen_visible_text):
|
||||
done = True
|
||||
|
||||
if saw_think_in_payload and not saw_visible_text_in_payload:
|
||||
self._saw_only_think_payloads = True
|
||||
|
||||
if intervene_text and done and not visible_snapshot:
|
||||
visible_snapshot = intervene_text
|
||||
|
||||
delta = self._compute_delta(self._rendered_text, visible_snapshot)
|
||||
self._rendered_text = visible_snapshot
|
||||
return delta, done
|
||||
|
||||
def should_fallback_to_dom_when_no_visible_content(self) -> bool:
|
||||
return bool(
|
||||
not self._has_seen_visible_text
|
||||
and (self._saw_hidden_repair_flow or self._saw_only_think_payloads)
|
||||
)
|
||||
|
||||
def should_wait_for_followup_stream(self) -> bool:
|
||||
return bool(self._awaiting_ppt_followup)
|
||||
|
||||
@classmethod
|
||||
def _looks_like_hidden_repair_text(cls, text: Any) -> bool:
|
||||
normalized = str(text or "").strip()
|
||||
if not normalized:
|
||||
return False
|
||||
|
||||
lowered = normalized.lower()
|
||||
if any(marker in lowered for marker in cls._HIDDEN_REPAIR_TEXT_PATTERNS):
|
||||
return True
|
||||
|
||||
if cls._THINK_REASONING_PREFIX_RE.search(normalized):
|
||||
if "<segment>" in normalized or "<context_summary>" in normalized:
|
||||
return True
|
||||
if "\\u003csegment\\u003e" in normalized or "\\u003ccontext_summary\\u003e" in normalized:
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
def export_debug_data(self, raw_response: str = "") -> Dict[str, Any]:
|
||||
return {
|
||||
"has_seen_visible_text": bool(self._has_seen_visible_text),
|
||||
"rendered_text_len": len(self._rendered_text or ""),
|
||||
"think_text_len": len(self._think_text or ""),
|
||||
"saw_hidden_repair_flow": bool(self._saw_hidden_repair_flow),
|
||||
"saw_only_think_payloads": bool(self._saw_only_think_payloads),
|
||||
"awaiting_ppt_followup": bool(self._awaiting_ppt_followup),
|
||||
"ppt_followup_command": self._ppt_followup_command,
|
||||
"deferred_followup_text_len": len(self._deferred_followup_text or ""),
|
||||
}
|
||||
|
||||
def _handle_system_error_item(self, item: Dict[str, Any], part: Dict[str, Any]) -> None:
|
||||
metadata = part.get("meta_data")
|
||||
if not isinstance(metadata, dict):
|
||||
metadata = {}
|
||||
command = str(
|
||||
metadata.get("failedCommand") or item.get("failedCommand") or ""
|
||||
).strip()
|
||||
if command not in self._PPT_FOLLOWUP_COMMANDS:
|
||||
return
|
||||
|
||||
self._awaiting_ppt_followup = True
|
||||
self._deferred_followup_text = ""
|
||||
self._ppt_followup_command = command
|
||||
|
||||
def _resolve_deferred_followup_text(self) -> str:
|
||||
snapshot = self._deferred_followup_text
|
||||
self._deferred_followup_text = ""
|
||||
if not snapshot or self._looks_like_ppt_template(snapshot):
|
||||
return ""
|
||||
self._awaiting_ppt_followup = False
|
||||
return snapshot
|
||||
|
||||
@classmethod
|
||||
def _looks_like_ppt_template(cls, text: Any) -> bool:
|
||||
normalized = str(text or "").strip().lower()
|
||||
if normalized.startswith("```json"):
|
||||
normalized = normalized[7:].lstrip()
|
||||
elif normalized.startswith("```"):
|
||||
normalized = normalized[3:].lstrip()
|
||||
if not normalized.startswith("{"):
|
||||
return False
|
||||
return sum(marker in normalized for marker in cls._PPT_TEMPLATE_MARKERS) >= 2
|
||||
|
||||
@classmethod
|
||||
def _extract_embedded_visible_snapshot(cls, text: Any) -> str:
|
||||
normalized = str(text or "")
|
||||
if not normalized:
|
||||
return ""
|
||||
|
||||
match = cls._EMBEDDED_REPLY_LABEL_RE.search(normalized)
|
||||
if not match:
|
||||
return ""
|
||||
|
||||
body = normalized[match.end():]
|
||||
if not body:
|
||||
return ""
|
||||
|
||||
fence_end = body.find("```")
|
||||
candidate = body[:fence_end] if fence_end >= 0 else body
|
||||
candidate = candidate.strip()
|
||||
if not candidate:
|
||||
return ""
|
||||
|
||||
lowered = candidate.lower()
|
||||
if lowered.startswith("<think"):
|
||||
return ""
|
||||
if any(marker in lowered for marker in cls._HIDDEN_REPAIR_TEXT_PATTERNS):
|
||||
return ""
|
||||
if not any(marker in lowered for marker in cls._VISIBLE_MARKERS):
|
||||
return ""
|
||||
return candidate
|
||||
|
||||
@staticmethod
|
||||
def _extract_intervene_text(last_error: Any) -> str:
|
||||
if not isinstance(last_error, dict):
|
||||
return ""
|
||||
text = last_error.get("intervene_text")
|
||||
if not isinstance(text, str):
|
||||
return ""
|
||||
return text.strip()
|
||||
|
||||
@staticmethod
|
||||
def _compute_delta(previous: str, current: str) -> str:
|
||||
if current == previous:
|
||||
return ""
|
||||
if previous and current.startswith(previous):
|
||||
return current[len(previous):]
|
||||
return current
|
||||
|
||||
@classmethod
|
||||
def get_id(cls) -> str:
|
||||
return "glm"
|
||||
|
||||
@classmethod
|
||||
def get_name(cls) -> str:
|
||||
return "GLM"
|
||||
|
||||
@classmethod
|
||||
def get_description(cls) -> str:
|
||||
return "Parse ChatGLM SSE streams and emit assistant text deltas while ignoring think/tool-call events"
|
||||
|
||||
@classmethod
|
||||
def get_supported_patterns(cls) -> List[str]:
|
||||
return ["/chatglm/backend-api/assistant/stream", "assistant/stream"]
|
||||
|
||||
|
||||
__all__ = ["GLMParser"]
|
||||
@@ -0,0 +1,529 @@
|
||||
"""
|
||||
grok_parser.py - Grok newline-delimited JSON stream parser.
|
||||
|
||||
Observed stream traits:
|
||||
- content-type: application/json
|
||||
- body is a growing sequence of JSON objects separated by newlines
|
||||
- visible answer tokens arrive as result.token with isThinking=false
|
||||
- final assistant snapshot arrives as result.modelResponse
|
||||
- completed reload/fetch snapshots can arrive as responses[*].message
|
||||
- generated images can be exposed via generatedImageUrls / imageAttachments
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any, Dict, Iterable, List, Tuple
|
||||
|
||||
from app.core.config import logger
|
||||
from .base import ResponseParser
|
||||
|
||||
|
||||
class GrokParser(ResponseParser):
|
||||
"""Parse Grok web NDJSON response streams."""
|
||||
|
||||
_IMAGE_URL_KEYS = {
|
||||
"url",
|
||||
"downloadurl",
|
||||
"download_url",
|
||||
"imageurl",
|
||||
"image_url",
|
||||
"imageuri",
|
||||
"image_uri",
|
||||
"uri",
|
||||
}
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._last_raw_length = 0
|
||||
self._pending = ""
|
||||
self._rendered_content = ""
|
||||
self._seen_image_refs: set[str] = set()
|
||||
self._media_generation_state: Dict[str, Any] = {}
|
||||
|
||||
def parse_chunk(self, raw_response: str) -> Dict[str, Any]:
|
||||
result: Dict[str, Any] = {
|
||||
"content": "",
|
||||
"images": [],
|
||||
"done": False,
|
||||
"error": None,
|
||||
}
|
||||
|
||||
try:
|
||||
if isinstance(raw_response, (bytes, bytearray)):
|
||||
raw_response = raw_response.decode("utf-8", errors="ignore")
|
||||
|
||||
snapshot = self._load_json_snapshot(raw_response)
|
||||
if snapshot is not None:
|
||||
snapshot_payload, snapshot_text = snapshot
|
||||
if snapshot_text == getattr(self, "_last_raw_response", ""):
|
||||
return result
|
||||
|
||||
self._last_raw_length = len(snapshot_text)
|
||||
self._last_raw_response = snapshot_text
|
||||
delta_content, images, done = self._consume_json_snapshot(snapshot_payload)
|
||||
if delta_content:
|
||||
result["content"] = delta_content
|
||||
if images:
|
||||
result["images"] = images
|
||||
result["done"] = done
|
||||
return result
|
||||
|
||||
if not isinstance(raw_response, str):
|
||||
raw_response = str(raw_response)
|
||||
|
||||
new_data = self._prepare_incremental_raw_response(raw_response)
|
||||
if not new_data:
|
||||
return result
|
||||
|
||||
delta_content, images, done = self._consume_new_data(new_data)
|
||||
if delta_content:
|
||||
result["content"] = delta_content
|
||||
if images:
|
||||
result["images"] = images
|
||||
result["done"] = done
|
||||
|
||||
except Exception as e:
|
||||
logger.debug(f"[GrokParser] parse exception: {e}")
|
||||
result["error"] = str(e)
|
||||
|
||||
return result
|
||||
|
||||
def reset(self) -> None:
|
||||
self._last_raw_length = 0
|
||||
self._last_raw_response = ""
|
||||
self._pending = ""
|
||||
self._rendered_content = ""
|
||||
self._seen_image_refs.clear()
|
||||
self._media_generation_state = {}
|
||||
|
||||
def get_media_generation_state(
|
||||
self,
|
||||
raw_response: str = "",
|
||||
parse_result: Dict[str, Any] | None = None,
|
||||
) -> Dict[str, Any]:
|
||||
return dict(self._media_generation_state or {})
|
||||
|
||||
def _consume_new_data(self, new_data: str) -> Tuple[str, List[Dict[str, Any]], bool]:
|
||||
previous_content = self._rendered_content
|
||||
complete_lines = self._extract_complete_lines(new_data)
|
||||
complete_lines.extend(self._drain_parseable_pending_line())
|
||||
if not complete_lines:
|
||||
return "", [], False
|
||||
|
||||
images: List[Dict[str, Any]] = []
|
||||
done = False
|
||||
|
||||
for raw_line in complete_lines:
|
||||
line = raw_line.strip()
|
||||
if not line:
|
||||
continue
|
||||
|
||||
try:
|
||||
payload = json.loads(line)
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
|
||||
line_images, line_done = self._consume_payload(payload)
|
||||
if line_images:
|
||||
images.extend(line_images)
|
||||
if line_done:
|
||||
done = True
|
||||
|
||||
delta = self._compute_delta(previous_content, self._rendered_content)
|
||||
return delta, images, done
|
||||
|
||||
def _consume_json_snapshot(self, payload: Any) -> Tuple[str, List[Dict[str, Any]], bool]:
|
||||
previous_content = self._rendered_content
|
||||
images, done = self._consume_payload(payload)
|
||||
delta = self._compute_delta(previous_content, self._rendered_content)
|
||||
return delta, images, done
|
||||
|
||||
@staticmethod
|
||||
def _load_json_snapshot(raw_response: Any) -> Tuple[Any, str] | None:
|
||||
if isinstance(raw_response, (dict, list)):
|
||||
snapshot_text = json.dumps(
|
||||
raw_response,
|
||||
ensure_ascii=False,
|
||||
sort_keys=True,
|
||||
default=str,
|
||||
)
|
||||
return raw_response, snapshot_text
|
||||
|
||||
if not isinstance(raw_response, str):
|
||||
return None
|
||||
|
||||
text = raw_response.strip()
|
||||
if not text or text[0] not in "[{":
|
||||
return None
|
||||
|
||||
try:
|
||||
payload = json.loads(text)
|
||||
except json.JSONDecodeError:
|
||||
return None
|
||||
|
||||
if not isinstance(payload, (dict, list)):
|
||||
return None
|
||||
|
||||
return payload, text
|
||||
|
||||
def _extract_complete_lines(self, new_data: str) -> List[str]:
|
||||
normalized = (self._pending + str(new_data or "")).replace("\r\n", "\n").replace("\r", "\n")
|
||||
if not normalized:
|
||||
return []
|
||||
|
||||
parts = normalized.split("\n")
|
||||
if normalized.endswith("\n"):
|
||||
self._pending = ""
|
||||
return parts[:-1]
|
||||
|
||||
self._pending = parts.pop() if parts else normalized
|
||||
return parts
|
||||
|
||||
def _drain_parseable_pending_line(self) -> List[str]:
|
||||
pending = str(self._pending or "").strip()
|
||||
if not pending:
|
||||
return []
|
||||
|
||||
try:
|
||||
json.loads(pending)
|
||||
except json.JSONDecodeError:
|
||||
return []
|
||||
|
||||
self._pending = ""
|
||||
return [pending]
|
||||
|
||||
def _consume_payload(self, payload: Any) -> Tuple[List[Dict[str, Any]], bool]:
|
||||
if isinstance(payload, list):
|
||||
if any(isinstance(item, dict) and "message" in item for item in payload):
|
||||
return self._consume_responses_snapshot(payload)
|
||||
|
||||
images: List[Dict[str, Any]] = []
|
||||
done = False
|
||||
for item in payload:
|
||||
item_images, item_done = self._consume_payload(item)
|
||||
if item_images:
|
||||
images.extend(item_images)
|
||||
if item_done:
|
||||
done = True
|
||||
return self._dedupe_images(images), done
|
||||
|
||||
if not isinstance(payload, dict):
|
||||
return [], False
|
||||
|
||||
responses = payload.get("responses")
|
||||
if isinstance(responses, list):
|
||||
return self._consume_responses_snapshot(responses)
|
||||
|
||||
if self._looks_like_response_snapshot(payload):
|
||||
return self._consume_response_snapshot(payload)
|
||||
|
||||
result = payload.get("result")
|
||||
if not isinstance(result, dict):
|
||||
return [], False
|
||||
|
||||
event_payload = self._unwrap_result_payload(result)
|
||||
if not isinstance(event_payload, dict):
|
||||
return [], False
|
||||
|
||||
self._consume_token_event(event_payload)
|
||||
|
||||
images = self._extract_image_items(event_payload)
|
||||
model_response = event_payload.get("modelResponse")
|
||||
done = False
|
||||
if isinstance(model_response, dict):
|
||||
self._consume_model_response(model_response)
|
||||
images.extend(self._extract_image_items(model_response))
|
||||
done = not bool(model_response.get("partial", False))
|
||||
|
||||
images = self._dedupe_images(images)
|
||||
if images:
|
||||
self._media_generation_state = {}
|
||||
else:
|
||||
self._update_media_generation_state(event_payload)
|
||||
|
||||
return images, done
|
||||
|
||||
def _consume_responses_snapshot(self, responses: List[Any]) -> Tuple[List[Dict[str, Any]], bool]:
|
||||
response = self._select_latest_assistant_response(responses)
|
||||
if response is None:
|
||||
return [], False
|
||||
return self._consume_response_snapshot(response)
|
||||
|
||||
def _consume_response_snapshot(self, response: Dict[str, Any]) -> Tuple[List[Dict[str, Any]], bool]:
|
||||
self._consume_model_response(response)
|
||||
images = self._dedupe_images(self._extract_image_items(response))
|
||||
done = not bool(response.get("partial", False))
|
||||
if images:
|
||||
self._media_generation_state = {}
|
||||
else:
|
||||
self._update_media_generation_state(response)
|
||||
return images, done
|
||||
|
||||
@classmethod
|
||||
def _select_latest_assistant_response(cls, responses: List[Any]) -> Dict[str, Any] | None:
|
||||
for response in reversed(responses):
|
||||
if not isinstance(response, dict):
|
||||
continue
|
||||
if cls._is_assistant_sender(response.get("sender")):
|
||||
return response
|
||||
return None
|
||||
|
||||
@classmethod
|
||||
def _looks_like_response_snapshot(cls, payload: Dict[str, Any]) -> bool:
|
||||
if "message" not in payload:
|
||||
return False
|
||||
sender = payload.get("sender")
|
||||
return sender in (None, "") or cls._is_assistant_sender(sender)
|
||||
|
||||
@staticmethod
|
||||
def _is_assistant_sender(sender: Any) -> bool:
|
||||
normalized = str(sender or "").strip().lower()
|
||||
return normalized in {"assistant", "bot"}
|
||||
|
||||
def _update_media_generation_state(self, event_payload: Dict[str, Any]) -> None:
|
||||
progress = event_payload.get("progressReport")
|
||||
if not isinstance(progress, dict):
|
||||
return
|
||||
|
||||
category = str(progress.get("category") or "").strip().upper()
|
||||
state = str(progress.get("state") or "").strip().upper()
|
||||
message = str(progress.get("message") or "").strip()
|
||||
|
||||
if "IMAGE" not in category:
|
||||
return
|
||||
|
||||
if any(token in state for token in ("COMPLETED", "COMPLETE", "SUCCEEDED", "SUCCESS", "FAILED", "ERROR", "CANCELED", "CANCELLED")):
|
||||
self._media_generation_state = {}
|
||||
return
|
||||
|
||||
if not any(token in state for token in ("PENDING", "STARTED", "RUNNING", "IN_PROGRESS")):
|
||||
return
|
||||
|
||||
hint_text = str(self._media_generation_state.get("hint_text") or "")
|
||||
hint_parts = [item for item in hint_text.split("\n\n") if item]
|
||||
if message and message not in hint_parts:
|
||||
hint_parts.append(message)
|
||||
|
||||
wait_timeout_seconds = self._media_generation_state.get("wait_timeout_seconds")
|
||||
self._media_generation_state = {
|
||||
"pending": True,
|
||||
"media_type": "image",
|
||||
"hint_text": "\n\n".join(hint_parts[:3]),
|
||||
"wait_timeout_seconds": max(float(wait_timeout_seconds or 0), 120.0),
|
||||
}
|
||||
|
||||
def _consume_token_event(self, result: Dict[str, Any]) -> None:
|
||||
token = result.get("token")
|
||||
if not isinstance(token, str) or not token:
|
||||
return
|
||||
|
||||
if bool(result.get("isThinking", False)):
|
||||
return
|
||||
|
||||
message_tag = str(result.get("messageTag") or "").strip().lower()
|
||||
if message_tag and message_tag not in {"final", "answer"}:
|
||||
return
|
||||
|
||||
self._rendered_content += token
|
||||
|
||||
def _consume_model_response(self, model_response: Dict[str, Any]) -> None:
|
||||
sender = str(model_response.get("sender") or "").strip().lower()
|
||||
if sender and sender not in {"assistant", "bot"}:
|
||||
return
|
||||
|
||||
message = model_response.get("message")
|
||||
if isinstance(message, str) and message:
|
||||
self._rendered_content = message
|
||||
|
||||
def _extract_image_items(self, container: Dict[str, Any]) -> List[Dict[str, Any]]:
|
||||
urls = self._collect_image_urls(container)
|
||||
image_items: List[Dict[str, Any]] = []
|
||||
for url in urls:
|
||||
if url in self._seen_image_refs:
|
||||
continue
|
||||
self._seen_image_refs.add(url)
|
||||
image_items.append(
|
||||
{
|
||||
"media_type": "image",
|
||||
"kind": "url",
|
||||
"url": url,
|
||||
"data_uri": None,
|
||||
"mime": None,
|
||||
"byte_size": None,
|
||||
"alt": "",
|
||||
"width": None,
|
||||
"height": None,
|
||||
"detected_at": datetime.now(timezone.utc).isoformat().replace("+00:00", "Z"),
|
||||
"source": "grok_stream",
|
||||
}
|
||||
)
|
||||
|
||||
return image_items
|
||||
|
||||
def _collect_image_urls(self, container: Dict[str, Any]) -> List[str]:
|
||||
urls: List[str] = []
|
||||
|
||||
generated = container.get("generatedImageUrls")
|
||||
if isinstance(generated, list):
|
||||
for item in generated:
|
||||
if isinstance(item, str) and self._looks_like_image_ref(item):
|
||||
urls.append(item)
|
||||
|
||||
attachments = container.get("imageAttachments")
|
||||
if isinstance(attachments, list):
|
||||
for item in attachments:
|
||||
urls.extend(self._extract_image_urls_from_value(item))
|
||||
|
||||
cards = container.get("cardAttachmentsJson")
|
||||
if isinstance(cards, list):
|
||||
for item in cards:
|
||||
parsed = item
|
||||
if isinstance(item, str):
|
||||
try:
|
||||
parsed = json.loads(item)
|
||||
except json.JSONDecodeError:
|
||||
parsed = item
|
||||
urls.extend(self._extract_image_urls_from_value(parsed))
|
||||
|
||||
deduped: List[str] = []
|
||||
seen = set()
|
||||
for url in urls:
|
||||
if url in seen:
|
||||
continue
|
||||
seen.add(url)
|
||||
deduped.append(url)
|
||||
|
||||
return deduped
|
||||
|
||||
@staticmethod
|
||||
def _unwrap_result_payload(result: Dict[str, Any]) -> Dict[str, Any]:
|
||||
response = result.get("response")
|
||||
if isinstance(response, dict):
|
||||
return response
|
||||
return result
|
||||
|
||||
def _extract_image_urls_from_value(self, value: Any) -> List[str]:
|
||||
urls: List[str] = []
|
||||
|
||||
if isinstance(value, str):
|
||||
if self._looks_like_image_ref(value):
|
||||
urls.append(value)
|
||||
return urls
|
||||
|
||||
if isinstance(value, list):
|
||||
for item in value:
|
||||
urls.extend(self._extract_image_urls_from_value(item))
|
||||
return urls
|
||||
|
||||
if not isinstance(value, dict):
|
||||
return urls
|
||||
|
||||
content_type = str(
|
||||
value.get("contentType") or value.get("mimeType") or value.get("mime") or ""
|
||||
).strip().lower()
|
||||
|
||||
for key, item in value.items():
|
||||
normalized_key = str(key or "").strip().lower()
|
||||
if normalized_key in self._IMAGE_URL_KEYS:
|
||||
if isinstance(item, str) and (self._looks_like_image_ref(item) or content_type.startswith("image/")):
|
||||
urls.append(item)
|
||||
continue
|
||||
|
||||
if normalized_key in {"images", "attachments", "items", "content", "data"}:
|
||||
urls.extend(self._extract_image_urls_from_value(item))
|
||||
|
||||
return urls
|
||||
|
||||
@staticmethod
|
||||
def _looks_like_image_ref(value: str) -> bool:
|
||||
text = str(value or "").strip()
|
||||
if not text:
|
||||
return False
|
||||
|
||||
lowered = text.lower()
|
||||
if lowered.startswith("data:image/"):
|
||||
return True
|
||||
|
||||
if not lowered.startswith(("http://", "https://")):
|
||||
return False
|
||||
|
||||
return any(marker in lowered for marker in ("/generated/", ".png", ".jpg", ".jpeg", ".webp", ".gif"))
|
||||
|
||||
@staticmethod
|
||||
def _compute_delta(previous: str, current: str) -> str:
|
||||
if current == previous:
|
||||
return ""
|
||||
if not previous:
|
||||
return current
|
||||
if current.startswith(previous):
|
||||
return current[len(previous) :]
|
||||
if previous.startswith(current):
|
||||
return ""
|
||||
|
||||
prefix_len = 0
|
||||
limit = min(len(previous), len(current))
|
||||
while prefix_len < limit and previous[prefix_len] == current[prefix_len]:
|
||||
prefix_len += 1
|
||||
|
||||
if prefix_len >= len(previous) // 2:
|
||||
return current[prefix_len:]
|
||||
return current
|
||||
|
||||
@staticmethod
|
||||
def _dedupe_images(images: Iterable[Dict[str, Any]]) -> List[Dict[str, Any]]:
|
||||
deduped: List[Dict[str, Any]] = []
|
||||
seen = set()
|
||||
|
||||
for item in images:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
key = (
|
||||
str(item.get("media_type") or ""),
|
||||
str(item.get("kind") or ""),
|
||||
str(item.get("url") or item.get("data_uri") or ""),
|
||||
)
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
deduped.append(item)
|
||||
|
||||
return deduped
|
||||
|
||||
def _iter_payloads(self, raw_response: str):
|
||||
self_pending = self._pending
|
||||
try:
|
||||
self._pending = ""
|
||||
lines = self._extract_complete_lines(raw_response)
|
||||
lines.extend(self._drain_parseable_pending_line())
|
||||
for raw_line in lines:
|
||||
line = raw_line.strip()
|
||||
if not line:
|
||||
continue
|
||||
try:
|
||||
payload = json.loads(line)
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
if isinstance(payload, dict):
|
||||
yield payload
|
||||
finally:
|
||||
self._pending = self_pending
|
||||
|
||||
@classmethod
|
||||
def get_id(cls) -> str:
|
||||
return "grok"
|
||||
|
||||
@classmethod
|
||||
def get_name(cls) -> str:
|
||||
return "Grok"
|
||||
|
||||
@classmethod
|
||||
def get_description(cls) -> str:
|
||||
return "Parse Grok NDJSON chat streams, ignore thinking tokens, and extract generated images"
|
||||
|
||||
@classmethod
|
||||
def get_supported_patterns(cls) -> List[str]:
|
||||
return ["/rest/app-chat/conversations/", "/responses", "/load-responses"]
|
||||
|
||||
|
||||
__all__ = ["GrokParser"]
|
||||
@@ -0,0 +1,250 @@
|
||||
"""
|
||||
kimi_parser.py - Kimi Connect streaming response parser.
|
||||
|
||||
Observed stream traits:
|
||||
- content-type: application/connect+json
|
||||
- each message is framed as 5 bytes of binary header + one JSON object payload
|
||||
- assistant text starts with op=set, mask=block.text
|
||||
- later deltas use op=append, mask=block.text.content
|
||||
- {"done": {}} or MESSAGE_STATUS_COMPLETED ends the stream
|
||||
|
||||
Notes:
|
||||
- In DevTools-exported `fullText` / `chunks.data`, the header bytes can be lossy once the
|
||||
raw binary stream is coerced into a Python string, so relying on the advertised payload
|
||||
length is brittle.
|
||||
- We therefore treat the first 5 bytes as opaque framing metadata and recover the payload
|
||||
by scanning for a complete JSON object boundary instead.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from typing import Any, Dict, List, Set, Tuple
|
||||
|
||||
from app.core.config import logger
|
||||
from .base import ResponseParser
|
||||
|
||||
|
||||
class KimiParser(ResponseParser):
|
||||
"""Parse Kimi connect+json streaming responses."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._last_raw = b""
|
||||
self._pending = b""
|
||||
self._seen_offsets: Set[int] = set()
|
||||
|
||||
def parse_chunk(self, raw_response: str) -> Dict[str, Any]:
|
||||
result: Dict[str, Any] = {
|
||||
"content": "",
|
||||
"images": [],
|
||||
"done": False,
|
||||
"error": None,
|
||||
}
|
||||
|
||||
try:
|
||||
raw_bytes = self._normalize_raw_bytes(raw_response)
|
||||
if not raw_bytes:
|
||||
return result
|
||||
|
||||
if self._last_raw and raw_bytes == self._last_raw:
|
||||
return result
|
||||
|
||||
if self._last_raw and raw_bytes.startswith(self._last_raw):
|
||||
new_data = raw_bytes[len(self._last_raw):]
|
||||
else:
|
||||
new_data = raw_bytes
|
||||
|
||||
self._last_raw = raw_bytes
|
||||
|
||||
delta_content, done = self._consume_new_data(new_data)
|
||||
if delta_content:
|
||||
result["content"] = delta_content
|
||||
result["done"] = done
|
||||
|
||||
except Exception as e:
|
||||
logger.debug(f"[KimiParser] parse exception: {e}")
|
||||
result["error"] = str(e)
|
||||
|
||||
return result
|
||||
|
||||
def reset(self) -> None:
|
||||
self._last_raw = b""
|
||||
self._pending = b""
|
||||
self._seen_offsets.clear()
|
||||
|
||||
@staticmethod
|
||||
def _normalize_raw_bytes(raw_response: Any) -> bytes:
|
||||
if isinstance(raw_response, bytes):
|
||||
return raw_response
|
||||
if isinstance(raw_response, bytearray):
|
||||
return bytes(raw_response)
|
||||
if isinstance(raw_response, str):
|
||||
if "\\u00" in raw_response and "\x00" not in raw_response:
|
||||
return KimiParser._decode_u00_escaped_bytes(raw_response)
|
||||
return raw_response.encode("utf-8", errors="ignore")
|
||||
return str(raw_response).encode("utf-8", errors="ignore")
|
||||
|
||||
@staticmethod
|
||||
def _decode_u00_escaped_bytes(raw_response: str) -> bytes:
|
||||
output = bytearray()
|
||||
i = 0
|
||||
|
||||
while i < len(raw_response):
|
||||
if (
|
||||
raw_response[i] == "\\"
|
||||
and i + 5 < len(raw_response)
|
||||
and raw_response[i + 1] == "u"
|
||||
and raw_response[i + 2:i + 4] == "00"
|
||||
):
|
||||
hex_part = raw_response[i + 4:i + 6]
|
||||
if all(ch in "0123456789abcdefABCDEF" for ch in hex_part):
|
||||
output.append(int(hex_part, 16))
|
||||
i += 6
|
||||
continue
|
||||
|
||||
output.extend(raw_response[i].encode("utf-8", errors="ignore"))
|
||||
i += 1
|
||||
|
||||
return bytes(output)
|
||||
|
||||
def _consume_new_data(self, new_data: bytes) -> Tuple[str, bool]:
|
||||
self._pending += new_data
|
||||
|
||||
content_parts: List[str] = []
|
||||
done = False
|
||||
|
||||
while True:
|
||||
frame = self._next_frame()
|
||||
if frame is None:
|
||||
break
|
||||
|
||||
payload_text, frame_done = frame
|
||||
if payload_text:
|
||||
content_parts.append(payload_text)
|
||||
if frame_done:
|
||||
done = True
|
||||
|
||||
return "".join(content_parts), done
|
||||
|
||||
def _next_frame(self) -> Tuple[str, bool] | None:
|
||||
if len(self._pending) < 6:
|
||||
return None
|
||||
|
||||
payload_end = self._find_json_payload_end(self._pending, start=5)
|
||||
if payload_end is None:
|
||||
return None
|
||||
|
||||
payload = self._pending[5:payload_end]
|
||||
self._pending = self._pending[payload_end:]
|
||||
|
||||
return self._parse_payload(payload)
|
||||
|
||||
@staticmethod
|
||||
def _find_json_payload_end(buffer: bytes, start: int) -> int | None:
|
||||
index = start
|
||||
limit = len(buffer)
|
||||
|
||||
while index < limit and chr(buffer[index]).isspace():
|
||||
index += 1
|
||||
|
||||
if index >= limit or buffer[index] != ord("{"):
|
||||
return None
|
||||
|
||||
depth = 0
|
||||
in_string = False
|
||||
escape = False
|
||||
|
||||
for pos in range(index, limit):
|
||||
byte = buffer[pos]
|
||||
|
||||
if in_string:
|
||||
if escape:
|
||||
escape = False
|
||||
elif byte == ord("\\"):
|
||||
escape = True
|
||||
elif byte == ord('"'):
|
||||
in_string = False
|
||||
continue
|
||||
|
||||
if byte == ord('"'):
|
||||
in_string = True
|
||||
elif byte == ord("{"):
|
||||
depth += 1
|
||||
elif byte == ord("}"):
|
||||
depth -= 1
|
||||
if depth == 0:
|
||||
return pos + 1
|
||||
|
||||
return None
|
||||
|
||||
def _parse_payload(self, payload: bytes) -> Tuple[str, bool]:
|
||||
payload_text = payload.decode("utf-8", errors="ignore").strip()
|
||||
if not payload_text:
|
||||
return "", False
|
||||
|
||||
try:
|
||||
data = json.loads(payload_text)
|
||||
except json.JSONDecodeError:
|
||||
return "", False
|
||||
|
||||
if not isinstance(data, dict):
|
||||
return "", False
|
||||
|
||||
if "done" in data:
|
||||
return "", True
|
||||
|
||||
if "heartbeat" in data:
|
||||
return "", False
|
||||
|
||||
event_offset = data.get("eventOffset")
|
||||
if isinstance(event_offset, int):
|
||||
if event_offset in self._seen_offsets:
|
||||
return "", False
|
||||
self._seen_offsets.add(event_offset)
|
||||
|
||||
op = data.get("op")
|
||||
mask = data.get("mask", "")
|
||||
|
||||
if op == "set" and mask == "block.text":
|
||||
return self._extract_block_text(data.get("block")), False
|
||||
|
||||
if op == "append" and mask == "block.text.content":
|
||||
return self._extract_block_text(data.get("block")), False
|
||||
|
||||
if op == "set" and mask == "message.status":
|
||||
message = data.get("message", {})
|
||||
if isinstance(message, dict) and message.get("status") == "MESSAGE_STATUS_COMPLETED":
|
||||
return "", True
|
||||
|
||||
return "", False
|
||||
|
||||
@staticmethod
|
||||
def _extract_block_text(block: Any) -> str:
|
||||
if not isinstance(block, dict):
|
||||
return ""
|
||||
|
||||
text_data = block.get("text")
|
||||
if not isinstance(text_data, dict):
|
||||
return ""
|
||||
|
||||
content = text_data.get("content", "")
|
||||
return content if isinstance(content, str) else ""
|
||||
|
||||
@classmethod
|
||||
def get_id(cls) -> str:
|
||||
return "kimi"
|
||||
|
||||
@classmethod
|
||||
def get_name(cls) -> str:
|
||||
return "Kimi"
|
||||
|
||||
@classmethod
|
||||
def get_description(cls) -> str:
|
||||
return "Parse Kimi connect+json streams and keep assistant text deltas"
|
||||
|
||||
@classmethod
|
||||
def get_supported_patterns(cls) -> List[str]:
|
||||
return ["/apiv2/kimi.gateway.chat.v1.ChatService/Chat", "/chat", "chat"]
|
||||
|
||||
|
||||
__all__ = ["KimiParser"]
|
||||
@@ -0,0 +1,424 @@
|
||||
"""
|
||||
lmarena_battle_side_parser.py - Arena.ai battle mode parsers.
|
||||
|
||||
The side parsers stream one selected side and finish when both battle sides are
|
||||
terminal. The winner parser buffers both sides and emits only the first
|
||||
completed side while still waiting for the battle response to finish normally.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from typing import Any, Dict, List
|
||||
|
||||
from app.core.config import logger
|
||||
from .base import ResponseParser
|
||||
from .lmarena_parser import LmarenaParser, _CP1252_TO_LATIN1, _extract_arena_image_items
|
||||
|
||||
|
||||
class _LmarenaBattleSideParser(ResponseParser):
|
||||
"""Shared implementation for left/right Arena battle text streams."""
|
||||
|
||||
SIDE_PREFIX = "a"
|
||||
SIDE_LABEL = "left"
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._accumulated = ""
|
||||
self._accumulated_reasoning = ""
|
||||
self._seen_image_refs: set[str] = set()
|
||||
self._completed_sides: set[str] = set()
|
||||
self._completed = False
|
||||
self._completion_side = ""
|
||||
|
||||
def parse_chunk(self, raw_response: str) -> Dict[str, Any]:
|
||||
result: Dict[str, Any] = {
|
||||
"content": "",
|
||||
"reasoning_content": "",
|
||||
"images": [],
|
||||
"done": False,
|
||||
"error": None,
|
||||
"selected_side": self.SIDE_LABEL,
|
||||
"completion_side": self._completion_side,
|
||||
}
|
||||
|
||||
if self._completed:
|
||||
result["done"] = True
|
||||
return result
|
||||
|
||||
if isinstance(raw_response, (bytes, bytearray)):
|
||||
raw_response = raw_response.decode("utf-8", errors="ignore")
|
||||
|
||||
if not raw_response or not isinstance(raw_response, str):
|
||||
return result
|
||||
|
||||
raw_response = self._fix_mojibake(raw_response)
|
||||
|
||||
try:
|
||||
content_parts: List[str] = []
|
||||
reasoning_parts: List[str] = []
|
||||
images: List[Dict[str, Any]] = []
|
||||
|
||||
for line in raw_response.split("\n"):
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
|
||||
colon_idx = line.find(":")
|
||||
if colon_idx < 1:
|
||||
continue
|
||||
|
||||
prefix = line[:colon_idx]
|
||||
payload = line[colon_idx + 1:]
|
||||
|
||||
if prefix == f"{self.SIDE_PREFIX}0":
|
||||
text = self._parse_text_chunk(payload)
|
||||
if text is not None:
|
||||
content_parts.append(text)
|
||||
elif prefix == f"{self.SIDE_PREFIX}g":
|
||||
text = self._parse_text_chunk(payload)
|
||||
if text is not None:
|
||||
reasoning_parts.append(text)
|
||||
elif prefix == f"{self.SIDE_PREFIX}2":
|
||||
images.extend(
|
||||
_extract_arena_image_items(
|
||||
payload,
|
||||
self._seen_image_refs,
|
||||
source=f"lmarena_battle_side_{self.SIDE_LABEL}_stream",
|
||||
)
|
||||
)
|
||||
elif prefix in {"ad", "bd"}:
|
||||
if self._is_finish_signal(payload):
|
||||
self._mark_terminal(prefix)
|
||||
elif prefix in {"ae", "be", "a3", "b3"}:
|
||||
error_msg = self._parse_error(payload)
|
||||
if error_msg:
|
||||
completion_side = self._mark_terminal(prefix)
|
||||
logger.debug(
|
||||
"[LmarenaBattleSideParser] stream error frame treated as completion "
|
||||
f"(selected={self.SIDE_LABEL}, side={completion_side}, error={error_msg[:160]})"
|
||||
)
|
||||
|
||||
new_content = "".join(content_parts)
|
||||
new_reasoning = "".join(reasoning_parts)
|
||||
|
||||
if new_content:
|
||||
delta, next_accumulated = LmarenaParser._content_delta(
|
||||
self._accumulated,
|
||||
new_content,
|
||||
append_disjoint=True,
|
||||
)
|
||||
if delta:
|
||||
result["content"] = delta
|
||||
self._accumulated = next_accumulated
|
||||
|
||||
if new_reasoning:
|
||||
delta_r, next_acc_r = LmarenaParser._content_delta(
|
||||
self._accumulated_reasoning,
|
||||
new_reasoning,
|
||||
append_disjoint=True,
|
||||
)
|
||||
if delta_r:
|
||||
result["reasoning_content"] = delta_r
|
||||
self._accumulated_reasoning = next_acc_r
|
||||
|
||||
if images:
|
||||
result["images"] = images
|
||||
|
||||
result["completion_side"] = self._completion_side
|
||||
|
||||
if len(self._completed_sides) >= 2:
|
||||
self._completed = True
|
||||
result["done"] = True
|
||||
result["completion_side"] = self._completion_side
|
||||
|
||||
except Exception as e:
|
||||
logger.debug(f"[LmarenaBattleSideParser] parse exception: {e}")
|
||||
result["error"] = str(e)
|
||||
|
||||
return result
|
||||
|
||||
def reset(self):
|
||||
self._accumulated = ""
|
||||
self._accumulated_reasoning = ""
|
||||
self._seen_image_refs.clear()
|
||||
self._completed_sides.clear()
|
||||
self._completed = False
|
||||
self._completion_side = ""
|
||||
|
||||
def should_fallback_to_dom_when_no_visible_content(self) -> bool:
|
||||
"""Arena 的心跳流可能因网络波动/长推理首字延迟提前静默,空结果必须交给 DOM 接管。"""
|
||||
return True
|
||||
|
||||
def _mark_terminal(self, prefix: str) -> str:
|
||||
side = "left" if prefix.startswith("a") else "right"
|
||||
self._completed_sides.add(side)
|
||||
if not self._completion_side:
|
||||
self._completion_side = side
|
||||
return side
|
||||
|
||||
@staticmethod
|
||||
def _fix_mojibake(text: str) -> str:
|
||||
try:
|
||||
mapped = text.translate(_CP1252_TO_LATIN1)
|
||||
return mapped.encode("latin-1").decode("utf-8")
|
||||
except (UnicodeEncodeError, UnicodeDecodeError):
|
||||
return text
|
||||
|
||||
@staticmethod
|
||||
def _parse_text_chunk(payload: str) -> str | None:
|
||||
try:
|
||||
value = json.loads(payload)
|
||||
if isinstance(value, str):
|
||||
return value
|
||||
except (json.JSONDecodeError, ValueError):
|
||||
pass
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _is_finish_signal(payload: str) -> bool:
|
||||
try:
|
||||
data = json.loads(payload)
|
||||
return isinstance(data, dict) and bool(data.get("finishReason"))
|
||||
except (json.JSONDecodeError, ValueError):
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def _parse_error(payload: str) -> str | None:
|
||||
try:
|
||||
data = json.loads(payload)
|
||||
if isinstance(data, dict):
|
||||
return data.get("message", str(data))
|
||||
return str(data)
|
||||
except (json.JSONDecodeError, ValueError):
|
||||
return payload.strip() if payload.strip() else None
|
||||
|
||||
@classmethod
|
||||
def get_supported_patterns(cls) -> List[str]:
|
||||
return ["nextjs-api/stream/create-evaluation"]
|
||||
|
||||
|
||||
class LmarenaBattleWinnerParser(ResponseParser):
|
||||
"""Emit only the first completed Arena battle side, without streaming."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._buffers = {"left": "", "right": ""}
|
||||
self._reasoning_buffers = {"left": "", "right": ""}
|
||||
self._seen_image_refs = {"left": set(), "right": set()}
|
||||
self._images = {"left": [], "right": []}
|
||||
self._completed_sides: set[str] = set()
|
||||
self._completed = False
|
||||
self._winner_side = ""
|
||||
self._emitted_winner = False
|
||||
|
||||
def parse_chunk(self, raw_response: str) -> Dict[str, Any]:
|
||||
result: Dict[str, Any] = {
|
||||
"content": "",
|
||||
"reasoning_content": "",
|
||||
"images": [],
|
||||
"done": False,
|
||||
"error": None,
|
||||
"selected_side": "winner",
|
||||
"winner_side": self._winner_side,
|
||||
}
|
||||
|
||||
if self._completed:
|
||||
result["done"] = True
|
||||
return result
|
||||
|
||||
if isinstance(raw_response, (bytes, bytearray)):
|
||||
raw_response = raw_response.decode("utf-8", errors="ignore")
|
||||
|
||||
if not raw_response or not isinstance(raw_response, str):
|
||||
return result
|
||||
|
||||
raw_response = _LmarenaBattleSideParser._fix_mojibake(raw_response)
|
||||
|
||||
try:
|
||||
content_parts = {"left": [], "right": []}
|
||||
reasoning_parts = {"left": [], "right": []}
|
||||
|
||||
for line in raw_response.split("\n"):
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
|
||||
colon_idx = line.find(":")
|
||||
if colon_idx < 1:
|
||||
continue
|
||||
|
||||
prefix = line[:colon_idx]
|
||||
payload = line[colon_idx + 1:]
|
||||
|
||||
if prefix == "a0":
|
||||
text = _LmarenaBattleSideParser._parse_text_chunk(payload)
|
||||
if text is not None:
|
||||
content_parts["left"].append(text)
|
||||
elif prefix == "b0":
|
||||
text = _LmarenaBattleSideParser._parse_text_chunk(payload)
|
||||
if text is not None:
|
||||
content_parts["right"].append(text)
|
||||
elif prefix == "ag":
|
||||
text = _LmarenaBattleSideParser._parse_text_chunk(payload)
|
||||
if text is not None:
|
||||
reasoning_parts["left"].append(text)
|
||||
elif prefix == "bg":
|
||||
text = _LmarenaBattleSideParser._parse_text_chunk(payload)
|
||||
if text is not None:
|
||||
reasoning_parts["right"].append(text)
|
||||
elif prefix == "a2":
|
||||
self._images["left"].extend(
|
||||
_extract_arena_image_items(
|
||||
payload,
|
||||
self._seen_image_refs["left"],
|
||||
source="lmarena_battle_winner_left_stream",
|
||||
)
|
||||
)
|
||||
elif prefix == "b2":
|
||||
self._images["right"].extend(
|
||||
_extract_arena_image_items(
|
||||
payload,
|
||||
self._seen_image_refs["right"],
|
||||
source="lmarena_battle_winner_right_stream",
|
||||
)
|
||||
)
|
||||
elif prefix in {"ad", "bd"}:
|
||||
if _LmarenaBattleSideParser._is_finish_signal(payload):
|
||||
self._mark_terminal(prefix)
|
||||
elif prefix in {"ae", "be", "a3", "b3"}:
|
||||
error_msg = _LmarenaBattleSideParser._parse_error(payload)
|
||||
if error_msg:
|
||||
winner_side = self._mark_terminal(prefix)
|
||||
logger.debug(
|
||||
"[LmarenaBattleWinnerParser] stream error frame treated as completion "
|
||||
f"(winner={winner_side}, error={error_msg[:160]})"
|
||||
)
|
||||
|
||||
for side, parts in content_parts.items():
|
||||
self._merge_text(side, "".join(parts))
|
||||
|
||||
for side, parts in reasoning_parts.items():
|
||||
self._merge_reasoning(side, "".join(parts))
|
||||
|
||||
if self._winner_side and not self._emitted_winner:
|
||||
self._emitted_winner = True
|
||||
result["content"] = self._buffers[self._winner_side]
|
||||
result["reasoning_content"] = self._reasoning_buffers[self._winner_side]
|
||||
result["images"] = self._images[self._winner_side]
|
||||
result["winner_side"] = self._winner_side
|
||||
|
||||
if len(self._completed_sides) >= 2:
|
||||
self._completed = True
|
||||
result["done"] = True
|
||||
result["winner_side"] = self._winner_side
|
||||
|
||||
except Exception as e:
|
||||
logger.debug(f"[LmarenaBattleWinnerParser] parse exception: {e}")
|
||||
result["error"] = str(e)
|
||||
|
||||
return result
|
||||
|
||||
def _merge_text(self, side: str, text: str) -> None:
|
||||
if not text:
|
||||
return
|
||||
|
||||
current = self._buffers[side]
|
||||
_delta, next_accumulated = LmarenaParser._content_delta(
|
||||
current,
|
||||
text,
|
||||
append_disjoint=True,
|
||||
)
|
||||
self._buffers[side] = next_accumulated
|
||||
|
||||
def _merge_reasoning(self, side: str, text: str) -> None:
|
||||
if not text:
|
||||
return
|
||||
|
||||
current = self._reasoning_buffers[side]
|
||||
_delta, next_accumulated = LmarenaParser._content_delta(
|
||||
current,
|
||||
text,
|
||||
append_disjoint=True,
|
||||
)
|
||||
self._reasoning_buffers[side] = next_accumulated
|
||||
|
||||
def reset(self):
|
||||
self._buffers = {"left": "", "right": ""}
|
||||
self._reasoning_buffers = {"left": "", "right": ""}
|
||||
self._seen_image_refs = {"left": set(), "right": set()}
|
||||
self._images = {"left": [], "right": []}
|
||||
self._completed_sides.clear()
|
||||
self._completed = False
|
||||
self._winner_side = ""
|
||||
self._emitted_winner = False
|
||||
|
||||
def should_fallback_to_dom_when_no_visible_content(self) -> bool:
|
||||
"""Arena 的心跳流可能因网络波动/长推理首字延迟提前静默,空结果必须交给 DOM 接管。"""
|
||||
return True
|
||||
|
||||
def _mark_terminal(self, prefix: str) -> str:
|
||||
side = "left" if prefix.startswith("a") else "right"
|
||||
self._completed_sides.add(side)
|
||||
if not self._winner_side:
|
||||
self._winner_side = side
|
||||
return side
|
||||
|
||||
@classmethod
|
||||
def get_supported_patterns(cls) -> List[str]:
|
||||
return ["nextjs-api/stream/create-evaluation"]
|
||||
|
||||
@classmethod
|
||||
def get_id(cls) -> str:
|
||||
return "lmarena_battle_winner"
|
||||
|
||||
@classmethod
|
||||
def get_name(cls) -> str:
|
||||
return "Arena.ai Battle Winner"
|
||||
|
||||
@classmethod
|
||||
def get_description(cls) -> str:
|
||||
return "Buffer both Arena.ai battle sides and emit only the first completed response"
|
||||
|
||||
|
||||
class LmarenaBattleSideLeftParser(_LmarenaBattleSideParser):
|
||||
"""Stream left/modelA text and finish when both battle sides finish."""
|
||||
|
||||
SIDE_PREFIX = "a"
|
||||
SIDE_LABEL = "left"
|
||||
|
||||
@classmethod
|
||||
def get_id(cls) -> str:
|
||||
return "lmarena_battle_side_left"
|
||||
|
||||
@classmethod
|
||||
def get_name(cls) -> str:
|
||||
return "Arena.ai Battle Side Left"
|
||||
|
||||
@classmethod
|
||||
def get_description(cls) -> str:
|
||||
return "Stream Arena.ai battle left/modelA text; finish when both sides complete"
|
||||
|
||||
|
||||
class LmarenaBattleSideRightParser(_LmarenaBattleSideParser):
|
||||
"""Stream right/modelB text and finish when both battle sides finish."""
|
||||
|
||||
SIDE_PREFIX = "b"
|
||||
SIDE_LABEL = "right"
|
||||
|
||||
@classmethod
|
||||
def get_id(cls) -> str:
|
||||
return "lmarena_battle_side_right"
|
||||
|
||||
@classmethod
|
||||
def get_name(cls) -> str:
|
||||
return "Arena.ai Battle Side Right"
|
||||
|
||||
@classmethod
|
||||
def get_description(cls) -> str:
|
||||
return "Stream Arena.ai battle right/modelB text; finish when both sides complete"
|
||||
|
||||
|
||||
__all__ = [
|
||||
"LmarenaBattleWinnerParser",
|
||||
"LmarenaBattleSideLeftParser",
|
||||
"LmarenaBattleSideRightParser",
|
||||
]
|
||||
@@ -0,0 +1,51 @@
|
||||
"""
|
||||
lmarena_image_side_left_parser.py - Arena.ai image battle left-side parser.
|
||||
|
||||
Purpose:
|
||||
- Reuse Arena side-by-side left channel parsing semantics.
|
||||
- Prefer quick DOM fallback when the network stream carries little or no
|
||||
user-visible text, which is common for image-generation flows where the
|
||||
rendered images are the real result.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import List
|
||||
|
||||
from .lmarena_side_left_parser import LmarenaSideLeftParser
|
||||
|
||||
|
||||
class LmarenaImageSideLeftParser(LmarenaSideLeftParser):
|
||||
"""
|
||||
Arena.ai image battle parser for the left/modelA channel.
|
||||
|
||||
The underlying stream protocol may still look like the standard Arena
|
||||
side-by-side SSE frames, but image flows often produce no visible text.
|
||||
In that case we prefer to fall back to DOM/image extraction quickly
|
||||
instead of treating an empty stream as success.
|
||||
"""
|
||||
|
||||
def should_fallback_to_dom_when_no_visible_content(self) -> bool:
|
||||
return True
|
||||
|
||||
@classmethod
|
||||
def get_id(cls) -> str:
|
||||
return "lmarena_image_side_left"
|
||||
|
||||
@classmethod
|
||||
def get_name(cls) -> str:
|
||||
return "Arena.ai Image Side Left"
|
||||
|
||||
@classmethod
|
||||
def get_description(cls) -> str:
|
||||
return "Parse Arena.ai image battle stream, keep left(modelA) and fall back to DOM when text is absent"
|
||||
|
||||
@classmethod
|
||||
def get_supported_patterns(cls) -> List[str]:
|
||||
return [
|
||||
"nextjs-api/stream/post-to-evaluation",
|
||||
"nextjs-api/stream/create-evaluation",
|
||||
]
|
||||
|
||||
|
||||
__all__ = ["LmarenaImageSideLeftParser"]
|
||||
@@ -0,0 +1,167 @@
|
||||
"""
|
||||
lmarena_image_side_right_parser.py - Arena.ai image battle right-side parser.
|
||||
|
||||
Purpose:
|
||||
- Keep the Arena image battle right/modelB stream channel.
|
||||
- Fall back to DOM/image extraction quickly when the stream has no visible text.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from typing import Any, Dict, List
|
||||
|
||||
from app.core.config import logger
|
||||
from .base import ResponseParser
|
||||
from .lmarena_parser import LmarenaParser, _CP1252_TO_LATIN1, _extract_arena_image_items
|
||||
|
||||
|
||||
class LmarenaImageSideRightParser(ResponseParser):
|
||||
"""Arena.ai image battle parser for the right/modelB channel."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._accumulated = ""
|
||||
self._seen_image_refs: set[str] = set()
|
||||
|
||||
def parse_chunk(self, raw_response: str) -> Dict[str, Any]:
|
||||
result: Dict[str, Any] = {
|
||||
"content": "",
|
||||
"images": [],
|
||||
"done": False,
|
||||
"error": None,
|
||||
}
|
||||
|
||||
if isinstance(raw_response, (bytes, bytearray)):
|
||||
raw_response = raw_response.decode("utf-8", errors="ignore")
|
||||
|
||||
if not raw_response or not isinstance(raw_response, str):
|
||||
return result
|
||||
|
||||
raw_response = self._fix_mojibake(raw_response)
|
||||
|
||||
try:
|
||||
content_parts: List[str] = []
|
||||
images: List[Dict[str, Any]] = []
|
||||
done = False
|
||||
|
||||
for line in raw_response.split("\n"):
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
|
||||
colon_idx = line.find(":")
|
||||
if colon_idx < 1:
|
||||
continue
|
||||
|
||||
prefix = line[:colon_idx]
|
||||
payload = line[colon_idx + 1:]
|
||||
|
||||
if prefix == "b0":
|
||||
text = self._parse_text_chunk(payload)
|
||||
if text is not None:
|
||||
content_parts.append(text)
|
||||
elif prefix == "b2":
|
||||
images.extend(
|
||||
_extract_arena_image_items(
|
||||
payload,
|
||||
self._seen_image_refs,
|
||||
source="lmarena_image_side_right_stream",
|
||||
)
|
||||
)
|
||||
elif prefix == "bd":
|
||||
if self._is_finish_signal(payload):
|
||||
done = True
|
||||
elif prefix in {"be", "b3"}:
|
||||
error_msg = self._parse_error(payload)
|
||||
if error_msg:
|
||||
result["error"] = error_msg
|
||||
done = True
|
||||
|
||||
new_content = "".join(content_parts)
|
||||
if new_content:
|
||||
delta, next_accumulated = LmarenaParser._content_delta(
|
||||
self._accumulated,
|
||||
new_content,
|
||||
append_disjoint=True,
|
||||
)
|
||||
if delta:
|
||||
result["content"] = delta
|
||||
self._accumulated = next_accumulated
|
||||
|
||||
if images:
|
||||
result["images"] = images
|
||||
result["done"] = done
|
||||
|
||||
except Exception as e:
|
||||
logger.debug(f"[LmarenaImageSideRightParser] parse exception: {e}")
|
||||
result["error"] = str(e)
|
||||
|
||||
return result
|
||||
|
||||
def reset(self):
|
||||
self._accumulated = ""
|
||||
self._seen_image_refs.clear()
|
||||
|
||||
def should_abort_on_error(self) -> bool:
|
||||
return True
|
||||
|
||||
def should_fallback_to_dom_when_no_visible_content(self) -> bool:
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
def _fix_mojibake(text: str) -> str:
|
||||
try:
|
||||
mapped = text.translate(_CP1252_TO_LATIN1)
|
||||
return mapped.encode("latin-1").decode("utf-8")
|
||||
except (UnicodeEncodeError, UnicodeDecodeError):
|
||||
return text
|
||||
|
||||
@staticmethod
|
||||
def _parse_text_chunk(payload: str) -> str | None:
|
||||
try:
|
||||
value = json.loads(payload)
|
||||
if isinstance(value, str):
|
||||
return value
|
||||
except (json.JSONDecodeError, ValueError):
|
||||
pass
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _is_finish_signal(payload: str) -> bool:
|
||||
try:
|
||||
data = json.loads(payload)
|
||||
return isinstance(data, dict) and bool(data.get("finishReason"))
|
||||
except (json.JSONDecodeError, ValueError):
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def _parse_error(payload: str) -> str | None:
|
||||
try:
|
||||
data = json.loads(payload)
|
||||
if isinstance(data, dict):
|
||||
return data.get("message", str(data))
|
||||
return str(data)
|
||||
except (json.JSONDecodeError, ValueError):
|
||||
return payload.strip() if payload.strip() else None
|
||||
|
||||
@classmethod
|
||||
def get_id(cls) -> str:
|
||||
return "lmarena_image_side_right"
|
||||
|
||||
@classmethod
|
||||
def get_name(cls) -> str:
|
||||
return "Arena.ai Image Side Right"
|
||||
|
||||
@classmethod
|
||||
def get_description(cls) -> str:
|
||||
return "Parse Arena.ai image battle stream, keep right(modelB) and fall back to DOM when text is absent"
|
||||
|
||||
@classmethod
|
||||
def get_supported_patterns(cls) -> List[str]:
|
||||
return [
|
||||
"nextjs-api/stream/post-to-evaluation",
|
||||
"nextjs-api/stream/create-evaluation",
|
||||
]
|
||||
|
||||
|
||||
__all__ = ["LmarenaImageSideRightParser"]
|
||||
@@ -0,0 +1,564 @@
|
||||
"""
|
||||
lmarena_parser.py - Arena.ai 响应解析器
|
||||
|
||||
响应格式:Vercel AI SDK Data Stream Protocol
|
||||
- 每行格式: {prefix}:{json_data}
|
||||
- a0: 文本增量(JSON 编码的字符串)
|
||||
- a2: 心跳/元数据(JSON 数组)
|
||||
- ad: 结束元数据(含 finishReason)
|
||||
- ae: 错误信息
|
||||
|
||||
编码问题:
|
||||
DrissionPage 通过 CDP 获取响应体时,UTF-8 字节被混合解码为
|
||||
cp1252 映射字符(如 0x92→' U+2019)和 latin-1 直通字符
|
||||
(如 0x90→U+0090),导致双重编码(mojibake)。
|
||||
需要两步修复:translate 统一到 latin-1 范围 → encode('latin-1') 还原字节
|
||||
|
||||
调用方式:
|
||||
NetworkMonitor 每次调用 parse_chunk 传入一个完整的 HTTP 响应体
|
||||
(DrissionPage listen.wait 是非流式的,一次性拿到整个 SSE body)
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any, Dict, List
|
||||
|
||||
from app.core.config import logger
|
||||
from .base import ResponseParser
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────
|
||||
# cp1252 → latin-1 逆映射表(模块级常量,只构建一次)
|
||||
# ─────────────────────────────────────────────────────
|
||||
# cp1252 在 0x80-0x9F 区间将部分字节映射到 U+0100 以上的 Unicode 字符,
|
||||
# 而 latin-1 将相同字节直通映射到 U+0080-U+009F。
|
||||
# 此表将 cp1252 的特殊映射还原为 latin-1 范围的等价字符,
|
||||
# 使后续 encode('latin-1') 能正确还原原始字节。
|
||||
#
|
||||
# 5 个未定义位置 (0x81, 0x8D, 0x8F, 0x90, 0x9D) 已经是
|
||||
# latin-1 直通映射 (U+0081 等),无需处理。
|
||||
_CP1252_TO_LATIN1 = str.maketrans({
|
||||
'\u20ac': '\x80', # €
|
||||
'\u201a': '\x82', # ‚
|
||||
'\u0192': '\x83', # ƒ
|
||||
'\u201e': '\x84', # „
|
||||
'\u2026': '\x85', # …
|
||||
'\u2020': '\x86', # †
|
||||
'\u2021': '\x87', # ‡
|
||||
'\u02c6': '\x88', # ˆ
|
||||
'\u2030': '\x89', # ‰
|
||||
'\u0160': '\x8a', # Š
|
||||
'\u2039': '\x8b', # ‹
|
||||
'\u0152': '\x8c', # Œ
|
||||
'\u017d': '\x8e', # Ž
|
||||
'\u2018': '\x91', # '
|
||||
'\u2019': '\x92', # '
|
||||
'\u201c': '\x93', # "
|
||||
'\u201d': '\x94', # "
|
||||
'\u2022': '\x95', # •
|
||||
'\u2013': '\x96', # –
|
||||
'\u2014': '\x97', # —
|
||||
'\u02dc': '\x98', # ˜
|
||||
'\u2122': '\x99', # ™
|
||||
'\u0161': '\x9a', # š
|
||||
'\u203a': '\x9b', # ›
|
||||
'\u0153': '\x9c', # œ
|
||||
'\u017e': '\x9e', # ž
|
||||
'\u0178': '\x9f', # Ÿ
|
||||
})
|
||||
|
||||
|
||||
def _looks_like_image_ref(value: str) -> bool:
|
||||
text = str(value or "").strip()
|
||||
if not text:
|
||||
return False
|
||||
lowered = text.lower()
|
||||
if lowered.startswith(("http://", "https://", "data:image/", "blob:")):
|
||||
return True
|
||||
return any(token in lowered for token in (".png", ".jpg", ".jpeg", ".webp", ".gif", ".bmp", ".svg", ".avif"))
|
||||
|
||||
|
||||
def _extract_arena_image_items(
|
||||
payload: str,
|
||||
seen_refs: set[str],
|
||||
*,
|
||||
source: str,
|
||||
) -> List[Dict[str, Any]]:
|
||||
items: List[Dict[str, Any]] = []
|
||||
|
||||
try:
|
||||
data = json.loads(payload)
|
||||
except (json.JSONDecodeError, ValueError):
|
||||
return items
|
||||
|
||||
entries = data if isinstance(data, list) else [data]
|
||||
for entry in entries:
|
||||
if not isinstance(entry, dict):
|
||||
continue
|
||||
|
||||
entry_type = str(entry.get("type") or "").strip().lower()
|
||||
if entry_type and entry_type != "image":
|
||||
continue
|
||||
|
||||
image_ref = ""
|
||||
for key in ("image", "url", "src"):
|
||||
candidate = str(entry.get(key) or "").strip()
|
||||
if _looks_like_image_ref(candidate):
|
||||
image_ref = candidate
|
||||
break
|
||||
|
||||
if not image_ref:
|
||||
continue
|
||||
|
||||
dedupe_key = f"image:{image_ref}"
|
||||
if dedupe_key in seen_refs:
|
||||
continue
|
||||
seen_refs.add(dedupe_key)
|
||||
|
||||
mime = str(entry.get("mimeType") or entry.get("mime") or "").strip() or None
|
||||
kind = "data_uri" if image_ref.startswith("data:image/") else "url"
|
||||
items.append(
|
||||
{
|
||||
"media_type": "image",
|
||||
"kind": kind,
|
||||
"url": image_ref if kind == "url" else None,
|
||||
"data_uri": image_ref if kind == "data_uri" else None,
|
||||
"mime": mime,
|
||||
"byte_size": None,
|
||||
"alt": str(entry.get("alt") or "").strip(),
|
||||
"width": entry.get("width"),
|
||||
"height": entry.get("height"),
|
||||
"detected_at": datetime.now(timezone.utc).isoformat().replace("+00:00", "Z"),
|
||||
"source": source,
|
||||
}
|
||||
)
|
||||
|
||||
return items
|
||||
|
||||
|
||||
class _LmarenaScanState:
|
||||
"""LmarenaParser 增量扫描的累计状态(只覆盖已提交的完整行 + 本次半行)。"""
|
||||
|
||||
__slots__ = (
|
||||
"content", "reasoning", "done", "error", "line_count", "prefix_counts",
|
||||
"unknown_prefix_count", "non_protocol_line_count", "parse_errors",
|
||||
"committed_len", "committed_raw",
|
||||
)
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.content = ""
|
||||
self.reasoning = ""
|
||||
self.done = False
|
||||
self.error = None
|
||||
self.line_count = 0
|
||||
self.prefix_counts: Dict[str, int] = {}
|
||||
self.unknown_prefix_count = 0
|
||||
self.non_protocol_line_count = 0
|
||||
self.parse_errors: list[Dict[str, Any]] = []
|
||||
self.committed_len = 0
|
||||
self.committed_raw = ""
|
||||
|
||||
def copy(self) -> "_LmarenaScanState":
|
||||
other = _LmarenaScanState()
|
||||
for name in self.__slots__:
|
||||
setattr(other, name, getattr(self, name))
|
||||
other.prefix_counts = dict(self.prefix_counts)
|
||||
other.parse_errors = list(self.parse_errors)
|
||||
return other
|
||||
|
||||
def add_parse_error(self, item: Dict[str, Any]) -> None:
|
||||
self.parse_errors.append(item)
|
||||
if len(self.parse_errors) > 16:
|
||||
del self.parse_errors[:-8]
|
||||
|
||||
|
||||
class LmarenaParser(ResponseParser):
|
||||
"""
|
||||
Arena.ai 响应解析器
|
||||
|
||||
URL 特征: /nextjs-api/stream/create-evaluation
|
||||
响应格式: Vercel AI SDK Data Stream Protocol (行分隔)
|
||||
"""
|
||||
|
||||
_PROTOCOL_PREFIXES = {"a0", "a2", "ad", "ae", "a3", "ag", "bg"}
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._accumulated = ""
|
||||
self._accumulated_reasoning = ""
|
||||
self._seen_image_refs: set[str] = set()
|
||||
self._last_debug_summary: Dict[str, Any] = {}
|
||||
self._last_debug_raw_signature = ""
|
||||
self._inc_state: "_LmarenaScanState | None" = None
|
||||
|
||||
@staticmethod
|
||||
def _content_delta(
|
||||
accumulated: str,
|
||||
candidate: str,
|
||||
*,
|
||||
append_disjoint: bool = False,
|
||||
) -> tuple[str, str]:
|
||||
"""Return unseen text and the new accumulated value for repeated SSE bodies."""
|
||||
if not candidate:
|
||||
return "", accumulated
|
||||
if not accumulated:
|
||||
return candidate, candidate
|
||||
if candidate == accumulated or accumulated.startswith(candidate):
|
||||
return "", accumulated
|
||||
if candidate.startswith(accumulated):
|
||||
return candidate[len(accumulated):], candidate
|
||||
|
||||
max_overlap = min(len(accumulated), len(candidate))
|
||||
for overlap in range(max_overlap, 0, -1):
|
||||
if accumulated.endswith(candidate[:overlap]):
|
||||
delta = candidate[overlap:]
|
||||
return delta, accumulated + delta
|
||||
|
||||
if append_disjoint:
|
||||
return candidate, accumulated + candidate
|
||||
|
||||
# A parser instance belongs to one response stream. A body that neither
|
||||
# extends nor overlaps the emitted text is a stale/non-monotonic replay;
|
||||
# emitting it wholesale would duplicate the response already sent.
|
||||
return "", accumulated
|
||||
|
||||
# ============ 增量扫描(P0-5) ============
|
||||
|
||||
# 置为 False 时每次整段解析(与旧实现逐字节一致),用于回归对照
|
||||
_INCREMENTAL = True
|
||||
|
||||
def _scan_response(self, raw_response: str) -> tuple["_LmarenaScanState", list]:
|
||||
"""返回 (扫描状态, 本次新发现的图片)。"""
|
||||
images: list[Dict[str, Any]] = []
|
||||
if not self._INCREMENTAL:
|
||||
# 旧行为:整段修复 mojibake 后逐行解析
|
||||
scan = _LmarenaScanState()
|
||||
self._scan_lines(self._fix_mojibake(raw_response), scan, images)
|
||||
return scan, images
|
||||
|
||||
cached = getattr(self, "_inc_state", None)
|
||||
if cached is not None and cached.committed_len <= len(raw_response) and raw_response.startswith(cached.committed_raw):
|
||||
scan = cached.copy()
|
||||
offset = cached.committed_len
|
||||
else:
|
||||
scan = _LmarenaScanState()
|
||||
offset = 0
|
||||
|
||||
last_newline = raw_response.rfind("\n", offset)
|
||||
if last_newline >= offset:
|
||||
committed_end = last_newline + 1
|
||||
# 完整行一定以 ASCII 换行结尾,不会截断多字节序列,可以按段修复 mojibake
|
||||
self._scan_lines(self._fix_mojibake(raw_response[offset:committed_end]), scan, images)
|
||||
scan.committed_len = committed_end
|
||||
scan.committed_raw = raw_response[:committed_end]
|
||||
self._inc_state = scan.copy()
|
||||
offset = committed_end
|
||||
elif cached is None or scan.committed_len == 0:
|
||||
self._inc_state = None
|
||||
|
||||
tail = raw_response[offset:]
|
||||
if tail:
|
||||
self._scan_lines(self._fix_mojibake(tail), scan, images)
|
||||
return scan, images
|
||||
|
||||
def _scan_lines(self, text: str, scan: "_LmarenaScanState", images: list) -> None:
|
||||
content_parts: list[str] = []
|
||||
reasoning_parts: list[str] = []
|
||||
for line in text.split("\n"):
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
scan.line_count += 1
|
||||
|
||||
colon_idx = line.find(":")
|
||||
if colon_idx < 1:
|
||||
scan.non_protocol_line_count += 1
|
||||
continue
|
||||
|
||||
prefix = line[:colon_idx]
|
||||
payload = line[colon_idx + 1:]
|
||||
if prefix in self._PROTOCOL_PREFIXES:
|
||||
scan.prefix_counts[prefix] = scan.prefix_counts.get(prefix, 0) + 1
|
||||
else:
|
||||
scan.unknown_prefix_count += 1
|
||||
|
||||
if prefix == "a0":
|
||||
text_chunk = self._parse_text_chunk(payload)
|
||||
if text_chunk is not None:
|
||||
content_parts.append(text_chunk)
|
||||
else:
|
||||
scan.add_parse_error({"prefix": prefix, "payload_preview": payload[:160]})
|
||||
|
||||
elif prefix in {"ag", "bg"}:
|
||||
text_chunk = self._parse_text_chunk(payload)
|
||||
if text_chunk is not None:
|
||||
reasoning_parts.append(text_chunk)
|
||||
else:
|
||||
scan.add_parse_error({"prefix": prefix, "payload_preview": payload[:160]})
|
||||
|
||||
elif prefix == "a2":
|
||||
images.extend(
|
||||
_extract_arena_image_items(
|
||||
payload,
|
||||
self._seen_image_refs,
|
||||
source="lmarena_stream",
|
||||
)
|
||||
)
|
||||
|
||||
elif prefix == "ad":
|
||||
if self._is_finish_signal(payload):
|
||||
scan.done = True
|
||||
|
||||
elif prefix in {"ae", "a3"}:
|
||||
error_msg = self._parse_error(payload)
|
||||
if error_msg:
|
||||
scan.error = error_msg
|
||||
scan.done = True
|
||||
if content_parts:
|
||||
scan.content += "".join(content_parts)
|
||||
if reasoning_parts:
|
||||
scan.reasoning += "".join(reasoning_parts)
|
||||
|
||||
# ============ 对外接口 ============
|
||||
|
||||
def parse_chunk(self, raw_response: str) -> Dict[str, Any]:
|
||||
"""
|
||||
解析完整的 HTTP 响应体(一次性包含所有 SSE 行)
|
||||
"""
|
||||
result: Dict[str, Any] = {
|
||||
"content": "",
|
||||
"reasoning_content": "",
|
||||
"images": [],
|
||||
"done": False,
|
||||
"error": None,
|
||||
}
|
||||
|
||||
if isinstance(raw_response, (bytes, bytearray)):
|
||||
raw_response = raw_response.decode("utf-8", errors="ignore")
|
||||
|
||||
if not raw_response or not isinstance(raw_response, str):
|
||||
return result
|
||||
|
||||
debug_raw_signature = self._debug_body_signature(raw_response)
|
||||
|
||||
try:
|
||||
# P0-5:增量解析。网络监听每次传入的是“不断增长的完整响应体”,旧实现
|
||||
# 每次都整段 split + json.loads,一个 160KB 的回答流累计要花数秒 CPU。
|
||||
# 这里把“已以换行结尾的完整行”的解析结果缓存下来,只解析新增部分;
|
||||
# 末尾未结束的半行每次重新解析、不入缓存。响应体不是上次的前缀扩展
|
||||
# (换了一条流)时回退为整段解析。
|
||||
scan, images = self._scan_response(raw_response)
|
||||
done = scan.done
|
||||
if scan.error:
|
||||
result["error"] = scan.error
|
||||
new_content = scan.content
|
||||
new_reasoning = scan.reasoning
|
||||
|
||||
if new_content:
|
||||
delta, next_accumulated = self._content_delta(self._accumulated, new_content)
|
||||
if delta:
|
||||
result["content"] = delta
|
||||
self._accumulated = next_accumulated
|
||||
|
||||
if new_reasoning:
|
||||
delta_r, next_acc_r = self._content_delta(self._accumulated_reasoning, new_reasoning)
|
||||
if delta_r:
|
||||
result["reasoning_content"] = delta_r
|
||||
self._accumulated_reasoning = next_acc_r
|
||||
|
||||
if images:
|
||||
result["images"] = images
|
||||
result["done"] = done
|
||||
self._last_debug_summary = {
|
||||
"raw_body_len": len(raw_response),
|
||||
"line_count": scan.line_count,
|
||||
"prefix_counts": dict(scan.prefix_counts),
|
||||
"unknown_prefix_count": scan.unknown_prefix_count,
|
||||
"non_protocol_line_count": scan.non_protocol_line_count,
|
||||
"content_candidate_len": len(new_content),
|
||||
"emitted_content_len": len(str(result.get("content") or "")),
|
||||
"accumulated_len": len(self._accumulated),
|
||||
"reasoning_candidate_len": len(new_reasoning),
|
||||
"emitted_reasoning_len": len(str(result.get("reasoning_content") or "")),
|
||||
"accumulated_reasoning_len": len(self._accumulated_reasoning),
|
||||
"done": bool(done),
|
||||
"error": str(result.get("error") or ""),
|
||||
"image_count": len(images),
|
||||
"parse_errors": scan.parse_errors[-8:],
|
||||
"incremental_offset": scan.committed_len if self._INCREMENTAL else 0,
|
||||
}
|
||||
self._last_debug_raw_signature = debug_raw_signature
|
||||
|
||||
except Exception as e:
|
||||
logger.debug(f"[LmarenaParser] 解析异常: {e}")
|
||||
result["error"] = str(e)
|
||||
self._inc_state = None
|
||||
self._last_debug_summary = {
|
||||
"raw_body_len": len(raw_response),
|
||||
"exception": str(e),
|
||||
}
|
||||
self._last_debug_raw_signature = debug_raw_signature
|
||||
|
||||
return result
|
||||
|
||||
def reset(self) -> None:
|
||||
"""重置状态"""
|
||||
self._accumulated = ""
|
||||
self._accumulated_reasoning = ""
|
||||
self._seen_image_refs.clear()
|
||||
self._last_debug_summary = {}
|
||||
self._last_debug_raw_signature = ""
|
||||
self._inc_state = None
|
||||
|
||||
def should_abort_on_error(self) -> bool:
|
||||
"""当 Arena 返回 ae: 或 a3: 等错误协议行时,立即终止工作流。"""
|
||||
return True
|
||||
|
||||
def should_fallback_to_dom_when_no_visible_content(self) -> bool:
|
||||
"""Arena 的心跳流可能因网络波动提前关闭,空结果必须交给 DOM 接管。"""
|
||||
return True
|
||||
|
||||
def export_debug_data(self, raw_response: str = "") -> Dict[str, Any]:
|
||||
"""Return a compact parser-side protocol summary for network_parser_debug."""
|
||||
summary = dict(self._last_debug_summary or {})
|
||||
if raw_response:
|
||||
text = self._normalize_debug_text(raw_response)
|
||||
raw_summary = self._summarize_protocol_lines(text)
|
||||
if self._last_debug_raw_signature == self._debug_body_signature(text):
|
||||
summary.update(raw_summary)
|
||||
else:
|
||||
summary = raw_summary
|
||||
summary.update({
|
||||
"accumulated_len": len(self._accumulated),
|
||||
"seen_image_refs": len(self._seen_image_refs),
|
||||
})
|
||||
return summary
|
||||
|
||||
# ============ 内部方法 ============
|
||||
|
||||
@staticmethod
|
||||
def _normalize_debug_text(raw_response: Any) -> str:
|
||||
if isinstance(raw_response, (bytes, bytearray)):
|
||||
return raw_response.decode("utf-8", errors="ignore")
|
||||
return str(raw_response or "")
|
||||
|
||||
@classmethod
|
||||
def _debug_body_signature(cls, raw_response: Any) -> str:
|
||||
text = cls._normalize_debug_text(raw_response)
|
||||
return f"{len(text)}:{text[:80]}:{text[-80:]}"
|
||||
|
||||
@classmethod
|
||||
def _summarize_protocol_lines(cls, raw_response: Any) -> Dict[str, Any]:
|
||||
text = cls._normalize_debug_text(raw_response)
|
||||
prefix_counts: Dict[str, int] = {}
|
||||
line_count = 0
|
||||
unknown_prefix_count = 0
|
||||
non_protocol_line_count = 0
|
||||
for raw_line in text.split("\n"):
|
||||
line = raw_line.strip()
|
||||
if not line:
|
||||
continue
|
||||
line_count += 1
|
||||
colon_idx = line.find(":")
|
||||
if colon_idx < 1:
|
||||
non_protocol_line_count += 1
|
||||
continue
|
||||
prefix = line[:colon_idx]
|
||||
if prefix in cls._PROTOCOL_PREFIXES:
|
||||
prefix_counts[prefix] = prefix_counts.get(prefix, 0) + 1
|
||||
else:
|
||||
unknown_prefix_count += 1
|
||||
return {
|
||||
"raw_body_len": len(text),
|
||||
"line_count": line_count,
|
||||
"prefix_counts": prefix_counts,
|
||||
"unknown_prefix_count": unknown_prefix_count,
|
||||
"non_protocol_line_count": non_protocol_line_count,
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _fix_mojibake(text: str) -> str:
|
||||
"""
|
||||
修复双重 UTF-8 编码(mojibake)
|
||||
|
||||
DrissionPage 通过 CDP 获取的响应体存在编码错误:
|
||||
UTF-8 字节被混合解码为 cp1252 + latin-1,导致:
|
||||
|
||||
原始 UTF-8: f0 9f 92 9e (💞)
|
||||
被 cp1252 解码: ð(F0) Ÿ(9F→U+0178) '(92→U+2019) ž(9E→U+017E)
|
||||
但 0x90 → U+0090(latin-1 直通,因 cp1252 未定义此位置)
|
||||
|
||||
cp1252 有 5 个未定义位置 (0x81,0x8D,0x8F,0x90,0x9D)
|
||||
走 latin-1 直通映射,导致 encode('cp1252') 和 encode('latin-1')
|
||||
都无法单独处理全部字符。
|
||||
|
||||
修复方案(两步):
|
||||
1. translate: 将 cp1252 特有字符 (U+2019等) 映射回 latin-1 范围 (U+0092等)
|
||||
2. encode('latin-1'): 所有字符现在都在 U+0000-U+00FF,可一一还原原始字节
|
||||
3. decode('utf-8'): 原始字节 → 正确文本
|
||||
"""
|
||||
try:
|
||||
mapped = text.translate(_CP1252_TO_LATIN1)
|
||||
return mapped.encode("latin-1").decode("utf-8")
|
||||
except (UnicodeDecodeError, UnicodeEncodeError):
|
||||
# 不是 mojibake(或只有部分是),返回原样
|
||||
return text
|
||||
|
||||
@staticmethod
|
||||
def _parse_text_chunk(payload: str) -> str | None:
|
||||
"""解析 a0 行的 payload"""
|
||||
try:
|
||||
value = json.loads(payload)
|
||||
if isinstance(value, str):
|
||||
return value
|
||||
except (json.JSONDecodeError, ValueError):
|
||||
pass
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _is_finish_signal(payload: str) -> bool:
|
||||
"""检查 ad 行是否表示结束"""
|
||||
try:
|
||||
data = json.loads(payload)
|
||||
if isinstance(data, dict) and data.get("finishReason"):
|
||||
return True
|
||||
except (json.JSONDecodeError, ValueError):
|
||||
pass
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def _parse_error(payload: str) -> str | None:
|
||||
"""解析 ae 行的错误信息"""
|
||||
try:
|
||||
data = json.loads(payload)
|
||||
if isinstance(data, dict):
|
||||
return data.get("message", str(data))
|
||||
return str(data)
|
||||
except (json.JSONDecodeError, ValueError):
|
||||
return payload.strip() if payload.strip() else None
|
||||
|
||||
# ============ 元数据 ============
|
||||
|
||||
@classmethod
|
||||
def get_id(cls) -> str:
|
||||
return "lmarena"
|
||||
|
||||
@classmethod
|
||||
def get_name(cls) -> str:
|
||||
return "Arena.ai"
|
||||
|
||||
@classmethod
|
||||
def get_description(cls) -> str:
|
||||
return "解析 Arena.ai 的流式响应 (Vercel AI SDK Data Stream Protocol)"
|
||||
|
||||
@classmethod
|
||||
def get_supported_patterns(cls) -> List[str]:
|
||||
return ["nextjs-api/stream/create-evaluation"]
|
||||
|
||||
|
||||
__all__ = [
|
||||
"LmarenaParser",
|
||||
"_CP1252_TO_LATIN1",
|
||||
"_extract_arena_image_items",
|
||||
]
|
||||
@@ -0,0 +1,197 @@
|
||||
"""
|
||||
lmarena_side_left_parser.py - Arena.ai side-by-side left-stream parser.
|
||||
|
||||
Protocol:
|
||||
- Vercel AI SDK data stream lines: "{prefix}:{payload}"
|
||||
- left(modelA): a0/ad/ae
|
||||
- right(modelB): b0/bd/be (ignored here)
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from typing import Any, Dict, List
|
||||
|
||||
from app.core.config import logger
|
||||
from .base import ResponseParser
|
||||
from .lmarena_parser import LmarenaParser, _CP1252_TO_LATIN1, _extract_arena_image_items
|
||||
|
||||
|
||||
class LmarenaSideLeftParser(ResponseParser):
|
||||
"""
|
||||
Parse Arena.ai side-by-side stream and keep only left/modelA channel.
|
||||
|
||||
This parser is intended for side mode where the left model is the target
|
||||
(for example gemini-3.1-pro-preview in your setup).
|
||||
"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._accumulated = ""
|
||||
self._accumulated_reasoning = ""
|
||||
self._seen_image_refs: set[str] = set()
|
||||
|
||||
def parse_chunk(self, raw_response: str) -> Dict[str, Any]:
|
||||
result: Dict[str, Any] = {
|
||||
"content": "",
|
||||
"reasoning_content": "",
|
||||
"images": [],
|
||||
"done": False,
|
||||
"error": None,
|
||||
}
|
||||
|
||||
if isinstance(raw_response, (bytes, bytearray)):
|
||||
raw_response = raw_response.decode("utf-8", errors="ignore")
|
||||
|
||||
if not raw_response or not isinstance(raw_response, str):
|
||||
return result
|
||||
|
||||
raw_response = self._fix_mojibake(raw_response)
|
||||
|
||||
try:
|
||||
content_parts: List[str] = []
|
||||
reasoning_parts: List[str] = []
|
||||
images: List[Dict[str, Any]] = []
|
||||
done = False
|
||||
|
||||
for line in raw_response.split("\n"):
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
|
||||
colon_idx = line.find(":")
|
||||
if colon_idx < 1:
|
||||
continue
|
||||
|
||||
prefix = line[:colon_idx]
|
||||
payload = line[colon_idx + 1:]
|
||||
|
||||
# left/modelA text delta
|
||||
if prefix == "a0":
|
||||
text = self._parse_text_chunk(payload)
|
||||
if text is not None:
|
||||
content_parts.append(text)
|
||||
elif prefix == "ag":
|
||||
text = self._parse_text_chunk(payload)
|
||||
if text is not None:
|
||||
reasoning_parts.append(text)
|
||||
elif prefix == "a2":
|
||||
images.extend(
|
||||
_extract_arena_image_items(
|
||||
payload,
|
||||
self._seen_image_refs,
|
||||
source="lmarena_side_left_stream",
|
||||
)
|
||||
)
|
||||
# left/modelA done
|
||||
elif prefix == "ad":
|
||||
if self._is_finish_signal(payload):
|
||||
done = True
|
||||
# left/modelA error frames (observed formats: ae / a3)
|
||||
elif prefix in {"ae", "a3"}:
|
||||
error_msg = self._parse_error(payload)
|
||||
if error_msg:
|
||||
result["error"] = error_msg
|
||||
done = True
|
||||
|
||||
new_content = "".join(content_parts)
|
||||
new_reasoning = "".join(reasoning_parts)
|
||||
|
||||
if new_content:
|
||||
delta, next_accumulated = LmarenaParser._content_delta(
|
||||
self._accumulated,
|
||||
new_content,
|
||||
append_disjoint=True,
|
||||
)
|
||||
if delta:
|
||||
result["content"] = delta
|
||||
self._accumulated = next_accumulated
|
||||
|
||||
if new_reasoning:
|
||||
delta_r, next_acc_r = LmarenaParser._content_delta(
|
||||
self._accumulated_reasoning,
|
||||
new_reasoning,
|
||||
append_disjoint=True,
|
||||
)
|
||||
if delta_r:
|
||||
result["reasoning_content"] = delta_r
|
||||
self._accumulated_reasoning = next_acc_r
|
||||
|
||||
if images:
|
||||
result["images"] = images
|
||||
result["done"] = done
|
||||
|
||||
except Exception as e:
|
||||
logger.debug(f"[LmarenaSideLeftParser] parse exception: {e}")
|
||||
result["error"] = str(e)
|
||||
|
||||
return result
|
||||
|
||||
def reset(self):
|
||||
self._accumulated = ""
|
||||
self._accumulated_reasoning = ""
|
||||
self._seen_image_refs.clear()
|
||||
|
||||
def should_fallback_to_dom_when_no_visible_content(self) -> bool:
|
||||
"""Arena 的心跳流可能因网络波动提前关闭,空结果必须交给 DOM 接管。"""
|
||||
return True
|
||||
|
||||
def should_abort_on_error(self) -> bool:
|
||||
# Side-by-side left mode only trusts the left stream. If the left channel
|
||||
# itself reports an error, we should fail fast instead of silently
|
||||
# falling back to whole-page DOM heuristics.
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
def _fix_mojibake(text: str) -> str:
|
||||
try:
|
||||
mapped = text.translate(_CP1252_TO_LATIN1)
|
||||
return mapped.encode("latin-1").decode("utf-8")
|
||||
except (UnicodeEncodeError, UnicodeDecodeError):
|
||||
return text
|
||||
|
||||
@staticmethod
|
||||
def _parse_text_chunk(payload: str) -> str | None:
|
||||
try:
|
||||
value = json.loads(payload)
|
||||
if isinstance(value, str):
|
||||
return value
|
||||
except (json.JSONDecodeError, ValueError):
|
||||
pass
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _is_finish_signal(payload: str) -> bool:
|
||||
try:
|
||||
data = json.loads(payload)
|
||||
return isinstance(data, dict) and bool(data.get("finishReason"))
|
||||
except (json.JSONDecodeError, ValueError):
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def _parse_error(payload: str) -> str | None:
|
||||
try:
|
||||
data = json.loads(payload)
|
||||
if isinstance(data, dict):
|
||||
return data.get("message", str(data))
|
||||
return str(data)
|
||||
except (json.JSONDecodeError, ValueError):
|
||||
return payload.strip() if payload.strip() else None
|
||||
|
||||
@classmethod
|
||||
def get_id(cls) -> str:
|
||||
return "lmarena_side_left"
|
||||
|
||||
@classmethod
|
||||
def get_name(cls) -> str:
|
||||
return "Arena.ai Side Left"
|
||||
|
||||
@classmethod
|
||||
def get_description(cls) -> str:
|
||||
return "Parse Arena.ai side-by-side stream, keep left(modelA) only"
|
||||
|
||||
@classmethod
|
||||
def get_supported_patterns(cls) -> List[str]:
|
||||
return ["nextjs-api/stream/create-evaluation"]
|
||||
|
||||
|
||||
__all__ = ["LmarenaSideLeftParser"]
|
||||
@@ -0,0 +1,338 @@
|
||||
"""
|
||||
mimo_parser.py - Xiaomi MiMo SSE response parser.
|
||||
|
||||
Observed stream traits:
|
||||
- POST /open-apis/bot/chat?... returns text/event-stream
|
||||
- answer chunks arrive as event:message with JSON data {"type":"text","content":"..."}
|
||||
- event:finish with {"content":"[DONE]"} ends the stream
|
||||
- reasoning text is wrapped inside <think>...</think> and should be ignored
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from typing import Any, Dict, List, Tuple
|
||||
|
||||
from app.core.config import logger
|
||||
from .base import ResponseParser
|
||||
|
||||
|
||||
class MimoParser(ResponseParser):
|
||||
"""Parse Xiaomi MiMo SSE streams and ignore think-phase content."""
|
||||
|
||||
_SUSPICIOUS_CHARS = set(
|
||||
"ÃÂâæåçèéêëìíîïðñòóôõöùúûüýþÿ"
|
||||
"€‚ƒ„…†‡ˆ‰Š‹ŒŽ‘’“”•–—˜™š›œžŸ"
|
||||
)
|
||||
_REVERSE_CP1252_BYTES = {
|
||||
0x20AC: 0x80,
|
||||
0x201A: 0x82,
|
||||
0x0192: 0x83,
|
||||
0x201E: 0x84,
|
||||
0x2026: 0x85,
|
||||
0x2020: 0x86,
|
||||
0x2021: 0x87,
|
||||
0x02C6: 0x88,
|
||||
0x2030: 0x89,
|
||||
0x0160: 0x8A,
|
||||
0x2039: 0x8B,
|
||||
0x0152: 0x8C,
|
||||
0x017D: 0x8E,
|
||||
0x2018: 0x91,
|
||||
0x2019: 0x92,
|
||||
0x201C: 0x93,
|
||||
0x201D: 0x94,
|
||||
0x2022: 0x95,
|
||||
0x2013: 0x96,
|
||||
0x2014: 0x97,
|
||||
0x02DC: 0x98,
|
||||
0x2122: 0x99,
|
||||
0x0161: 0x9A,
|
||||
0x203A: 0x9B,
|
||||
0x0153: 0x9C,
|
||||
0x017E: 0x9E,
|
||||
0x0178: 0x9F,
|
||||
}
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._last_raw_length = 0
|
||||
self._pending = ""
|
||||
self._inside_think = False
|
||||
self._has_seen_visible_text = False
|
||||
|
||||
def reset(self) -> None:
|
||||
self._last_raw_length = 0
|
||||
self._last_raw_response = ""
|
||||
self._pending = ""
|
||||
self._inside_think = False
|
||||
self._has_seen_visible_text = False
|
||||
|
||||
def parse_chunk(self, raw_response: str) -> Dict[str, Any]:
|
||||
result: Dict[str, Any] = {
|
||||
"content": "",
|
||||
"images": [],
|
||||
"done": False,
|
||||
"error": None,
|
||||
}
|
||||
|
||||
try:
|
||||
if isinstance(raw_response, (bytes, bytearray)):
|
||||
raw_response = raw_response.decode("utf-8", errors="ignore")
|
||||
elif not isinstance(raw_response, str):
|
||||
raw_response = str(raw_response)
|
||||
|
||||
new_data = self._prepare_incremental_raw_response(raw_response)
|
||||
if not new_data:
|
||||
return result
|
||||
|
||||
delta_content, done = self._consume_new_data(new_data)
|
||||
if delta_content:
|
||||
result["content"] = delta_content
|
||||
result["done"] = done
|
||||
|
||||
except Exception as e:
|
||||
logger.debug(f"[MimoParser] parse exception: {e}")
|
||||
result["error"] = str(e)
|
||||
|
||||
return result
|
||||
|
||||
def _consume_new_data(self, new_data: str) -> Tuple[str, bool]:
|
||||
normalized = (self._pending + new_data).replace("\r\n", "\n")
|
||||
if not normalized:
|
||||
return "", False
|
||||
|
||||
blocks = normalized.split("\n\n")
|
||||
if normalized.endswith("\n\n"):
|
||||
self._pending = ""
|
||||
complete_blocks = [block for block in blocks if block.strip()]
|
||||
else:
|
||||
self._pending = blocks.pop() if blocks else normalized
|
||||
complete_blocks = [block for block in blocks if block.strip()]
|
||||
|
||||
content_parts: List[str] = []
|
||||
done = False
|
||||
|
||||
for block in complete_blocks:
|
||||
block_content, block_done = self._parse_event_block(block)
|
||||
if block_content:
|
||||
content_parts.append(block_content)
|
||||
if block_done:
|
||||
done = True
|
||||
|
||||
return "".join(content_parts), done
|
||||
|
||||
def _parse_event_block(self, block: str) -> Tuple[str, bool]:
|
||||
event_name = ""
|
||||
data_lines: List[str] = []
|
||||
|
||||
for raw_line in block.split("\n"):
|
||||
line = raw_line.strip()
|
||||
if not line:
|
||||
continue
|
||||
if line.startswith("event:"):
|
||||
event_name = line[6:].strip()
|
||||
elif line.startswith("data:"):
|
||||
data_lines.append(line[5:].strip())
|
||||
|
||||
if event_name == "finish":
|
||||
return "", self._has_seen_visible_text
|
||||
if event_name != "message":
|
||||
return "", False
|
||||
|
||||
payload = "\n".join(data_lines).strip()
|
||||
if not payload:
|
||||
return "", False
|
||||
|
||||
try:
|
||||
data = json.loads(payload)
|
||||
except json.JSONDecodeError:
|
||||
return "", False
|
||||
|
||||
if not isinstance(data, dict):
|
||||
return "", False
|
||||
if str(data.get("type") or "").strip().lower() != "text":
|
||||
return "", False
|
||||
|
||||
text = data.get("content", "")
|
||||
if not isinstance(text, str) or not text:
|
||||
return "", False
|
||||
|
||||
visible_text = self._strip_think_content(text)
|
||||
visible_text = self._repair_mojibake_text(visible_text)
|
||||
if visible_text:
|
||||
self._has_seen_visible_text = True
|
||||
return visible_text, False
|
||||
|
||||
def _strip_think_content(self, text: str) -> str:
|
||||
if not text:
|
||||
return ""
|
||||
|
||||
normalized = text.replace("\x00", "")
|
||||
visible_parts: List[str] = []
|
||||
cursor = 0
|
||||
|
||||
while cursor < len(normalized):
|
||||
if self._inside_think:
|
||||
end_idx = normalized.find("</think>", cursor)
|
||||
if end_idx == -1:
|
||||
return "".join(visible_parts)
|
||||
self._inside_think = False
|
||||
cursor = end_idx + len("</think>")
|
||||
continue
|
||||
|
||||
start_idx = normalized.find("<think>", cursor)
|
||||
if start_idx == -1:
|
||||
visible_parts.append(normalized[cursor:])
|
||||
break
|
||||
|
||||
if start_idx > cursor:
|
||||
visible_parts.append(normalized[cursor:start_idx])
|
||||
self._inside_think = True
|
||||
cursor = start_idx + len("<think>")
|
||||
|
||||
return "".join(visible_parts)
|
||||
|
||||
@classmethod
|
||||
def _count_cjk(cls, text: str) -> int:
|
||||
total = 0
|
||||
for ch in text or "":
|
||||
code = ord(ch)
|
||||
if (
|
||||
0x4E00 <= code <= 0x9FFF
|
||||
or 0x3400 <= code <= 0x4DBF
|
||||
or 0xF900 <= code <= 0xFAFF
|
||||
):
|
||||
total += 1
|
||||
return total
|
||||
|
||||
@classmethod
|
||||
def _count_suspicious(cls, text: str) -> int:
|
||||
return sum(1 for ch in (text or "") if ch in cls._SUSPICIOUS_CHARS)
|
||||
|
||||
@classmethod
|
||||
def _is_byte_like_char(cls, ch: str) -> bool:
|
||||
code = ord(ch)
|
||||
return code in cls._REVERSE_CP1252_BYTES or 0 <= code <= 0xFF
|
||||
|
||||
@classmethod
|
||||
def _has_high_byte_signal(cls, text: str) -> bool:
|
||||
for ch in text or "":
|
||||
code = ord(ch)
|
||||
if code in cls._REVERSE_CP1252_BYTES or 0x80 <= code <= 0xFF:
|
||||
return True
|
||||
return False
|
||||
|
||||
@classmethod
|
||||
def _looks_like_utf8_mojibake(cls, text: str) -> bool:
|
||||
if not text or len(text) < 2:
|
||||
return False
|
||||
|
||||
cjk = cls._count_cjk(text)
|
||||
if cjk == 0:
|
||||
return cls._has_high_byte_signal(text)
|
||||
|
||||
suspicious = cls._count_suspicious(text)
|
||||
return suspicious >= 1 and cls._has_high_byte_signal(text)
|
||||
|
||||
@classmethod
|
||||
def _quality_score(cls, text: str) -> tuple[int, int, int]:
|
||||
return (
|
||||
cls._count_cjk(text),
|
||||
-cls._count_suspicious(text),
|
||||
-(text.count("\ufffd") if text else 0),
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def _try_redecode(cls, text: str) -> str:
|
||||
try:
|
||||
payload = bytearray()
|
||||
for ch in text:
|
||||
code = ord(ch)
|
||||
if code in cls._REVERSE_CP1252_BYTES:
|
||||
payload.append(cls._REVERSE_CP1252_BYTES[code])
|
||||
continue
|
||||
if 0 <= code <= 0xFF:
|
||||
payload.append(code)
|
||||
continue
|
||||
return text
|
||||
return bytes(payload).decode("utf-8")
|
||||
except Exception:
|
||||
return text
|
||||
|
||||
@classmethod
|
||||
def _repair_segmentwise(cls, text: str) -> str:
|
||||
if not text:
|
||||
return text
|
||||
|
||||
pieces: List[str] = []
|
||||
buffer: List[str] = []
|
||||
|
||||
def _flush_buffer() -> None:
|
||||
if not buffer:
|
||||
return
|
||||
segment = "".join(buffer)
|
||||
buffer.clear()
|
||||
|
||||
if (
|
||||
len(segment) >= 2
|
||||
and cls._count_cjk(segment) == 0
|
||||
and cls._has_high_byte_signal(segment)
|
||||
):
|
||||
candidate = cls._try_redecode(segment)
|
||||
if cls._quality_score(candidate) > cls._quality_score(segment):
|
||||
pieces.append(candidate)
|
||||
return
|
||||
pieces.append(segment)
|
||||
|
||||
for ch in text:
|
||||
if cls._is_byte_like_char(ch):
|
||||
buffer.append(ch)
|
||||
else:
|
||||
_flush_buffer()
|
||||
pieces.append(ch)
|
||||
|
||||
_flush_buffer()
|
||||
return "".join(pieces)
|
||||
|
||||
@classmethod
|
||||
def _repair_mojibake_text(cls, text: str) -> str:
|
||||
if not text or not cls._looks_like_utf8_mojibake(text):
|
||||
return text
|
||||
|
||||
best = text
|
||||
best_score = cls._quality_score(text)
|
||||
|
||||
candidate = cls._try_redecode(text)
|
||||
if candidate != best:
|
||||
candidate_score = cls._quality_score(candidate)
|
||||
if candidate_score > best_score:
|
||||
best = candidate
|
||||
best_score = candidate_score
|
||||
|
||||
candidate = cls._repair_segmentwise(best)
|
||||
if candidate != best:
|
||||
candidate_score = cls._quality_score(candidate)
|
||||
if candidate_score > best_score:
|
||||
best = candidate
|
||||
best_score = candidate_score
|
||||
|
||||
return best
|
||||
|
||||
@classmethod
|
||||
def get_id(cls) -> str:
|
||||
return "mimo"
|
||||
|
||||
@classmethod
|
||||
def get_name(cls) -> str:
|
||||
return "Xiaomi MiMo"
|
||||
|
||||
@classmethod
|
||||
def get_description(cls) -> str:
|
||||
return "Parse Xiaomi MiMo SSE streams and ignore think-phase text"
|
||||
|
||||
@classmethod
|
||||
def get_supported_patterns(cls) -> List[str]:
|
||||
return ["**/open-apis/bot/chat**"]
|
||||
|
||||
|
||||
__all__ = ["MimoParser"]
|
||||
@@ -0,0 +1,338 @@
|
||||
"""
|
||||
mimo_parser.py - Xiaomi MiMo SSE response parser.
|
||||
|
||||
Observed stream traits:
|
||||
- POST /open-apis/bot/chat?... returns text/event-stream
|
||||
- answer chunks arrive as event:message with JSON data {"type":"text","content":"..."}
|
||||
- event:finish with {"content":"[DONE]"} ends the stream
|
||||
- reasoning text is wrapped inside <think>...</think> and should be ignored
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from typing import Any, Dict, List, Tuple
|
||||
|
||||
from app.core.config import logger
|
||||
from .base import ResponseParser
|
||||
|
||||
|
||||
class MimoParser(ResponseParser):
|
||||
"""Parse Xiaomi MiMo SSE streams and ignore think-phase content."""
|
||||
|
||||
_SUSPICIOUS_CHARS = set(
|
||||
"ÃÂâæåçèéêëìíîïðñòóôõöùúûüýþÿ"
|
||||
"€‚ƒ„…†‡ˆ‰Š‹ŒŽ‘’“”•–—˜™š›œžŸ"
|
||||
)
|
||||
_REVERSE_CP1252_BYTES = {
|
||||
0x20AC: 0x80,
|
||||
0x201A: 0x82,
|
||||
0x0192: 0x83,
|
||||
0x201E: 0x84,
|
||||
0x2026: 0x85,
|
||||
0x2020: 0x86,
|
||||
0x2021: 0x87,
|
||||
0x02C6: 0x88,
|
||||
0x2030: 0x89,
|
||||
0x0160: 0x8A,
|
||||
0x2039: 0x8B,
|
||||
0x0152: 0x8C,
|
||||
0x017D: 0x8E,
|
||||
0x2018: 0x91,
|
||||
0x2019: 0x92,
|
||||
0x201C: 0x93,
|
||||
0x201D: 0x94,
|
||||
0x2022: 0x95,
|
||||
0x2013: 0x96,
|
||||
0x2014: 0x97,
|
||||
0x02DC: 0x98,
|
||||
0x2122: 0x99,
|
||||
0x0161: 0x9A,
|
||||
0x203A: 0x9B,
|
||||
0x0153: 0x9C,
|
||||
0x017E: 0x9E,
|
||||
0x0178: 0x9F,
|
||||
}
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._last_raw_length = 0
|
||||
self._pending = ""
|
||||
self._inside_think = False
|
||||
self._has_seen_visible_text = False
|
||||
|
||||
def reset(self) -> None:
|
||||
self._last_raw_length = 0
|
||||
self._last_raw_response = ""
|
||||
self._pending = ""
|
||||
self._inside_think = False
|
||||
self._has_seen_visible_text = False
|
||||
|
||||
def parse_chunk(self, raw_response: str) -> Dict[str, Any]:
|
||||
result: Dict[str, Any] = {
|
||||
"content": "",
|
||||
"images": [],
|
||||
"done": False,
|
||||
"error": None,
|
||||
}
|
||||
|
||||
try:
|
||||
if isinstance(raw_response, (bytes, bytearray)):
|
||||
raw_response = raw_response.decode("utf-8", errors="ignore")
|
||||
elif not isinstance(raw_response, str):
|
||||
raw_response = str(raw_response)
|
||||
|
||||
new_data = self._prepare_incremental_raw_response(raw_response)
|
||||
if not new_data:
|
||||
return result
|
||||
|
||||
delta_content, done = self._consume_new_data(new_data)
|
||||
if delta_content:
|
||||
result["content"] = delta_content
|
||||
result["done"] = done
|
||||
|
||||
except Exception as e:
|
||||
logger.debug(f"[MimoParser] parse exception: {e}")
|
||||
result["error"] = str(e)
|
||||
|
||||
return result
|
||||
|
||||
def _consume_new_data(self, new_data: str) -> Tuple[str, bool]:
|
||||
normalized = (self._pending + new_data).replace("\r\n", "\n")
|
||||
if not normalized:
|
||||
return "", False
|
||||
|
||||
blocks = normalized.split("\n\n")
|
||||
if normalized.endswith("\n\n"):
|
||||
self._pending = ""
|
||||
complete_blocks = [block for block in blocks if block.strip()]
|
||||
else:
|
||||
self._pending = blocks.pop() if blocks else normalized
|
||||
complete_blocks = [block for block in blocks if block.strip()]
|
||||
|
||||
content_parts: List[str] = []
|
||||
done = False
|
||||
|
||||
for block in complete_blocks:
|
||||
block_content, block_done = self._parse_event_block(block)
|
||||
if block_content:
|
||||
content_parts.append(block_content)
|
||||
if block_done:
|
||||
done = True
|
||||
|
||||
return "".join(content_parts), done
|
||||
|
||||
def _parse_event_block(self, block: str) -> Tuple[str, bool]:
|
||||
event_name = ""
|
||||
data_lines: List[str] = []
|
||||
|
||||
for raw_line in block.split("\n"):
|
||||
line = raw_line.strip()
|
||||
if not line:
|
||||
continue
|
||||
if line.startswith("event:"):
|
||||
event_name = line[6:].strip()
|
||||
elif line.startswith("data:"):
|
||||
data_lines.append(line[5:].strip())
|
||||
|
||||
if event_name == "finish":
|
||||
return "", self._has_seen_visible_text
|
||||
if event_name != "message":
|
||||
return "", False
|
||||
|
||||
payload = "\n".join(data_lines).strip()
|
||||
if not payload:
|
||||
return "", False
|
||||
|
||||
try:
|
||||
data = json.loads(payload)
|
||||
except json.JSONDecodeError:
|
||||
return "", False
|
||||
|
||||
if not isinstance(data, dict):
|
||||
return "", False
|
||||
if str(data.get("type") or "").strip().lower() != "text":
|
||||
return "", False
|
||||
|
||||
text = data.get("content", "")
|
||||
if not isinstance(text, str) or not text:
|
||||
return "", False
|
||||
|
||||
visible_text = self._strip_think_content(text)
|
||||
visible_text = self._repair_mojibake_text(visible_text)
|
||||
if visible_text:
|
||||
self._has_seen_visible_text = True
|
||||
return visible_text, False
|
||||
|
||||
def _strip_think_content(self, text: str) -> str:
|
||||
if not text:
|
||||
return ""
|
||||
|
||||
normalized = text.replace("\x00", "")
|
||||
visible_parts: List[str] = []
|
||||
cursor = 0
|
||||
|
||||
while cursor < len(normalized):
|
||||
if self._inside_think:
|
||||
end_idx = normalized.find("</think>", cursor)
|
||||
if end_idx == -1:
|
||||
return "".join(visible_parts)
|
||||
self._inside_think = False
|
||||
cursor = end_idx + len("</think>")
|
||||
continue
|
||||
|
||||
start_idx = normalized.find("<think>", cursor)
|
||||
if start_idx == -1:
|
||||
visible_parts.append(normalized[cursor:])
|
||||
break
|
||||
|
||||
if start_idx > cursor:
|
||||
visible_parts.append(normalized[cursor:start_idx])
|
||||
self._inside_think = True
|
||||
cursor = start_idx + len("<think>")
|
||||
|
||||
return "".join(visible_parts)
|
||||
|
||||
@classmethod
|
||||
def _count_cjk(cls, text: str) -> int:
|
||||
total = 0
|
||||
for ch in text or "":
|
||||
code = ord(ch)
|
||||
if (
|
||||
0x4E00 <= code <= 0x9FFF
|
||||
or 0x3400 <= code <= 0x4DBF
|
||||
or 0xF900 <= code <= 0xFAFF
|
||||
):
|
||||
total += 1
|
||||
return total
|
||||
|
||||
@classmethod
|
||||
def _count_suspicious(cls, text: str) -> int:
|
||||
return sum(1 for ch in (text or "") if ch in cls._SUSPICIOUS_CHARS)
|
||||
|
||||
@classmethod
|
||||
def _is_byte_like_char(cls, ch: str) -> bool:
|
||||
code = ord(ch)
|
||||
return code in cls._REVERSE_CP1252_BYTES or 0 <= code <= 0xFF
|
||||
|
||||
@classmethod
|
||||
def _has_high_byte_signal(cls, text: str) -> bool:
|
||||
for ch in text or "":
|
||||
code = ord(ch)
|
||||
if code in cls._REVERSE_CP1252_BYTES or 0x80 <= code <= 0xFF:
|
||||
return True
|
||||
return False
|
||||
|
||||
@classmethod
|
||||
def _looks_like_utf8_mojibake(cls, text: str) -> bool:
|
||||
if not text or len(text) < 2:
|
||||
return False
|
||||
|
||||
cjk = cls._count_cjk(text)
|
||||
if cjk == 0:
|
||||
return cls._has_high_byte_signal(text)
|
||||
|
||||
suspicious = cls._count_suspicious(text)
|
||||
return suspicious >= 1 and cls._has_high_byte_signal(text)
|
||||
|
||||
@classmethod
|
||||
def _quality_score(cls, text: str) -> tuple[int, int, int]:
|
||||
return (
|
||||
cls._count_cjk(text),
|
||||
-cls._count_suspicious(text),
|
||||
-(text.count("\ufffd") if text else 0),
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def _try_redecode(cls, text: str) -> str:
|
||||
try:
|
||||
payload = bytearray()
|
||||
for ch in text:
|
||||
code = ord(ch)
|
||||
if code in cls._REVERSE_CP1252_BYTES:
|
||||
payload.append(cls._REVERSE_CP1252_BYTES[code])
|
||||
continue
|
||||
if 0 <= code <= 0xFF:
|
||||
payload.append(code)
|
||||
continue
|
||||
return text
|
||||
return bytes(payload).decode("utf-8")
|
||||
except Exception:
|
||||
return text
|
||||
|
||||
@classmethod
|
||||
def _repair_segmentwise(cls, text: str) -> str:
|
||||
if not text:
|
||||
return text
|
||||
|
||||
pieces: List[str] = []
|
||||
buffer: List[str] = []
|
||||
|
||||
def _flush_buffer() -> None:
|
||||
if not buffer:
|
||||
return
|
||||
segment = "".join(buffer)
|
||||
buffer.clear()
|
||||
|
||||
if (
|
||||
len(segment) >= 2
|
||||
and cls._count_cjk(segment) == 0
|
||||
and cls._has_high_byte_signal(segment)
|
||||
):
|
||||
candidate = cls._try_redecode(segment)
|
||||
if cls._quality_score(candidate) > cls._quality_score(segment):
|
||||
pieces.append(candidate)
|
||||
return
|
||||
pieces.append(segment)
|
||||
|
||||
for ch in text:
|
||||
if cls._is_byte_like_char(ch):
|
||||
buffer.append(ch)
|
||||
else:
|
||||
_flush_buffer()
|
||||
pieces.append(ch)
|
||||
|
||||
_flush_buffer()
|
||||
return "".join(pieces)
|
||||
|
||||
@classmethod
|
||||
def _repair_mojibake_text(cls, text: str) -> str:
|
||||
if not text or not cls._looks_like_utf8_mojibake(text):
|
||||
return text
|
||||
|
||||
best = text
|
||||
best_score = cls._quality_score(text)
|
||||
|
||||
candidate = cls._try_redecode(text)
|
||||
if candidate != best:
|
||||
candidate_score = cls._quality_score(candidate)
|
||||
if candidate_score > best_score:
|
||||
best = candidate
|
||||
best_score = candidate_score
|
||||
|
||||
candidate = cls._repair_segmentwise(best)
|
||||
if candidate != best:
|
||||
candidate_score = cls._quality_score(candidate)
|
||||
if candidate_score > best_score:
|
||||
best = candidate
|
||||
best_score = candidate_score
|
||||
|
||||
return best
|
||||
|
||||
@classmethod
|
||||
def get_id(cls) -> str:
|
||||
return "mimo"
|
||||
|
||||
@classmethod
|
||||
def get_name(cls) -> str:
|
||||
return "Xiaomi MiMo"
|
||||
|
||||
@classmethod
|
||||
def get_description(cls) -> str:
|
||||
return "Parse Xiaomi MiMo SSE streams and ignore think-phase text"
|
||||
|
||||
@classmethod
|
||||
def get_supported_patterns(cls) -> List[str]:
|
||||
return ["**/open-apis/bot/chat**"]
|
||||
|
||||
|
||||
__all__ = ["MimoParser"]
|
||||
@@ -0,0 +1,197 @@
|
||||
"""
|
||||
qwen_parser.py - Qwen SSE response parser.
|
||||
|
||||
Observed stream traits:
|
||||
- data: {"response.created": ...} announces the response id
|
||||
- answer tokens are in choices[*].delta.content with phase=answer
|
||||
- thinking_summary chunks should be ignored
|
||||
- choices[*].delta.status=finished ends the answer stream
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from typing import Dict, Any, List, Tuple
|
||||
|
||||
from app.core.config import logger
|
||||
from .base import ResponseParser
|
||||
|
||||
|
||||
class QwenParser(ResponseParser):
|
||||
"""Parse Qwen web SSE responses."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._last_raw_length = 0
|
||||
self._pending = ""
|
||||
self._has_seen_answer_text = False
|
||||
self._accumulated_reasoning = ""
|
||||
|
||||
def parse_chunk(self, raw_response: str) -> Dict[str, Any]:
|
||||
result: Dict[str, Any] = {
|
||||
"content": "",
|
||||
"reasoning_content": "",
|
||||
"images": [],
|
||||
"done": False,
|
||||
"error": None,
|
||||
}
|
||||
|
||||
try:
|
||||
if isinstance(raw_response, (bytes, bytearray)):
|
||||
raw_response = raw_response.decode("utf-8", errors="ignore")
|
||||
elif not isinstance(raw_response, str):
|
||||
raw_response = str(raw_response)
|
||||
|
||||
new_data = self._prepare_incremental_raw_response(raw_response)
|
||||
if not new_data:
|
||||
return result
|
||||
|
||||
delta_content, delta_reasoning, done = self._consume_new_data(new_data)
|
||||
if delta_content:
|
||||
result["content"] = delta_content
|
||||
if delta_reasoning:
|
||||
result["reasoning_content"] = delta_reasoning
|
||||
result["done"] = done
|
||||
|
||||
except Exception as e:
|
||||
logger.debug(f"[QwenParser] parse exception: {e}")
|
||||
result["error"] = str(e)
|
||||
|
||||
return result
|
||||
|
||||
def reset(self) -> None:
|
||||
self._last_raw_length = 0
|
||||
self._last_raw_response = ""
|
||||
self._pending = ""
|
||||
self._has_seen_answer_text = False
|
||||
self._accumulated_reasoning = ""
|
||||
|
||||
def _consume_new_data(self, new_data: str) -> Tuple[str, str, bool]:
|
||||
normalized = (self._pending + new_data).replace("\r\n", "\n")
|
||||
if not normalized:
|
||||
return "", "", False
|
||||
|
||||
blocks = normalized.split("\n\n")
|
||||
if normalized.endswith("\n\n"):
|
||||
self._pending = ""
|
||||
complete_blocks = [block for block in blocks if block.strip()]
|
||||
else:
|
||||
self._pending = blocks.pop() if blocks else normalized
|
||||
complete_blocks = [block for block in blocks if block.strip()]
|
||||
|
||||
content_parts: List[str] = []
|
||||
reasoning_parts: List[str] = []
|
||||
done = False
|
||||
|
||||
for block in complete_blocks:
|
||||
block_content, block_reasoning, block_done = self._parse_event_block(block)
|
||||
if block_content:
|
||||
content_parts.append(block_content)
|
||||
if block_reasoning:
|
||||
reasoning_parts.append(block_reasoning)
|
||||
if block_done:
|
||||
done = True
|
||||
|
||||
new_reasoning = "\n\n".join(reasoning_parts)
|
||||
delta_reasoning = ""
|
||||
if new_reasoning:
|
||||
delta_reasoning, self._accumulated_reasoning = self._reasoning_delta(
|
||||
self._accumulated_reasoning, new_reasoning
|
||||
)
|
||||
|
||||
return "".join(content_parts), delta_reasoning, done
|
||||
|
||||
@staticmethod
|
||||
def _reasoning_delta(accumulated: str, candidate: str) -> Tuple[str, str]:
|
||||
if not candidate:
|
||||
return "", accumulated
|
||||
if not accumulated:
|
||||
return candidate, candidate
|
||||
if candidate == accumulated or accumulated.startswith(candidate):
|
||||
return "", accumulated
|
||||
if candidate.startswith(accumulated):
|
||||
return candidate[len(accumulated):], candidate
|
||||
|
||||
max_overlap = min(len(accumulated), len(candidate))
|
||||
for overlap in range(max_overlap, 0, -1):
|
||||
if accumulated.endswith(candidate[:overlap]):
|
||||
delta = candidate[overlap:]
|
||||
return delta, accumulated + delta
|
||||
|
||||
return candidate, accumulated + candidate
|
||||
|
||||
def _parse_event_block(self, block: str) -> Tuple[str, str, bool]:
|
||||
data_lines: List[str] = []
|
||||
|
||||
for raw_line in block.split("\n"):
|
||||
line = raw_line.strip()
|
||||
if line.startswith("data:"):
|
||||
data_lines.append(line[5:].strip())
|
||||
|
||||
payload = "\n".join(data_lines).strip()
|
||||
if not payload:
|
||||
return "", "", False
|
||||
|
||||
try:
|
||||
data = json.loads(payload)
|
||||
except json.JSONDecodeError:
|
||||
return "", "", False
|
||||
|
||||
choices = data.get("choices")
|
||||
if not isinstance(choices, list):
|
||||
return "", "", False
|
||||
|
||||
content_parts: List[str] = []
|
||||
reasoning_parts: List[str] = []
|
||||
done = False
|
||||
|
||||
for choice in choices:
|
||||
if not isinstance(choice, dict):
|
||||
continue
|
||||
delta = choice.get("delta")
|
||||
if not isinstance(delta, dict):
|
||||
continue
|
||||
|
||||
phase = delta.get("phase")
|
||||
|
||||
if phase == "thinking_summary":
|
||||
extra = delta.get("extra")
|
||||
if isinstance(extra, dict):
|
||||
summary_thought = extra.get("summary_thought")
|
||||
if isinstance(summary_thought, dict):
|
||||
thought_list = summary_thought.get("content")
|
||||
if isinstance(thought_list, list):
|
||||
reasoning_parts.append("\n\n".join([str(x) for x in thought_list if x]))
|
||||
|
||||
elif phase == "answer":
|
||||
text = delta.get("content", "")
|
||||
if isinstance(text, str) and text:
|
||||
content_parts.append(text)
|
||||
self._has_seen_answer_text = True
|
||||
|
||||
if delta.get("status") == "finished" and self._has_seen_answer_text:
|
||||
done = True
|
||||
|
||||
return "".join(content_parts), "\n\n".join(reasoning_parts), done
|
||||
|
||||
@classmethod
|
||||
def get_id(cls) -> str:
|
||||
return "qwen"
|
||||
|
||||
@classmethod
|
||||
def get_name(cls) -> str:
|
||||
return "Qwen"
|
||||
|
||||
@classmethod
|
||||
def get_description(cls) -> str:
|
||||
return "Parse Qwen SSE streams and keep only answer-phase text"
|
||||
|
||||
def should_fallback_to_dom_when_no_visible_content(self) -> bool:
|
||||
"""Use the rendered reply when Qwen's captured SSE stalls in its thinking phase."""
|
||||
return True
|
||||
|
||||
@classmethod
|
||||
def get_supported_patterns(cls) -> List[str]:
|
||||
return ["**/api/v2/chat/completions**"]
|
||||
|
||||
|
||||
__all__ = ["QwenParser"]
|
||||
@@ -0,0 +1,120 @@
|
||||
"""
|
||||
app/core/parsers/registry.py - 解析器注册中心
|
||||
|
||||
提供解析器的动态注册、查找、实例化功能
|
||||
"""
|
||||
|
||||
import importlib
|
||||
import sys
|
||||
from typing import Dict, Type, Optional, List
|
||||
|
||||
from app.core.config import logger
|
||||
from .base import ResponseParser
|
||||
|
||||
|
||||
class ParserRegistry:
|
||||
"""
|
||||
解析器注册中心
|
||||
|
||||
职责:
|
||||
1. 管理所有可用的响应解析器
|
||||
2. 提供注册机制
|
||||
3. 根据 ID 获取解析器实例
|
||||
"""
|
||||
|
||||
_parsers: Dict[str, Type[ResponseParser]] = {}
|
||||
|
||||
@classmethod
|
||||
def register_class(cls, parser_class: Type[ResponseParser], parser_id: Optional[str] = None):
|
||||
"""
|
||||
注册解析器类
|
||||
|
||||
Args:
|
||||
parser_class: 解析器类
|
||||
parser_id: 解析器 ID(可选,默认使用 get_id())
|
||||
"""
|
||||
pid = parser_id or parser_class.get_id()
|
||||
cls._parsers[pid] = parser_class
|
||||
logger.info(f"已注册响应解析器: {pid} -> {parser_class.__name__}")
|
||||
|
||||
@classmethod
|
||||
def get(cls, parser_id: str) -> ResponseParser:
|
||||
"""
|
||||
获取解析器实例
|
||||
|
||||
Args:
|
||||
parser_id: 解析器 ID
|
||||
|
||||
Returns:
|
||||
解析器实例
|
||||
|
||||
Raises:
|
||||
ValueError: 解析器 ID 不存在
|
||||
"""
|
||||
if parser_id not in cls._parsers:
|
||||
raise ValueError(
|
||||
f"未知的解析器 ID: {parser_id},可用的解析器: {list(cls._parsers.keys())}"
|
||||
)
|
||||
|
||||
parser_class = cls._parsers[parser_id]
|
||||
return parser_class()
|
||||
|
||||
@classmethod
|
||||
def exists(cls, parser_id: str) -> bool:
|
||||
"""检查解析器是否存在"""
|
||||
return parser_id in cls._parsers
|
||||
|
||||
@classmethod
|
||||
def list_all(cls) -> List[Dict[str, str]]:
|
||||
"""
|
||||
列出所有已注册的解析器
|
||||
|
||||
Returns:
|
||||
解析器信息列表
|
||||
"""
|
||||
result = []
|
||||
for pid, parser_class in cls._parsers.items():
|
||||
info = {
|
||||
"id": pid,
|
||||
"name": parser_class.get_name(),
|
||||
"description": parser_class.get_description(),
|
||||
"patterns": parser_class.get_supported_patterns(),
|
||||
}
|
||||
result.append(info)
|
||||
|
||||
return result
|
||||
|
||||
@classmethod
|
||||
def load_from_module(cls, module_path: str, class_name: str, parser_id: str):
|
||||
"""
|
||||
动态加载并注册解析器类
|
||||
|
||||
Args:
|
||||
module_path: 模块路径(如 "app.core.parsers.custom")
|
||||
class_name: 类名(如 "CustomParser")
|
||||
parser_id: 注册的 ID
|
||||
|
||||
Raises:
|
||||
ImportError: 模块不存在
|
||||
AttributeError: 类不存在
|
||||
"""
|
||||
try:
|
||||
importlib.invalidate_caches()
|
||||
if module_path in sys.modules:
|
||||
module = importlib.reload(sys.modules[module_path])
|
||||
else:
|
||||
module = importlib.import_module(module_path)
|
||||
parser_class = getattr(module, class_name)
|
||||
|
||||
if not issubclass(parser_class, ResponseParser):
|
||||
raise TypeError(f"{class_name} 不是 ResponseParser 的子类")
|
||||
|
||||
cls.register_class(parser_class, parser_id)
|
||||
logger.info(f"动态加载解析器成功: {module_path}.{class_name} -> {parser_id}")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"动态加载解析器失败: {module_path}.{class_name} - {e}")
|
||||
raise
|
||||
|
||||
|
||||
__all__ = ['ParserRegistry']
|
||||
Reference in new issue
Block a user