Initial commit

This commit is contained in:
Alex committed 2026-11-02 22:54:14 +08:00
commit 57fc3525eb
7311 files changed
+1291398

No files matched your search

+74
View File
@@ -0,0 +1,74 @@
"""
app/core/parsers - 网络响应解析器
职责:
- 定义响应解析的标准接口
- 提供注册机制支持多站点适配
- 解析增量响应数据
"""
from .base import ResponseParser
from .registry import ParserRegistry
from .gemini_parser import GeminiParser
from .chatgpt_parser import ChatGPTParser
from .deepseek_parser import DeepSeekParser
from .aistudio_parser import AIStudioParser
from .doubao_parser import DoubaoParser
from .claude_parser import ClaudeParser
from .kimi_parser import KimiParser
from .glm_parser import GLMParser
from .qwen_parser import QwenParser
from .mimo_parser import MimoParser
from .lmarena_parser import LmarenaParser
from .lmarena_side_left_parser import LmarenaSideLeftParser
from .lmarena_battle_side_parser import (
LmarenaBattleSideLeftParser,
LmarenaBattleSideRightParser,
LmarenaBattleWinnerParser,
)
from .lmarena_image_side_left_parser import LmarenaImageSideLeftParser
from .lmarena_image_side_right_parser import LmarenaImageSideRightParser
from .grok_parser import GrokParser
# 自动注册内置解析器
ParserRegistry.register_class(GeminiParser)
ParserRegistry.register_class(ChatGPTParser)
ParserRegistry.register_class(DeepSeekParser)
ParserRegistry.register_class(AIStudioParser)
ParserRegistry.register_class(DoubaoParser)
ParserRegistry.register_class(ClaudeParser)
ParserRegistry.register_class(KimiParser)
ParserRegistry.register_class(GLMParser)
ParserRegistry.register_class(QwenParser)
ParserRegistry.register_class(MimoParser)
ParserRegistry.register_class(LmarenaParser)
ParserRegistry.register_class(LmarenaSideLeftParser)
ParserRegistry.register_class(LmarenaBattleWinnerParser)
ParserRegistry.register_class(LmarenaBattleSideLeftParser)
ParserRegistry.register_class(LmarenaBattleSideRightParser)
ParserRegistry.register_class(LmarenaImageSideLeftParser)
ParserRegistry.register_class(LmarenaImageSideRightParser)
ParserRegistry.register_class(GrokParser)
__all__ = [
'ResponseParser',
'ParserRegistry',
'GeminiParser',
'ChatGPTParser',
'DeepSeekParser',
'AIStudioParser',
'DoubaoParser',
'ClaudeParser',
'KimiParser',
'GLMParser',
'QwenParser',
'MimoParser',
'LmarenaParser',
'LmarenaSideLeftParser',
'LmarenaBattleWinnerParser',
'LmarenaBattleSideLeftParser',
'LmarenaBattleSideRightParser',
'LmarenaImageSideLeftParser',
'LmarenaImageSideRightParser',
'GrokParser',
]
Binary file not shown.
+514
View File
@@ -0,0 +1,514 @@
"""
aistudio_parser.py - Google AI Studio 响应解析器
响应格式特征:
- Content-Type: application/json+protobuf
- 嵌套数组结构
- 增量文本模式
- 包含 thinking 内容(需过滤)
"""
from __future__ import annotations
import json
import re
from typing import Any, Dict, List, Tuple
from app.core.config import logger
from .base import ResponseParser
class AIStudioParser(ResponseParser):
"""
Google AI Studio (MakerSuite) 响应解析器
URL 特征: MakerSuiteService/GenerateContent
响应格式: JSON+Protobuf (嵌套数组)
"""
_VISIBLE_TEXT_BLOCK_RE = re.compile(
r'null,"((?:\\.|[^"\\])*)"\]\],"model"'
)
_VISIBLE_TEXT_BLOCK_PREFIX = 'null,"'
_OBFUSCATED_PAYLOAD_RE = re.compile(r"^[A-Za-z0-9+/=_-]+$")
_CJK_RE = re.compile(r"[\u3400-\u9fff]")
def __init__(self):
self._accumulated_content = ""
self._accumulated_reasoning = ""
self._is_done = False
self._last_raw_length = 0
self._pending_raw = ""
self._visible_text_scan_pending = ""
self._visible_text_fallback_content = ""
def parse_chunk(self, raw_response: str) -> Dict[str, Any]:
"""
解析响应(返回完整内容的增量)
"""
result = {
"content": "",
"reasoning_content": "",
"images": [],
"done": False,
"error": None
}
try:
if isinstance(raw_response, bytes):
raw_response = raw_response.decode("utf-8", errors="ignore")
elif not isinstance(raw_response, str):
raw_response = str(raw_response)
new_data = self._prepare_incremental_raw_response(raw_response)
if not new_data:
return result
payloads = self._decode_payloads(new_data, incremental=True)
content = ""
is_done = False
reasoning = ""
for data in payloads:
current_content, current_reasoning, current_done = self._extract_content(data)
if current_content:
content = current_content
if current_reasoning:
reasoning = current_reasoning
if current_done:
is_done = True
if not content:
content = self._extract_visible_text_from_increment(new_data)
if content:
if content.startswith(self._accumulated_content):
delta = content[len(self._accumulated_content):]
self._accumulated_content = content
else:
delta = content
self._accumulated_content += content
if delta:
result["content"] = delta
if reasoning:
if reasoning.startswith(self._accumulated_reasoning):
delta_r = reasoning[len(self._accumulated_reasoning):]
self._accumulated_reasoning = reasoning
else:
delta_r = reasoning
self._accumulated_reasoning += reasoning
if delta_r:
result["reasoning_content"] = delta_r
result["done"] = is_done
except json.JSONDecodeError as e:
logger.debug(f"[AIStudioParser] JSON 解析失败: {e}")
result["error"] = str(e)
except Exception as e:
logger.debug(f"[AIStudioParser] 解析异常: {e}")
result["error"] = str(e)
return result
def reset(self):
"""重置状态"""
self._accumulated_content = ""
self._accumulated_reasoning = ""
self._is_done = False
self._last_raw_length = 0
self._last_raw_response = ""
self._pending_raw = ""
self._visible_text_scan_pending = ""
self._visible_text_fallback_content = ""
@staticmethod
def _normalize_raw_text(raw_response: Any) -> str:
if isinstance(raw_response, bytes):
raw_response = raw_response.decode("utf-8", errors="ignore")
elif not isinstance(raw_response, str):
raw_response = str(raw_response)
text = raw_response.lstrip("\ufeff")
stripped = text.lstrip()
if stripped.startswith(")]}'"):
stripped = stripped[4:].lstrip("\r\n")
return stripped
def _decode_payloads(self, raw_response: Any, incremental: bool = False) -> List[Any]:
if isinstance(raw_response, (list, dict)):
return [raw_response]
normalized = self._normalize_raw_text(raw_response)
if not normalized:
return []
if incremental:
normalized = f"{self._pending_raw}{normalized}"
try:
payload = json.loads(normalized)
if incremental:
self._pending_raw = ""
return [payload]
except json.JSONDecodeError as exc:
payloads, remainder = self._decode_json_prefix_values(normalized)
if incremental:
self._pending_raw = remainder
if payloads:
return payloads
repaired = self._try_repair_unclosed_json(normalized)
if repaired is not None:
return [repaired]
logger.debug_throttled(
f"aistudio.incomplete_json.{id(self)}",
"[AIStudioParser] JSON 未完整,继续等待流增长 "
f"(body_len={len(normalized)}, error={exc})",
interval_sec=3.0,
)
return []
@staticmethod
def _try_repair_unclosed_json(text: str) -> Any:
stripped = text.strip()
if not stripped:
return None
for pad in range(0, 30):
try:
attempt = stripped + (']' * pad)
return json.loads(attempt)
except Exception:
pass
try:
attempt = stripped + '"]' + (']' * pad)
return json.loads(attempt)
except Exception:
pass
return None
@classmethod
def _extract_visible_text_from_raw_body(cls, raw_response: Any) -> str:
raw_text = cls._normalize_raw_text(raw_response)
if not raw_text:
return ""
pieces, _last_match_end = cls._scan_visible_text_matches(raw_text)
return "".join(pieces)
@classmethod
def _scan_visible_text_matches(cls, raw_text: str) -> Tuple[List[str], int]:
pieces: List[str] = []
last_match_end = 0
for match in cls._VISIBLE_TEXT_BLOCK_RE.finditer(raw_text):
last_match_end = match.end()
piece = cls._decode_json_fragment(match.group(1))
if piece and cls._looks_like_visible_text(piece):
pieces.append(piece)
return pieces, last_match_end
@classmethod
def _build_visible_text_scan_pending(cls, scan_text: str, last_match_end: int) -> str:
remainder = scan_text[last_match_end:] if last_match_end > 0 else scan_text
marker_index = remainder.rfind(cls._VISIBLE_TEXT_BLOCK_PREFIX)
if marker_index >= 0:
return remainder[marker_index:]
keep_chars = max(0, len(cls._VISIBLE_TEXT_BLOCK_PREFIX) - 1)
return remainder[-keep_chars:] if keep_chars else ""
def _extract_visible_text_from_increment(self, new_data: str) -> str:
if not new_data and not self._visible_text_scan_pending:
return self._visible_text_fallback_content
normalized_new_data = self._normalize_raw_text(new_data)
scan_text = f"{self._visible_text_scan_pending}{normalized_new_data}"
if not scan_text:
return self._visible_text_fallback_content
pieces, last_match_end = self._scan_visible_text_matches(scan_text)
if pieces:
self._visible_text_fallback_content += "".join(pieces)
self._visible_text_scan_pending = self._build_visible_text_scan_pending(
scan_text,
last_match_end,
)
return self._visible_text_fallback_content
@classmethod
def _looks_like_obfuscated_payload(cls, value: Any) -> bool:
text = str(value or "").strip()
if len(text) < 96:
return False
if any(ch.isspace() for ch in text):
return False
if cls._CJK_RE.search(text):
return False
if not cls._OBFUSCATED_PAYLOAD_RE.fullmatch(text):
return False
if not any(ch in "+/=_-" for ch in text):
return False
distinct_chars = len(set(text))
if distinct_chars < 20:
return False
alpha_num_ratio = sum(ch.isalnum() for ch in text) / len(text)
return alpha_num_ratio >= 0.85
@classmethod
def _looks_like_visible_text(cls, value: Any) -> bool:
text = str(value or "")
stripped = text.strip()
if not stripped:
return False
if cls._looks_like_obfuscated_payload(stripped):
return False
if cls._CJK_RE.search(stripped):
return True
if any(ch.isspace() for ch in stripped):
return True
if stripped.startswith(("{", "[", "<", "```", "http://", "https://")):
return True
punctuation_count = sum(
ch in '.,!?;:()[]{}<>"\'`~@#$%^&*\\|'
for ch in stripped
)
if punctuation_count > 0:
return True
return len(stripped) <= 80
@classmethod
def _pick_visible_text_candidate(cls, values: List[Any]) -> str:
for item in values:
if not isinstance(item, str):
continue
if cls._looks_like_visible_text(item):
return item
return ""
@staticmethod
def _decode_json_fragment(value: str) -> str:
text = str(value or "")
if not text:
return ""
try:
decoded = json.loads(f'"{text}"')
return decoded if isinstance(decoded, str) else ""
except json.JSONDecodeError:
return (
text.replace("\\n", "\n")
.replace("\\r", "\r")
.replace("\\t", "\t")
.replace('\\"', '"')
.replace("\\\\", "\\")
)
@staticmethod
def _decode_json_prefix_values(raw_text: str) -> Tuple[List[Any], str]:
decoder = json.JSONDecoder()
values: List[Any] = []
index = 0
length = len(raw_text)
while index < length:
while index < length and raw_text[index].isspace():
index += 1
if index >= length:
break
try:
value, next_index = decoder.raw_decode(raw_text, index)
except json.JSONDecodeError:
return values, raw_text[index:]
values.append(value)
index = next_index
return values, ""
@staticmethod
def _normalize_outer_blocks(data: Any) -> List[Any]:
if not isinstance(data, list) or not data:
return []
if any(isinstance(item, list) for item in data):
if (
len(data) == 1
and isinstance(data[0], list)
and any(isinstance(item, list) for item in data[0])
):
return data[0]
return data
return []
def _extract_content(self, data: Any) -> Tuple[str, str, bool]:
"""
从响应数据中提取文本内容与思考过程
Returns:
(accumulated_text, accumulated_reasoning, is_done)
"""
try:
outer = self._normalize_outer_blocks(data)
if not outer:
return "", "", False
accumulated = ""
accumulated_reasoning = ""
is_done = False
for block in outer:
if not isinstance(block, list):
continue
# 检查是否是统计块(结束标志之一)
if self._is_stats_block(block):
is_done = True
continue
# 提取文本
text, block_done, is_thinking = self._extract_block_content(block)
if is_thinking:
if text:
accumulated_reasoning += text
else:
if text:
accumulated += text
if block_done:
is_done = True
return accumulated, accumulated_reasoning, is_done
except Exception as e:
logger.debug(f"[AIStudioParser] _extract_content 异常: {e}")
return "", "", False
def _is_stats_block(self, block: list) -> bool:
"""检查是否是统计块(响应结束)"""
try:
# 格式: [null, null, null, [timestamp, ...]]
if len(block) >= 4:
if block[0] is None and block[1] is None and block[2] is None:
if isinstance(block[3], list):
return True
return False
except:
return False
def _extract_block_content(self, block: list) -> tuple:
"""
从单个块提取内容
Returns:
(text, is_done, is_thinking)
"""
try:
# 路径: block[0][0][0][0][0]
# thinking: [13 items, None, "**Thinking...", ..., 1]
# 正常内容: [2 items, None, "文本内容"]
if not isinstance(block, list) or len(block) == 0:
return "", False, False
level1 = block[0]
if not isinstance(level1, list) or len(level1) == 0:
return "", False, False
level2 = level1[0]
if not isinstance(level2, list) or len(level2) == 0:
return "", False, False
level3 = level2[0]
if not isinstance(level3, list) or len(level3) == 0:
return "", False, False
level4 = level3[0]
if not isinstance(level4, list) or len(level4) == 0:
return "", False, False
# 这里就是 content_arr
content_arr = level4[0]
if not isinstance(content_arr, list):
return "", False, False
# 提取文本:只接受像“可见正文”的字符串,避免把内部高熵 payload 误当回复。
text = ""
primary_text = content_arr[1] if len(content_arr) > 1 else ""
if self._looks_like_visible_text(primary_text):
text = primary_text
else:
text = self._pick_visible_text_candidate(content_arr[2:])
if not text:
suppressed_payload = next(
(
item for item in content_arr[2:]
if isinstance(item, str)
and self._looks_like_obfuscated_payload(item)
),
"",
)
if suppressed_payload:
logger.debug_throttled(
f"aistudio.hidden_payload.{id(self)}",
"[AIStudioParser] 忽略疑似内部高熵 payload "
f"(len={len(str(suppressed_payload).strip())})",
interval_sec=3.0,
)
# 判断是否是 thinking
# thinking 块特征:len >= 13 且索引 12 == 1
is_thinking = False
if len(content_arr) >= 13:
if len(content_arr) > 12 and content_arr[12] == 1:
is_thinking = True
# 检查是否结束
# 1. level2 的第二个元素是 1
is_done = False
if len(level2) > 1 and level2[1] == 1:
is_done = True
# 2. 有长 token 字符串(索引 13 或 14)
if len(content_arr) > 13 and isinstance(content_arr[13], str):
if len(content_arr[13]) > 50:
is_done = True
return text, is_done, is_thinking
except Exception as e:
logger.debug(f"[AIStudioParser] _extract_block_content 异常: {e}")
return "", False, False
# ============ 元数据 ============
@classmethod
def get_id(cls) -> str:
return "aistudio"
@classmethod
def get_name(cls) -> str:
return "Google AI Studio"
@classmethod
def get_description(cls) -> str:
return "解析 Google AI Studio (MakerSuite) 的 GenerateContent 响应"
@classmethod
def get_supported_patterns(cls) -> List[str]:
return ["MakerSuiteService/GenerateContent", "GenerateContent"]
def should_fallback_to_dom_when_no_visible_content(self) -> bool:
return True
__all__ = ['AIStudioParser']
+180
View File
@@ -0,0 +1,180 @@
"""
app/core/parsers/base.py - 响应解析器基类
定义网络响应解析的标准接口(支持增量响应)
"""
from abc import ABC, abstractmethod
from typing import Dict, List, Optional, Any
class ResponseParser(ABC):
"""
响应解析器抽象基类
职责:
- 解析网络拦截到的原始响应体
- 支持增量响应(流式传输)
- 提取文本内容和图片
"""
# ============ 抽象方法(子类必须实现)============
@abstractmethod
def parse_chunk(self, raw_response: str) -> Dict[str, Any]:
"""
解析单次响应数据(增量)
Args:
raw_response: 原始响应体(字符串)
Returns:
{
"content": str, # 本次增量的文本内容
"images": List[Dict], # 本次增量的图片(可选)
"done": bool, # 是否为最后一块数据
"error": Optional[str] # 解析错误信息
}
注意:
- content 为空字符串表示本次无新增文本
- images 为空列表表示本次无新增图片
- done=True 表示流式传输结束
- error 不为 None 表示解析失败
"""
pass
@abstractmethod
def reset(self):
"""
重置解析器状态
用于新一轮对话开始时清空累积状态
"""
pass
def _prepare_incremental_raw_response(self, raw_response: str) -> str:
"""
Return the append-only delta for a full raw response snapshot.
Network listeners usually pass a growing full body to parsers. When the
browser replaces that snapshot with another response of the same or
greater length, slicing by length alone loses content. This helper
preserves the fast append path and resets parser state when the new
snapshot is not a prefix extension of the previous one.
"""
if not isinstance(raw_response, str):
raw_response = str(raw_response)
previous = getattr(self, "_last_raw_response", "")
try:
previous_len = int(getattr(self, "_last_raw_length", 0) or 0)
except Exception:
previous_len = 0
if previous_len > 0 and previous:
if raw_response == previous:
return ""
if raw_response.startswith(previous):
new_data = raw_response[len(previous):]
else:
self.reset()
new_data = raw_response
else:
new_data = raw_response
self._last_raw_length = len(raw_response)
self._last_raw_response = raw_response
return new_data
# ============ 可选覆盖方法 ============
def validate_response(self, raw_response: str) -> bool:
"""
验证响应格式是否匹配此解析器
Args:
raw_response: 原始响应体
Returns:
True 表示格式匹配,False 表示不匹配
"""
try:
result = self.parse_chunk(raw_response)
return result.get("error") is None
except Exception:
return False
def should_abort_on_error(self) -> bool:
"""
当 parse_chunk 返回 error 时,是否应立即终止当前工作流。
默认仅记录并继续,交给具体解析器按需升级为硬失败。
"""
return False
def should_fallback_to_dom_when_no_visible_content(self) -> bool:
"""
网络流存在但长期拿不到可见正文时,是否优先回退 DOM 监听。
适用于网络协议里混入推理/中间态,而页面真实正文由前端二次拼装的站点。
"""
return False
def should_require_explicit_done(self) -> bool:
"""Whether a stream must expose its protocol-level completion event."""
return False
def should_wait_for_replacement_stream_on_incomplete_capture(self) -> bool:
"""Whether an incomplete captured response may be followed by another stream."""
return False
def get_media_generation_state(
self,
raw_response: str = "",
parse_result: Optional[Dict[str, Any]] = None,
) -> Dict[str, Any]:
"""
提取通用的媒体生成状态。
返回约定:
{
"pending": bool,
"media_type": "image" | "audio" | "video" | "",
"hint_text": str,
"wait_timeout_seconds": float | int | None,
}
通用工作流层只消费这些字段,不关心站点私有协议细节。
"""
return {}
# ============ 元数据接口 ============
@classmethod
def get_id(cls) -> str:
"""解析器唯一标识符"""
return cls.__name__.lower().replace('parser', '')
@classmethod
def get_name(cls) -> str:
"""解析器显示名称"""
return cls.__name__
@classmethod
def get_description(cls) -> str:
"""解析器描述"""
return "未提供描述"
@classmethod
def get_supported_patterns(cls) -> List[str]:
"""
返回此解析器支持的 URL 匹配模式
Returns:
URL 子串列表(用于 page.listen.start())
"""
return []
__all__ = ['ResponseParser']
+645
View File
@@ -0,0 +1,645 @@
"""
chatgpt_parser.py - ChatGPT 响应解析器
响应格式特征:
- SSE (Server-Sent Events) 流式响应
- 使用 v1 delta 编码
- 增量文本通过 event: delta 事件传递
- 结束标志: data: [DONE]
"""
import json
import re
from typing import Dict, Any, List, Optional
from app.core.config import logger
from .base import ResponseParser
class ChatGPTParser(ResponseParser):
"""
ChatGPT API 响应解析器
URL 特征: /backend-api/f/conversation
响应格式: SSE (text/event-stream) with v1 delta encoding
"""
def __init__(self):
self._accumulated_content = ""
self._last_raw_length = 0
self._last_raw_response = ""
self._rendered_content = ""
self._message_parts: Dict[int, str] = {}
self._message_parts_joined = ""
self._message_parts_dirty = False
self._content_references: List[Dict[str, Any]] = []
self._content_references_sorted: List[Dict[str, Any]] = []
self._content_references_dirty = False
self._active_part_index: Optional[int] = None
self._pending_sse_chunk = ""
self._ref_start = "\ue200"
self._ref_end = "\ue201"
self._ref_sep = "\ue202"
self._media_generation_state: Dict[str, Any] = {}
self._inline_ref_pattern = re.compile(
rf"{re.escape(self._ref_start)}([^{re.escape(self._ref_sep)}{re.escape(self._ref_end)}]+)"
rf"{re.escape(self._ref_sep)}(.*?){re.escape(self._ref_end)}",
re.DOTALL,
)
self._dangling_ref_pattern = re.compile(
rf"{re.escape(self._ref_start)}[^{re.escape(self._ref_end)}]*$",
re.DOTALL,
)
def parse_chunk(self, raw_response: str) -> Dict[str, Any]:
"""
解析SSE响应流(返回增量)
"""
result = {
"content": "",
"images": [],
"done": False,
"error": None
}
try:
# 确保是字符串
if isinstance(raw_response, bytes):
raw_response = raw_response.decode('utf-8', errors='ignore')
if not isinstance(raw_response, str):
raw_response = str(raw_response)
# 只处理新增部分;非前缀新快照会在 helper 内重置解析状态。
new_data = self._prepare_incremental_raw_response(raw_response)
if not new_data:
return result
# 解析新增的SSE事件
delta_content, done = self._parse_sse_chunk(new_data)
if delta_content:
result["content"] = delta_content
self._accumulated_content += delta_content
# 检测结束标志
if done:
result["done"] = True
except Exception as e:
logger.debug(f"[ChatGPTParser] 解析异常: {e}")
result["error"] = str(e)
return result
def _parse_sse_chunk(self, chunk: str) -> tuple[str, bool]:
"""解析SSE数据块,提取文本增量"""
chunk = self._extract_complete_sse_chunk(chunk)
if not chunk:
return "", False
previous_content = self._rendered_content
done = False
for current_event, data in self._iter_sse_json_events(chunk):
if not isinstance(data, dict):
continue
self._consume_event(current_event, data)
for raw_line in str(chunk or "").split('\n'):
line = raw_line.strip()
if line.startswith("data:") and line[5:].strip() == "[DONE]":
done = True
break
current_content = self._render_visible_text()
self._rendered_content = current_content
return self._compute_delta(previous_content, current_content), done
def _extract_complete_sse_chunk(self, chunk: str) -> str:
"""Buffer partial SSE frames until a blank line terminates the event."""
combined = f"{self._pending_sse_chunk}{str(chunk or '')}"
if not combined:
return ""
normalized = combined.replace("\r\n", "\n").replace("\r", "\n")
parts = normalized.split("\n\n")
if normalized.endswith("\n\n"):
self._pending_sse_chunk = ""
complete_parts = parts[:-1]
elif normalized.rstrip("\n").endswith("data: [DONE]"):
self._pending_sse_chunk = ""
complete_parts = [normalized.rstrip("\n")]
else:
self._pending_sse_chunk = parts.pop() if parts else normalized
complete_parts = parts
if not complete_parts:
return ""
return "\n\n".join(complete_parts) + "\n\n"
def _iter_sse_json_events(self, chunk: str):
"""遍历 SSE 中可解析为 JSON 的事件。"""
current_event = None
for raw_line in str(chunk or "").split('\n'):
line = raw_line.strip()
if not line:
current_event = None
continue
if line.startswith('event:'):
current_event = line[6:].strip()
continue
if not line.startswith('data:'):
continue
data_str = line[5:].strip()
if not data_str or data_str == "[DONE]":
continue
try:
payload = json.loads(data_str)
except json.JSONDecodeError:
continue
yield current_event, payload
def _consume_event(self, event_name: Optional[str], data: Dict[str, Any]) -> None:
self._update_media_generation_state(event_name, data)
if event_name != "delta":
return
operations: List[Dict[str, Any]] = []
if data.get("o") == "patch" and isinstance(data.get("v"), list):
operations = [item for item in data.get("v", []) if isinstance(item, dict)]
elif isinstance(data.get("v"), list):
operations = [item for item in data.get("v", []) if isinstance(item, dict)]
elif isinstance(data.get("p"), str) and data.get("o") is not None:
operations = [data]
if operations:
for operation in operations:
self._apply_operation(operation)
return
value = data.get("v")
if isinstance(value, str) and self._active_part_index is not None:
self._append_message_part(self._active_part_index, value)
return
self._sync_message_snapshot_from_payload(data)
def _sync_message_snapshot_from_payload(self, payload: Dict[str, Any]) -> None:
message = None
value = payload.get("v")
if isinstance(value, dict) and isinstance(value.get("message"), dict):
message = value.get("message") or {}
elif isinstance(payload.get("message"), dict):
message = payload.get("message") or {}
if not isinstance(message, dict):
return
author = message.get("author") or {}
if str(author.get("role") or "").strip().lower() != "assistant":
return
content = message.get("content") or {}
if not isinstance(content, dict):
return
if str(content.get("content_type") or "").strip().lower() != "text":
return
parts = content.get("parts")
if isinstance(parts, list):
self._set_message_parts(parts)
self._active_part_index = max(self._message_parts.keys(), default=0)
metadata = message.get("metadata") or {}
refs = metadata.get("content_references")
if isinstance(refs, list):
self._content_references = [
dict(item) for item in refs if isinstance(item, dict)
]
self._mark_content_references_dirty()
def _apply_operation(self, operation: Dict[str, Any]) -> None:
path = str(operation.get("p") or "")
action = str(operation.get("o") or "").strip().lower()
value = operation.get("v")
if self._apply_content_part_operation(path, action, value):
return
if self._apply_content_reference_operation(path, action, value):
return
def _apply_content_part_operation(self, path: str, action: str, value: Any) -> bool:
if not path.startswith("/message/content/parts"):
return False
if path == "/message/content/parts":
if action in {"add", "replace"} and isinstance(value, list):
self._set_message_parts(value)
self._active_part_index = max(self._message_parts.keys(), default=0)
elif action == "append" and isinstance(value, list):
start_index = max(self._message_parts.keys(), default=-1) + 1
for offset, part in enumerate(value):
if isinstance(part, str):
self._set_message_part(start_index + offset, part)
self._active_part_index = max(self._message_parts.keys(), default=0)
elif action == "remove":
self._clear_message_parts()
self._active_part_index = None
return True
match = re.match(r"^/message/content/parts/(\d+)$", path)
if not match:
return True
index = int(match.group(1))
if action == "append" and isinstance(value, str):
self._append_message_part(index, value)
self._active_part_index = index
elif action in {"add", "replace"}:
if isinstance(value, str):
self._set_message_part(index, value)
self._active_part_index = index
elif value is None:
self._set_message_part(index, "")
self._active_part_index = index
elif action == "remove":
self._remove_message_part(index)
if self._active_part_index == index:
self._active_part_index = max(self._message_parts.keys(), default=None)
return True
def _apply_content_reference_operation(self, path: str, action: str, value: Any) -> bool:
if not path.startswith("/message/metadata/content_references"):
return False
match = re.match(r"^/message/metadata/content_references(?:/(\d+)(?:/(.+))?)?$", path)
if not match:
return True
index_text, subpath = match.groups()
if index_text is None:
if action in {"add", "replace"} and isinstance(value, list):
self._content_references = [
dict(item) for item in value if isinstance(item, dict)
]
self._mark_content_references_dirty()
elif action == "append":
changed = False
items = value if isinstance(value, list) else [value]
for item in items:
if isinstance(item, dict):
self._content_references.append(dict(item))
changed = True
if changed:
self._mark_content_references_dirty()
elif action == "remove":
self._clear_content_references()
return True
index = int(index_text)
if self._ensure_reference_index(index):
self._mark_content_references_dirty()
if not subpath:
if action == "remove":
if 0 <= index < len(self._content_references):
self._content_references.pop(index)
self._mark_content_references_dirty()
elif action in {"add", "replace"} and isinstance(value, dict):
self._content_references[index] = dict(value)
self._mark_content_references_dirty()
elif action == "append" and isinstance(value, dict):
self._content_references[index].update(value)
self._mark_content_references_dirty()
return True
target = self._content_references[index]
self._apply_nested_patch(target, subpath.split("/"), action, value)
self._mark_content_references_dirty()
return True
def _ensure_reference_index(self, index: int) -> bool:
changed = False
while len(self._content_references) <= index:
self._content_references.append({})
changed = True
return changed
def _apply_nested_patch(
self,
container: Dict[str, Any],
path_segments: List[str],
action: str,
value: Any,
) -> None:
current = container
for segment in path_segments[:-1]:
next_value = current.get(segment)
if not isinstance(next_value, dict):
next_value = {}
current[segment] = next_value
current = next_value
key = path_segments[-1]
existing = current.get(key)
if action == "append":
if isinstance(existing, str) and isinstance(value, str):
current[key] = existing + value
elif isinstance(existing, list) and isinstance(value, list):
current[key] = existing + value
elif isinstance(existing, dict) and isinstance(value, dict):
merged = dict(existing)
merged.update(value)
current[key] = merged
elif existing is None:
current[key] = value
else:
current[key] = value
elif action in {"add", "replace"}:
current[key] = value
elif action == "remove":
current.pop(key, None)
def _render_visible_text(self) -> str:
if not self._message_parts:
return self._rendered_content
text = self._render_message_parts()
text = self._replace_content_references(text)
text = self._strip_private_markup(text)
return text
def _set_message_parts(self, parts: List[Any]) -> None:
self._message_parts = {
index: str(part)
for index, part in enumerate(parts)
if isinstance(part, str)
}
self._mark_message_parts_dirty()
def _set_message_part(self, index: int, value: str) -> None:
self._message_parts[index] = str(value)
self._mark_message_parts_dirty()
def _append_message_part(self, index: int, value: str) -> None:
if not value:
return
self._message_parts[index] = self._message_parts.get(index, "") + str(value)
self._mark_message_parts_dirty()
def _remove_message_part(self, index: int) -> None:
if index in self._message_parts:
self._message_parts.pop(index, None)
self._mark_message_parts_dirty()
def _clear_message_parts(self) -> None:
if self._message_parts or self._message_parts_joined:
self._message_parts = {}
self._message_parts_joined = ""
self._message_parts_dirty = False
def _mark_message_parts_dirty(self) -> None:
self._message_parts_dirty = True
def _render_message_parts(self) -> str:
if self._message_parts_dirty:
self._message_parts_joined = "".join(
self._message_parts[index] for index in sorted(self._message_parts)
)
self._message_parts_dirty = False
return self._message_parts_joined
def _mark_content_references_dirty(self) -> None:
self._content_references_dirty = True
def _clear_content_references(self) -> None:
if self._content_references or self._content_references_sorted:
self._content_references = []
self._content_references_sorted = []
self._content_references_dirty = False
def _sorted_content_references(self) -> List[Dict[str, Any]]:
if self._content_references_dirty:
self._content_references_sorted = sorted(
(
ref for ref in self._content_references
if isinstance(ref, dict) and str(ref.get("matched_text") or "")
),
key=lambda ref: len(str(ref.get("matched_text") or "")),
reverse=True,
)
self._content_references_dirty = False
return self._content_references_sorted
def _replace_content_references(self, text: str) -> str:
rendered = text
for ref in self._sorted_content_references():
matched_text = str(ref.get("matched_text") or "")
if not matched_text or matched_text not in rendered:
continue
rendered = rendered.replace(
matched_text,
self._build_reference_replacement(ref),
)
rendered = self._inline_ref_pattern.sub(
lambda match: self._decode_inline_reference(match.group(1), match.group(2)),
rendered,
)
return rendered
def _build_reference_replacement(self, ref: Dict[str, Any]) -> str:
ref_type = str(ref.get("type") or "").strip().lower()
if ref_type == "hidden":
return ""
label = ""
for key in ("alt", "prompt_text", "name"):
value = str(ref.get(key) or "").strip()
if value:
label = value
break
if not label:
matched_text = str(ref.get("matched_text") or "")
if matched_text:
label = self._decode_inline_reference_from_text(matched_text)
safe_urls = ref.get("safe_urls")
first_url = ""
if isinstance(safe_urls, list):
for item in safe_urls:
candidate = str(item or "").strip()
if candidate:
first_url = candidate
break
if first_url and label:
if first_url in label or "](" in label or label.startswith("http://") or label.startswith("https://"):
return label
return f"{label} ({first_url})"
if label:
return label
if first_url:
return first_url
return ""
def _decode_inline_reference_from_text(self, text: str) -> str:
match = self._inline_ref_pattern.search(str(text or ""))
if not match:
return ""
return self._decode_inline_reference(match.group(1), match.group(2))
def _decode_inline_reference(self, ref_kind: str, payload: str) -> str:
kind = str(ref_kind or "").strip().lower()
payload_text = str(payload or "")
if kind == "entity":
try:
decoded = json.loads(payload_text)
if isinstance(decoded, list):
for index in (1, 0, 2):
if index < len(decoded):
value = str(decoded[index] or "").strip()
if value:
return value
except Exception:
return ""
return ""
def _strip_private_markup(self, text: str) -> str:
cleaned = self._dangling_ref_pattern.sub("", text)
cleaned = cleaned.replace(self._ref_start, "")
cleaned = cleaned.replace(self._ref_sep, "")
cleaned = cleaned.replace(self._ref_end, "")
return cleaned
def _update_media_generation_state(self, event_name: Optional[str], payload: Dict[str, Any]) -> None:
pending = False
media_type = str(self._media_generation_state.get("media_type") or "")
hint_parts = [
item for item in str(self._media_generation_state.get("hint_text") or "").split("\n\n")
if item
]
wait_timeout_seconds = self._media_generation_state.get("wait_timeout_seconds")
payload_type = str(payload.get("type") or "").strip().lower()
if payload_type == "server_ste_metadata":
metadata = payload.get("metadata") or {}
if str(metadata.get("turn_use_case") or "").strip().lower() == "image gen":
pending = True
media_type = media_type or "image"
wait_timeout_seconds = max(float(wait_timeout_seconds or 0), 90.0)
message = None
if event_name == "delta":
value = payload.get("v")
if isinstance(value, dict) and isinstance(value.get("message"), dict):
message = value.get("message") or {}
elif isinstance(payload.get("message"), dict):
message = payload.get("message") or {}
if isinstance(message, dict):
metadata = message.get("metadata") or {}
content = message.get("content") or {}
if str(metadata.get("image_gen_task_id") or "").strip():
pending = True
media_type = media_type or "image"
wait_timeout_seconds = max(float(wait_timeout_seconds or 0), 90.0)
if bool(metadata.get("image_gen_multi_stream")):
pending = True
media_type = media_type or "image"
wait_timeout_seconds = max(float(wait_timeout_seconds or 0), 90.0)
for field in ("ui_card_title", "ui_card_description"):
value = str(metadata.get(field) or "").strip()
if value and value not in hint_parts:
hint_parts.append(value)
if isinstance(content, dict):
parts = content.get("parts")
if isinstance(parts, list):
for item in parts:
value = str(item or "").strip() if isinstance(item, str) else ""
if value and value not in hint_parts:
hint_parts.append(value)
if pending and media_type:
self._media_generation_state = {
"pending": True,
"media_type": media_type,
"hint_text": "\n\n".join(hint_parts[:3]),
"wait_timeout_seconds": wait_timeout_seconds,
}
@staticmethod
def _compute_delta(previous: str, current: str) -> str:
if not current or current == previous:
return ""
if not previous:
return current
if current.startswith(previous):
return current[len(previous):]
prefix_len = 0
max_prefix = min(len(previous), len(current))
while prefix_len < max_prefix and previous[prefix_len] == current[prefix_len]:
prefix_len += 1
return current[prefix_len:]
def reset(self):
"""重置状态"""
self._accumulated_content = ""
self._last_raw_length = 0
self._last_raw_response = ""
self._rendered_content = ""
self._message_parts = {}
self._message_parts_joined = ""
self._message_parts_dirty = False
self._content_references = []
self._content_references_sorted = []
self._content_references_dirty = False
self._active_part_index = None
self._pending_sse_chunk = ""
self._media_generation_state = {}
def get_media_generation_state(
self,
raw_response: str = "",
parse_result: Optional[Dict[str, Any]] = None,
) -> Dict[str, Any]:
return dict(self._media_generation_state or {})
@classmethod
def get_id(cls) -> str:
return "chatgpt"
@classmethod
def get_name(cls) -> str:
return "ChatGPT API"
@classmethod
def get_description(cls) -> str:
return "解析 ChatGPT 的 API 响应(SSE流式,v1 delta编码)"
@classmethod
def get_supported_patterns(cls) -> List[str]:
return ["**/backend-api/f/conversation**"]
__all__ = ['ChatGPTParser']
+179
View File
@@ -0,0 +1,179 @@
"""
claude_parser.py - Claude SSE response parser.
Observed stream traits:
- event: content_block_delta with delta.type=text_delta carries answer text
- event: content_block_delta with delta.type=thinking_delta carries Extended Thinking CoT text
- event: message_stop marks stream completion
"""
from __future__ import annotations
import json
from typing import Any, Dict, List, Tuple
from app.core.config import logger
from .base import ResponseParser
class ClaudeParser(ResponseParser):
"""Parse Claude web SSE responses."""
def __init__(self) -> None:
self._last_raw_length = 0
self._pending = ""
def parse_chunk(self, raw_response: str) -> Dict[str, Any]:
result: Dict[str, Any] = {
"content": "",
"reasoning_content": "",
"images": [],
"done": False,
"error": None,
}
try:
if isinstance(raw_response, (bytes, bytearray)):
raw_response = raw_response.decode("utf-8", errors="ignore")
elif not isinstance(raw_response, str):
raw_response = str(raw_response)
new_data = self._prepare_incremental_raw_response(raw_response)
if not new_data:
return result
delta_content, delta_reasoning, done = self._consume_new_data(new_data)
if delta_content:
result["content"] = delta_content
if delta_reasoning:
result["reasoning_content"] = delta_reasoning
result["done"] = done
except Exception as e:
logger.debug(f"[ClaudeParser] parse exception: {e}")
result["error"] = str(e)
return result
def reset(self) -> None:
self._last_raw_length = 0
self._last_raw_response = ""
self._pending = ""
def _consume_new_data(self, new_data: str) -> Tuple[str, str, bool]:
# Browser stream snapshots can use either CRLF or lone CR line endings.
normalized = (self._pending + new_data).replace("\r\n", "\n").replace("\r", "\n")
if not normalized:
return "", "", False
blocks = normalized.split("\n\n")
if normalized.endswith("\n\n"):
self._pending = ""
complete_blocks = [block for block in blocks if block.strip()]
else:
self._pending = blocks.pop() if blocks else normalized
complete_blocks = [block for block in blocks if block.strip()]
content_parts: List[str] = []
reasoning_parts: List[str] = []
done = False
for block in complete_blocks:
block_content, block_reasoning, block_done = self._parse_event_block(block)
if block_content:
content_parts.append(block_content)
if block_reasoning:
reasoning_parts.append(block_reasoning)
if block_done:
done = True
return "".join(content_parts), "".join(reasoning_parts), done
def _parse_event_block(self, block: str) -> Tuple[str, str, bool]:
event_name = ""
data_lines: List[str] = []
for raw_line in block.split("\n"):
line = raw_line.strip()
if not line:
continue
if line.startswith("event:"):
event_name = line[6:].strip()
elif line.startswith("data:"):
data_lines.append(line[5:].strip())
if event_name == "message_stop":
return "", "", True
payload = "\n".join(data_lines).strip()
if not payload:
return "", "", False
try:
data = json.loads(payload)
except json.JSONDecodeError:
return "", "", False
# Some response wrappers omit the SSE event line but preserve the
# protocol event type in the JSON payload.
if not event_name:
event_name = str(data.get("type") or "").strip()
if event_name == "message_stop":
return "", "", True
if event_name == "content_block_start":
cb = data.get("content_block", {})
if isinstance(cb, dict):
cb_type = cb.get("type")
if cb_type == "text" and isinstance(cb.get("text"), str):
return cb.get("text", ""), "", False
elif cb_type == "thinking" and isinstance(cb.get("thinking"), str):
return "", cb.get("thinking", ""), False
if event_name == "content_block_delta":
delta = data.get("delta", {})
if isinstance(delta, dict):
delta_type = delta.get("type")
if delta_type == "text_delta":
text = delta.get("text", "")
return (text, "", False) if isinstance(text, str) else ("", "", False)
elif delta_type == "thinking_delta":
thinking_text = delta.get("thinking", "")
return ("", thinking_text, False) if isinstance(thinking_text, str) else ("", "", False)
elif delta_type == "thinking_summary_delta":
summary_text = delta.get("summary", delta.get("thinking", ""))
return ("", summary_text, False) if isinstance(summary_text, str) else ("", "", False)
return "", "", False
@classmethod
def get_id(cls) -> str:
return "claude"
@classmethod
def get_name(cls) -> str:
return "Claude"
@classmethod
def get_description(cls) -> str:
return "Parse Claude SSE streams including Extended Thinking blocks"
def should_require_explicit_done(self) -> bool:
# Claude's stream object may report capture completion after an
# intermediate snapshot; message_stop is the reliable protocol end.
return True
def should_wait_for_replacement_stream_on_incomplete_capture(self) -> bool:
return True
@classmethod
def get_supported_patterns(cls) -> List[str]:
# The conversation path is also used by /title and other JSON requests.
# Match the actual SSE completion endpoint so auxiliary requests cannot
# replace the active stream.
return ["/completion"]
__all__ = ["ClaudeParser"]
+488
View File
@@ -0,0 +1,488 @@
"""
deepseek_parser.py - DeepSeek SSE response parser.
Observed stream traits:
- data: {"v": {"response": {"fragments": [...]}}} initializes fragment state
- thinking mode sends THINK fragments first and answer text later in RESPONSE fragments
- path-less data: {"v": "..."} continues the current fragment's content
- response/status or quasi_status=FINISHED ends the stream
"""
from __future__ import annotations
import json
from typing import Any, Dict, List, Tuple
from app.core.config import logger
from .base import ResponseParser
class DeepSeekParser(ResponseParser):
"""Parse DeepSeek web SSE streams and ignore THINK fragments."""
def __init__(self) -> None:
self._last_raw_length = 0
self._pending = ""
self._fragment_types: List[str] = []
self._fragment_contents: List[str] = []
self._current_fragment_type = ""
self._current_fragment_index = -1
self._debug_records: List[Dict[str, Any]] = []
self._emit_buffer = ""
self._has_seen_visible_response_text = False
def parse_chunk(self, raw_response: str) -> Dict[str, Any]:
result: Dict[str, Any] = {
"content": "",
"images": [],
"done": False,
"error": None,
}
try:
if isinstance(raw_response, (bytes, bytearray)):
raw_response = raw_response.decode("utf-8", errors="ignore")
elif not isinstance(raw_response, str):
raw_response = str(raw_response)
new_data = self._prepare_incremental_raw_response(raw_response)
if not new_data:
return result
delta_content, done = self._consume_new_data(new_data)
safe_content = self._drain_safe_output(delta_content, force=done)
if safe_content:
result["content"] = safe_content
result["done"] = done
except Exception as e:
logger.debug(f"[DeepSeekParser] parse exception: {e}")
result["error"] = str(e)
return result
def reset(self) -> None:
self._last_raw_length = 0
self._last_raw_response = ""
self._pending = ""
self._fragment_types = []
self._fragment_contents = []
self._current_fragment_type = ""
self._current_fragment_index = -1
self._debug_records = []
self._emit_buffer = ""
self._has_seen_visible_response_text = False
def _consume_new_data(self, new_data: str) -> Tuple[str, bool]:
normalized = (self._pending + new_data).replace("\r\n", "\n")
if not normalized:
return "", False
blocks = normalized.split("\n\n")
if normalized.endswith("\n\n"):
self._pending = ""
complete_blocks = [block for block in blocks if block.strip()]
else:
self._pending = blocks.pop() if blocks else normalized
complete_blocks = [block for block in blocks if block.strip()]
content_parts: List[str] = []
done = False
for block in complete_blocks:
block_content, block_done = self._parse_event_block(block)
if block_content:
content_parts.append(block_content)
if block_done:
done = True
return "".join(content_parts), done
def _parse_event_block(self, block: str) -> Tuple[str, bool]:
event_name = ""
data_lines: List[str] = []
for raw_line in block.split("\n"):
line = raw_line.strip()
if not line:
continue
if line.startswith("event:"):
event_name = line[6:].strip()
elif line.startswith("data:"):
data_lines.append(line[5:].strip())
if event_name in {"ready", "update_session", "title"}:
return "", False
if event_name in {"finish", "close"}:
return "", self._can_accept_done_signal()
payload = "\n".join(data_lines).strip()
if not payload or payload == "{}":
return "", False
try:
data = json.loads(payload)
except json.JSONDecodeError:
return "", False
content, done = self._extract_content(data)
self._record_debug_event(event_name, data, content, done)
return content, done
def _extract_content(self, data: Dict[str, Any]) -> Tuple[str, bool]:
path = str(data.get("p", "") or "")
operation = str(data.get("o", "") or "")
value = data.get("v")
if operation == "BATCH" and isinstance(value, list):
return self._extract_batch_content(value)
if isinstance(value, dict):
response = value.get("response")
if isinstance(response, dict):
return self._extract_response_object(response)
if self._is_done_signal(path, value):
return "", self._can_accept_done_signal()
if self._is_fragment_append(path, operation, value):
return self._register_fragments(value), False
if self._is_fragment_content_patch(path, value):
fragment_index = self._resolve_fragment_index(path)
fragment_type = self._resolve_fragment_type(path)
delta_text = self._apply_fragment_content_update(fragment_index, value)
if self._should_emit_fragment_text(fragment_type):
self._mark_visible_response_text(fragment_type, delta_text)
return delta_text, False
return "", False
if not path and not operation and isinstance(value, str):
self._append_to_current_fragment(value)
if self._should_emit_fragment_text(self._current_fragment_type):
self._mark_visible_response_text(self._current_fragment_type, value)
return value, False
return "", False
return "", False
def _extract_batch_content(self, operations: List[Dict[str, Any]]) -> Tuple[str, bool]:
content_parts: List[str] = []
saw_done_signal = False
for op in operations:
if not isinstance(op, dict):
continue
path = str(op.get("p", "") or "")
operation = str(op.get("o", "") or "")
value = op.get("v")
if self._is_done_signal(path, value):
saw_done_signal = True
continue
if self._is_fragment_append(path, operation, value):
fragment_content = self._register_fragments(value)
if fragment_content:
content_parts.append(fragment_content)
continue
if self._is_fragment_content_patch(path, value):
fragment_index = self._resolve_fragment_index(path)
fragment_type = self._resolve_fragment_type(path)
delta_text = self._apply_fragment_content_update(fragment_index, value)
if self._should_emit_fragment_text(fragment_type):
self._mark_visible_response_text(fragment_type, delta_text)
content_parts.append(delta_text)
return "".join(content_parts), bool(saw_done_signal and self._can_accept_done_signal())
def _extract_response_object(self, response: Dict[str, Any]) -> Tuple[str, bool]:
fragments = response.get("fragments")
content = self._register_fragments(fragments) if isinstance(fragments, list) else ""
done = False
if self._can_accept_done_signal():
if self._is_done_signal("response/status", response.get("status")):
done = True
if self._is_done_signal("response/quasi_status", response.get("quasi_status")):
done = True
return content, done
def _register_fragments(self, fragments: List[Dict[str, Any]]) -> str:
content_parts: List[str] = []
for fragment in fragments:
if not isinstance(fragment, dict):
continue
fragment_type = str(fragment.get("type", "") or "").upper()
self._fragment_types.append(fragment_type)
self._fragment_contents.append("")
self._current_fragment_index = len(self._fragment_types) - 1
if fragment_type:
self._current_fragment_type = fragment_type
text = fragment.get("content", "")
if isinstance(text, str):
self._fragment_contents[self._current_fragment_index] = text
if text and self._should_emit_fragment_text(fragment_type):
self._mark_visible_response_text(fragment_type, text)
content_parts.append(text)
return "".join(content_parts)
def _resolve_fragment_index(self, path: str) -> int:
marker = "response/fragments/"
if marker not in path or not self._fragment_types:
return self._current_fragment_index
index_part = path.split(marker, 1)[1].split("/", 1)[0]
if index_part == "-1":
return len(self._fragment_types) - 1
try:
index = int(index_part)
except ValueError:
return self._current_fragment_index
if index < 0:
index += len(self._fragment_types)
if 0 <= index < len(self._fragment_types):
return index
return self._current_fragment_index
def _resolve_fragment_type(self, path: str) -> str:
fragment_index = self._resolve_fragment_index(path)
if 0 <= fragment_index < len(self._fragment_types):
fragment_type = self._fragment_types[fragment_index]
self._current_fragment_index = fragment_index
self._current_fragment_type = fragment_type
return fragment_type
return self._current_fragment_type
def _append_to_current_fragment(self, delta_text: str) -> None:
if not isinstance(delta_text, str) or not delta_text:
return
if 0 <= self._current_fragment_index < len(self._fragment_contents):
self._fragment_contents[self._current_fragment_index] += delta_text
def _apply_fragment_content_update(self, fragment_index: int, incoming_text: str) -> str:
if not isinstance(incoming_text, str) or not incoming_text:
return ""
if not (0 <= fragment_index < len(self._fragment_contents)):
return incoming_text
previous_text = self._fragment_contents[fragment_index]
new_text, delta_text = self._merge_fragment_text(previous_text, incoming_text)
self._fragment_contents[fragment_index] = new_text
self._current_fragment_index = fragment_index
return delta_text
@staticmethod
def _merge_fragment_text(previous_text: str, incoming_text: str) -> Tuple[str, str]:
previous = previous_text if isinstance(previous_text, str) else ""
incoming = incoming_text if isinstance(incoming_text, str) else ""
if not previous:
return incoming, incoming
if not incoming:
return previous, ""
if incoming.startswith(previous):
return incoming, incoming[len(previous) :]
if previous.endswith(incoming):
return previous, ""
common_prefix_len = 0
max_prefix_len = min(len(previous), len(incoming))
while common_prefix_len < max_prefix_len and previous[common_prefix_len] == incoming[common_prefix_len]:
common_prefix_len += 1
if common_prefix_len > 0:
return incoming, incoming[common_prefix_len:]
return previous + incoming, incoming
@staticmethod
def _is_fragment_append(path: str, operation: str, value: Any) -> bool:
if not isinstance(value, list):
return False
return operation == "APPEND" and path in {"response/fragments", "fragments"}
@staticmethod
def _is_fragment_content_patch(path: str, value: Any) -> bool:
return isinstance(value, str) and "fragments/" in path and path.endswith("/content")
@staticmethod
def _is_done_signal(path: str, value: Any) -> bool:
if not isinstance(value, str):
return False
if value.upper() != "FINISHED":
return False
return path.endswith("status") or path.endswith("quasi_status")
def _should_emit_fragment_text(self, fragment_type: str) -> bool:
if not self._fragment_types:
return True
return fragment_type == "RESPONSE"
def _mark_visible_response_text(self, fragment_type: str, text: str) -> None:
if not isinstance(text, str) or not text:
return
if self._should_emit_fragment_text(fragment_type):
self._has_seen_visible_response_text = True
def _can_accept_done_signal(self) -> bool:
return bool(self._has_seen_visible_response_text)
def _drain_safe_output(self, delta_text: str, force: bool = False) -> str:
if delta_text:
self._emit_buffer += delta_text
if not self._emit_buffer:
return ""
if force:
flushed = self._emit_buffer
self._emit_buffer = ""
return flushed
safe_index = self._find_last_safe_emit_index(self._emit_buffer)
if safe_index <= 0:
return ""
flushed = self._emit_buffer[:safe_index]
self._emit_buffer = self._emit_buffer[safe_index:]
return flushed
@staticmethod
def _find_last_safe_emit_index(text: str) -> int:
if not text:
return 0
inside_tag = False
quote_char = ""
last_safe_index = 0
for index, char in enumerate(text):
if inside_tag:
if quote_char:
if char == quote_char:
quote_char = ""
continue
if char in {'"', "'"}:
quote_char = char
continue
if char == ">":
inside_tag = False
last_safe_index = index + 1
continue
if char == "<":
inside_tag = True
continue
last_safe_index = index + 1
if inside_tag:
last_open = text.rfind("<")
if last_open != -1:
return last_open
return last_safe_index
def export_debug_data(self, raw_response: str = "") -> Dict[str, Any]:
return {
"parser": self.get_id(),
"fragment_types": list(self._fragment_types),
"fragment_lengths": [len(item) for item in self._fragment_contents],
"current_fragment_type": self._current_fragment_type,
"current_fragment_index": self._current_fragment_index,
"has_seen_visible_response_text": bool(self._has_seen_visible_response_text),
"debug_records": self._debug_records[-120:],
"raw_response_preview": str(raw_response or "")[:12000],
}
def export_debug_snapshot(self, raw_response: str = "") -> str:
import time
from pathlib import Path
payload = {
**self.export_debug_data(raw_response),
"exported_at": int(time.time()),
}
path = Path("logs") / "deepseek_parser_debug.json"
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(
json.dumps(payload, ensure_ascii=False, indent=2) + "\n",
encoding="utf-8",
newline="\n",
)
return str(path)
def _record_debug_event(self, event_name: str, data: Dict[str, Any], content: str, done: bool) -> None:
value = data.get("v")
path = str(data.get("p", "") or "")
operation = str(data.get("o", "") or "")
record: Dict[str, Any] = {
"event": str(event_name or ""),
"path": path,
"op": operation,
"done": bool(done),
"content_len": len(content or ""),
"content_preview": str(content or "")[:240],
"value_type": type(value).__name__,
"fragment_count": len(self._fragment_types),
"current_fragment_type": self._current_fragment_type,
"current_fragment_index": self._current_fragment_index,
}
if isinstance(value, str):
record["value_len"] = len(value)
record["value_preview"] = value[:240]
elif isinstance(value, list):
record["value_len"] = len(value)
elif isinstance(value, dict):
response = value.get("response") if isinstance(value.get("response"), dict) else {}
fragments = response.get("fragments") if isinstance(response.get("fragments"), list) else []
record["response_keys"] = list(response.keys())[:12]
record["response_fragment_count"] = len(fragments)
if fragments:
record["response_fragment_types"] = [
str(fragment.get("type", "") or "").upper()
for fragment in fragments[:8]
if isinstance(fragment, dict)
]
self._debug_records.append(record)
if len(self._debug_records) > 200:
self._debug_records = self._debug_records[-200:]
@classmethod
def get_id(cls) -> str:
return "deepseek"
@classmethod
def get_name(cls) -> str:
return "DeepSeek API"
@classmethod
def get_description(cls) -> str:
return "Parse DeepSeek SSE streams and ignore THINK fragments"
@classmethod
def get_supported_patterns(cls) -> List[str]:
return ["**/api/v0/chat/completion**"]
__all__ = ["DeepSeekParser"]
+693
View File
@@ -0,0 +1,693 @@
"""
doubao_parser.py - Doubao SSE response parser.
Observed stream traits:
- event: STREAM_MSG_NOTIFY carries the first assistant block
- event: STREAM_CHUNK may carry extra content_block patches
- event: CHUNK_DELTA carries plain text deltas
- event: SSE_REPLY_END marks the answer as finished
"""
from __future__ import annotations
import json
import re
from typing import Any, Dict, List, Tuple
from app.core.config import logger
from .base import ResponseParser
class DoubaoParser(ResponseParser):
"""Parse Doubao's SSE response stream."""
# 生成图片以 image_ori / image_thumb 等字段挂在 SSE 事件里,
# 同一张图的各尺寸共享 tos 路径(差异仅在 ~tplv 模板与签名参数)。
_IMAGE_FIELD_RE = re.compile(
r'"(image_ori|image_ori_raw|image_preview_resize|image_preview|image_thumb)"\s*:\s*'
r'\{[^{}]*?"url"\s*:\s*"(https?:\\?/\\?/[^"\s]+)"'
)
_IMAGE_KEY_PRIORITY = {
"image_ori": 0,
"image_ori_raw": 1,
"image_preview": 2,
"image_preview_resize": 3,
"image_thumb": 4,
}
def __init__(self) -> None:
self._last_raw = ""
self._pending = ""
self._block_texts: Dict[str, str] = {}
self._last_full_message = ""
self._assembled_text = ""
self._has_seen_visible_text = False
self._seen_image_keys: set = set()
def _collect_image_urls(self, new_data: str) -> List[str]:
"""从新的 SSE 数据中提取生成图片 URL,按图片去重、尺寸优先级取最优。"""
found: Dict[str, Tuple[int, str]] = {}
for match in self._IMAGE_FIELD_RE.finditer(new_data):
key, raw_url = match.group(1), match.group(2)
url = raw_url.replace(r"\/", "/")
# 去掉主机与签名参数/裁剪模板,同一张图的不同 CDN 副本视为一条
base_key = re.sub(r"^https?://[^/]+/", "", url).split("?", 1)[0].split("~tplv", 1)[0]
priority = self._IMAGE_KEY_PRIORITY.get(key, 99)
current = found.get(base_key)
if current is None or priority < current[0]:
found[base_key] = (priority, url)
fresh: List[str] = []
for base_key, (_, url) in found.items():
if base_key in self._seen_image_keys:
continue
self._seen_image_keys.add(base_key)
fresh.append(url)
return fresh
def parse_chunk(self, raw_response: str) -> Dict[str, Any]:
result: Dict[str, Any] = {
"content": "",
"images": [],
"done": False,
"error": None,
}
try:
if isinstance(raw_response, (bytes, bytearray)):
raw_response = raw_response.decode("utf-8", errors="ignore")
elif not isinstance(raw_response, str):
raw_response = str(raw_response)
if not raw_response:
return result
if self._last_raw and raw_response == self._last_raw:
return result
if self._last_raw and raw_response.startswith(self._last_raw):
new_data = raw_response[len(self._last_raw):]
else:
new_data = raw_response
self._last_raw = raw_response
result["images"] = self._collect_image_urls(new_data)
direct_content, direct_done = self._parse_direct_payload(new_data)
if direct_content or direct_done:
repaired = self._repair_text(direct_content)
result["content"] = repaired
if repaired:
self._assembled_text += repaired
self._has_seen_visible_text = True
result["done"] = bool(direct_done and self._can_accept_done_signal(repaired))
return result
delta_content, done = self._consume_new_data(new_data)
if delta_content:
repaired = self._repair_text(delta_content)
if repaired != delta_content and self._assembled_text.endswith(delta_content):
self._assembled_text = self._assembled_text[:-len(delta_content)] + repaired
result["content"] = repaired
result["done"] = done
except Exception as e:
logger.debug(f"[DoubaoParser] parse exception: {e}")
result["error"] = str(e)
return result
def reset(self) -> None:
self._last_raw = ""
self._pending = ""
self._block_texts.clear()
self._last_full_message = ""
self._assembled_text = ""
self._has_seen_visible_text = False
self._seen_image_keys.clear()
def _consume_new_data(self, new_data: str) -> Tuple[str, bool]:
normalized = (self._pending + new_data).replace("\r\n", "\n")
if not normalized:
return "", False
blocks = normalized.split("\n\n")
if normalized.endswith("\n\n"):
self._pending = ""
complete_blocks = [block for block in blocks if block.strip()]
else:
self._pending = blocks.pop() if blocks else normalized
complete_blocks = [block for block in blocks if block.strip()]
content_parts: List[str] = []
done = False
for block in complete_blocks:
block_content, block_done = self._parse_event_block(block)
if block_content:
content_parts.append(block_content)
self._assembled_text += block_content
self._has_seen_visible_text = True
if block_done:
done = True
return "".join(content_parts), done
def _parse_direct_payload(self, payload: str) -> Tuple[str, bool]:
stripped = (payload or "").strip()
if not stripped or not stripped.startswith("{"):
return "", False
try:
data = json.loads(stripped)
except json.JSONDecodeError:
return "", False
logger.debug(f"[DoubaoParser][DirectJSON] {self._describe_direct_json(data)}")
embedded_stream = self._extract_embedded_stream_payload(data)
if embedded_stream:
logger.debug(
"[DoubaoParser][DirectJSON] branch=embedded_stream "
f"payload_len={len(embedded_stream)}"
)
return self._consume_new_data(embedded_stream)
logger.debug("[DoubaoParser][DirectJSON] branch=direct_content")
return self._extract_direct_json_content(data)
@staticmethod
def _extract_embedded_stream_payload(data: Any) -> str:
if not isinstance(data, dict):
return ""
containers: List[Dict[str, Any]] = [data]
body = data.get("body")
if isinstance(body, dict):
containers.append(body)
for container in containers:
for stream_key in ("_stream", "stream"):
stream_data = container.get(stream_key)
if not isinstance(stream_data, dict):
continue
full_text = stream_data.get("fullText")
if isinstance(full_text, str) and full_text:
return full_text
chunks = stream_data.get("chunks")
if not isinstance(chunks, list):
continue
parts: List[str] = []
for chunk in chunks:
if not isinstance(chunk, dict):
continue
chunk_data = chunk.get("data")
if isinstance(chunk_data, str) and chunk_data:
parts.append(chunk_data)
if parts:
return "".join(parts)
return ""
def _parse_event_block(self, block: str) -> Tuple[str, bool]:
event_name = ""
data_lines: List[str] = []
for raw_line in block.split("\n"):
line = raw_line.strip()
if not line:
continue
if line.startswith("event:"):
event_name = line[6:].strip()
elif line.startswith("data:"):
data_lines.append(line[5:].strip())
payload = "\n".join(data_lines).strip()
if not event_name:
return "", False
if not payload:
return "", False
if event_name in {"STREAM_FINISH", "STREAM_END", "DONE", "COMPLETION"}:
logger.debug(f"[DoubaoParser][SSE] event={event_name} done_only=1")
return "", self._can_accept_done_signal()
if event_name == "CHUNK_DELTA":
text = self._extract_chunk_delta(payload)
text = self._repair_text(text)
if text:
logger.debug(
f"[DoubaoParser][SSE] event=CHUNK_DELTA text_len={len(text)} "
f"preview={text[:60]!r}"
)
return text, False
if event_name == "STREAM_MSG_NOTIFY":
text = self._extract_stream_msg_notify(payload)
text = self._repair_text(text)
if text:
logger.debug(
f"[DoubaoParser][SSE] event=STREAM_MSG_NOTIFY text_len={len(text)} "
f"preview={text[:60]!r}"
)
return text, False
if event_name == "STREAM_CHUNK":
text, done = self._extract_stream_chunk(payload)
text = self._repair_text(text)
if text or done:
logger.debug(
f"[DoubaoParser][SSE] event=STREAM_CHUNK text_len={len(text)} "
f"done={done} preview={text[:60]!r}"
)
return text, bool(done and self._can_accept_done_signal(text))
if event_name == "FULL_MSG_NOTIFY":
text = self._extract_full_msg_notify(payload)
text = self._repair_text(text)
if text:
logger.debug(
f"[DoubaoParser][SSE] event=FULL_MSG_NOTIFY text_len={len(text)} "
f"preview={text[:60]!r}"
)
return text, False
if event_name == "SSE_REPLY_END":
text, done = self._extract_reply_end(payload)
text = self._repair_text(text)
logger.debug(
f"[DoubaoParser][SSE] event=SSE_REPLY_END text_len={len(text)} "
f"done={done} preview={text[:60]!r}"
)
return text, bool(done and self._can_accept_done_signal(text))
return "", False
def _extract_direct_json_content(self, data: Any) -> Tuple[str, bool]:
if not isinstance(data, dict):
return "", False
role = data.get("role")
content = data.get("content")
if role == "assistant" and isinstance(content, str) and content:
logger.debug(
"[DoubaoParser][DirectJSON] content_source=role.content "
f"len={len(content)}"
)
return self._extract_full_message_delta(content), True
message = data.get("message")
if isinstance(message, dict):
role = message.get("role")
content = message.get("content")
if role == "assistant" and isinstance(content, str) and content:
logger.debug(
"[DoubaoParser][DirectJSON] content_source=message.content "
f"len={len(content)}"
)
return self._extract_full_message_delta(content), True
choices = data.get("choices")
if isinstance(choices, list):
parts: List[str] = []
for choice in choices:
if not isinstance(choice, dict):
continue
delta = choice.get("delta")
if isinstance(delta, dict):
text = delta.get("content")
if isinstance(text, str) and text:
parts.append(text)
message_obj = choice.get("message")
if isinstance(message_obj, dict):
text = message_obj.get("content")
if isinstance(text, str) and text:
parts.append(text)
if parts:
logger.debug(
"[DoubaoParser][DirectJSON] content_source=choices "
f"parts={len(parts)} total_len={len(''.join(parts))}"
)
return self._extract_full_message_delta("".join(parts)), bool(data.get("done", True))
return "", False
@staticmethod
def _describe_direct_json(data: Any) -> str:
if not isinstance(data, dict):
return f"type={type(data).__name__}"
def _keys_of(obj: Any) -> list[str]:
if isinstance(obj, dict):
return [str(k) for k in list(obj.keys())[:8]]
return []
message = data.get("message")
body = data.get("body")
message_content = message.get("content") if isinstance(message, dict) else None
message_content_len = len(message_content) if isinstance(message_content, str) else 0
choices = data.get("choices")
choices_len = len(choices) if isinstance(choices, list) else 0
return (
f"keys={_keys_of(data)}, "
f"stream_keys={_keys_of(data.get('stream'))}, "
f"_stream_keys={_keys_of(data.get('_stream'))}, "
f"body_keys={_keys_of(body)}, "
f"message_keys={_keys_of(message)}, "
f"message_content_len={message_content_len}, "
f"choices_len={choices_len}"
)
@staticmethod
def _extract_chunk_delta(payload: str) -> str:
try:
data = json.loads(payload)
except json.JSONDecodeError:
return ""
text = data.get("text", "")
return text if isinstance(text, str) else ""
def _extract_stream_msg_notify(self, payload: str) -> str:
try:
data = json.loads(payload)
except json.JSONDecodeError:
return ""
content = data.get("content", {})
return self._extract_content_blocks(content.get("content_block"))
def _extract_full_msg_notify(self, payload: str) -> str:
try:
data = json.loads(payload)
except json.JSONDecodeError:
return ""
message = data.get("message")
if not isinstance(message, dict):
return ""
if message.get("user_type") != 2:
return ""
content_block_text = self._extract_content_blocks(message.get("content_block"))
if content_block_text:
return content_block_text
raw_content = message.get("content")
if not isinstance(raw_content, str) or not raw_content.strip():
return ""
try:
parsed_content = json.loads(raw_content)
except json.JSONDecodeError:
return ""
return self._extract_content_blocks(parsed_content)
def _extract_stream_chunk(self, payload: str) -> Tuple[str, bool]:
try:
data = json.loads(payload)
except json.JSONDecodeError:
return "", False
content_parts: List[str] = []
done = False
for patch in data.get("patch_op", []) or []:
if not isinstance(patch, dict):
continue
patch_value = patch.get("patch_value")
if not isinstance(patch_value, dict):
continue
text = self._extract_content_blocks(patch_value.get("content_block"))
if text:
content_parts.append(text)
ext = patch_value.get("ext")
if isinstance(ext, dict) and str(ext.get("is_finish", "")) == "1":
done = True
return "".join(content_parts), done
def _extract_reply_end(self, payload: str) -> Tuple[str, bool]:
try:
data = json.loads(payload)
except json.JSONDecodeError:
return "", True
if self._assembled_text:
return "", True
finish_attr = data.get("msg_finish_attr")
if not isinstance(finish_attr, dict):
return "", True
brief = finish_attr.get("brief")
if not isinstance(brief, str) or not brief:
return "", True
return self._extract_full_message_delta(brief), True
def _can_accept_done_signal(self, pending_text: str = "") -> bool:
return bool(
self._has_seen_visible_text
or self._assembled_text
or str(pending_text or "")
)
@staticmethod
def _extract_text_from_block(block: Any) -> Tuple[str, str]:
if not isinstance(block, dict):
return "", ""
block_id = block.get("block_id") or block.get("id") or ""
if not isinstance(block_id, str):
block_id = str(block_id)
content = block.get("content")
if not isinstance(content, dict):
return block_id, ""
text_block = content.get("text_block")
if not isinstance(text_block, dict):
return block_id, ""
text = text_block.get("text", "")
return block_id, text if isinstance(text, str) else ""
def _extract_content_blocks(self, blocks: Any) -> str:
if not isinstance(blocks, list):
return ""
parts: List[str] = []
for block in blocks:
block_id, text = self._extract_text_from_block(block)
if not text:
continue
previous = self._block_texts.get(block_id, "")
if previous and text.startswith(previous):
delta = text[len(previous):]
elif previous == text:
delta = ""
else:
delta = text
self._block_texts[block_id] = text
if delta:
parts.append(delta)
return "".join(parts)
def _extract_full_message_delta(self, text: str) -> str:
if not text:
return ""
previous = self._last_full_message
self._last_full_message = text
if self._assembled_text and text.startswith(self._assembled_text):
return text[len(self._assembled_text):]
if previous and text.startswith(previous):
return text[len(previous):]
if previous == text:
return ""
return text
@staticmethod
def _repair_text(text: str) -> str:
if not text:
return text
repaired_whole = DoubaoParser._repair_text_candidate(text)
if repaired_whole != text:
return repaired_whole
repaired_segments: List[str] = []
changed = False
buffer = ""
buffer_is_suspicious: bool | None = None
def flush() -> None:
nonlocal buffer, buffer_is_suspicious, changed
if not buffer:
return
if buffer_is_suspicious:
repaired = DoubaoParser._repair_text_candidate(buffer)
if repaired != buffer:
changed = True
repaired_segments.append(repaired)
else:
repaired_segments.append(buffer)
buffer = ""
buffer_is_suspicious = None
for ch in text:
is_suspicious = DoubaoParser._is_suspicious_char(ch)
if buffer_is_suspicious is None:
buffer = ch
buffer_is_suspicious = is_suspicious
continue
if is_suspicious == buffer_is_suspicious:
buffer += ch
continue
flush()
buffer = ch
buffer_is_suspicious = is_suspicious
flush()
if changed:
repaired_text = "".join(repaired_segments)
if repaired_text != text:
logger.debug(
"[DoubaoParser][Repair] repaired mixed mojibake "
f"(before_len={len(text)}, after_len={len(repaired_text)})"
)
return repaired_text
return text
@staticmethod
def _repair_text_candidate(text: str) -> str:
if not text or not DoubaoParser._should_try_repair_text(text):
return text
for source_encoding in ("latin1", "cp1252"):
repaired = DoubaoParser._try_redecode_text(text, source_encoding)
if not repaired:
continue
if DoubaoParser._looks_more_readable(repaired, text):
logger.debug(
"[DoubaoParser][Repair] repaired mojibake "
f"(enc={source_encoding}, before_len={len(text)}, after_len={len(repaired)})"
)
return repaired
return text
@staticmethod
def _should_try_repair_text(text: str) -> bool:
if not text:
return False
if any(0x4E00 <= ord(ch) <= 0x9FFF or ord(ch) > 0xFFFF for ch in text):
return False
cp1252_extra = {
0x0152, 0x0153, 0x0160, 0x0161, 0x0178, 0x017D, 0x017E,
0x02C6, 0x02DC, 0x2013, 0x2014, 0x2018, 0x2019, 0x201A,
0x201C, 0x201D, 0x201E, 0x2020, 0x2021, 0x2022, 0x2026,
0x2030, 0x2039, 0x203A, 0x20AC, 0x2122,
}
suspicious_count = 0
for ch in text:
if DoubaoParser._is_suspicious_char(ch, cp1252_extra):
suspicious_count += 1
return suspicious_count >= max(2, len(text) // 4)
@staticmethod
def _is_suspicious_char(ch: str, cp1252_extra: set[int] | None = None) -> bool:
if not ch:
return False
if cp1252_extra is None:
cp1252_extra = {
0x0152, 0x0153, 0x0160, 0x0161, 0x0178, 0x017D, 0x017E,
0x02C6, 0x02DC, 0x2013, 0x2014, 0x2018, 0x2019, 0x201A,
0x201C, 0x201D, 0x201E, 0x2020, 0x2021, 0x2022, 0x2026,
0x2030, 0x2039, 0x203A, 0x20AC, 0x2122,
}
code = ord(ch)
return 0x80 <= code <= 0x00FF or code in cp1252_extra
@staticmethod
def _try_redecode_text(text: str, source_encoding: str) -> str:
data = bytearray()
for ch in text:
code = ord(ch)
if code <= 0xFF:
data.append(code)
continue
try:
encoded = ch.encode(source_encoding)
except Exception:
return ""
if len(encoded) != 1:
return ""
data.extend(encoded)
try:
return bytes(data).decode("utf-8")
except UnicodeDecodeError:
return bytes(data).decode("utf-8", errors="ignore")
@staticmethod
def _looks_more_readable(candidate: str, original: str) -> bool:
readable_punct = set(".,!?;:'\"()[]{}-_/~`@#$%^&*+=<>,。!?:;()【】《》、~“”‘’…")
def score(value: str) -> int:
total = 0
for ch in value:
code = ord(ch)
if 0x4E00 <= code <= 0x9FFF:
total += 2
elif code > 0xFFFF:
total += 2
elif ch.isascii() and (ch.isalnum() or ch.isspace() or ch in readable_punct):
total += 1
elif ch in readable_punct or ch.isspace():
total += 1
return total
return score(candidate) > score(original)
@classmethod
def get_id(cls) -> str:
return "doubao"
@classmethod
def get_name(cls) -> str:
return "Doubao"
@classmethod
def get_description(cls) -> str:
return "Parse Doubao SSE streams with STREAM_MSG_NOTIFY / CHUNK_DELTA events"
@classmethod
def get_supported_patterns(cls) -> List[str]:
return ["**/chat/completion**"]
__all__ = ["DoubaoParser"]
+723
View File
@@ -0,0 +1,723 @@
"""
gemini_parser.py - Gemini StreamGenerate 响应解析器
响应格式特征
1. 安全前缀: )]}'\n
2. 多块格式: <len>\n[[JSON]]
3. 文本路径: outer[0][2] ➜ json.loads ➜ inner[4][0][1][0]
4. 文本是“累积”模式:每块都返回从开头到当前的完整文本
5. 结束标志: [["di",...]] / [["e",...]] / [["af.httprm",...]]
"""
from __future__ import annotations
import json
import re
from typing import Any, Dict, List, Optional, Tuple
from app.core.config import logger
from .base import ResponseParser
# ----------------------------------------------------------
# 私有工具
# ----------------------------------------------------------
_ESCAPE_FIXER = re.compile(r"\\([<>`#*_\[\]()])") # 常见转义符号:HTML/CSS/Markdown
_EOL_FIXER = re.compile(r"\\\\n") # \\n → \n
_STANDALONE_BS = re.compile(r"\\\n") # backslash + real LF
_REASONING_HINTS = (
"i'm now ",
"i am now ",
"i've ",
"i have ",
"i must ",
"i clarified",
"i've clarified",
"i have clarified",
"user's request",
"assistant persona",
"brainstorm",
"analyzing",
"analysis",
"focusing on",
"clarifying",
"refining",
"crafting",
"delivering",
"defining",
"prioritizing",
"interpr",
)
_REASONING_PROCESS_HINTS = (
"define",
"defined",
"defining",
"develop",
"developing",
"draft",
"drafting",
"craft",
"crafted",
"crafting",
"focus",
"focusing",
"clarif",
"refin",
"break",
"breaking",
"apply",
"applying",
"confirm",
"confirming",
)
_REASONING_META_HINTS = (
"task",
"request",
"user",
"intent",
"persona",
"story",
"parameter",
"format",
"constraint",
"component",
"narrative",
"title",
"scene",
"style",
"protagonist",
"plot",
"character",
"dialogue",
"ending",
"climax",
"structure",
)
_GEMINI_GENERATED_MEDIA_URL_RE = re.compile(
r"https?://(?:[\w.-]+\.)?(?:googleusercontent\.com|lh3\.googleusercontent\.com)/[^\s\"'<>\]\)\}\\]+",
re.IGNORECASE,
)
_GEMINI_PLACEHOLDER_MEDIA_RE = re.compile(
r"https?://(?:[\w.-]+\.)?googleusercontent\.com/(?:image_generation_content|generated_music_content)/\d+",
re.IGNORECASE,
)
_RENDER_START_RE = re.compile(r"<\s*render\b", re.IGNORECASE)
_RENDER_CLOSE_RE = re.compile(r"<\s*/\s*render\s*>", re.IGNORECASE)
_MEDIA_SCAN_TAIL_CHARS = 4096
_MEDIA_SCAN_DELIMITERS = (" ", "\n", "\r", "\t", '"', "'", "<", ">", "\\")
def _clean_escaped(text: str) -> str:
"""
Gemini 返回的字符串仍可能残留多余的反斜杠:
1. \\<ctx\\> → <ctx>
2. \\\\n → \n → 换行
3. \\`code\\` → `code`
4. 反斜杠 + 真 \n → 真 \n
"""
if "\\" not in text:
return text
# 1) 处理 \n / \\n
text = _STANDALONE_BS.sub("\n", text) # \ + 真换行
text = _EOL_FIXER.sub(r"\n", text) # \\n → \n
# 2) 去除转义符号前缀(只对 < > ` 三个常见 HTML 符号)
text = _ESCAPE_FIXER.sub(r"\1", text)
return text
# ----------------------------------------------------------
# 解析器主体
# ----------------------------------------------------------
class GeminiParser(ResponseParser):
"""
Google Gemini StreamGenerate 响应解析器
"""
def __init__(self) -> None:
self._last_len = 0 # 已发送给上层的字符数
self._full_cache = "" # 最新完整文本
self._last_reasoning_len = 0
self._full_reasoning_cache = ""
self._last_raw_length = 0
self._pending_raw = ""
self._seen_media_refs: set[str] = set()
self._media_scan_tail = ""
# ---------- 对外接口 ---------- #
def parse_chunk(self, raw: str | bytes) -> Dict[str, Any]:
"""
解析 *单个* HTTP chunk,返回增量内容
"""
if isinstance(raw, bytes):
raw = raw.decode("utf-8", errors="ignore")
elif not isinstance(raw, str):
raw = str(raw)
try:
previous_len = int(getattr(self, "_last_raw_length", 0) or 0)
previous_raw = getattr(self, "_last_raw_response", "")
will_reset = bool(previous_len and previous_raw and not raw.startswith(previous_raw))
is_stream_start = (previous_len == 0 and not self._pending_raw) or will_reset
new_data = self._prepare_incremental_raw_response(raw)
if not new_data:
return {
"content": "",
"reasoning_content": "",
"images": [],
"done": False,
"error": None,
"unclosed_render_output": self._has_unclosed_render_output(
self._full_cache
),
}
full_txt, full_reasoning, done = self._parse_incremental(
new_data,
is_stream_start=is_stream_start,
)
images = self._extract_generated_images_from_increment(new_data)
except Exception as exc: # pragma: no cover
logger.debug(f"[GeminiParser] 解析异常: {exc}")
return {"content": "", "reasoning_content": "", "images": [], "done": False, "error": str(exc)}
delta = ""
if full_txt is not None and len(full_txt) > self._last_len:
delta = full_txt[self._last_len :]
self._last_len = len(full_txt)
self._full_cache = full_txt
delta_reasoning = ""
if full_reasoning is not None and len(full_reasoning) > self._last_reasoning_len:
delta_reasoning = full_reasoning[self._last_reasoning_len :]
self._last_reasoning_len = len(full_reasoning)
self._full_reasoning_cache = full_reasoning
return {
"content": delta,
"reasoning_content": delta_reasoning,
"images": images,
"done": done,
"error": None,
"unclosed_render_output": self._has_unclosed_render_output(
self._full_cache
),
}
def reset(self) -> None:
self._last_len = 0
self._full_cache = ""
self._last_reasoning_len = 0
self._full_reasoning_cache = ""
self._last_raw_length = 0
self._last_raw_response = ""
self._pending_raw = ""
self._seen_media_refs.clear()
self._media_scan_tail = ""
@staticmethod
def _list_get(value: Any, index: int) -> Any:
if isinstance(value, list) and 0 <= index < len(value):
return value[index]
return None
@classmethod
def _nested_list_get(cls, value: Any, *path: int) -> Any:
current = value
for index in path:
current = cls._list_get(current, index)
if current is None:
return None
return current
@staticmethod
def _looks_like_content(value: Any) -> bool:
if not isinstance(value, str):
return False
text = value.strip()
if not text:
return False
lowered = text.lower()
if lowered.startswith(("r_", "c_", "rc_")) and len(text) <= 80:
return False
if text.startswith(("http://", "https://", "data:image", "blob:")):
return True
if "\n" in text or "\r" in text:
return True
if "<" in text and ">" in text:
return True
if any("\u4e00" <= ch <= "\u9fff" for ch in text):
return True
alpha_count = sum(ch.isalpha() for ch in text)
return len(text) >= 16 and alpha_count >= 6
@classmethod
def _iter_content_candidates(cls, value: Any):
if isinstance(value, str):
if cls._looks_like_content(value):
yield value
return
if isinstance(value, list):
for item in value:
yield from cls._iter_content_candidates(item)
return
if isinstance(value, dict):
for item in value.values():
yield from cls._iter_content_candidates(item)
@staticmethod
def _decode_content_text(content: str) -> str:
try:
decoded = json.loads(f'"{content}"')
if isinstance(decoded, str):
content = decoded
except json.JSONDecodeError:
content = _clean_escaped(content)
return _clean_escaped(content)
@staticmethod
def _has_unclosed_render_output(text: str) -> bool:
text = str(text or "")
if not text:
return False
stripped = text.lstrip()
if not _RENDER_START_RE.match(stripped):
return False
render_starts = list(_RENDER_START_RE.finditer(text))
if not render_starts:
return False
render_closes = list(_RENDER_CLOSE_RE.finditer(text))
last_open_at = render_starts[-1].start()
last_close_at = render_closes[-1].start() if render_closes else -1
return last_open_at > last_close_at
@staticmethod
def _looks_like_reasoning(value: str) -> bool:
text = str(value or "").strip()
if not text:
return False
lowered = " ".join(text.lower().split())
if not lowered:
return False
looks_like_structured_answer = text.startswith(
("{", "```json", "```", "<render", "<segment", "<kaitou", "<scene")
) or any(token in text for token in ('"role"', '"content"', '"tool_calls"', '"decision"'))
if text.startswith("**") and any(hint in lowered for hint in _REASONING_HINTS):
return True
if (
not looks_like_structured_answer
and lowered.startswith(("i'm ", "i am ", "i've ", "i have "))
and any(hint in lowered for hint in _REASONING_PROCESS_HINTS)
and any(hint in lowered for hint in _REASONING_META_HINTS)
):
return True
return any(
marker in lowered
for marker in (
"i'm now carefully",
"i am now carefully",
"i'm now focused",
"i am now focused",
"i've established",
"i have established",
"the user's compositional request",
"the user's intent",
"helpful assistant persona",
)
)
@classmethod
def _score_candidate(cls, value: str) -> int:
text = str(value or "").strip()
if not text:
return -10_000
score = 0
lowered = text.lower()
if text.startswith(("{", "```json", "```", "<render", "<segment", "<kaitou", "<scene")):
score += 120
if any(token in text for token in ('"role"', '"content"', '"tool_calls"', '"decision"', '"name"', '"description"')):
score += 80
if any("\u4e00" <= ch <= "\u9fff" for ch in text):
score += 60
if text.startswith(("http://", "https://", "data:image", "blob:")):
score += 40
if "\n" in text or "\r" in text:
score += 10
if len(text) >= 80:
score += 10
elif len(text) >= 24:
score += 5
if text.startswith("**"):
score -= 30
if lowered.startswith(("r_", "c_", "rc_")) and len(text) <= 80:
score -= 200
return score
@classmethod
def _select_best_candidate(cls, value: Any) -> Optional[str]:
seen: set[str] = set()
best_text: Optional[str] = None
best_score: Optional[int] = None
for candidate in cls._iter_content_candidates(value):
decoded = cls._decode_content_text(candidate)
normalized = decoded.strip()
if not normalized or normalized in seen:
continue
seen.add(normalized)
score = cls._score_candidate(decoded)
if (
best_text is None
or best_score is None
or score > best_score
or (score == best_score and len(decoded) > len(best_text))
):
best_text = decoded
best_score = score
if best_text is None or best_score is None or best_score <= 0:
return None
return best_text
@classmethod
def _extract_direct_answer(cls, inner: Any) -> Optional[str]:
content_block = cls._list_get(inner, 4)
if not isinstance(content_block, list):
return None
best_text: Optional[str] = None
best_score: Optional[int] = None
for item in content_block:
direct_payload = cls._list_get(item, 1)
if direct_payload is None:
continue
candidate = cls._select_best_candidate(direct_payload)
if not candidate:
continue
score = cls._score_candidate(candidate)
if (
best_text is None
or best_score is None
or score > best_score
or (score == best_score and len(candidate) > len(best_text))
):
best_text = candidate
best_score = score
return best_text
@classmethod
def _extract_thinking(cls, inner: Any) -> Optional[str]:
try:
content_block = cls._list_get(inner, 4)
if not isinstance(content_block, list) or not content_block:
return None
for item in content_block:
if not isinstance(item, list):
continue
candidate = item
if isinstance(item, list) and item and isinstance(item[0], list):
candidate = item[0]
if not isinstance(candidate, list):
continue
# 1. 优先读取标准 37 槽位(零开销命中)
if len(candidate) > 37 and isinstance(candidate[37], list) and candidate[37]:
thinking_data = candidate[37]
if (
isinstance(thinking_data, list)
and thinking_data
and isinstance(thinking_data[0], list)
and thinking_data[0]
and isinstance(thinking_data[0][0], str)
):
text = thinking_data[0][0].strip()
if text:
return text
# 2. 备用:结构化特征扫描(抵御未来索引偏移变动)
for sub in candidate:
if (
isinstance(sub, list)
and len(sub) >= 2
and isinstance(sub[0], list)
and sub[0]
and isinstance(sub[0][0], str)
):
text = sub[0][0].strip()
if text and (
text.startswith(("**Initiating", "**Assessing", "**Thinking", "**Outlining", "**Dissecting", "**Refining"))
or "I'm " in text or "I've " in text or "Initiating " in text
):
return text
except Exception:
pass
return None
# ---------- 内部逻辑 ---------- #
def _parse_incremental(
self,
raw_text: str,
*,
is_stream_start: bool = False,
) -> Tuple[Optional[str], Optional[str], bool]:
"""
解析 Gemini 的 *增量* HTTP body 片段,返回 (完整文本, 完整思考过程, 是否结束)
"""
combined = f"{self._pending_raw}{str(raw_text or '')}"
if not combined:
return None, None, False
# 仅在全新流的首个片段上清理安全前缀。
if is_stream_start:
combined = combined.lstrip(")]}'\n")
lines = combined.splitlines(keepends=True)
full_content: Optional[str] = None
full_reasoning: Optional[str] = None
done = False
pending_start = len(combined)
i = 0
cursor = 0
while i < len(lines):
line = lines[i]
line_start = cursor
line_end = line_start + len(line)
meta = line.strip()
if not meta: # 空行
cursor = line_end
i += 1
continue
if meta.isdigit(): # 长度行
if i + 1 >= len(lines):
pending_start = line_start
break
json_block = lines[i + 1].rstrip("\r\n")
try:
outer = json.loads(json_block)
except json.JSONDecodeError:
pending_start = line_start
break
if self._is_end_signal(outer):
done = True
else:
content, reasoning = self._extract_content(outer)
if content:
full_content = content
if reasoning:
full_reasoning = reasoning
cursor = line_end + len(lines[i + 1])
i += 2
else:
cursor = line_end
i += 1
self._pending_raw = combined[pending_start:] if pending_start < len(combined) else ""
return full_content, full_reasoning, done
@staticmethod
def _is_end_signal(data: list) -> bool:
"""
结束块格式:
[["di", 123]] / [["e",...]] / [["af.httprm",...]]
"""
if (
isinstance(data, list)
and data
and isinstance(data[0], list)
and data[0]
and data[0][0] in ("di", "e", "af.httprm")
):
return True
return False
# -------- 核心:提取 & 转义修复 -------- #
def _extract_content(self, outer: list) -> Tuple[Optional[str], Optional[str]]:
"""
outer → inner → (content, reasoning_content)
"""
try:
first = outer[0]
if not (isinstance(first, list) and len(first) >= 3 and first[0] == "wrb.fr"):
return None, None
inner_raw: str = first[2]
if not isinstance(inner_raw, str):
return None, None
inner = json.loads(inner_raw) # type: ignore[arg-type]
content = self._extract_direct_answer(inner)
reasoning = self._extract_thinking(inner)
return content, reasoning
except Exception as exc: # pragma: no cover
logger.debug(f"[GeminiParser] 提取失败: {exc}")
return None, None
def should_fallback_to_dom_when_no_visible_content(self) -> bool:
return True
def get_media_generation_state(
self,
raw_response: str = "",
parse_result: Optional[Dict[str, Any]] = None,
) -> Dict[str, Any]:
raw_text = str(raw_response or "")
if not raw_text:
return {}
parse_images = parse_result.get("images") if isinstance(parse_result, dict) else None
if parse_images or self._seen_media_refs or self._has_generated_image(raw_text):
return {}
hint_parts: List[str] = []
lowered = raw_text.lower()
wait_timeout_seconds = None
if "nano banana" in lowered:
hint_parts.append("正在加载 Nano Banana 2...")
wait_timeout_seconds = max(float(wait_timeout_seconds or 0), 90.0)
if "image_generation_content/" in lowered:
hint_parts.append("image_generation_content")
wait_timeout_seconds = max(float(wait_timeout_seconds or 0), 90.0)
if not hint_parts:
return {}
deduped_hint_parts: List[str] = []
seen = set()
for item in hint_parts:
if item in seen:
continue
seen.add(item)
deduped_hint_parts.append(item)
return {
"pending": True,
"media_type": "image",
"hint_text": "\n\n".join(deduped_hint_parts[:3]),
"wait_timeout_seconds": wait_timeout_seconds,
}
@staticmethod
def _build_media_scan_tail(raw_text: str) -> str:
text = str(raw_text or "")
if not text:
return ""
last_delimiter = -1
for delimiter in _MEDIA_SCAN_DELIMITERS:
last_delimiter = max(last_delimiter, text.rfind(delimiter))
tail = text[last_delimiter + 1:] if last_delimiter >= 0 else text
if len(tail) > _MEDIA_SCAN_TAIL_CHARS:
tail = tail[-_MEDIA_SCAN_TAIL_CHARS:]
return tail
@classmethod
def _iter_generated_image_urls(cls, raw_response: str):
raw_text = str(raw_response or "")
if not raw_text:
return
for match in _GEMINI_GENERATED_MEDIA_URL_RE.finditer(raw_text):
url = str(match.group(0) or "").strip().rstrip(".,;:!?")
if not url:
continue
if _GEMINI_PLACEHOLDER_MEDIA_RE.fullmatch(url):
continue
lowered = url.lower()
if "/gg/" not in lowered and "/gg-dl/" not in lowered:
continue
if not any(ext in lowered for ext in (".png", ".jpg", ".jpeg", ".webp", ".gif", ".bmp", ".avif")):
continue
yield url
@classmethod
def _has_generated_image(cls, raw_response: str) -> bool:
return any(cls._iter_generated_image_urls(raw_response))
def _extract_generated_images_from_increment(self, new_data: str) -> List[Dict[str, Any]]:
scan_text = f"{self._media_scan_tail}{str(new_data or '')}"
images = self._extract_generated_images(scan_text)
self._media_scan_tail = self._build_media_scan_tail(scan_text)
return images
def _extract_generated_images(self, raw_response: str, *, mark_seen: bool = True) -> List[Dict[str, Any]]:
local_seen: set[str] = set()
images: List[Dict[str, Any]] = []
for url in self._iter_generated_image_urls(raw_response):
if url in local_seen:
continue
local_seen.add(url)
if mark_seen:
if url in self._seen_media_refs:
continue
self._seen_media_refs.add(url)
images.append(
{
"media_type": "image",
"kind": "url",
"url": url,
"data_uri": None,
"mime": None,
"byte_size": None,
"source": "gemini_stream",
}
)
return images
# ---------- 元数据 ---------- #
@classmethod
def get_id(cls) -> str:
return "gemini"
@classmethod
def get_name(cls) -> str:
return "Gemini StreamGenerate"
@classmethod
def get_description(cls) -> str:
return "解析 Google Gemini 的 StreamGenerate 响应"
@classmethod
def get_supported_patterns(cls) -> List[str]:
return ["StreamGenerate"]
__all__ = ["GeminiParser"]
+384
View File
@@ -0,0 +1,384 @@
"""
glm_parser.py - ChatGLM SSE response parser.
Observed stream traits:
- content-type: text/event-stream
- each SSE block is carried in a data: line with a JSON payload
- think/tool_calls/text/system_error are all emitted through parts[*].content[*]
- text payloads are full rendered text snapshots, not pure deltas
- stream ends when a text part status becomes finish or top-level status becomes finish
"""
from __future__ import annotations
import json
import re
from typing import Any, Dict, List
from app.core.config import logger
from .base import ResponseParser
class GLMParser(ResponseParser):
"""Parse ChatGLM SSE streams while ignoring think/tool-call noise."""
_TERMINAL_STATUSES = {"finish", "finished", "intervene", "intervened"}
_PPT_FOLLOWUP_COMMANDS = {"change_mode_to_engine_ppt"}
_PPT_TEMPLATE_MARKERS = ('"theme"', '"background"', '"palette"')
_HIDDEN_REPAIR_TEXT_PATTERNS = (
"xml-style tool call",
"could not be parsed into a valid declared tool",
"could not be parsed into valid tool_calls",
"validation errors",
"the rejected assistant reply was",
)
_THINK_REASONING_PREFIX_RE = re.compile(
r"^\s*(let me\b|looking at\b|the user has\b|让我|我来|先分析|仔细分析)",
re.IGNORECASE,
)
_EMBEDDED_REPLY_LABEL_RE = re.compile(
r"(?:the rejected assistant reply was|the rejected response was|被拒绝的响应是|被拒绝的助手回复是)\s*[::]?\s*```",
re.IGNORECASE,
)
_VISIBLE_MARKERS = ("<render", "<segment", "<context_summary", "<q>", "<act>", "<inner>", "<aside>", "<meme")
def __init__(self) -> None:
self._last_raw_length = 0
self._pending = ""
self._rendered_text = ""
self._think_text = ""
self._has_seen_visible_text = False
self._saw_hidden_repair_flow = False
self._saw_only_think_payloads = False
self._awaiting_ppt_followup = False
self._deferred_followup_text = ""
self._ppt_followup_command = ""
def reset(self) -> None:
self._last_raw_length = 0
self._last_raw_response = ""
self._pending = ""
self._rendered_text = ""
self._think_text = ""
self._has_seen_visible_text = False
self._saw_hidden_repair_flow = False
self._saw_only_think_payloads = False
self._awaiting_ppt_followup = False
self._deferred_followup_text = ""
self._ppt_followup_command = ""
def prepare_for_followup_stream(self) -> None:
awaiting_ppt_followup = self._awaiting_ppt_followup
ppt_followup_command = self._ppt_followup_command
self.reset()
self._awaiting_ppt_followup = awaiting_ppt_followup
self._ppt_followup_command = ppt_followup_command
def parse_chunk(self, raw_response: str) -> Dict[str, Any]:
result: Dict[str, Any] = {
"content": "",
"images": [],
"done": False,
"error": None,
}
try:
if isinstance(raw_response, (bytes, bytearray)):
raw_response = raw_response.decode("utf-8", errors="ignore")
elif not isinstance(raw_response, str):
raw_response = str(raw_response)
new_data = self._prepare_incremental_raw_response(raw_response)
if not new_data:
return result
content, done = self._consume_new_data(new_data)
if content:
result["content"] = content
result["done"] = done
except Exception as e:
logger.debug(f"[GLMParser] parse exception: {e}")
result["error"] = str(e)
return result
def _consume_new_data(self, new_data: str) -> tuple[str, bool]:
normalized = (self._pending + new_data).replace("\r\n", "\n")
if not normalized:
return "", False
blocks = normalized.split("\n\n")
if normalized.endswith("\n\n"):
self._pending = ""
complete_blocks = [block for block in blocks if block.strip()]
else:
self._pending = blocks.pop() if blocks else normalized
complete_blocks = [block for block in blocks if block.strip()]
content_parts: List[str] = []
done = False
for block in complete_blocks:
block_content, block_done = self._parse_event_block(block)
if block_content:
content_parts.append(block_content)
if block_done:
done = True
return "".join(content_parts), done
def _parse_event_block(self, block: str) -> tuple[str, bool]:
data_lines: List[str] = []
for raw_line in block.split("\n"):
line = raw_line.strip()
if not line:
continue
if line.startswith("data:"):
data_lines.append(line[5:].strip())
payload = "\n".join(data_lines).strip()
if not payload:
return "", False
try:
data = json.loads(payload)
except json.JSONDecodeError:
return "", False
if not isinstance(data, dict):
return "", False
return self._extract_payload(data)
def _extract_payload(self, data: Dict[str, Any]) -> tuple[str, bool]:
top_status = str(data.get("status") or "").strip().lower()
last_error = data.get("last_error")
intervene_text = self._extract_intervene_text(last_error)
if intervene_text and self._looks_like_hidden_repair_text(intervene_text):
intervene_text = ""
parts = data.get("parts")
if not isinstance(parts, list) or not parts:
if self._awaiting_ppt_followup and top_status in self._TERMINAL_STATUSES:
snapshot = self._resolve_deferred_followup_text()
if snapshot:
delta = self._compute_delta(self._rendered_text, snapshot)
self._rendered_text = snapshot
self._has_seen_visible_text = True
return delta, True
return "", False
if intervene_text and top_status in self._TERMINAL_STATUSES:
delta = self._compute_delta(self._rendered_text, intervene_text)
self._rendered_text = intervene_text
if intervene_text:
self._has_seen_visible_text = True
return delta, True
return "", bool(self._has_seen_visible_text and top_status in self._TERMINAL_STATUSES)
visible_snapshot = self._rendered_text
saw_visible_text_in_payload = False
saw_think_in_payload = False
done = bool(self._has_seen_visible_text and top_status in self._TERMINAL_STATUSES)
for part in parts:
if not isinstance(part, dict):
continue
part_status = str(part.get("status") or "").strip().lower()
content_items = part.get("content")
if not isinstance(content_items, list):
continue
for item in content_items:
if not isinstance(item, dict):
continue
item_type = str(item.get("type") or "").strip().lower()
if item_type == "system_error":
self._handle_system_error_item(item, part)
continue
if item_type == "think":
think_text = str(item.get("think") or self._think_text)
self._think_text = think_text
saw_think_in_payload = True
if self._awaiting_ppt_followup and think_text.strip():
self._awaiting_ppt_followup = False
self._deferred_followup_text = ""
embedded_snapshot = self._extract_embedded_visible_snapshot(think_text)
if embedded_snapshot:
visible_snapshot = embedded_snapshot
saw_visible_text_in_payload = True
self._has_seen_visible_text = True
if self._looks_like_hidden_repair_text(think_text):
self._saw_hidden_repair_flow = True
continue
if item_type == "tool_calls":
continue
if item_type != "text":
continue
snapshot = str(item.get("text") or "")
if self._awaiting_ppt_followup:
if snapshot:
self._deferred_followup_text = snapshot
if part_status in self._TERMINAL_STATUSES:
snapshot = self._resolve_deferred_followup_text()
if snapshot:
visible_snapshot = snapshot
saw_visible_text_in_payload = True
self._has_seen_visible_text = True
done = True
continue
if snapshot:
visible_snapshot = snapshot
saw_visible_text_in_payload = True
self._has_seen_visible_text = True
if part_status in self._TERMINAL_STATUSES and (snapshot or self._has_seen_visible_text):
done = True
if saw_think_in_payload and not saw_visible_text_in_payload:
self._saw_only_think_payloads = True
if intervene_text and done and not visible_snapshot:
visible_snapshot = intervene_text
delta = self._compute_delta(self._rendered_text, visible_snapshot)
self._rendered_text = visible_snapshot
return delta, done
def should_fallback_to_dom_when_no_visible_content(self) -> bool:
return bool(
not self._has_seen_visible_text
and (self._saw_hidden_repair_flow or self._saw_only_think_payloads)
)
def should_wait_for_followup_stream(self) -> bool:
return bool(self._awaiting_ppt_followup)
@classmethod
def _looks_like_hidden_repair_text(cls, text: Any) -> bool:
normalized = str(text or "").strip()
if not normalized:
return False
lowered = normalized.lower()
if any(marker in lowered for marker in cls._HIDDEN_REPAIR_TEXT_PATTERNS):
return True
if cls._THINK_REASONING_PREFIX_RE.search(normalized):
if "<segment>" in normalized or "<context_summary>" in normalized:
return True
if "\\u003csegment\\u003e" in normalized or "\\u003ccontext_summary\\u003e" in normalized:
return True
return False
def export_debug_data(self, raw_response: str = "") -> Dict[str, Any]:
return {
"has_seen_visible_text": bool(self._has_seen_visible_text),
"rendered_text_len": len(self._rendered_text or ""),
"think_text_len": len(self._think_text or ""),
"saw_hidden_repair_flow": bool(self._saw_hidden_repair_flow),
"saw_only_think_payloads": bool(self._saw_only_think_payloads),
"awaiting_ppt_followup": bool(self._awaiting_ppt_followup),
"ppt_followup_command": self._ppt_followup_command,
"deferred_followup_text_len": len(self._deferred_followup_text or ""),
}
def _handle_system_error_item(self, item: Dict[str, Any], part: Dict[str, Any]) -> None:
metadata = part.get("meta_data")
if not isinstance(metadata, dict):
metadata = {}
command = str(
metadata.get("failedCommand") or item.get("failedCommand") or ""
).strip()
if command not in self._PPT_FOLLOWUP_COMMANDS:
return
self._awaiting_ppt_followup = True
self._deferred_followup_text = ""
self._ppt_followup_command = command
def _resolve_deferred_followup_text(self) -> str:
snapshot = self._deferred_followup_text
self._deferred_followup_text = ""
if not snapshot or self._looks_like_ppt_template(snapshot):
return ""
self._awaiting_ppt_followup = False
return snapshot
@classmethod
def _looks_like_ppt_template(cls, text: Any) -> bool:
normalized = str(text or "").strip().lower()
if normalized.startswith("```json"):
normalized = normalized[7:].lstrip()
elif normalized.startswith("```"):
normalized = normalized[3:].lstrip()
if not normalized.startswith("{"):
return False
return sum(marker in normalized for marker in cls._PPT_TEMPLATE_MARKERS) >= 2
@classmethod
def _extract_embedded_visible_snapshot(cls, text: Any) -> str:
normalized = str(text or "")
if not normalized:
return ""
match = cls._EMBEDDED_REPLY_LABEL_RE.search(normalized)
if not match:
return ""
body = normalized[match.end():]
if not body:
return ""
fence_end = body.find("```")
candidate = body[:fence_end] if fence_end >= 0 else body
candidate = candidate.strip()
if not candidate:
return ""
lowered = candidate.lower()
if lowered.startswith("<think"):
return ""
if any(marker in lowered for marker in cls._HIDDEN_REPAIR_TEXT_PATTERNS):
return ""
if not any(marker in lowered for marker in cls._VISIBLE_MARKERS):
return ""
return candidate
@staticmethod
def _extract_intervene_text(last_error: Any) -> str:
if not isinstance(last_error, dict):
return ""
text = last_error.get("intervene_text")
if not isinstance(text, str):
return ""
return text.strip()
@staticmethod
def _compute_delta(previous: str, current: str) -> str:
if current == previous:
return ""
if previous and current.startswith(previous):
return current[len(previous):]
return current
@classmethod
def get_id(cls) -> str:
return "glm"
@classmethod
def get_name(cls) -> str:
return "GLM"
@classmethod
def get_description(cls) -> str:
return "Parse ChatGLM SSE streams and emit assistant text deltas while ignoring think/tool-call events"
@classmethod
def get_supported_patterns(cls) -> List[str]:
return ["/chatglm/backend-api/assistant/stream", "assistant/stream"]
__all__ = ["GLMParser"]
+529
View File
@@ -0,0 +1,529 @@
"""
grok_parser.py - Grok newline-delimited JSON stream parser.
Observed stream traits:
- content-type: application/json
- body is a growing sequence of JSON objects separated by newlines
- visible answer tokens arrive as result.token with isThinking=false
- final assistant snapshot arrives as result.modelResponse
- completed reload/fetch snapshots can arrive as responses[*].message
- generated images can be exposed via generatedImageUrls / imageAttachments
"""
from __future__ import annotations
import json
from datetime import datetime, timezone
from typing import Any, Dict, Iterable, List, Tuple
from app.core.config import logger
from .base import ResponseParser
class GrokParser(ResponseParser):
"""Parse Grok web NDJSON response streams."""
_IMAGE_URL_KEYS = {
"url",
"downloadurl",
"download_url",
"imageurl",
"image_url",
"imageuri",
"image_uri",
"uri",
}
def __init__(self) -> None:
self._last_raw_length = 0
self._pending = ""
self._rendered_content = ""
self._seen_image_refs: set[str] = set()
self._media_generation_state: Dict[str, Any] = {}
def parse_chunk(self, raw_response: str) -> Dict[str, Any]:
result: Dict[str, Any] = {
"content": "",
"images": [],
"done": False,
"error": None,
}
try:
if isinstance(raw_response, (bytes, bytearray)):
raw_response = raw_response.decode("utf-8", errors="ignore")
snapshot = self._load_json_snapshot(raw_response)
if snapshot is not None:
snapshot_payload, snapshot_text = snapshot
if snapshot_text == getattr(self, "_last_raw_response", ""):
return result
self._last_raw_length = len(snapshot_text)
self._last_raw_response = snapshot_text
delta_content, images, done = self._consume_json_snapshot(snapshot_payload)
if delta_content:
result["content"] = delta_content
if images:
result["images"] = images
result["done"] = done
return result
if not isinstance(raw_response, str):
raw_response = str(raw_response)
new_data = self._prepare_incremental_raw_response(raw_response)
if not new_data:
return result
delta_content, images, done = self._consume_new_data(new_data)
if delta_content:
result["content"] = delta_content
if images:
result["images"] = images
result["done"] = done
except Exception as e:
logger.debug(f"[GrokParser] parse exception: {e}")
result["error"] = str(e)
return result
def reset(self) -> None:
self._last_raw_length = 0
self._last_raw_response = ""
self._pending = ""
self._rendered_content = ""
self._seen_image_refs.clear()
self._media_generation_state = {}
def get_media_generation_state(
self,
raw_response: str = "",
parse_result: Dict[str, Any] | None = None,
) -> Dict[str, Any]:
return dict(self._media_generation_state or {})
def _consume_new_data(self, new_data: str) -> Tuple[str, List[Dict[str, Any]], bool]:
previous_content = self._rendered_content
complete_lines = self._extract_complete_lines(new_data)
complete_lines.extend(self._drain_parseable_pending_line())
if not complete_lines:
return "", [], False
images: List[Dict[str, Any]] = []
done = False
for raw_line in complete_lines:
line = raw_line.strip()
if not line:
continue
try:
payload = json.loads(line)
except json.JSONDecodeError:
continue
line_images, line_done = self._consume_payload(payload)
if line_images:
images.extend(line_images)
if line_done:
done = True
delta = self._compute_delta(previous_content, self._rendered_content)
return delta, images, done
def _consume_json_snapshot(self, payload: Any) -> Tuple[str, List[Dict[str, Any]], bool]:
previous_content = self._rendered_content
images, done = self._consume_payload(payload)
delta = self._compute_delta(previous_content, self._rendered_content)
return delta, images, done
@staticmethod
def _load_json_snapshot(raw_response: Any) -> Tuple[Any, str] | None:
if isinstance(raw_response, (dict, list)):
snapshot_text = json.dumps(
raw_response,
ensure_ascii=False,
sort_keys=True,
default=str,
)
return raw_response, snapshot_text
if not isinstance(raw_response, str):
return None
text = raw_response.strip()
if not text or text[0] not in "[{":
return None
try:
payload = json.loads(text)
except json.JSONDecodeError:
return None
if not isinstance(payload, (dict, list)):
return None
return payload, text
def _extract_complete_lines(self, new_data: str) -> List[str]:
normalized = (self._pending + str(new_data or "")).replace("\r\n", "\n").replace("\r", "\n")
if not normalized:
return []
parts = normalized.split("\n")
if normalized.endswith("\n"):
self._pending = ""
return parts[:-1]
self._pending = parts.pop() if parts else normalized
return parts
def _drain_parseable_pending_line(self) -> List[str]:
pending = str(self._pending or "").strip()
if not pending:
return []
try:
json.loads(pending)
except json.JSONDecodeError:
return []
self._pending = ""
return [pending]
def _consume_payload(self, payload: Any) -> Tuple[List[Dict[str, Any]], bool]:
if isinstance(payload, list):
if any(isinstance(item, dict) and "message" in item for item in payload):
return self._consume_responses_snapshot(payload)
images: List[Dict[str, Any]] = []
done = False
for item in payload:
item_images, item_done = self._consume_payload(item)
if item_images:
images.extend(item_images)
if item_done:
done = True
return self._dedupe_images(images), done
if not isinstance(payload, dict):
return [], False
responses = payload.get("responses")
if isinstance(responses, list):
return self._consume_responses_snapshot(responses)
if self._looks_like_response_snapshot(payload):
return self._consume_response_snapshot(payload)
result = payload.get("result")
if not isinstance(result, dict):
return [], False
event_payload = self._unwrap_result_payload(result)
if not isinstance(event_payload, dict):
return [], False
self._consume_token_event(event_payload)
images = self._extract_image_items(event_payload)
model_response = event_payload.get("modelResponse")
done = False
if isinstance(model_response, dict):
self._consume_model_response(model_response)
images.extend(self._extract_image_items(model_response))
done = not bool(model_response.get("partial", False))
images = self._dedupe_images(images)
if images:
self._media_generation_state = {}
else:
self._update_media_generation_state(event_payload)
return images, done
def _consume_responses_snapshot(self, responses: List[Any]) -> Tuple[List[Dict[str, Any]], bool]:
response = self._select_latest_assistant_response(responses)
if response is None:
return [], False
return self._consume_response_snapshot(response)
def _consume_response_snapshot(self, response: Dict[str, Any]) -> Tuple[List[Dict[str, Any]], bool]:
self._consume_model_response(response)
images = self._dedupe_images(self._extract_image_items(response))
done = not bool(response.get("partial", False))
if images:
self._media_generation_state = {}
else:
self._update_media_generation_state(response)
return images, done
@classmethod
def _select_latest_assistant_response(cls, responses: List[Any]) -> Dict[str, Any] | None:
for response in reversed(responses):
if not isinstance(response, dict):
continue
if cls._is_assistant_sender(response.get("sender")):
return response
return None
@classmethod
def _looks_like_response_snapshot(cls, payload: Dict[str, Any]) -> bool:
if "message" not in payload:
return False
sender = payload.get("sender")
return sender in (None, "") or cls._is_assistant_sender(sender)
@staticmethod
def _is_assistant_sender(sender: Any) -> bool:
normalized = str(sender or "").strip().lower()
return normalized in {"assistant", "bot"}
def _update_media_generation_state(self, event_payload: Dict[str, Any]) -> None:
progress = event_payload.get("progressReport")
if not isinstance(progress, dict):
return
category = str(progress.get("category") or "").strip().upper()
state = str(progress.get("state") or "").strip().upper()
message = str(progress.get("message") or "").strip()
if "IMAGE" not in category:
return
if any(token in state for token in ("COMPLETED", "COMPLETE", "SUCCEEDED", "SUCCESS", "FAILED", "ERROR", "CANCELED", "CANCELLED")):
self._media_generation_state = {}
return
if not any(token in state for token in ("PENDING", "STARTED", "RUNNING", "IN_PROGRESS")):
return
hint_text = str(self._media_generation_state.get("hint_text") or "")
hint_parts = [item for item in hint_text.split("\n\n") if item]
if message and message not in hint_parts:
hint_parts.append(message)
wait_timeout_seconds = self._media_generation_state.get("wait_timeout_seconds")
self._media_generation_state = {
"pending": True,
"media_type": "image",
"hint_text": "\n\n".join(hint_parts[:3]),
"wait_timeout_seconds": max(float(wait_timeout_seconds or 0), 120.0),
}
def _consume_token_event(self, result: Dict[str, Any]) -> None:
token = result.get("token")
if not isinstance(token, str) or not token:
return
if bool(result.get("isThinking", False)):
return
message_tag = str(result.get("messageTag") or "").strip().lower()
if message_tag and message_tag not in {"final", "answer"}:
return
self._rendered_content += token
def _consume_model_response(self, model_response: Dict[str, Any]) -> None:
sender = str(model_response.get("sender") or "").strip().lower()
if sender and sender not in {"assistant", "bot"}:
return
message = model_response.get("message")
if isinstance(message, str) and message:
self._rendered_content = message
def _extract_image_items(self, container: Dict[str, Any]) -> List[Dict[str, Any]]:
urls = self._collect_image_urls(container)
image_items: List[Dict[str, Any]] = []
for url in urls:
if url in self._seen_image_refs:
continue
self._seen_image_refs.add(url)
image_items.append(
{
"media_type": "image",
"kind": "url",
"url": url,
"data_uri": None,
"mime": None,
"byte_size": None,
"alt": "",
"width": None,
"height": None,
"detected_at": datetime.now(timezone.utc).isoformat().replace("+00:00", "Z"),
"source": "grok_stream",
}
)
return image_items
def _collect_image_urls(self, container: Dict[str, Any]) -> List[str]:
urls: List[str] = []
generated = container.get("generatedImageUrls")
if isinstance(generated, list):
for item in generated:
if isinstance(item, str) and self._looks_like_image_ref(item):
urls.append(item)
attachments = container.get("imageAttachments")
if isinstance(attachments, list):
for item in attachments:
urls.extend(self._extract_image_urls_from_value(item))
cards = container.get("cardAttachmentsJson")
if isinstance(cards, list):
for item in cards:
parsed = item
if isinstance(item, str):
try:
parsed = json.loads(item)
except json.JSONDecodeError:
parsed = item
urls.extend(self._extract_image_urls_from_value(parsed))
deduped: List[str] = []
seen = set()
for url in urls:
if url in seen:
continue
seen.add(url)
deduped.append(url)
return deduped
@staticmethod
def _unwrap_result_payload(result: Dict[str, Any]) -> Dict[str, Any]:
response = result.get("response")
if isinstance(response, dict):
return response
return result
def _extract_image_urls_from_value(self, value: Any) -> List[str]:
urls: List[str] = []
if isinstance(value, str):
if self._looks_like_image_ref(value):
urls.append(value)
return urls
if isinstance(value, list):
for item in value:
urls.extend(self._extract_image_urls_from_value(item))
return urls
if not isinstance(value, dict):
return urls
content_type = str(
value.get("contentType") or value.get("mimeType") or value.get("mime") or ""
).strip().lower()
for key, item in value.items():
normalized_key = str(key or "").strip().lower()
if normalized_key in self._IMAGE_URL_KEYS:
if isinstance(item, str) and (self._looks_like_image_ref(item) or content_type.startswith("image/")):
urls.append(item)
continue
if normalized_key in {"images", "attachments", "items", "content", "data"}:
urls.extend(self._extract_image_urls_from_value(item))
return urls
@staticmethod
def _looks_like_image_ref(value: str) -> bool:
text = str(value or "").strip()
if not text:
return False
lowered = text.lower()
if lowered.startswith("data:image/"):
return True
if not lowered.startswith(("http://", "https://")):
return False
return any(marker in lowered for marker in ("/generated/", ".png", ".jpg", ".jpeg", ".webp", ".gif"))
@staticmethod
def _compute_delta(previous: str, current: str) -> str:
if current == previous:
return ""
if not previous:
return current
if current.startswith(previous):
return current[len(previous) :]
if previous.startswith(current):
return ""
prefix_len = 0
limit = min(len(previous), len(current))
while prefix_len < limit and previous[prefix_len] == current[prefix_len]:
prefix_len += 1
if prefix_len >= len(previous) // 2:
return current[prefix_len:]
return current
@staticmethod
def _dedupe_images(images: Iterable[Dict[str, Any]]) -> List[Dict[str, Any]]:
deduped: List[Dict[str, Any]] = []
seen = set()
for item in images:
if not isinstance(item, dict):
continue
key = (
str(item.get("media_type") or ""),
str(item.get("kind") or ""),
str(item.get("url") or item.get("data_uri") or ""),
)
if key in seen:
continue
seen.add(key)
deduped.append(item)
return deduped
def _iter_payloads(self, raw_response: str):
self_pending = self._pending
try:
self._pending = ""
lines = self._extract_complete_lines(raw_response)
lines.extend(self._drain_parseable_pending_line())
for raw_line in lines:
line = raw_line.strip()
if not line:
continue
try:
payload = json.loads(line)
except json.JSONDecodeError:
continue
if isinstance(payload, dict):
yield payload
finally:
self._pending = self_pending
@classmethod
def get_id(cls) -> str:
return "grok"
@classmethod
def get_name(cls) -> str:
return "Grok"
@classmethod
def get_description(cls) -> str:
return "Parse Grok NDJSON chat streams, ignore thinking tokens, and extract generated images"
@classmethod
def get_supported_patterns(cls) -> List[str]:
return ["/rest/app-chat/conversations/", "/responses", "/load-responses"]
__all__ = ["GrokParser"]
+250
View File
@@ -0,0 +1,250 @@
"""
kimi_parser.py - Kimi Connect streaming response parser.
Observed stream traits:
- content-type: application/connect+json
- each message is framed as 5 bytes of binary header + one JSON object payload
- assistant text starts with op=set, mask=block.text
- later deltas use op=append, mask=block.text.content
- {"done": {}} or MESSAGE_STATUS_COMPLETED ends the stream
Notes:
- In DevTools-exported `fullText` / `chunks.data`, the header bytes can be lossy once the
raw binary stream is coerced into a Python string, so relying on the advertised payload
length is brittle.
- We therefore treat the first 5 bytes as opaque framing metadata and recover the payload
by scanning for a complete JSON object boundary instead.
"""
from __future__ import annotations
import json
from typing import Any, Dict, List, Set, Tuple
from app.core.config import logger
from .base import ResponseParser
class KimiParser(ResponseParser):
"""Parse Kimi connect+json streaming responses."""
def __init__(self) -> None:
self._last_raw = b""
self._pending = b""
self._seen_offsets: Set[int] = set()
def parse_chunk(self, raw_response: str) -> Dict[str, Any]:
result: Dict[str, Any] = {
"content": "",
"images": [],
"done": False,
"error": None,
}
try:
raw_bytes = self._normalize_raw_bytes(raw_response)
if not raw_bytes:
return result
if self._last_raw and raw_bytes == self._last_raw:
return result
if self._last_raw and raw_bytes.startswith(self._last_raw):
new_data = raw_bytes[len(self._last_raw):]
else:
new_data = raw_bytes
self._last_raw = raw_bytes
delta_content, done = self._consume_new_data(new_data)
if delta_content:
result["content"] = delta_content
result["done"] = done
except Exception as e:
logger.debug(f"[KimiParser] parse exception: {e}")
result["error"] = str(e)
return result
def reset(self) -> None:
self._last_raw = b""
self._pending = b""
self._seen_offsets.clear()
@staticmethod
def _normalize_raw_bytes(raw_response: Any) -> bytes:
if isinstance(raw_response, bytes):
return raw_response
if isinstance(raw_response, bytearray):
return bytes(raw_response)
if isinstance(raw_response, str):
if "\\u00" in raw_response and "\x00" not in raw_response:
return KimiParser._decode_u00_escaped_bytes(raw_response)
return raw_response.encode("utf-8", errors="ignore")
return str(raw_response).encode("utf-8", errors="ignore")
@staticmethod
def _decode_u00_escaped_bytes(raw_response: str) -> bytes:
output = bytearray()
i = 0
while i < len(raw_response):
if (
raw_response[i] == "\\"
and i + 5 < len(raw_response)
and raw_response[i + 1] == "u"
and raw_response[i + 2:i + 4] == "00"
):
hex_part = raw_response[i + 4:i + 6]
if all(ch in "0123456789abcdefABCDEF" for ch in hex_part):
output.append(int(hex_part, 16))
i += 6
continue
output.extend(raw_response[i].encode("utf-8", errors="ignore"))
i += 1
return bytes(output)
def _consume_new_data(self, new_data: bytes) -> Tuple[str, bool]:
self._pending += new_data
content_parts: List[str] = []
done = False
while True:
frame = self._next_frame()
if frame is None:
break
payload_text, frame_done = frame
if payload_text:
content_parts.append(payload_text)
if frame_done:
done = True
return "".join(content_parts), done
def _next_frame(self) -> Tuple[str, bool] | None:
if len(self._pending) < 6:
return None
payload_end = self._find_json_payload_end(self._pending, start=5)
if payload_end is None:
return None
payload = self._pending[5:payload_end]
self._pending = self._pending[payload_end:]
return self._parse_payload(payload)
@staticmethod
def _find_json_payload_end(buffer: bytes, start: int) -> int | None:
index = start
limit = len(buffer)
while index < limit and chr(buffer[index]).isspace():
index += 1
if index >= limit or buffer[index] != ord("{"):
return None
depth = 0
in_string = False
escape = False
for pos in range(index, limit):
byte = buffer[pos]
if in_string:
if escape:
escape = False
elif byte == ord("\\"):
escape = True
elif byte == ord('"'):
in_string = False
continue
if byte == ord('"'):
in_string = True
elif byte == ord("{"):
depth += 1
elif byte == ord("}"):
depth -= 1
if depth == 0:
return pos + 1
return None
def _parse_payload(self, payload: bytes) -> Tuple[str, bool]:
payload_text = payload.decode("utf-8", errors="ignore").strip()
if not payload_text:
return "", False
try:
data = json.loads(payload_text)
except json.JSONDecodeError:
return "", False
if not isinstance(data, dict):
return "", False
if "done" in data:
return "", True
if "heartbeat" in data:
return "", False
event_offset = data.get("eventOffset")
if isinstance(event_offset, int):
if event_offset in self._seen_offsets:
return "", False
self._seen_offsets.add(event_offset)
op = data.get("op")
mask = data.get("mask", "")
if op == "set" and mask == "block.text":
return self._extract_block_text(data.get("block")), False
if op == "append" and mask == "block.text.content":
return self._extract_block_text(data.get("block")), False
if op == "set" and mask == "message.status":
message = data.get("message", {})
if isinstance(message, dict) and message.get("status") == "MESSAGE_STATUS_COMPLETED":
return "", True
return "", False
@staticmethod
def _extract_block_text(block: Any) -> str:
if not isinstance(block, dict):
return ""
text_data = block.get("text")
if not isinstance(text_data, dict):
return ""
content = text_data.get("content", "")
return content if isinstance(content, str) else ""
@classmethod
def get_id(cls) -> str:
return "kimi"
@classmethod
def get_name(cls) -> str:
return "Kimi"
@classmethod
def get_description(cls) -> str:
return "Parse Kimi connect+json streams and keep assistant text deltas"
@classmethod
def get_supported_patterns(cls) -> List[str]:
return ["/apiv2/kimi.gateway.chat.v1.ChatService/Chat", "/chat", "chat"]
__all__ = ["KimiParser"]
@@ -0,0 +1,424 @@
"""
lmarena_battle_side_parser.py - Arena.ai battle mode parsers.
The side parsers stream one selected side and finish when both battle sides are
terminal. The winner parser buffers both sides and emits only the first
completed side while still waiting for the battle response to finish normally.
"""
from __future__ import annotations
import json
from typing import Any, Dict, List
from app.core.config import logger
from .base import ResponseParser
from .lmarena_parser import LmarenaParser, _CP1252_TO_LATIN1, _extract_arena_image_items
class _LmarenaBattleSideParser(ResponseParser):
"""Shared implementation for left/right Arena battle text streams."""
SIDE_PREFIX = "a"
SIDE_LABEL = "left"
def __init__(self) -> None:
self._accumulated = ""
self._accumulated_reasoning = ""
self._seen_image_refs: set[str] = set()
self._completed_sides: set[str] = set()
self._completed = False
self._completion_side = ""
def parse_chunk(self, raw_response: str) -> Dict[str, Any]:
result: Dict[str, Any] = {
"content": "",
"reasoning_content": "",
"images": [],
"done": False,
"error": None,
"selected_side": self.SIDE_LABEL,
"completion_side": self._completion_side,
}
if self._completed:
result["done"] = True
return result
if isinstance(raw_response, (bytes, bytearray)):
raw_response = raw_response.decode("utf-8", errors="ignore")
if not raw_response or not isinstance(raw_response, str):
return result
raw_response = self._fix_mojibake(raw_response)
try:
content_parts: List[str] = []
reasoning_parts: List[str] = []
images: List[Dict[str, Any]] = []
for line in raw_response.split("\n"):
line = line.strip()
if not line:
continue
colon_idx = line.find(":")
if colon_idx < 1:
continue
prefix = line[:colon_idx]
payload = line[colon_idx + 1:]
if prefix == f"{self.SIDE_PREFIX}0":
text = self._parse_text_chunk(payload)
if text is not None:
content_parts.append(text)
elif prefix == f"{self.SIDE_PREFIX}g":
text = self._parse_text_chunk(payload)
if text is not None:
reasoning_parts.append(text)
elif prefix == f"{self.SIDE_PREFIX}2":
images.extend(
_extract_arena_image_items(
payload,
self._seen_image_refs,
source=f"lmarena_battle_side_{self.SIDE_LABEL}_stream",
)
)
elif prefix in {"ad", "bd"}:
if self._is_finish_signal(payload):
self._mark_terminal(prefix)
elif prefix in {"ae", "be", "a3", "b3"}:
error_msg = self._parse_error(payload)
if error_msg:
completion_side = self._mark_terminal(prefix)
logger.debug(
"[LmarenaBattleSideParser] stream error frame treated as completion "
f"(selected={self.SIDE_LABEL}, side={completion_side}, error={error_msg[:160]})"
)
new_content = "".join(content_parts)
new_reasoning = "".join(reasoning_parts)
if new_content:
delta, next_accumulated = LmarenaParser._content_delta(
self._accumulated,
new_content,
append_disjoint=True,
)
if delta:
result["content"] = delta
self._accumulated = next_accumulated
if new_reasoning:
delta_r, next_acc_r = LmarenaParser._content_delta(
self._accumulated_reasoning,
new_reasoning,
append_disjoint=True,
)
if delta_r:
result["reasoning_content"] = delta_r
self._accumulated_reasoning = next_acc_r
if images:
result["images"] = images
result["completion_side"] = self._completion_side
if len(self._completed_sides) >= 2:
self._completed = True
result["done"] = True
result["completion_side"] = self._completion_side
except Exception as e:
logger.debug(f"[LmarenaBattleSideParser] parse exception: {e}")
result["error"] = str(e)
return result
def reset(self):
self._accumulated = ""
self._accumulated_reasoning = ""
self._seen_image_refs.clear()
self._completed_sides.clear()
self._completed = False
self._completion_side = ""
def should_fallback_to_dom_when_no_visible_content(self) -> bool:
"""Arena 的心跳流可能因网络波动/长推理首字延迟提前静默,空结果必须交给 DOM 接管。"""
return True
def _mark_terminal(self, prefix: str) -> str:
side = "left" if prefix.startswith("a") else "right"
self._completed_sides.add(side)
if not self._completion_side:
self._completion_side = side
return side
@staticmethod
def _fix_mojibake(text: str) -> str:
try:
mapped = text.translate(_CP1252_TO_LATIN1)
return mapped.encode("latin-1").decode("utf-8")
except (UnicodeEncodeError, UnicodeDecodeError):
return text
@staticmethod
def _parse_text_chunk(payload: str) -> str | None:
try:
value = json.loads(payload)
if isinstance(value, str):
return value
except (json.JSONDecodeError, ValueError):
pass
return None
@staticmethod
def _is_finish_signal(payload: str) -> bool:
try:
data = json.loads(payload)
return isinstance(data, dict) and bool(data.get("finishReason"))
except (json.JSONDecodeError, ValueError):
return False
@staticmethod
def _parse_error(payload: str) -> str | None:
try:
data = json.loads(payload)
if isinstance(data, dict):
return data.get("message", str(data))
return str(data)
except (json.JSONDecodeError, ValueError):
return payload.strip() if payload.strip() else None
@classmethod
def get_supported_patterns(cls) -> List[str]:
return ["nextjs-api/stream/create-evaluation"]
class LmarenaBattleWinnerParser(ResponseParser):
"""Emit only the first completed Arena battle side, without streaming."""
def __init__(self) -> None:
self._buffers = {"left": "", "right": ""}
self._reasoning_buffers = {"left": "", "right": ""}
self._seen_image_refs = {"left": set(), "right": set()}
self._images = {"left": [], "right": []}
self._completed_sides: set[str] = set()
self._completed = False
self._winner_side = ""
self._emitted_winner = False
def parse_chunk(self, raw_response: str) -> Dict[str, Any]:
result: Dict[str, Any] = {
"content": "",
"reasoning_content": "",
"images": [],
"done": False,
"error": None,
"selected_side": "winner",
"winner_side": self._winner_side,
}
if self._completed:
result["done"] = True
return result
if isinstance(raw_response, (bytes, bytearray)):
raw_response = raw_response.decode("utf-8", errors="ignore")
if not raw_response or not isinstance(raw_response, str):
return result
raw_response = _LmarenaBattleSideParser._fix_mojibake(raw_response)
try:
content_parts = {"left": [], "right": []}
reasoning_parts = {"left": [], "right": []}
for line in raw_response.split("\n"):
line = line.strip()
if not line:
continue
colon_idx = line.find(":")
if colon_idx < 1:
continue
prefix = line[:colon_idx]
payload = line[colon_idx + 1:]
if prefix == "a0":
text = _LmarenaBattleSideParser._parse_text_chunk(payload)
if text is not None:
content_parts["left"].append(text)
elif prefix == "b0":
text = _LmarenaBattleSideParser._parse_text_chunk(payload)
if text is not None:
content_parts["right"].append(text)
elif prefix == "ag":
text = _LmarenaBattleSideParser._parse_text_chunk(payload)
if text is not None:
reasoning_parts["left"].append(text)
elif prefix == "bg":
text = _LmarenaBattleSideParser._parse_text_chunk(payload)
if text is not None:
reasoning_parts["right"].append(text)
elif prefix == "a2":
self._images["left"].extend(
_extract_arena_image_items(
payload,
self._seen_image_refs["left"],
source="lmarena_battle_winner_left_stream",
)
)
elif prefix == "b2":
self._images["right"].extend(
_extract_arena_image_items(
payload,
self._seen_image_refs["right"],
source="lmarena_battle_winner_right_stream",
)
)
elif prefix in {"ad", "bd"}:
if _LmarenaBattleSideParser._is_finish_signal(payload):
self._mark_terminal(prefix)
elif prefix in {"ae", "be", "a3", "b3"}:
error_msg = _LmarenaBattleSideParser._parse_error(payload)
if error_msg:
winner_side = self._mark_terminal(prefix)
logger.debug(
"[LmarenaBattleWinnerParser] stream error frame treated as completion "
f"(winner={winner_side}, error={error_msg[:160]})"
)
for side, parts in content_parts.items():
self._merge_text(side, "".join(parts))
for side, parts in reasoning_parts.items():
self._merge_reasoning(side, "".join(parts))
if self._winner_side and not self._emitted_winner:
self._emitted_winner = True
result["content"] = self._buffers[self._winner_side]
result["reasoning_content"] = self._reasoning_buffers[self._winner_side]
result["images"] = self._images[self._winner_side]
result["winner_side"] = self._winner_side
if len(self._completed_sides) >= 2:
self._completed = True
result["done"] = True
result["winner_side"] = self._winner_side
except Exception as e:
logger.debug(f"[LmarenaBattleWinnerParser] parse exception: {e}")
result["error"] = str(e)
return result
def _merge_text(self, side: str, text: str) -> None:
if not text:
return
current = self._buffers[side]
_delta, next_accumulated = LmarenaParser._content_delta(
current,
text,
append_disjoint=True,
)
self._buffers[side] = next_accumulated
def _merge_reasoning(self, side: str, text: str) -> None:
if not text:
return
current = self._reasoning_buffers[side]
_delta, next_accumulated = LmarenaParser._content_delta(
current,
text,
append_disjoint=True,
)
self._reasoning_buffers[side] = next_accumulated
def reset(self):
self._buffers = {"left": "", "right": ""}
self._reasoning_buffers = {"left": "", "right": ""}
self._seen_image_refs = {"left": set(), "right": set()}
self._images = {"left": [], "right": []}
self._completed_sides.clear()
self._completed = False
self._winner_side = ""
self._emitted_winner = False
def should_fallback_to_dom_when_no_visible_content(self) -> bool:
"""Arena 的心跳流可能因网络波动/长推理首字延迟提前静默,空结果必须交给 DOM 接管。"""
return True
def _mark_terminal(self, prefix: str) -> str:
side = "left" if prefix.startswith("a") else "right"
self._completed_sides.add(side)
if not self._winner_side:
self._winner_side = side
return side
@classmethod
def get_supported_patterns(cls) -> List[str]:
return ["nextjs-api/stream/create-evaluation"]
@classmethod
def get_id(cls) -> str:
return "lmarena_battle_winner"
@classmethod
def get_name(cls) -> str:
return "Arena.ai Battle Winner"
@classmethod
def get_description(cls) -> str:
return "Buffer both Arena.ai battle sides and emit only the first completed response"
class LmarenaBattleSideLeftParser(_LmarenaBattleSideParser):
"""Stream left/modelA text and finish when both battle sides finish."""
SIDE_PREFIX = "a"
SIDE_LABEL = "left"
@classmethod
def get_id(cls) -> str:
return "lmarena_battle_side_left"
@classmethod
def get_name(cls) -> str:
return "Arena.ai Battle Side Left"
@classmethod
def get_description(cls) -> str:
return "Stream Arena.ai battle left/modelA text; finish when both sides complete"
class LmarenaBattleSideRightParser(_LmarenaBattleSideParser):
"""Stream right/modelB text and finish when both battle sides finish."""
SIDE_PREFIX = "b"
SIDE_LABEL = "right"
@classmethod
def get_id(cls) -> str:
return "lmarena_battle_side_right"
@classmethod
def get_name(cls) -> str:
return "Arena.ai Battle Side Right"
@classmethod
def get_description(cls) -> str:
return "Stream Arena.ai battle right/modelB text; finish when both sides complete"
__all__ = [
"LmarenaBattleWinnerParser",
"LmarenaBattleSideLeftParser",
"LmarenaBattleSideRightParser",
]
@@ -0,0 +1,51 @@
"""
lmarena_image_side_left_parser.py - Arena.ai image battle left-side parser.
Purpose:
- Reuse Arena side-by-side left channel parsing semantics.
- Prefer quick DOM fallback when the network stream carries little or no
user-visible text, which is common for image-generation flows where the
rendered images are the real result.
"""
from __future__ import annotations
from typing import List
from .lmarena_side_left_parser import LmarenaSideLeftParser
class LmarenaImageSideLeftParser(LmarenaSideLeftParser):
"""
Arena.ai image battle parser for the left/modelA channel.
The underlying stream protocol may still look like the standard Arena
side-by-side SSE frames, but image flows often produce no visible text.
In that case we prefer to fall back to DOM/image extraction quickly
instead of treating an empty stream as success.
"""
def should_fallback_to_dom_when_no_visible_content(self) -> bool:
return True
@classmethod
def get_id(cls) -> str:
return "lmarena_image_side_left"
@classmethod
def get_name(cls) -> str:
return "Arena.ai Image Side Left"
@classmethod
def get_description(cls) -> str:
return "Parse Arena.ai image battle stream, keep left(modelA) and fall back to DOM when text is absent"
@classmethod
def get_supported_patterns(cls) -> List[str]:
return [
"nextjs-api/stream/post-to-evaluation",
"nextjs-api/stream/create-evaluation",
]
__all__ = ["LmarenaImageSideLeftParser"]
@@ -0,0 +1,167 @@
"""
lmarena_image_side_right_parser.py - Arena.ai image battle right-side parser.
Purpose:
- Keep the Arena image battle right/modelB stream channel.
- Fall back to DOM/image extraction quickly when the stream has no visible text.
"""
from __future__ import annotations
import json
from typing import Any, Dict, List
from app.core.config import logger
from .base import ResponseParser
from .lmarena_parser import LmarenaParser, _CP1252_TO_LATIN1, _extract_arena_image_items
class LmarenaImageSideRightParser(ResponseParser):
"""Arena.ai image battle parser for the right/modelB channel."""
def __init__(self) -> None:
self._accumulated = ""
self._seen_image_refs: set[str] = set()
def parse_chunk(self, raw_response: str) -> Dict[str, Any]:
result: Dict[str, Any] = {
"content": "",
"images": [],
"done": False,
"error": None,
}
if isinstance(raw_response, (bytes, bytearray)):
raw_response = raw_response.decode("utf-8", errors="ignore")
if not raw_response or not isinstance(raw_response, str):
return result
raw_response = self._fix_mojibake(raw_response)
try:
content_parts: List[str] = []
images: List[Dict[str, Any]] = []
done = False
for line in raw_response.split("\n"):
line = line.strip()
if not line:
continue
colon_idx = line.find(":")
if colon_idx < 1:
continue
prefix = line[:colon_idx]
payload = line[colon_idx + 1:]
if prefix == "b0":
text = self._parse_text_chunk(payload)
if text is not None:
content_parts.append(text)
elif prefix == "b2":
images.extend(
_extract_arena_image_items(
payload,
self._seen_image_refs,
source="lmarena_image_side_right_stream",
)
)
elif prefix == "bd":
if self._is_finish_signal(payload):
done = True
elif prefix in {"be", "b3"}:
error_msg = self._parse_error(payload)
if error_msg:
result["error"] = error_msg
done = True
new_content = "".join(content_parts)
if new_content:
delta, next_accumulated = LmarenaParser._content_delta(
self._accumulated,
new_content,
append_disjoint=True,
)
if delta:
result["content"] = delta
self._accumulated = next_accumulated
if images:
result["images"] = images
result["done"] = done
except Exception as e:
logger.debug(f"[LmarenaImageSideRightParser] parse exception: {e}")
result["error"] = str(e)
return result
def reset(self):
self._accumulated = ""
self._seen_image_refs.clear()
def should_abort_on_error(self) -> bool:
return True
def should_fallback_to_dom_when_no_visible_content(self) -> bool:
return True
@staticmethod
def _fix_mojibake(text: str) -> str:
try:
mapped = text.translate(_CP1252_TO_LATIN1)
return mapped.encode("latin-1").decode("utf-8")
except (UnicodeEncodeError, UnicodeDecodeError):
return text
@staticmethod
def _parse_text_chunk(payload: str) -> str | None:
try:
value = json.loads(payload)
if isinstance(value, str):
return value
except (json.JSONDecodeError, ValueError):
pass
return None
@staticmethod
def _is_finish_signal(payload: str) -> bool:
try:
data = json.loads(payload)
return isinstance(data, dict) and bool(data.get("finishReason"))
except (json.JSONDecodeError, ValueError):
return False
@staticmethod
def _parse_error(payload: str) -> str | None:
try:
data = json.loads(payload)
if isinstance(data, dict):
return data.get("message", str(data))
return str(data)
except (json.JSONDecodeError, ValueError):
return payload.strip() if payload.strip() else None
@classmethod
def get_id(cls) -> str:
return "lmarena_image_side_right"
@classmethod
def get_name(cls) -> str:
return "Arena.ai Image Side Right"
@classmethod
def get_description(cls) -> str:
return "Parse Arena.ai image battle stream, keep right(modelB) and fall back to DOM when text is absent"
@classmethod
def get_supported_patterns(cls) -> List[str]:
return [
"nextjs-api/stream/post-to-evaluation",
"nextjs-api/stream/create-evaluation",
]
__all__ = ["LmarenaImageSideRightParser"]
+564
View File
@@ -0,0 +1,564 @@
"""
lmarena_parser.py - Arena.ai 响应解析器
响应格式:Vercel AI SDK Data Stream Protocol
- 每行格式: {prefix}:{json_data}
- a0: 文本增量(JSON 编码的字符串)
- a2: 心跳/元数据(JSON 数组)
- ad: 结束元数据(含 finishReason)
- ae: 错误信息
编码问题:
DrissionPage 通过 CDP 获取响应体时,UTF-8 字节被混合解码为
cp1252 映射字符(如 0x92→' U+2019)和 latin-1 直通字符
(如 0x90→U+0090),导致双重编码(mojibake)。
需要两步修复:translate 统一到 latin-1 范围 → encode('latin-1') 还原字节
调用方式:
NetworkMonitor 每次调用 parse_chunk 传入一个完整的 HTTP 响应体
(DrissionPage listen.wait 是非流式的,一次性拿到整个 SSE body)
"""
from __future__ import annotations
import json
from datetime import datetime, timezone
from typing import Any, Dict, List
from app.core.config import logger
from .base import ResponseParser
# ─────────────────────────────────────────────────────
# cp1252 → latin-1 逆映射表(模块级常量,只构建一次)
# ─────────────────────────────────────────────────────
# cp1252 在 0x80-0x9F 区间将部分字节映射到 U+0100 以上的 Unicode 字符,
# 而 latin-1 将相同字节直通映射到 U+0080-U+009F。
# 此表将 cp1252 的特殊映射还原为 latin-1 范围的等价字符,
# 使后续 encode('latin-1') 能正确还原原始字节。
#
# 5 个未定义位置 (0x81, 0x8D, 0x8F, 0x90, 0x9D) 已经是
# latin-1 直通映射 (U+0081 等),无需处理。
_CP1252_TO_LATIN1 = str.maketrans({
'\u20ac': '\x80', # €
'\u201a': '\x82', # ‚
'\u0192': '\x83', # ƒ
'\u201e': '\x84', # „
'\u2026': '\x85', # …
'\u2020': '\x86', # †
'\u2021': '\x87', # ‡
'\u02c6': '\x88', # ˆ
'\u2030': '\x89', # ‰
'\u0160': '\x8a', # Š
'\u2039': '\x8b', # ‹
'\u0152': '\x8c', # Œ
'\u017d': '\x8e', # Ž
'\u2018': '\x91', # '
'\u2019': '\x92', # '
'\u201c': '\x93', # "
'\u201d': '\x94', # "
'\u2022': '\x95', # •
'\u2013': '\x96', # –
'\u2014': '\x97', # —
'\u02dc': '\x98', # ˜
'\u2122': '\x99', # ™
'\u0161': '\x9a', # š
'\u203a': '\x9b', # ›
'\u0153': '\x9c', # œ
'\u017e': '\x9e', # ž
'\u0178': '\x9f', # Ÿ
})
def _looks_like_image_ref(value: str) -> bool:
text = str(value or "").strip()
if not text:
return False
lowered = text.lower()
if lowered.startswith(("http://", "https://", "data:image/", "blob:")):
return True
return any(token in lowered for token in (".png", ".jpg", ".jpeg", ".webp", ".gif", ".bmp", ".svg", ".avif"))
def _extract_arena_image_items(
payload: str,
seen_refs: set[str],
*,
source: str,
) -> List[Dict[str, Any]]:
items: List[Dict[str, Any]] = []
try:
data = json.loads(payload)
except (json.JSONDecodeError, ValueError):
return items
entries = data if isinstance(data, list) else [data]
for entry in entries:
if not isinstance(entry, dict):
continue
entry_type = str(entry.get("type") or "").strip().lower()
if entry_type and entry_type != "image":
continue
image_ref = ""
for key in ("image", "url", "src"):
candidate = str(entry.get(key) or "").strip()
if _looks_like_image_ref(candidate):
image_ref = candidate
break
if not image_ref:
continue
dedupe_key = f"image:{image_ref}"
if dedupe_key in seen_refs:
continue
seen_refs.add(dedupe_key)
mime = str(entry.get("mimeType") or entry.get("mime") or "").strip() or None
kind = "data_uri" if image_ref.startswith("data:image/") else "url"
items.append(
{
"media_type": "image",
"kind": kind,
"url": image_ref if kind == "url" else None,
"data_uri": image_ref if kind == "data_uri" else None,
"mime": mime,
"byte_size": None,
"alt": str(entry.get("alt") or "").strip(),
"width": entry.get("width"),
"height": entry.get("height"),
"detected_at": datetime.now(timezone.utc).isoformat().replace("+00:00", "Z"),
"source": source,
}
)
return items
class _LmarenaScanState:
"""LmarenaParser 增量扫描的累计状态(只覆盖已提交的完整行 + 本次半行)。"""
__slots__ = (
"content", "reasoning", "done", "error", "line_count", "prefix_counts",
"unknown_prefix_count", "non_protocol_line_count", "parse_errors",
"committed_len", "committed_raw",
)
def __init__(self) -> None:
self.content = ""
self.reasoning = ""
self.done = False
self.error = None
self.line_count = 0
self.prefix_counts: Dict[str, int] = {}
self.unknown_prefix_count = 0
self.non_protocol_line_count = 0
self.parse_errors: list[Dict[str, Any]] = []
self.committed_len = 0
self.committed_raw = ""
def copy(self) -> "_LmarenaScanState":
other = _LmarenaScanState()
for name in self.__slots__:
setattr(other, name, getattr(self, name))
other.prefix_counts = dict(self.prefix_counts)
other.parse_errors = list(self.parse_errors)
return other
def add_parse_error(self, item: Dict[str, Any]) -> None:
self.parse_errors.append(item)
if len(self.parse_errors) > 16:
del self.parse_errors[:-8]
class LmarenaParser(ResponseParser):
"""
Arena.ai 响应解析器
URL 特征: /nextjs-api/stream/create-evaluation
响应格式: Vercel AI SDK Data Stream Protocol (行分隔)
"""
_PROTOCOL_PREFIXES = {"a0", "a2", "ad", "ae", "a3", "ag", "bg"}
def __init__(self) -> None:
self._accumulated = ""
self._accumulated_reasoning = ""
self._seen_image_refs: set[str] = set()
self._last_debug_summary: Dict[str, Any] = {}
self._last_debug_raw_signature = ""
self._inc_state: "_LmarenaScanState | None" = None
@staticmethod
def _content_delta(
accumulated: str,
candidate: str,
*,
append_disjoint: bool = False,
) -> tuple[str, str]:
"""Return unseen text and the new accumulated value for repeated SSE bodies."""
if not candidate:
return "", accumulated
if not accumulated:
return candidate, candidate
if candidate == accumulated or accumulated.startswith(candidate):
return "", accumulated
if candidate.startswith(accumulated):
return candidate[len(accumulated):], candidate
max_overlap = min(len(accumulated), len(candidate))
for overlap in range(max_overlap, 0, -1):
if accumulated.endswith(candidate[:overlap]):
delta = candidate[overlap:]
return delta, accumulated + delta
if append_disjoint:
return candidate, accumulated + candidate
# A parser instance belongs to one response stream. A body that neither
# extends nor overlaps the emitted text is a stale/non-monotonic replay;
# emitting it wholesale would duplicate the response already sent.
return "", accumulated
# ============ 增量扫描(P0-5) ============
# 置为 False 时每次整段解析(与旧实现逐字节一致),用于回归对照
_INCREMENTAL = True
def _scan_response(self, raw_response: str) -> tuple["_LmarenaScanState", list]:
"""返回 (扫描状态, 本次新发现的图片)。"""
images: list[Dict[str, Any]] = []
if not self._INCREMENTAL:
# 旧行为:整段修复 mojibake 后逐行解析
scan = _LmarenaScanState()
self._scan_lines(self._fix_mojibake(raw_response), scan, images)
return scan, images
cached = getattr(self, "_inc_state", None)
if cached is not None and cached.committed_len <= len(raw_response) and raw_response.startswith(cached.committed_raw):
scan = cached.copy()
offset = cached.committed_len
else:
scan = _LmarenaScanState()
offset = 0
last_newline = raw_response.rfind("\n", offset)
if last_newline >= offset:
committed_end = last_newline + 1
# 完整行一定以 ASCII 换行结尾,不会截断多字节序列,可以按段修复 mojibake
self._scan_lines(self._fix_mojibake(raw_response[offset:committed_end]), scan, images)
scan.committed_len = committed_end
scan.committed_raw = raw_response[:committed_end]
self._inc_state = scan.copy()
offset = committed_end
elif cached is None or scan.committed_len == 0:
self._inc_state = None
tail = raw_response[offset:]
if tail:
self._scan_lines(self._fix_mojibake(tail), scan, images)
return scan, images
def _scan_lines(self, text: str, scan: "_LmarenaScanState", images: list) -> None:
content_parts: list[str] = []
reasoning_parts: list[str] = []
for line in text.split("\n"):
line = line.strip()
if not line:
continue
scan.line_count += 1
colon_idx = line.find(":")
if colon_idx < 1:
scan.non_protocol_line_count += 1
continue
prefix = line[:colon_idx]
payload = line[colon_idx + 1:]
if prefix in self._PROTOCOL_PREFIXES:
scan.prefix_counts[prefix] = scan.prefix_counts.get(prefix, 0) + 1
else:
scan.unknown_prefix_count += 1
if prefix == "a0":
text_chunk = self._parse_text_chunk(payload)
if text_chunk is not None:
content_parts.append(text_chunk)
else:
scan.add_parse_error({"prefix": prefix, "payload_preview": payload[:160]})
elif prefix in {"ag", "bg"}:
text_chunk = self._parse_text_chunk(payload)
if text_chunk is not None:
reasoning_parts.append(text_chunk)
else:
scan.add_parse_error({"prefix": prefix, "payload_preview": payload[:160]})
elif prefix == "a2":
images.extend(
_extract_arena_image_items(
payload,
self._seen_image_refs,
source="lmarena_stream",
)
)
elif prefix == "ad":
if self._is_finish_signal(payload):
scan.done = True
elif prefix in {"ae", "a3"}:
error_msg = self._parse_error(payload)
if error_msg:
scan.error = error_msg
scan.done = True
if content_parts:
scan.content += "".join(content_parts)
if reasoning_parts:
scan.reasoning += "".join(reasoning_parts)
# ============ 对外接口 ============
def parse_chunk(self, raw_response: str) -> Dict[str, Any]:
"""
解析完整的 HTTP 响应体(一次性包含所有 SSE 行)
"""
result: Dict[str, Any] = {
"content": "",
"reasoning_content": "",
"images": [],
"done": False,
"error": None,
}
if isinstance(raw_response, (bytes, bytearray)):
raw_response = raw_response.decode("utf-8", errors="ignore")
if not raw_response or not isinstance(raw_response, str):
return result
debug_raw_signature = self._debug_body_signature(raw_response)
try:
# P0-5:增量解析。网络监听每次传入的是“不断增长的完整响应体”,旧实现
# 每次都整段 split + json.loads,一个 160KB 的回答流累计要花数秒 CPU。
# 这里把“已以换行结尾的完整行”的解析结果缓存下来,只解析新增部分;
# 末尾未结束的半行每次重新解析、不入缓存。响应体不是上次的前缀扩展
# (换了一条流)时回退为整段解析。
scan, images = self._scan_response(raw_response)
done = scan.done
if scan.error:
result["error"] = scan.error
new_content = scan.content
new_reasoning = scan.reasoning
if new_content:
delta, next_accumulated = self._content_delta(self._accumulated, new_content)
if delta:
result["content"] = delta
self._accumulated = next_accumulated
if new_reasoning:
delta_r, next_acc_r = self._content_delta(self._accumulated_reasoning, new_reasoning)
if delta_r:
result["reasoning_content"] = delta_r
self._accumulated_reasoning = next_acc_r
if images:
result["images"] = images
result["done"] = done
self._last_debug_summary = {
"raw_body_len": len(raw_response),
"line_count": scan.line_count,
"prefix_counts": dict(scan.prefix_counts),
"unknown_prefix_count": scan.unknown_prefix_count,
"non_protocol_line_count": scan.non_protocol_line_count,
"content_candidate_len": len(new_content),
"emitted_content_len": len(str(result.get("content") or "")),
"accumulated_len": len(self._accumulated),
"reasoning_candidate_len": len(new_reasoning),
"emitted_reasoning_len": len(str(result.get("reasoning_content") or "")),
"accumulated_reasoning_len": len(self._accumulated_reasoning),
"done": bool(done),
"error": str(result.get("error") or ""),
"image_count": len(images),
"parse_errors": scan.parse_errors[-8:],
"incremental_offset": scan.committed_len if self._INCREMENTAL else 0,
}
self._last_debug_raw_signature = debug_raw_signature
except Exception as e:
logger.debug(f"[LmarenaParser] 解析异常: {e}")
result["error"] = str(e)
self._inc_state = None
self._last_debug_summary = {
"raw_body_len": len(raw_response),
"exception": str(e),
}
self._last_debug_raw_signature = debug_raw_signature
return result
def reset(self) -> None:
"""重置状态"""
self._accumulated = ""
self._accumulated_reasoning = ""
self._seen_image_refs.clear()
self._last_debug_summary = {}
self._last_debug_raw_signature = ""
self._inc_state = None
def should_abort_on_error(self) -> bool:
"""当 Arena 返回 ae: 或 a3: 等错误协议行时,立即终止工作流。"""
return True
def should_fallback_to_dom_when_no_visible_content(self) -> bool:
"""Arena 的心跳流可能因网络波动提前关闭,空结果必须交给 DOM 接管。"""
return True
def export_debug_data(self, raw_response: str = "") -> Dict[str, Any]:
"""Return a compact parser-side protocol summary for network_parser_debug."""
summary = dict(self._last_debug_summary or {})
if raw_response:
text = self._normalize_debug_text(raw_response)
raw_summary = self._summarize_protocol_lines(text)
if self._last_debug_raw_signature == self._debug_body_signature(text):
summary.update(raw_summary)
else:
summary = raw_summary
summary.update({
"accumulated_len": len(self._accumulated),
"seen_image_refs": len(self._seen_image_refs),
})
return summary
# ============ 内部方法 ============
@staticmethod
def _normalize_debug_text(raw_response: Any) -> str:
if isinstance(raw_response, (bytes, bytearray)):
return raw_response.decode("utf-8", errors="ignore")
return str(raw_response or "")
@classmethod
def _debug_body_signature(cls, raw_response: Any) -> str:
text = cls._normalize_debug_text(raw_response)
return f"{len(text)}:{text[:80]}:{text[-80:]}"
@classmethod
def _summarize_protocol_lines(cls, raw_response: Any) -> Dict[str, Any]:
text = cls._normalize_debug_text(raw_response)
prefix_counts: Dict[str, int] = {}
line_count = 0
unknown_prefix_count = 0
non_protocol_line_count = 0
for raw_line in text.split("\n"):
line = raw_line.strip()
if not line:
continue
line_count += 1
colon_idx = line.find(":")
if colon_idx < 1:
non_protocol_line_count += 1
continue
prefix = line[:colon_idx]
if prefix in cls._PROTOCOL_PREFIXES:
prefix_counts[prefix] = prefix_counts.get(prefix, 0) + 1
else:
unknown_prefix_count += 1
return {
"raw_body_len": len(text),
"line_count": line_count,
"prefix_counts": prefix_counts,
"unknown_prefix_count": unknown_prefix_count,
"non_protocol_line_count": non_protocol_line_count,
}
@staticmethod
def _fix_mojibake(text: str) -> str:
"""
修复双重 UTF-8 编码(mojibake)
DrissionPage 通过 CDP 获取的响应体存在编码错误:
UTF-8 字节被混合解码为 cp1252 + latin-1,导致:
原始 UTF-8: f0 9f 92 9e (💞)
被 cp1252 解码: ð(F0) Ÿ(9F→U+0178) '(92→U+2019) ž(9E→U+017E)
但 0x90 → U+0090(latin-1 直通,因 cp1252 未定义此位置)
cp1252 有 5 个未定义位置 (0x81,0x8D,0x8F,0x90,0x9D)
走 latin-1 直通映射,导致 encode('cp1252') 和 encode('latin-1')
都无法单独处理全部字符。
修复方案(两步):
1. translate: 将 cp1252 特有字符 (U+2019等) 映射回 latin-1 范围 (U+0092等)
2. encode('latin-1'): 所有字符现在都在 U+0000-U+00FF,可一一还原原始字节
3. decode('utf-8'): 原始字节 → 正确文本
"""
try:
mapped = text.translate(_CP1252_TO_LATIN1)
return mapped.encode("latin-1").decode("utf-8")
except (UnicodeDecodeError, UnicodeEncodeError):
# 不是 mojibake(或只有部分是),返回原样
return text
@staticmethod
def _parse_text_chunk(payload: str) -> str | None:
"""解析 a0 行的 payload"""
try:
value = json.loads(payload)
if isinstance(value, str):
return value
except (json.JSONDecodeError, ValueError):
pass
return None
@staticmethod
def _is_finish_signal(payload: str) -> bool:
"""检查 ad 行是否表示结束"""
try:
data = json.loads(payload)
if isinstance(data, dict) and data.get("finishReason"):
return True
except (json.JSONDecodeError, ValueError):
pass
return False
@staticmethod
def _parse_error(payload: str) -> str | None:
"""解析 ae 行的错误信息"""
try:
data = json.loads(payload)
if isinstance(data, dict):
return data.get("message", str(data))
return str(data)
except (json.JSONDecodeError, ValueError):
return payload.strip() if payload.strip() else None
# ============ 元数据 ============
@classmethod
def get_id(cls) -> str:
return "lmarena"
@classmethod
def get_name(cls) -> str:
return "Arena.ai"
@classmethod
def get_description(cls) -> str:
return "解析 Arena.ai 的流式响应 (Vercel AI SDK Data Stream Protocol)"
@classmethod
def get_supported_patterns(cls) -> List[str]:
return ["nextjs-api/stream/create-evaluation"]
__all__ = [
"LmarenaParser",
"_CP1252_TO_LATIN1",
"_extract_arena_image_items",
]
@@ -0,0 +1,197 @@
"""
lmarena_side_left_parser.py - Arena.ai side-by-side left-stream parser.
Protocol:
- Vercel AI SDK data stream lines: "{prefix}:{payload}"
- left(modelA): a0/ad/ae
- right(modelB): b0/bd/be (ignored here)
"""
from __future__ import annotations
import json
from typing import Any, Dict, List
from app.core.config import logger
from .base import ResponseParser
from .lmarena_parser import LmarenaParser, _CP1252_TO_LATIN1, _extract_arena_image_items
class LmarenaSideLeftParser(ResponseParser):
"""
Parse Arena.ai side-by-side stream and keep only left/modelA channel.
This parser is intended for side mode where the left model is the target
(for example gemini-3.1-pro-preview in your setup).
"""
def __init__(self) -> None:
self._accumulated = ""
self._accumulated_reasoning = ""
self._seen_image_refs: set[str] = set()
def parse_chunk(self, raw_response: str) -> Dict[str, Any]:
result: Dict[str, Any] = {
"content": "",
"reasoning_content": "",
"images": [],
"done": False,
"error": None,
}
if isinstance(raw_response, (bytes, bytearray)):
raw_response = raw_response.decode("utf-8", errors="ignore")
if not raw_response or not isinstance(raw_response, str):
return result
raw_response = self._fix_mojibake(raw_response)
try:
content_parts: List[str] = []
reasoning_parts: List[str] = []
images: List[Dict[str, Any]] = []
done = False
for line in raw_response.split("\n"):
line = line.strip()
if not line:
continue
colon_idx = line.find(":")
if colon_idx < 1:
continue
prefix = line[:colon_idx]
payload = line[colon_idx + 1:]
# left/modelA text delta
if prefix == "a0":
text = self._parse_text_chunk(payload)
if text is not None:
content_parts.append(text)
elif prefix == "ag":
text = self._parse_text_chunk(payload)
if text is not None:
reasoning_parts.append(text)
elif prefix == "a2":
images.extend(
_extract_arena_image_items(
payload,
self._seen_image_refs,
source="lmarena_side_left_stream",
)
)
# left/modelA done
elif prefix == "ad":
if self._is_finish_signal(payload):
done = True
# left/modelA error frames (observed formats: ae / a3)
elif prefix in {"ae", "a3"}:
error_msg = self._parse_error(payload)
if error_msg:
result["error"] = error_msg
done = True
new_content = "".join(content_parts)
new_reasoning = "".join(reasoning_parts)
if new_content:
delta, next_accumulated = LmarenaParser._content_delta(
self._accumulated,
new_content,
append_disjoint=True,
)
if delta:
result["content"] = delta
self._accumulated = next_accumulated
if new_reasoning:
delta_r, next_acc_r = LmarenaParser._content_delta(
self._accumulated_reasoning,
new_reasoning,
append_disjoint=True,
)
if delta_r:
result["reasoning_content"] = delta_r
self._accumulated_reasoning = next_acc_r
if images:
result["images"] = images
result["done"] = done
except Exception as e:
logger.debug(f"[LmarenaSideLeftParser] parse exception: {e}")
result["error"] = str(e)
return result
def reset(self):
self._accumulated = ""
self._accumulated_reasoning = ""
self._seen_image_refs.clear()
def should_fallback_to_dom_when_no_visible_content(self) -> bool:
"""Arena 的心跳流可能因网络波动提前关闭,空结果必须交给 DOM 接管。"""
return True
def should_abort_on_error(self) -> bool:
# Side-by-side left mode only trusts the left stream. If the left channel
# itself reports an error, we should fail fast instead of silently
# falling back to whole-page DOM heuristics.
return True
@staticmethod
def _fix_mojibake(text: str) -> str:
try:
mapped = text.translate(_CP1252_TO_LATIN1)
return mapped.encode("latin-1").decode("utf-8")
except (UnicodeEncodeError, UnicodeDecodeError):
return text
@staticmethod
def _parse_text_chunk(payload: str) -> str | None:
try:
value = json.loads(payload)
if isinstance(value, str):
return value
except (json.JSONDecodeError, ValueError):
pass
return None
@staticmethod
def _is_finish_signal(payload: str) -> bool:
try:
data = json.loads(payload)
return isinstance(data, dict) and bool(data.get("finishReason"))
except (json.JSONDecodeError, ValueError):
return False
@staticmethod
def _parse_error(payload: str) -> str | None:
try:
data = json.loads(payload)
if isinstance(data, dict):
return data.get("message", str(data))
return str(data)
except (json.JSONDecodeError, ValueError):
return payload.strip() if payload.strip() else None
@classmethod
def get_id(cls) -> str:
return "lmarena_side_left"
@classmethod
def get_name(cls) -> str:
return "Arena.ai Side Left"
@classmethod
def get_description(cls) -> str:
return "Parse Arena.ai side-by-side stream, keep left(modelA) only"
@classmethod
def get_supported_patterns(cls) -> List[str]:
return ["nextjs-api/stream/create-evaluation"]
__all__ = ["LmarenaSideLeftParser"]
+338
View File
@@ -0,0 +1,338 @@
"""
mimo_parser.py - Xiaomi MiMo SSE response parser.
Observed stream traits:
- POST /open-apis/bot/chat?... returns text/event-stream
- answer chunks arrive as event:message with JSON data {"type":"text","content":"..."}
- event:finish with {"content":"[DONE]"} ends the stream
- reasoning text is wrapped inside <think>...</think> and should be ignored
"""
from __future__ import annotations
import json
from typing import Any, Dict, List, Tuple
from app.core.config import logger
from .base import ResponseParser
class MimoParser(ResponseParser):
"""Parse Xiaomi MiMo SSE streams and ignore think-phase content."""
_SUSPICIOUS_CHARS = set(
"ÃÂâæåçèéêëìíîïðñòóôõöùúûüýþÿ"
"€‚ƒ„…†‡ˆ‰Š‹ŒŽ‘’“”•–—˜™š›œžŸ"
)
_REVERSE_CP1252_BYTES = {
0x20AC: 0x80,
0x201A: 0x82,
0x0192: 0x83,
0x201E: 0x84,
0x2026: 0x85,
0x2020: 0x86,
0x2021: 0x87,
0x02C6: 0x88,
0x2030: 0x89,
0x0160: 0x8A,
0x2039: 0x8B,
0x0152: 0x8C,
0x017D: 0x8E,
0x2018: 0x91,
0x2019: 0x92,
0x201C: 0x93,
0x201D: 0x94,
0x2022: 0x95,
0x2013: 0x96,
0x2014: 0x97,
0x02DC: 0x98,
0x2122: 0x99,
0x0161: 0x9A,
0x203A: 0x9B,
0x0153: 0x9C,
0x017E: 0x9E,
0x0178: 0x9F,
}
def __init__(self) -> None:
self._last_raw_length = 0
self._pending = ""
self._inside_think = False
self._has_seen_visible_text = False
def reset(self) -> None:
self._last_raw_length = 0
self._last_raw_response = ""
self._pending = ""
self._inside_think = False
self._has_seen_visible_text = False
def parse_chunk(self, raw_response: str) -> Dict[str, Any]:
result: Dict[str, Any] = {
"content": "",
"images": [],
"done": False,
"error": None,
}
try:
if isinstance(raw_response, (bytes, bytearray)):
raw_response = raw_response.decode("utf-8", errors="ignore")
elif not isinstance(raw_response, str):
raw_response = str(raw_response)
new_data = self._prepare_incremental_raw_response(raw_response)
if not new_data:
return result
delta_content, done = self._consume_new_data(new_data)
if delta_content:
result["content"] = delta_content
result["done"] = done
except Exception as e:
logger.debug(f"[MimoParser] parse exception: {e}")
result["error"] = str(e)
return result
def _consume_new_data(self, new_data: str) -> Tuple[str, bool]:
normalized = (self._pending + new_data).replace("\r\n", "\n")
if not normalized:
return "", False
blocks = normalized.split("\n\n")
if normalized.endswith("\n\n"):
self._pending = ""
complete_blocks = [block for block in blocks if block.strip()]
else:
self._pending = blocks.pop() if blocks else normalized
complete_blocks = [block for block in blocks if block.strip()]
content_parts: List[str] = []
done = False
for block in complete_blocks:
block_content, block_done = self._parse_event_block(block)
if block_content:
content_parts.append(block_content)
if block_done:
done = True
return "".join(content_parts), done
def _parse_event_block(self, block: str) -> Tuple[str, bool]:
event_name = ""
data_lines: List[str] = []
for raw_line in block.split("\n"):
line = raw_line.strip()
if not line:
continue
if line.startswith("event:"):
event_name = line[6:].strip()
elif line.startswith("data:"):
data_lines.append(line[5:].strip())
if event_name == "finish":
return "", self._has_seen_visible_text
if event_name != "message":
return "", False
payload = "\n".join(data_lines).strip()
if not payload:
return "", False
try:
data = json.loads(payload)
except json.JSONDecodeError:
return "", False
if not isinstance(data, dict):
return "", False
if str(data.get("type") or "").strip().lower() != "text":
return "", False
text = data.get("content", "")
if not isinstance(text, str) or not text:
return "", False
visible_text = self._strip_think_content(text)
visible_text = self._repair_mojibake_text(visible_text)
if visible_text:
self._has_seen_visible_text = True
return visible_text, False
def _strip_think_content(self, text: str) -> str:
if not text:
return ""
normalized = text.replace("\x00", "")
visible_parts: List[str] = []
cursor = 0
while cursor < len(normalized):
if self._inside_think:
end_idx = normalized.find("</think>", cursor)
if end_idx == -1:
return "".join(visible_parts)
self._inside_think = False
cursor = end_idx + len("</think>")
continue
start_idx = normalized.find("<think>", cursor)
if start_idx == -1:
visible_parts.append(normalized[cursor:])
break
if start_idx > cursor:
visible_parts.append(normalized[cursor:start_idx])
self._inside_think = True
cursor = start_idx + len("<think>")
return "".join(visible_parts)
@classmethod
def _count_cjk(cls, text: str) -> int:
total = 0
for ch in text or "":
code = ord(ch)
if (
0x4E00 <= code <= 0x9FFF
or 0x3400 <= code <= 0x4DBF
or 0xF900 <= code <= 0xFAFF
):
total += 1
return total
@classmethod
def _count_suspicious(cls, text: str) -> int:
return sum(1 for ch in (text or "") if ch in cls._SUSPICIOUS_CHARS)
@classmethod
def _is_byte_like_char(cls, ch: str) -> bool:
code = ord(ch)
return code in cls._REVERSE_CP1252_BYTES or 0 <= code <= 0xFF
@classmethod
def _has_high_byte_signal(cls, text: str) -> bool:
for ch in text or "":
code = ord(ch)
if code in cls._REVERSE_CP1252_BYTES or 0x80 <= code <= 0xFF:
return True
return False
@classmethod
def _looks_like_utf8_mojibake(cls, text: str) -> bool:
if not text or len(text) < 2:
return False
cjk = cls._count_cjk(text)
if cjk == 0:
return cls._has_high_byte_signal(text)
suspicious = cls._count_suspicious(text)
return suspicious >= 1 and cls._has_high_byte_signal(text)
@classmethod
def _quality_score(cls, text: str) -> tuple[int, int, int]:
return (
cls._count_cjk(text),
-cls._count_suspicious(text),
-(text.count("\ufffd") if text else 0),
)
@classmethod
def _try_redecode(cls, text: str) -> str:
try:
payload = bytearray()
for ch in text:
code = ord(ch)
if code in cls._REVERSE_CP1252_BYTES:
payload.append(cls._REVERSE_CP1252_BYTES[code])
continue
if 0 <= code <= 0xFF:
payload.append(code)
continue
return text
return bytes(payload).decode("utf-8")
except Exception:
return text
@classmethod
def _repair_segmentwise(cls, text: str) -> str:
if not text:
return text
pieces: List[str] = []
buffer: List[str] = []
def _flush_buffer() -> None:
if not buffer:
return
segment = "".join(buffer)
buffer.clear()
if (
len(segment) >= 2
and cls._count_cjk(segment) == 0
and cls._has_high_byte_signal(segment)
):
candidate = cls._try_redecode(segment)
if cls._quality_score(candidate) > cls._quality_score(segment):
pieces.append(candidate)
return
pieces.append(segment)
for ch in text:
if cls._is_byte_like_char(ch):
buffer.append(ch)
else:
_flush_buffer()
pieces.append(ch)
_flush_buffer()
return "".join(pieces)
@classmethod
def _repair_mojibake_text(cls, text: str) -> str:
if not text or not cls._looks_like_utf8_mojibake(text):
return text
best = text
best_score = cls._quality_score(text)
candidate = cls._try_redecode(text)
if candidate != best:
candidate_score = cls._quality_score(candidate)
if candidate_score > best_score:
best = candidate
best_score = candidate_score
candidate = cls._repair_segmentwise(best)
if candidate != best:
candidate_score = cls._quality_score(candidate)
if candidate_score > best_score:
best = candidate
best_score = candidate_score
return best
@classmethod
def get_id(cls) -> str:
return "mimo"
@classmethod
def get_name(cls) -> str:
return "Xiaomi MiMo"
@classmethod
def get_description(cls) -> str:
return "Parse Xiaomi MiMo SSE streams and ignore think-phase text"
@classmethod
def get_supported_patterns(cls) -> List[str]:
return ["**/open-apis/bot/chat**"]
__all__ = ["MimoParser"]
+338
View File
@@ -0,0 +1,338 @@
"""
mimo_parser.py - Xiaomi MiMo SSE response parser.
Observed stream traits:
- POST /open-apis/bot/chat?... returns text/event-stream
- answer chunks arrive as event:message with JSON data {"type":"text","content":"..."}
- event:finish with {"content":"[DONE]"} ends the stream
- reasoning text is wrapped inside <think>...</think> and should be ignored
"""
from __future__ import annotations
import json
from typing import Any, Dict, List, Tuple
from app.core.config import logger
from .base import ResponseParser
class MimoParser(ResponseParser):
"""Parse Xiaomi MiMo SSE streams and ignore think-phase content."""
_SUSPICIOUS_CHARS = set(
"ÃÂâæåçèéêëìíîïðñòóôõöùúûüýþÿ"
"€‚ƒ„…†‡ˆ‰Š‹ŒŽ‘’“”•–—˜™š›œžŸ"
)
_REVERSE_CP1252_BYTES = {
0x20AC: 0x80,
0x201A: 0x82,
0x0192: 0x83,
0x201E: 0x84,
0x2026: 0x85,
0x2020: 0x86,
0x2021: 0x87,
0x02C6: 0x88,
0x2030: 0x89,
0x0160: 0x8A,
0x2039: 0x8B,
0x0152: 0x8C,
0x017D: 0x8E,
0x2018: 0x91,
0x2019: 0x92,
0x201C: 0x93,
0x201D: 0x94,
0x2022: 0x95,
0x2013: 0x96,
0x2014: 0x97,
0x02DC: 0x98,
0x2122: 0x99,
0x0161: 0x9A,
0x203A: 0x9B,
0x0153: 0x9C,
0x017E: 0x9E,
0x0178: 0x9F,
}
def __init__(self) -> None:
self._last_raw_length = 0
self._pending = ""
self._inside_think = False
self._has_seen_visible_text = False
def reset(self) -> None:
self._last_raw_length = 0
self._last_raw_response = ""
self._pending = ""
self._inside_think = False
self._has_seen_visible_text = False
def parse_chunk(self, raw_response: str) -> Dict[str, Any]:
result: Dict[str, Any] = {
"content": "",
"images": [],
"done": False,
"error": None,
}
try:
if isinstance(raw_response, (bytes, bytearray)):
raw_response = raw_response.decode("utf-8", errors="ignore")
elif not isinstance(raw_response, str):
raw_response = str(raw_response)
new_data = self._prepare_incremental_raw_response(raw_response)
if not new_data:
return result
delta_content, done = self._consume_new_data(new_data)
if delta_content:
result["content"] = delta_content
result["done"] = done
except Exception as e:
logger.debug(f"[MimoParser] parse exception: {e}")
result["error"] = str(e)
return result
def _consume_new_data(self, new_data: str) -> Tuple[str, bool]:
normalized = (self._pending + new_data).replace("\r\n", "\n")
if not normalized:
return "", False
blocks = normalized.split("\n\n")
if normalized.endswith("\n\n"):
self._pending = ""
complete_blocks = [block for block in blocks if block.strip()]
else:
self._pending = blocks.pop() if blocks else normalized
complete_blocks = [block for block in blocks if block.strip()]
content_parts: List[str] = []
done = False
for block in complete_blocks:
block_content, block_done = self._parse_event_block(block)
if block_content:
content_parts.append(block_content)
if block_done:
done = True
return "".join(content_parts), done
def _parse_event_block(self, block: str) -> Tuple[str, bool]:
event_name = ""
data_lines: List[str] = []
for raw_line in block.split("\n"):
line = raw_line.strip()
if not line:
continue
if line.startswith("event:"):
event_name = line[6:].strip()
elif line.startswith("data:"):
data_lines.append(line[5:].strip())
if event_name == "finish":
return "", self._has_seen_visible_text
if event_name != "message":
return "", False
payload = "\n".join(data_lines).strip()
if not payload:
return "", False
try:
data = json.loads(payload)
except json.JSONDecodeError:
return "", False
if not isinstance(data, dict):
return "", False
if str(data.get("type") or "").strip().lower() != "text":
return "", False
text = data.get("content", "")
if not isinstance(text, str) or not text:
return "", False
visible_text = self._strip_think_content(text)
visible_text = self._repair_mojibake_text(visible_text)
if visible_text:
self._has_seen_visible_text = True
return visible_text, False
def _strip_think_content(self, text: str) -> str:
if not text:
return ""
normalized = text.replace("\x00", "")
visible_parts: List[str] = []
cursor = 0
while cursor < len(normalized):
if self._inside_think:
end_idx = normalized.find("</think>", cursor)
if end_idx == -1:
return "".join(visible_parts)
self._inside_think = False
cursor = end_idx + len("</think>")
continue
start_idx = normalized.find("<think>", cursor)
if start_idx == -1:
visible_parts.append(normalized[cursor:])
break
if start_idx > cursor:
visible_parts.append(normalized[cursor:start_idx])
self._inside_think = True
cursor = start_idx + len("<think>")
return "".join(visible_parts)
@classmethod
def _count_cjk(cls, text: str) -> int:
total = 0
for ch in text or "":
code = ord(ch)
if (
0x4E00 <= code <= 0x9FFF
or 0x3400 <= code <= 0x4DBF
or 0xF900 <= code <= 0xFAFF
):
total += 1
return total
@classmethod
def _count_suspicious(cls, text: str) -> int:
return sum(1 for ch in (text or "") if ch in cls._SUSPICIOUS_CHARS)
@classmethod
def _is_byte_like_char(cls, ch: str) -> bool:
code = ord(ch)
return code in cls._REVERSE_CP1252_BYTES or 0 <= code <= 0xFF
@classmethod
def _has_high_byte_signal(cls, text: str) -> bool:
for ch in text or "":
code = ord(ch)
if code in cls._REVERSE_CP1252_BYTES or 0x80 <= code <= 0xFF:
return True
return False
@classmethod
def _looks_like_utf8_mojibake(cls, text: str) -> bool:
if not text or len(text) < 2:
return False
cjk = cls._count_cjk(text)
if cjk == 0:
return cls._has_high_byte_signal(text)
suspicious = cls._count_suspicious(text)
return suspicious >= 1 and cls._has_high_byte_signal(text)
@classmethod
def _quality_score(cls, text: str) -> tuple[int, int, int]:
return (
cls._count_cjk(text),
-cls._count_suspicious(text),
-(text.count("\ufffd") if text else 0),
)
@classmethod
def _try_redecode(cls, text: str) -> str:
try:
payload = bytearray()
for ch in text:
code = ord(ch)
if code in cls._REVERSE_CP1252_BYTES:
payload.append(cls._REVERSE_CP1252_BYTES[code])
continue
if 0 <= code <= 0xFF:
payload.append(code)
continue
return text
return bytes(payload).decode("utf-8")
except Exception:
return text
@classmethod
def _repair_segmentwise(cls, text: str) -> str:
if not text:
return text
pieces: List[str] = []
buffer: List[str] = []
def _flush_buffer() -> None:
if not buffer:
return
segment = "".join(buffer)
buffer.clear()
if (
len(segment) >= 2
and cls._count_cjk(segment) == 0
and cls._has_high_byte_signal(segment)
):
candidate = cls._try_redecode(segment)
if cls._quality_score(candidate) > cls._quality_score(segment):
pieces.append(candidate)
return
pieces.append(segment)
for ch in text:
if cls._is_byte_like_char(ch):
buffer.append(ch)
else:
_flush_buffer()
pieces.append(ch)
_flush_buffer()
return "".join(pieces)
@classmethod
def _repair_mojibake_text(cls, text: str) -> str:
if not text or not cls._looks_like_utf8_mojibake(text):
return text
best = text
best_score = cls._quality_score(text)
candidate = cls._try_redecode(text)
if candidate != best:
candidate_score = cls._quality_score(candidate)
if candidate_score > best_score:
best = candidate
best_score = candidate_score
candidate = cls._repair_segmentwise(best)
if candidate != best:
candidate_score = cls._quality_score(candidate)
if candidate_score > best_score:
best = candidate
best_score = candidate_score
return best
@classmethod
def get_id(cls) -> str:
return "mimo"
@classmethod
def get_name(cls) -> str:
return "Xiaomi MiMo"
@classmethod
def get_description(cls) -> str:
return "Parse Xiaomi MiMo SSE streams and ignore think-phase text"
@classmethod
def get_supported_patterns(cls) -> List[str]:
return ["**/open-apis/bot/chat**"]
__all__ = ["MimoParser"]
+197
View File
@@ -0,0 +1,197 @@
"""
qwen_parser.py - Qwen SSE response parser.
Observed stream traits:
- data: {"response.created": ...} announces the response id
- answer tokens are in choices[*].delta.content with phase=answer
- thinking_summary chunks should be ignored
- choices[*].delta.status=finished ends the answer stream
"""
from __future__ import annotations
import json
from typing import Dict, Any, List, Tuple
from app.core.config import logger
from .base import ResponseParser
class QwenParser(ResponseParser):
"""Parse Qwen web SSE responses."""
def __init__(self) -> None:
self._last_raw_length = 0
self._pending = ""
self._has_seen_answer_text = False
self._accumulated_reasoning = ""
def parse_chunk(self, raw_response: str) -> Dict[str, Any]:
result: Dict[str, Any] = {
"content": "",
"reasoning_content": "",
"images": [],
"done": False,
"error": None,
}
try:
if isinstance(raw_response, (bytes, bytearray)):
raw_response = raw_response.decode("utf-8", errors="ignore")
elif not isinstance(raw_response, str):
raw_response = str(raw_response)
new_data = self._prepare_incremental_raw_response(raw_response)
if not new_data:
return result
delta_content, delta_reasoning, done = self._consume_new_data(new_data)
if delta_content:
result["content"] = delta_content
if delta_reasoning:
result["reasoning_content"] = delta_reasoning
result["done"] = done
except Exception as e:
logger.debug(f"[QwenParser] parse exception: {e}")
result["error"] = str(e)
return result
def reset(self) -> None:
self._last_raw_length = 0
self._last_raw_response = ""
self._pending = ""
self._has_seen_answer_text = False
self._accumulated_reasoning = ""
def _consume_new_data(self, new_data: str) -> Tuple[str, str, bool]:
normalized = (self._pending + new_data).replace("\r\n", "\n")
if not normalized:
return "", "", False
blocks = normalized.split("\n\n")
if normalized.endswith("\n\n"):
self._pending = ""
complete_blocks = [block for block in blocks if block.strip()]
else:
self._pending = blocks.pop() if blocks else normalized
complete_blocks = [block for block in blocks if block.strip()]
content_parts: List[str] = []
reasoning_parts: List[str] = []
done = False
for block in complete_blocks:
block_content, block_reasoning, block_done = self._parse_event_block(block)
if block_content:
content_parts.append(block_content)
if block_reasoning:
reasoning_parts.append(block_reasoning)
if block_done:
done = True
new_reasoning = "\n\n".join(reasoning_parts)
delta_reasoning = ""
if new_reasoning:
delta_reasoning, self._accumulated_reasoning = self._reasoning_delta(
self._accumulated_reasoning, new_reasoning
)
return "".join(content_parts), delta_reasoning, done
@staticmethod
def _reasoning_delta(accumulated: str, candidate: str) -> Tuple[str, str]:
if not candidate:
return "", accumulated
if not accumulated:
return candidate, candidate
if candidate == accumulated or accumulated.startswith(candidate):
return "", accumulated
if candidate.startswith(accumulated):
return candidate[len(accumulated):], candidate
max_overlap = min(len(accumulated), len(candidate))
for overlap in range(max_overlap, 0, -1):
if accumulated.endswith(candidate[:overlap]):
delta = candidate[overlap:]
return delta, accumulated + delta
return candidate, accumulated + candidate
def _parse_event_block(self, block: str) -> Tuple[str, str, bool]:
data_lines: List[str] = []
for raw_line in block.split("\n"):
line = raw_line.strip()
if line.startswith("data:"):
data_lines.append(line[5:].strip())
payload = "\n".join(data_lines).strip()
if not payload:
return "", "", False
try:
data = json.loads(payload)
except json.JSONDecodeError:
return "", "", False
choices = data.get("choices")
if not isinstance(choices, list):
return "", "", False
content_parts: List[str] = []
reasoning_parts: List[str] = []
done = False
for choice in choices:
if not isinstance(choice, dict):
continue
delta = choice.get("delta")
if not isinstance(delta, dict):
continue
phase = delta.get("phase")
if phase == "thinking_summary":
extra = delta.get("extra")
if isinstance(extra, dict):
summary_thought = extra.get("summary_thought")
if isinstance(summary_thought, dict):
thought_list = summary_thought.get("content")
if isinstance(thought_list, list):
reasoning_parts.append("\n\n".join([str(x) for x in thought_list if x]))
elif phase == "answer":
text = delta.get("content", "")
if isinstance(text, str) and text:
content_parts.append(text)
self._has_seen_answer_text = True
if delta.get("status") == "finished" and self._has_seen_answer_text:
done = True
return "".join(content_parts), "\n\n".join(reasoning_parts), done
@classmethod
def get_id(cls) -> str:
return "qwen"
@classmethod
def get_name(cls) -> str:
return "Qwen"
@classmethod
def get_description(cls) -> str:
return "Parse Qwen SSE streams and keep only answer-phase text"
def should_fallback_to_dom_when_no_visible_content(self) -> bool:
"""Use the rendered reply when Qwen's captured SSE stalls in its thinking phase."""
return True
@classmethod
def get_supported_patterns(cls) -> List[str]:
return ["**/api/v2/chat/completions**"]
__all__ = ["QwenParser"]
+120
View File
@@ -0,0 +1,120 @@
"""
app/core/parsers/registry.py - 解析器注册中心
提供解析器的动态注册、查找、实例化功能
"""
import importlib
import sys
from typing import Dict, Type, Optional, List
from app.core.config import logger
from .base import ResponseParser
class ParserRegistry:
"""
解析器注册中心
职责:
1. 管理所有可用的响应解析器
2. 提供注册机制
3. 根据 ID 获取解析器实例
"""
_parsers: Dict[str, Type[ResponseParser]] = {}
@classmethod
def register_class(cls, parser_class: Type[ResponseParser], parser_id: Optional[str] = None):
"""
注册解析器类
Args:
parser_class: 解析器类
parser_id: 解析器 ID(可选,默认使用 get_id())
"""
pid = parser_id or parser_class.get_id()
cls._parsers[pid] = parser_class
logger.info(f"已注册响应解析器: {pid} -> {parser_class.__name__}")
@classmethod
def get(cls, parser_id: str) -> ResponseParser:
"""
获取解析器实例
Args:
parser_id: 解析器 ID
Returns:
解析器实例
Raises:
ValueError: 解析器 ID 不存在
"""
if parser_id not in cls._parsers:
raise ValueError(
f"未知的解析器 ID: {parser_id},可用的解析器: {list(cls._parsers.keys())}"
)
parser_class = cls._parsers[parser_id]
return parser_class()
@classmethod
def exists(cls, parser_id: str) -> bool:
"""检查解析器是否存在"""
return parser_id in cls._parsers
@classmethod
def list_all(cls) -> List[Dict[str, str]]:
"""
列出所有已注册的解析器
Returns:
解析器信息列表
"""
result = []
for pid, parser_class in cls._parsers.items():
info = {
"id": pid,
"name": parser_class.get_name(),
"description": parser_class.get_description(),
"patterns": parser_class.get_supported_patterns(),
}
result.append(info)
return result
@classmethod
def load_from_module(cls, module_path: str, class_name: str, parser_id: str):
"""
动态加载并注册解析器类
Args:
module_path: 模块路径(如 "app.core.parsers.custom")
class_name: 类名(如 "CustomParser")
parser_id: 注册的 ID
Raises:
ImportError: 模块不存在
AttributeError: 类不存在
"""
try:
importlib.invalidate_caches()
if module_path in sys.modules:
module = importlib.reload(sys.modules[module_path])
else:
module = importlib.import_module(module_path)
parser_class = getattr(module, class_name)
if not issubclass(parser_class, ResponseParser):
raise TypeError(f"{class_name} 不是 ResponseParser 的子类")
cls.register_class(parser_class, parser_id)
logger.info(f"动态加载解析器成功: {module_path}.{class_name} -> {parser_id}")
except Exception as e:
logger.error(f"动态加载解析器失败: {module_path}.{class_name} - {e}")
raise
__all__ = ['ParserRegistry']