515 lines
17 KiB
Python
515 lines
17 KiB
Python
"""
|
||
aistudio_parser.py - Google AI Studio 响应解析器
|
||
|
||
响应格式特征:
|
||
- Content-Type: application/json+protobuf
|
||
- 嵌套数组结构
|
||
- 增量文本模式
|
||
- 包含 thinking 内容(需过滤)
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import json
|
||
import re
|
||
from typing import Any, Dict, List, Tuple
|
||
|
||
from app.core.config import logger
|
||
from .base import ResponseParser
|
||
|
||
|
||
class AIStudioParser(ResponseParser):
|
||
"""
|
||
Google AI Studio (MakerSuite) 响应解析器
|
||
|
||
URL 特征: MakerSuiteService/GenerateContent
|
||
响应格式: JSON+Protobuf (嵌套数组)
|
||
"""
|
||
|
||
_VISIBLE_TEXT_BLOCK_RE = re.compile(
|
||
r'null,"((?:\\.|[^"\\])*)"\]\],"model"'
|
||
)
|
||
_VISIBLE_TEXT_BLOCK_PREFIX = 'null,"'
|
||
_OBFUSCATED_PAYLOAD_RE = re.compile(r"^[A-Za-z0-9+/=_-]+$")
|
||
_CJK_RE = re.compile(r"[\u3400-\u9fff]")
|
||
|
||
def __init__(self):
|
||
self._accumulated_content = ""
|
||
self._accumulated_reasoning = ""
|
||
self._is_done = False
|
||
self._last_raw_length = 0
|
||
self._pending_raw = ""
|
||
self._visible_text_scan_pending = ""
|
||
self._visible_text_fallback_content = ""
|
||
|
||
def parse_chunk(self, raw_response: str) -> Dict[str, Any]:
|
||
"""
|
||
解析响应(返回完整内容的增量)
|
||
"""
|
||
result = {
|
||
"content": "",
|
||
"reasoning_content": "",
|
||
"images": [],
|
||
"done": False,
|
||
"error": None
|
||
}
|
||
|
||
try:
|
||
if isinstance(raw_response, bytes):
|
||
raw_response = raw_response.decode("utf-8", errors="ignore")
|
||
elif not isinstance(raw_response, str):
|
||
raw_response = str(raw_response)
|
||
|
||
new_data = self._prepare_incremental_raw_response(raw_response)
|
||
if not new_data:
|
||
return result
|
||
|
||
payloads = self._decode_payloads(new_data, incremental=True)
|
||
content = ""
|
||
is_done = False
|
||
|
||
reasoning = ""
|
||
for data in payloads:
|
||
current_content, current_reasoning, current_done = self._extract_content(data)
|
||
if current_content:
|
||
content = current_content
|
||
if current_reasoning:
|
||
reasoning = current_reasoning
|
||
if current_done:
|
||
is_done = True
|
||
|
||
if not content:
|
||
content = self._extract_visible_text_from_increment(new_data)
|
||
|
||
if content:
|
||
if content.startswith(self._accumulated_content):
|
||
delta = content[len(self._accumulated_content):]
|
||
self._accumulated_content = content
|
||
else:
|
||
delta = content
|
||
self._accumulated_content += content
|
||
if delta:
|
||
result["content"] = delta
|
||
|
||
if reasoning:
|
||
if reasoning.startswith(self._accumulated_reasoning):
|
||
delta_r = reasoning[len(self._accumulated_reasoning):]
|
||
self._accumulated_reasoning = reasoning
|
||
else:
|
||
delta_r = reasoning
|
||
self._accumulated_reasoning += reasoning
|
||
if delta_r:
|
||
result["reasoning_content"] = delta_r
|
||
|
||
result["done"] = is_done
|
||
|
||
except json.JSONDecodeError as e:
|
||
logger.debug(f"[AIStudioParser] JSON 解析失败: {e}")
|
||
result["error"] = str(e)
|
||
except Exception as e:
|
||
logger.debug(f"[AIStudioParser] 解析异常: {e}")
|
||
result["error"] = str(e)
|
||
|
||
return result
|
||
|
||
def reset(self):
|
||
"""重置状态"""
|
||
self._accumulated_content = ""
|
||
self._accumulated_reasoning = ""
|
||
self._is_done = False
|
||
self._last_raw_length = 0
|
||
self._last_raw_response = ""
|
||
self._pending_raw = ""
|
||
self._visible_text_scan_pending = ""
|
||
self._visible_text_fallback_content = ""
|
||
|
||
@staticmethod
|
||
def _normalize_raw_text(raw_response: Any) -> str:
|
||
if isinstance(raw_response, bytes):
|
||
raw_response = raw_response.decode("utf-8", errors="ignore")
|
||
elif not isinstance(raw_response, str):
|
||
raw_response = str(raw_response)
|
||
|
||
text = raw_response.lstrip("\ufeff")
|
||
stripped = text.lstrip()
|
||
if stripped.startswith(")]}'"):
|
||
stripped = stripped[4:].lstrip("\r\n")
|
||
return stripped
|
||
|
||
def _decode_payloads(self, raw_response: Any, incremental: bool = False) -> List[Any]:
|
||
if isinstance(raw_response, (list, dict)):
|
||
return [raw_response]
|
||
|
||
normalized = self._normalize_raw_text(raw_response)
|
||
if not normalized:
|
||
return []
|
||
|
||
if incremental:
|
||
normalized = f"{self._pending_raw}{normalized}"
|
||
|
||
try:
|
||
payload = json.loads(normalized)
|
||
if incremental:
|
||
self._pending_raw = ""
|
||
return [payload]
|
||
except json.JSONDecodeError as exc:
|
||
payloads, remainder = self._decode_json_prefix_values(normalized)
|
||
if incremental:
|
||
self._pending_raw = remainder
|
||
if payloads:
|
||
return payloads
|
||
|
||
repaired = self._try_repair_unclosed_json(normalized)
|
||
if repaired is not None:
|
||
return [repaired]
|
||
|
||
logger.debug_throttled(
|
||
f"aistudio.incomplete_json.{id(self)}",
|
||
"[AIStudioParser] JSON 未完整,继续等待流增长 "
|
||
f"(body_len={len(normalized)}, error={exc})",
|
||
interval_sec=3.0,
|
||
)
|
||
return []
|
||
|
||
@staticmethod
|
||
def _try_repair_unclosed_json(text: str) -> Any:
|
||
stripped = text.strip()
|
||
if not stripped:
|
||
return None
|
||
for pad in range(0, 30):
|
||
try:
|
||
attempt = stripped + (']' * pad)
|
||
return json.loads(attempt)
|
||
except Exception:
|
||
pass
|
||
try:
|
||
attempt = stripped + '"]' + (']' * pad)
|
||
return json.loads(attempt)
|
||
except Exception:
|
||
pass
|
||
return None
|
||
|
||
@classmethod
|
||
def _extract_visible_text_from_raw_body(cls, raw_response: Any) -> str:
|
||
raw_text = cls._normalize_raw_text(raw_response)
|
||
if not raw_text:
|
||
return ""
|
||
|
||
pieces, _last_match_end = cls._scan_visible_text_matches(raw_text)
|
||
return "".join(pieces)
|
||
|
||
@classmethod
|
||
def _scan_visible_text_matches(cls, raw_text: str) -> Tuple[List[str], int]:
|
||
pieces: List[str] = []
|
||
last_match_end = 0
|
||
for match in cls._VISIBLE_TEXT_BLOCK_RE.finditer(raw_text):
|
||
last_match_end = match.end()
|
||
piece = cls._decode_json_fragment(match.group(1))
|
||
if piece and cls._looks_like_visible_text(piece):
|
||
pieces.append(piece)
|
||
|
||
return pieces, last_match_end
|
||
|
||
@classmethod
|
||
def _build_visible_text_scan_pending(cls, scan_text: str, last_match_end: int) -> str:
|
||
remainder = scan_text[last_match_end:] if last_match_end > 0 else scan_text
|
||
marker_index = remainder.rfind(cls._VISIBLE_TEXT_BLOCK_PREFIX)
|
||
if marker_index >= 0:
|
||
return remainder[marker_index:]
|
||
|
||
keep_chars = max(0, len(cls._VISIBLE_TEXT_BLOCK_PREFIX) - 1)
|
||
return remainder[-keep_chars:] if keep_chars else ""
|
||
|
||
def _extract_visible_text_from_increment(self, new_data: str) -> str:
|
||
if not new_data and not self._visible_text_scan_pending:
|
||
return self._visible_text_fallback_content
|
||
|
||
normalized_new_data = self._normalize_raw_text(new_data)
|
||
scan_text = f"{self._visible_text_scan_pending}{normalized_new_data}"
|
||
if not scan_text:
|
||
return self._visible_text_fallback_content
|
||
|
||
pieces, last_match_end = self._scan_visible_text_matches(scan_text)
|
||
if pieces:
|
||
self._visible_text_fallback_content += "".join(pieces)
|
||
self._visible_text_scan_pending = self._build_visible_text_scan_pending(
|
||
scan_text,
|
||
last_match_end,
|
||
)
|
||
return self._visible_text_fallback_content
|
||
|
||
@classmethod
|
||
def _looks_like_obfuscated_payload(cls, value: Any) -> bool:
|
||
text = str(value or "").strip()
|
||
if len(text) < 96:
|
||
return False
|
||
if any(ch.isspace() for ch in text):
|
||
return False
|
||
if cls._CJK_RE.search(text):
|
||
return False
|
||
if not cls._OBFUSCATED_PAYLOAD_RE.fullmatch(text):
|
||
return False
|
||
if not any(ch in "+/=_-" for ch in text):
|
||
return False
|
||
|
||
distinct_chars = len(set(text))
|
||
if distinct_chars < 20:
|
||
return False
|
||
|
||
alpha_num_ratio = sum(ch.isalnum() for ch in text) / len(text)
|
||
return alpha_num_ratio >= 0.85
|
||
|
||
@classmethod
|
||
def _looks_like_visible_text(cls, value: Any) -> bool:
|
||
text = str(value or "")
|
||
stripped = text.strip()
|
||
if not stripped:
|
||
return False
|
||
if cls._looks_like_obfuscated_payload(stripped):
|
||
return False
|
||
if cls._CJK_RE.search(stripped):
|
||
return True
|
||
if any(ch.isspace() for ch in stripped):
|
||
return True
|
||
if stripped.startswith(("{", "[", "<", "```", "http://", "https://")):
|
||
return True
|
||
|
||
punctuation_count = sum(
|
||
ch in '.,!?;:()[]{}<>"\'`~@#$%^&*\\|'
|
||
for ch in stripped
|
||
)
|
||
if punctuation_count > 0:
|
||
return True
|
||
|
||
return len(stripped) <= 80
|
||
|
||
@classmethod
|
||
def _pick_visible_text_candidate(cls, values: List[Any]) -> str:
|
||
for item in values:
|
||
if not isinstance(item, str):
|
||
continue
|
||
if cls._looks_like_visible_text(item):
|
||
return item
|
||
return ""
|
||
|
||
@staticmethod
|
||
def _decode_json_fragment(value: str) -> str:
|
||
text = str(value or "")
|
||
if not text:
|
||
return ""
|
||
|
||
try:
|
||
decoded = json.loads(f'"{text}"')
|
||
return decoded if isinstance(decoded, str) else ""
|
||
except json.JSONDecodeError:
|
||
return (
|
||
text.replace("\\n", "\n")
|
||
.replace("\\r", "\r")
|
||
.replace("\\t", "\t")
|
||
.replace('\\"', '"')
|
||
.replace("\\\\", "\\")
|
||
)
|
||
|
||
@staticmethod
|
||
def _decode_json_prefix_values(raw_text: str) -> Tuple[List[Any], str]:
|
||
decoder = json.JSONDecoder()
|
||
values: List[Any] = []
|
||
index = 0
|
||
length = len(raw_text)
|
||
|
||
while index < length:
|
||
while index < length and raw_text[index].isspace():
|
||
index += 1
|
||
if index >= length:
|
||
break
|
||
|
||
try:
|
||
value, next_index = decoder.raw_decode(raw_text, index)
|
||
except json.JSONDecodeError:
|
||
return values, raw_text[index:]
|
||
|
||
values.append(value)
|
||
index = next_index
|
||
|
||
return values, ""
|
||
|
||
@staticmethod
|
||
def _normalize_outer_blocks(data: Any) -> List[Any]:
|
||
if not isinstance(data, list) or not data:
|
||
return []
|
||
|
||
if any(isinstance(item, list) for item in data):
|
||
if (
|
||
len(data) == 1
|
||
and isinstance(data[0], list)
|
||
and any(isinstance(item, list) for item in data[0])
|
||
):
|
||
return data[0]
|
||
return data
|
||
|
||
return []
|
||
|
||
def _extract_content(self, data: Any) -> Tuple[str, str, bool]:
|
||
"""
|
||
从响应数据中提取文本内容与思考过程
|
||
|
||
Returns:
|
||
(accumulated_text, accumulated_reasoning, is_done)
|
||
"""
|
||
try:
|
||
outer = self._normalize_outer_blocks(data)
|
||
if not outer:
|
||
return "", "", False
|
||
|
||
accumulated = ""
|
||
accumulated_reasoning = ""
|
||
is_done = False
|
||
|
||
for block in outer:
|
||
if not isinstance(block, list):
|
||
continue
|
||
|
||
# 检查是否是统计块(结束标志之一)
|
||
if self._is_stats_block(block):
|
||
is_done = True
|
||
continue
|
||
|
||
# 提取文本
|
||
text, block_done, is_thinking = self._extract_block_content(block)
|
||
|
||
if is_thinking:
|
||
if text:
|
||
accumulated_reasoning += text
|
||
else:
|
||
if text:
|
||
accumulated += text
|
||
|
||
if block_done:
|
||
is_done = True
|
||
|
||
return accumulated, accumulated_reasoning, is_done
|
||
|
||
except Exception as e:
|
||
logger.debug(f"[AIStudioParser] _extract_content 异常: {e}")
|
||
return "", "", False
|
||
|
||
def _is_stats_block(self, block: list) -> bool:
|
||
"""检查是否是统计块(响应结束)"""
|
||
try:
|
||
# 格式: [null, null, null, [timestamp, ...]]
|
||
if len(block) >= 4:
|
||
if block[0] is None and block[1] is None and block[2] is None:
|
||
if isinstance(block[3], list):
|
||
return True
|
||
return False
|
||
except:
|
||
return False
|
||
|
||
def _extract_block_content(self, block: list) -> tuple:
|
||
"""
|
||
从单个块提取内容
|
||
|
||
Returns:
|
||
(text, is_done, is_thinking)
|
||
"""
|
||
try:
|
||
# 路径: block[0][0][0][0][0]
|
||
# thinking: [13 items, None, "**Thinking...", ..., 1]
|
||
# 正常内容: [2 items, None, "文本内容"]
|
||
|
||
if not isinstance(block, list) or len(block) == 0:
|
||
return "", False, False
|
||
|
||
level1 = block[0]
|
||
if not isinstance(level1, list) or len(level1) == 0:
|
||
return "", False, False
|
||
|
||
level2 = level1[0]
|
||
if not isinstance(level2, list) or len(level2) == 0:
|
||
return "", False, False
|
||
|
||
level3 = level2[0]
|
||
if not isinstance(level3, list) or len(level3) == 0:
|
||
return "", False, False
|
||
|
||
level4 = level3[0]
|
||
if not isinstance(level4, list) or len(level4) == 0:
|
||
return "", False, False
|
||
|
||
# 这里就是 content_arr
|
||
content_arr = level4[0]
|
||
if not isinstance(content_arr, list):
|
||
return "", False, False
|
||
|
||
# 提取文本:只接受像“可见正文”的字符串,避免把内部高熵 payload 误当回复。
|
||
text = ""
|
||
primary_text = content_arr[1] if len(content_arr) > 1 else ""
|
||
if self._looks_like_visible_text(primary_text):
|
||
text = primary_text
|
||
else:
|
||
text = self._pick_visible_text_candidate(content_arr[2:])
|
||
if not text:
|
||
suppressed_payload = next(
|
||
(
|
||
item for item in content_arr[2:]
|
||
if isinstance(item, str)
|
||
and self._looks_like_obfuscated_payload(item)
|
||
),
|
||
"",
|
||
)
|
||
if suppressed_payload:
|
||
logger.debug_throttled(
|
||
f"aistudio.hidden_payload.{id(self)}",
|
||
"[AIStudioParser] 忽略疑似内部高熵 payload "
|
||
f"(len={len(str(suppressed_payload).strip())})",
|
||
interval_sec=3.0,
|
||
)
|
||
|
||
# 判断是否是 thinking
|
||
# thinking 块特征:len >= 13 且索引 12 == 1
|
||
is_thinking = False
|
||
if len(content_arr) >= 13:
|
||
if len(content_arr) > 12 and content_arr[12] == 1:
|
||
is_thinking = True
|
||
|
||
# 检查是否结束
|
||
# 1. level2 的第二个元素是 1
|
||
is_done = False
|
||
if len(level2) > 1 and level2[1] == 1:
|
||
is_done = True
|
||
|
||
# 2. 有长 token 字符串(索引 13 或 14)
|
||
if len(content_arr) > 13 and isinstance(content_arr[13], str):
|
||
if len(content_arr[13]) > 50:
|
||
is_done = True
|
||
|
||
return text, is_done, is_thinking
|
||
|
||
except Exception as e:
|
||
logger.debug(f"[AIStudioParser] _extract_block_content 异常: {e}")
|
||
return "", False, False
|
||
|
||
# ============ 元数据 ============
|
||
|
||
@classmethod
|
||
def get_id(cls) -> str:
|
||
return "aistudio"
|
||
|
||
@classmethod
|
||
def get_name(cls) -> str:
|
||
return "Google AI Studio"
|
||
|
||
@classmethod
|
||
def get_description(cls) -> str:
|
||
return "解析 Google AI Studio (MakerSuite) 的 GenerateContent 响应"
|
||
|
||
@classmethod
|
||
def get_supported_patterns(cls) -> List[str]:
|
||
return ["MakerSuiteService/GenerateContent", "GenerateContent"]
|
||
|
||
def should_fallback_to_dom_when_no_visible_content(self) -> bool:
|
||
return True
|
||
|
||
|
||
__all__ = ['AIStudioParser']
|