Files
2026-11-02 22:54:14 +08:00

515 lines
17 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""
aistudio_parser.py - Google AI Studio 响应解析器
响应格式特征:
- Content-Type: application/json+protobuf
- 嵌套数组结构
- 增量文本模式
- 包含 thinking 内容(需过滤)
"""
from __future__ import annotations
import json
import re
from typing import Any, Dict, List, Tuple
from app.core.config import logger
from .base import ResponseParser
class AIStudioParser(ResponseParser):
"""
Google AI Studio (MakerSuite) 响应解析器
URL 特征: MakerSuiteService/GenerateContent
响应格式: JSON+Protobuf (嵌套数组)
"""
_VISIBLE_TEXT_BLOCK_RE = re.compile(
r'null,"((?:\\.|[^"\\])*)"\]\],"model"'
)
_VISIBLE_TEXT_BLOCK_PREFIX = 'null,"'
_OBFUSCATED_PAYLOAD_RE = re.compile(r"^[A-Za-z0-9+/=_-]+$")
_CJK_RE = re.compile(r"[\u3400-\u9fff]")
def __init__(self):
self._accumulated_content = ""
self._accumulated_reasoning = ""
self._is_done = False
self._last_raw_length = 0
self._pending_raw = ""
self._visible_text_scan_pending = ""
self._visible_text_fallback_content = ""
def parse_chunk(self, raw_response: str) -> Dict[str, Any]:
"""
解析响应(返回完整内容的增量)
"""
result = {
"content": "",
"reasoning_content": "",
"images": [],
"done": False,
"error": None
}
try:
if isinstance(raw_response, bytes):
raw_response = raw_response.decode("utf-8", errors="ignore")
elif not isinstance(raw_response, str):
raw_response = str(raw_response)
new_data = self._prepare_incremental_raw_response(raw_response)
if not new_data:
return result
payloads = self._decode_payloads(new_data, incremental=True)
content = ""
is_done = False
reasoning = ""
for data in payloads:
current_content, current_reasoning, current_done = self._extract_content(data)
if current_content:
content = current_content
if current_reasoning:
reasoning = current_reasoning
if current_done:
is_done = True
if not content:
content = self._extract_visible_text_from_increment(new_data)
if content:
if content.startswith(self._accumulated_content):
delta = content[len(self._accumulated_content):]
self._accumulated_content = content
else:
delta = content
self._accumulated_content += content
if delta:
result["content"] = delta
if reasoning:
if reasoning.startswith(self._accumulated_reasoning):
delta_r = reasoning[len(self._accumulated_reasoning):]
self._accumulated_reasoning = reasoning
else:
delta_r = reasoning
self._accumulated_reasoning += reasoning
if delta_r:
result["reasoning_content"] = delta_r
result["done"] = is_done
except json.JSONDecodeError as e:
logger.debug(f"[AIStudioParser] JSON 解析失败: {e}")
result["error"] = str(e)
except Exception as e:
logger.debug(f"[AIStudioParser] 解析异常: {e}")
result["error"] = str(e)
return result
def reset(self):
"""重置状态"""
self._accumulated_content = ""
self._accumulated_reasoning = ""
self._is_done = False
self._last_raw_length = 0
self._last_raw_response = ""
self._pending_raw = ""
self._visible_text_scan_pending = ""
self._visible_text_fallback_content = ""
@staticmethod
def _normalize_raw_text(raw_response: Any) -> str:
if isinstance(raw_response, bytes):
raw_response = raw_response.decode("utf-8", errors="ignore")
elif not isinstance(raw_response, str):
raw_response = str(raw_response)
text = raw_response.lstrip("\ufeff")
stripped = text.lstrip()
if stripped.startswith(")]}'"):
stripped = stripped[4:].lstrip("\r\n")
return stripped
def _decode_payloads(self, raw_response: Any, incremental: bool = False) -> List[Any]:
if isinstance(raw_response, (list, dict)):
return [raw_response]
normalized = self._normalize_raw_text(raw_response)
if not normalized:
return []
if incremental:
normalized = f"{self._pending_raw}{normalized}"
try:
payload = json.loads(normalized)
if incremental:
self._pending_raw = ""
return [payload]
except json.JSONDecodeError as exc:
payloads, remainder = self._decode_json_prefix_values(normalized)
if incremental:
self._pending_raw = remainder
if payloads:
return payloads
repaired = self._try_repair_unclosed_json(normalized)
if repaired is not None:
return [repaired]
logger.debug_throttled(
f"aistudio.incomplete_json.{id(self)}",
"[AIStudioParser] JSON 未完整,继续等待流增长 "
f"(body_len={len(normalized)}, error={exc})",
interval_sec=3.0,
)
return []
@staticmethod
def _try_repair_unclosed_json(text: str) -> Any:
stripped = text.strip()
if not stripped:
return None
for pad in range(0, 30):
try:
attempt = stripped + (']' * pad)
return json.loads(attempt)
except Exception:
pass
try:
attempt = stripped + '"]' + (']' * pad)
return json.loads(attempt)
except Exception:
pass
return None
@classmethod
def _extract_visible_text_from_raw_body(cls, raw_response: Any) -> str:
raw_text = cls._normalize_raw_text(raw_response)
if not raw_text:
return ""
pieces, _last_match_end = cls._scan_visible_text_matches(raw_text)
return "".join(pieces)
@classmethod
def _scan_visible_text_matches(cls, raw_text: str) -> Tuple[List[str], int]:
pieces: List[str] = []
last_match_end = 0
for match in cls._VISIBLE_TEXT_BLOCK_RE.finditer(raw_text):
last_match_end = match.end()
piece = cls._decode_json_fragment(match.group(1))
if piece and cls._looks_like_visible_text(piece):
pieces.append(piece)
return pieces, last_match_end
@classmethod
def _build_visible_text_scan_pending(cls, scan_text: str, last_match_end: int) -> str:
remainder = scan_text[last_match_end:] if last_match_end > 0 else scan_text
marker_index = remainder.rfind(cls._VISIBLE_TEXT_BLOCK_PREFIX)
if marker_index >= 0:
return remainder[marker_index:]
keep_chars = max(0, len(cls._VISIBLE_TEXT_BLOCK_PREFIX) - 1)
return remainder[-keep_chars:] if keep_chars else ""
def _extract_visible_text_from_increment(self, new_data: str) -> str:
if not new_data and not self._visible_text_scan_pending:
return self._visible_text_fallback_content
normalized_new_data = self._normalize_raw_text(new_data)
scan_text = f"{self._visible_text_scan_pending}{normalized_new_data}"
if not scan_text:
return self._visible_text_fallback_content
pieces, last_match_end = self._scan_visible_text_matches(scan_text)
if pieces:
self._visible_text_fallback_content += "".join(pieces)
self._visible_text_scan_pending = self._build_visible_text_scan_pending(
scan_text,
last_match_end,
)
return self._visible_text_fallback_content
@classmethod
def _looks_like_obfuscated_payload(cls, value: Any) -> bool:
text = str(value or "").strip()
if len(text) < 96:
return False
if any(ch.isspace() for ch in text):
return False
if cls._CJK_RE.search(text):
return False
if not cls._OBFUSCATED_PAYLOAD_RE.fullmatch(text):
return False
if not any(ch in "+/=_-" for ch in text):
return False
distinct_chars = len(set(text))
if distinct_chars < 20:
return False
alpha_num_ratio = sum(ch.isalnum() for ch in text) / len(text)
return alpha_num_ratio >= 0.85
@classmethod
def _looks_like_visible_text(cls, value: Any) -> bool:
text = str(value or "")
stripped = text.strip()
if not stripped:
return False
if cls._looks_like_obfuscated_payload(stripped):
return False
if cls._CJK_RE.search(stripped):
return True
if any(ch.isspace() for ch in stripped):
return True
if stripped.startswith(("{", "[", "<", "```", "http://", "https://")):
return True
punctuation_count = sum(
ch in '.,!?;:()[]{}<>"\'`~@#$%^&*\\|'
for ch in stripped
)
if punctuation_count > 0:
return True
return len(stripped) <= 80
@classmethod
def _pick_visible_text_candidate(cls, values: List[Any]) -> str:
for item in values:
if not isinstance(item, str):
continue
if cls._looks_like_visible_text(item):
return item
return ""
@staticmethod
def _decode_json_fragment(value: str) -> str:
text = str(value or "")
if not text:
return ""
try:
decoded = json.loads(f'"{text}"')
return decoded if isinstance(decoded, str) else ""
except json.JSONDecodeError:
return (
text.replace("\\n", "\n")
.replace("\\r", "\r")
.replace("\\t", "\t")
.replace('\\"', '"')
.replace("\\\\", "\\")
)
@staticmethod
def _decode_json_prefix_values(raw_text: str) -> Tuple[List[Any], str]:
decoder = json.JSONDecoder()
values: List[Any] = []
index = 0
length = len(raw_text)
while index < length:
while index < length and raw_text[index].isspace():
index += 1
if index >= length:
break
try:
value, next_index = decoder.raw_decode(raw_text, index)
except json.JSONDecodeError:
return values, raw_text[index:]
values.append(value)
index = next_index
return values, ""
@staticmethod
def _normalize_outer_blocks(data: Any) -> List[Any]:
if not isinstance(data, list) or not data:
return []
if any(isinstance(item, list) for item in data):
if (
len(data) == 1
and isinstance(data[0], list)
and any(isinstance(item, list) for item in data[0])
):
return data[0]
return data
return []
def _extract_content(self, data: Any) -> Tuple[str, str, bool]:
"""
从响应数据中提取文本内容与思考过程
Returns:
(accumulated_text, accumulated_reasoning, is_done)
"""
try:
outer = self._normalize_outer_blocks(data)
if not outer:
return "", "", False
accumulated = ""
accumulated_reasoning = ""
is_done = False
for block in outer:
if not isinstance(block, list):
continue
# 检查是否是统计块(结束标志之一)
if self._is_stats_block(block):
is_done = True
continue
# 提取文本
text, block_done, is_thinking = self._extract_block_content(block)
if is_thinking:
if text:
accumulated_reasoning += text
else:
if text:
accumulated += text
if block_done:
is_done = True
return accumulated, accumulated_reasoning, is_done
except Exception as e:
logger.debug(f"[AIStudioParser] _extract_content 异常: {e}")
return "", "", False
def _is_stats_block(self, block: list) -> bool:
"""检查是否是统计块(响应结束)"""
try:
# 格式: [null, null, null, [timestamp, ...]]
if len(block) >= 4:
if block[0] is None and block[1] is None and block[2] is None:
if isinstance(block[3], list):
return True
return False
except:
return False
def _extract_block_content(self, block: list) -> tuple:
"""
从单个块提取内容
Returns:
(text, is_done, is_thinking)
"""
try:
# 路径: block[0][0][0][0][0]
# thinking: [13 items, None, "**Thinking...", ..., 1]
# 正常内容: [2 items, None, "文本内容"]
if not isinstance(block, list) or len(block) == 0:
return "", False, False
level1 = block[0]
if not isinstance(level1, list) or len(level1) == 0:
return "", False, False
level2 = level1[0]
if not isinstance(level2, list) or len(level2) == 0:
return "", False, False
level3 = level2[0]
if not isinstance(level3, list) or len(level3) == 0:
return "", False, False
level4 = level3[0]
if not isinstance(level4, list) or len(level4) == 0:
return "", False, False
# 这里就是 content_arr
content_arr = level4[0]
if not isinstance(content_arr, list):
return "", False, False
# 提取文本:只接受像“可见正文”的字符串,避免把内部高熵 payload 误当回复。
text = ""
primary_text = content_arr[1] if len(content_arr) > 1 else ""
if self._looks_like_visible_text(primary_text):
text = primary_text
else:
text = self._pick_visible_text_candidate(content_arr[2:])
if not text:
suppressed_payload = next(
(
item for item in content_arr[2:]
if isinstance(item, str)
and self._looks_like_obfuscated_payload(item)
),
"",
)
if suppressed_payload:
logger.debug_throttled(
f"aistudio.hidden_payload.{id(self)}",
"[AIStudioParser] 忽略疑似内部高熵 payload "
f"(len={len(str(suppressed_payload).strip())})",
interval_sec=3.0,
)
# 判断是否是 thinking
# thinking 块特征:len >= 13 且索引 12 == 1
is_thinking = False
if len(content_arr) >= 13:
if len(content_arr) > 12 and content_arr[12] == 1:
is_thinking = True
# 检查是否结束
# 1. level2 的第二个元素是 1
is_done = False
if len(level2) > 1 and level2[1] == 1:
is_done = True
# 2. 有长 token 字符串(索引 13 或 14)
if len(content_arr) > 13 and isinstance(content_arr[13], str):
if len(content_arr[13]) > 50:
is_done = True
return text, is_done, is_thinking
except Exception as e:
logger.debug(f"[AIStudioParser] _extract_block_content 异常: {e}")
return "", False, False
# ============ 元数据 ============
@classmethod
def get_id(cls) -> str:
return "aistudio"
@classmethod
def get_name(cls) -> str:
return "Google AI Studio"
@classmethod
def get_description(cls) -> str:
return "解析 Google AI Studio (MakerSuite) 的 GenerateContent 响应"
@classmethod
def get_supported_patterns(cls) -> List[str]:
return ["MakerSuiteService/GenerateContent", "GenerateContent"]
def should_fallback_to_dom_when_no_visible_content(self) -> bool:
return True
__all__ = ['AIStudioParser']