Initial commit

This commit is contained in:
Alex committed 2026-11-02 22:54:14 +08:00
commit 57fc3525eb
7311 files changed
+1291398

No files matched your search

+514
View File
@@ -0,0 +1,514 @@
"""
aistudio_parser.py - Google AI Studio 响应解析器
响应格式特征:
- Content-Type: application/json+protobuf
- 嵌套数组结构
- 增量文本模式
- 包含 thinking 内容(需过滤)
"""
from __future__ import annotations
import json
import re
from typing import Any, Dict, List, Tuple
from app.core.config import logger
from .base import ResponseParser
class AIStudioParser(ResponseParser):
"""
Google AI Studio (MakerSuite) 响应解析器
URL 特征: MakerSuiteService/GenerateContent
响应格式: JSON+Protobuf (嵌套数组)
"""
_VISIBLE_TEXT_BLOCK_RE = re.compile(
r'null,"((?:\\.|[^"\\])*)"\]\],"model"'
)
_VISIBLE_TEXT_BLOCK_PREFIX = 'null,"'
_OBFUSCATED_PAYLOAD_RE = re.compile(r"^[A-Za-z0-9+/=_-]+$")
_CJK_RE = re.compile(r"[\u3400-\u9fff]")
def __init__(self):
self._accumulated_content = ""
self._accumulated_reasoning = ""
self._is_done = False
self._last_raw_length = 0
self._pending_raw = ""
self._visible_text_scan_pending = ""
self._visible_text_fallback_content = ""
def parse_chunk(self, raw_response: str) -> Dict[str, Any]:
"""
解析响应(返回完整内容的增量)
"""
result = {
"content": "",
"reasoning_content": "",
"images": [],
"done": False,
"error": None
}
try:
if isinstance(raw_response, bytes):
raw_response = raw_response.decode("utf-8", errors="ignore")
elif not isinstance(raw_response, str):
raw_response = str(raw_response)
new_data = self._prepare_incremental_raw_response(raw_response)
if not new_data:
return result
payloads = self._decode_payloads(new_data, incremental=True)
content = ""
is_done = False
reasoning = ""
for data in payloads:
current_content, current_reasoning, current_done = self._extract_content(data)
if current_content:
content = current_content
if current_reasoning:
reasoning = current_reasoning
if current_done:
is_done = True
if not content:
content = self._extract_visible_text_from_increment(new_data)
if content:
if content.startswith(self._accumulated_content):
delta = content[len(self._accumulated_content):]
self._accumulated_content = content
else:
delta = content
self._accumulated_content += content
if delta:
result["content"] = delta
if reasoning:
if reasoning.startswith(self._accumulated_reasoning):
delta_r = reasoning[len(self._accumulated_reasoning):]
self._accumulated_reasoning = reasoning
else:
delta_r = reasoning
self._accumulated_reasoning += reasoning
if delta_r:
result["reasoning_content"] = delta_r
result["done"] = is_done
except json.JSONDecodeError as e:
logger.debug(f"[AIStudioParser] JSON 解析失败: {e}")
result["error"] = str(e)
except Exception as e:
logger.debug(f"[AIStudioParser] 解析异常: {e}")
result["error"] = str(e)
return result
def reset(self):
"""重置状态"""
self._accumulated_content = ""
self._accumulated_reasoning = ""
self._is_done = False
self._last_raw_length = 0
self._last_raw_response = ""
self._pending_raw = ""
self._visible_text_scan_pending = ""
self._visible_text_fallback_content = ""
@staticmethod
def _normalize_raw_text(raw_response: Any) -> str:
if isinstance(raw_response, bytes):
raw_response = raw_response.decode("utf-8", errors="ignore")
elif not isinstance(raw_response, str):
raw_response = str(raw_response)
text = raw_response.lstrip("\ufeff")
stripped = text.lstrip()
if stripped.startswith(")]}'"):
stripped = stripped[4:].lstrip("\r\n")
return stripped
def _decode_payloads(self, raw_response: Any, incremental: bool = False) -> List[Any]:
if isinstance(raw_response, (list, dict)):
return [raw_response]
normalized = self._normalize_raw_text(raw_response)
if not normalized:
return []
if incremental:
normalized = f"{self._pending_raw}{normalized}"
try:
payload = json.loads(normalized)
if incremental:
self._pending_raw = ""
return [payload]
except json.JSONDecodeError as exc:
payloads, remainder = self._decode_json_prefix_values(normalized)
if incremental:
self._pending_raw = remainder
if payloads:
return payloads
repaired = self._try_repair_unclosed_json(normalized)
if repaired is not None:
return [repaired]
logger.debug_throttled(
f"aistudio.incomplete_json.{id(self)}",
"[AIStudioParser] JSON 未完整,继续等待流增长 "
f"(body_len={len(normalized)}, error={exc})",
interval_sec=3.0,
)
return []
@staticmethod
def _try_repair_unclosed_json(text: str) -> Any:
stripped = text.strip()
if not stripped:
return None
for pad in range(0, 30):
try:
attempt = stripped + (']' * pad)
return json.loads(attempt)
except Exception:
pass
try:
attempt = stripped + '"]' + (']' * pad)
return json.loads(attempt)
except Exception:
pass
return None
@classmethod
def _extract_visible_text_from_raw_body(cls, raw_response: Any) -> str:
raw_text = cls._normalize_raw_text(raw_response)
if not raw_text:
return ""
pieces, _last_match_end = cls._scan_visible_text_matches(raw_text)
return "".join(pieces)
@classmethod
def _scan_visible_text_matches(cls, raw_text: str) -> Tuple[List[str], int]:
pieces: List[str] = []
last_match_end = 0
for match in cls._VISIBLE_TEXT_BLOCK_RE.finditer(raw_text):
last_match_end = match.end()
piece = cls._decode_json_fragment(match.group(1))
if piece and cls._looks_like_visible_text(piece):
pieces.append(piece)
return pieces, last_match_end
@classmethod
def _build_visible_text_scan_pending(cls, scan_text: str, last_match_end: int) -> str:
remainder = scan_text[last_match_end:] if last_match_end > 0 else scan_text
marker_index = remainder.rfind(cls._VISIBLE_TEXT_BLOCK_PREFIX)
if marker_index >= 0:
return remainder[marker_index:]
keep_chars = max(0, len(cls._VISIBLE_TEXT_BLOCK_PREFIX) - 1)
return remainder[-keep_chars:] if keep_chars else ""
def _extract_visible_text_from_increment(self, new_data: str) -> str:
if not new_data and not self._visible_text_scan_pending:
return self._visible_text_fallback_content
normalized_new_data = self._normalize_raw_text(new_data)
scan_text = f"{self._visible_text_scan_pending}{normalized_new_data}"
if not scan_text:
return self._visible_text_fallback_content
pieces, last_match_end = self._scan_visible_text_matches(scan_text)
if pieces:
self._visible_text_fallback_content += "".join(pieces)
self._visible_text_scan_pending = self._build_visible_text_scan_pending(
scan_text,
last_match_end,
)
return self._visible_text_fallback_content
@classmethod
def _looks_like_obfuscated_payload(cls, value: Any) -> bool:
text = str(value or "").strip()
if len(text) < 96:
return False
if any(ch.isspace() for ch in text):
return False
if cls._CJK_RE.search(text):
return False
if not cls._OBFUSCATED_PAYLOAD_RE.fullmatch(text):
return False
if not any(ch in "+/=_-" for ch in text):
return False
distinct_chars = len(set(text))
if distinct_chars < 20:
return False
alpha_num_ratio = sum(ch.isalnum() for ch in text) / len(text)
return alpha_num_ratio >= 0.85
@classmethod
def _looks_like_visible_text(cls, value: Any) -> bool:
text = str(value or "")
stripped = text.strip()
if not stripped:
return False
if cls._looks_like_obfuscated_payload(stripped):
return False
if cls._CJK_RE.search(stripped):
return True
if any(ch.isspace() for ch in stripped):
return True
if stripped.startswith(("{", "[", "<", "```", "http://", "https://")):
return True
punctuation_count = sum(
ch in '.,!?;:()[]{}<>"\'`~@#$%^&*\\|'
for ch in stripped
)
if punctuation_count > 0:
return True
return len(stripped) <= 80
@classmethod
def _pick_visible_text_candidate(cls, values: List[Any]) -> str:
for item in values:
if not isinstance(item, str):
continue
if cls._looks_like_visible_text(item):
return item
return ""
@staticmethod
def _decode_json_fragment(value: str) -> str:
text = str(value or "")
if not text:
return ""
try:
decoded = json.loads(f'"{text}"')
return decoded if isinstance(decoded, str) else ""
except json.JSONDecodeError:
return (
text.replace("\\n", "\n")
.replace("\\r", "\r")
.replace("\\t", "\t")
.replace('\\"', '"')
.replace("\\\\", "\\")
)
@staticmethod
def _decode_json_prefix_values(raw_text: str) -> Tuple[List[Any], str]:
decoder = json.JSONDecoder()
values: List[Any] = []
index = 0
length = len(raw_text)
while index < length:
while index < length and raw_text[index].isspace():
index += 1
if index >= length:
break
try:
value, next_index = decoder.raw_decode(raw_text, index)
except json.JSONDecodeError:
return values, raw_text[index:]
values.append(value)
index = next_index
return values, ""
@staticmethod
def _normalize_outer_blocks(data: Any) -> List[Any]:
if not isinstance(data, list) or not data:
return []
if any(isinstance(item, list) for item in data):
if (
len(data) == 1
and isinstance(data[0], list)
and any(isinstance(item, list) for item in data[0])
):
return data[0]
return data
return []
def _extract_content(self, data: Any) -> Tuple[str, str, bool]:
"""
从响应数据中提取文本内容与思考过程
Returns:
(accumulated_text, accumulated_reasoning, is_done)
"""
try:
outer = self._normalize_outer_blocks(data)
if not outer:
return "", "", False
accumulated = ""
accumulated_reasoning = ""
is_done = False
for block in outer:
if not isinstance(block, list):
continue
# 检查是否是统计块(结束标志之一)
if self._is_stats_block(block):
is_done = True
continue
# 提取文本
text, block_done, is_thinking = self._extract_block_content(block)
if is_thinking:
if text:
accumulated_reasoning += text
else:
if text:
accumulated += text
if block_done:
is_done = True
return accumulated, accumulated_reasoning, is_done
except Exception as e:
logger.debug(f"[AIStudioParser] _extract_content 异常: {e}")
return "", "", False
def _is_stats_block(self, block: list) -> bool:
"""检查是否是统计块(响应结束)"""
try:
# 格式: [null, null, null, [timestamp, ...]]
if len(block) >= 4:
if block[0] is None and block[1] is None and block[2] is None:
if isinstance(block[3], list):
return True
return False
except:
return False
def _extract_block_content(self, block: list) -> tuple:
"""
从单个块提取内容
Returns:
(text, is_done, is_thinking)
"""
try:
# 路径: block[0][0][0][0][0]
# thinking: [13 items, None, "**Thinking...", ..., 1]
# 正常内容: [2 items, None, "文本内容"]
if not isinstance(block, list) or len(block) == 0:
return "", False, False
level1 = block[0]
if not isinstance(level1, list) or len(level1) == 0:
return "", False, False
level2 = level1[0]
if not isinstance(level2, list) or len(level2) == 0:
return "", False, False
level3 = level2[0]
if not isinstance(level3, list) or len(level3) == 0:
return "", False, False
level4 = level3[0]
if not isinstance(level4, list) or len(level4) == 0:
return "", False, False
# 这里就是 content_arr
content_arr = level4[0]
if not isinstance(content_arr, list):
return "", False, False
# 提取文本:只接受像“可见正文”的字符串,避免把内部高熵 payload 误当回复。
text = ""
primary_text = content_arr[1] if len(content_arr) > 1 else ""
if self._looks_like_visible_text(primary_text):
text = primary_text
else:
text = self._pick_visible_text_candidate(content_arr[2:])
if not text:
suppressed_payload = next(
(
item for item in content_arr[2:]
if isinstance(item, str)
and self._looks_like_obfuscated_payload(item)
),
"",
)
if suppressed_payload:
logger.debug_throttled(
f"aistudio.hidden_payload.{id(self)}",
"[AIStudioParser] 忽略疑似内部高熵 payload "
f"(len={len(str(suppressed_payload).strip())})",
interval_sec=3.0,
)
# 判断是否是 thinking
# thinking 块特征:len >= 13 且索引 12 == 1
is_thinking = False
if len(content_arr) >= 13:
if len(content_arr) > 12 and content_arr[12] == 1:
is_thinking = True
# 检查是否结束
# 1. level2 的第二个元素是 1
is_done = False
if len(level2) > 1 and level2[1] == 1:
is_done = True
# 2. 有长 token 字符串(索引 13 或 14)
if len(content_arr) > 13 and isinstance(content_arr[13], str):
if len(content_arr[13]) > 50:
is_done = True
return text, is_done, is_thinking
except Exception as e:
logger.debug(f"[AIStudioParser] _extract_block_content 异常: {e}")
return "", False, False
# ============ 元数据 ============
@classmethod
def get_id(cls) -> str:
return "aistudio"
@classmethod
def get_name(cls) -> str:
return "Google AI Studio"
@classmethod
def get_description(cls) -> str:
return "解析 Google AI Studio (MakerSuite) 的 GenerateContent 响应"
@classmethod
def get_supported_patterns(cls) -> List[str]:
return ["MakerSuiteService/GenerateContent", "GenerateContent"]
def should_fallback_to_dom_when_no_visible_content(self) -> bool:
return True
__all__ = ['AIStudioParser']