Initial commit
This commit is contained in:
commit
57fc3525eb
7311 files changed
+1291398
No files matched your search
@@ -0,0 +1,514 @@
|
||||
"""
|
||||
aistudio_parser.py - Google AI Studio 响应解析器
|
||||
|
||||
响应格式特征:
|
||||
- Content-Type: application/json+protobuf
|
||||
- 嵌套数组结构
|
||||
- 增量文本模式
|
||||
- 包含 thinking 内容(需过滤)
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
from typing import Any, Dict, List, Tuple
|
||||
|
||||
from app.core.config import logger
|
||||
from .base import ResponseParser
|
||||
|
||||
|
||||
class AIStudioParser(ResponseParser):
|
||||
"""
|
||||
Google AI Studio (MakerSuite) 响应解析器
|
||||
|
||||
URL 特征: MakerSuiteService/GenerateContent
|
||||
响应格式: JSON+Protobuf (嵌套数组)
|
||||
"""
|
||||
|
||||
_VISIBLE_TEXT_BLOCK_RE = re.compile(
|
||||
r'null,"((?:\\.|[^"\\])*)"\]\],"model"'
|
||||
)
|
||||
_VISIBLE_TEXT_BLOCK_PREFIX = 'null,"'
|
||||
_OBFUSCATED_PAYLOAD_RE = re.compile(r"^[A-Za-z0-9+/=_-]+$")
|
||||
_CJK_RE = re.compile(r"[\u3400-\u9fff]")
|
||||
|
||||
def __init__(self):
|
||||
self._accumulated_content = ""
|
||||
self._accumulated_reasoning = ""
|
||||
self._is_done = False
|
||||
self._last_raw_length = 0
|
||||
self._pending_raw = ""
|
||||
self._visible_text_scan_pending = ""
|
||||
self._visible_text_fallback_content = ""
|
||||
|
||||
def parse_chunk(self, raw_response: str) -> Dict[str, Any]:
|
||||
"""
|
||||
解析响应(返回完整内容的增量)
|
||||
"""
|
||||
result = {
|
||||
"content": "",
|
||||
"reasoning_content": "",
|
||||
"images": [],
|
||||
"done": False,
|
||||
"error": None
|
||||
}
|
||||
|
||||
try:
|
||||
if isinstance(raw_response, bytes):
|
||||
raw_response = raw_response.decode("utf-8", errors="ignore")
|
||||
elif not isinstance(raw_response, str):
|
||||
raw_response = str(raw_response)
|
||||
|
||||
new_data = self._prepare_incremental_raw_response(raw_response)
|
||||
if not new_data:
|
||||
return result
|
||||
|
||||
payloads = self._decode_payloads(new_data, incremental=True)
|
||||
content = ""
|
||||
is_done = False
|
||||
|
||||
reasoning = ""
|
||||
for data in payloads:
|
||||
current_content, current_reasoning, current_done = self._extract_content(data)
|
||||
if current_content:
|
||||
content = current_content
|
||||
if current_reasoning:
|
||||
reasoning = current_reasoning
|
||||
if current_done:
|
||||
is_done = True
|
||||
|
||||
if not content:
|
||||
content = self._extract_visible_text_from_increment(new_data)
|
||||
|
||||
if content:
|
||||
if content.startswith(self._accumulated_content):
|
||||
delta = content[len(self._accumulated_content):]
|
||||
self._accumulated_content = content
|
||||
else:
|
||||
delta = content
|
||||
self._accumulated_content += content
|
||||
if delta:
|
||||
result["content"] = delta
|
||||
|
||||
if reasoning:
|
||||
if reasoning.startswith(self._accumulated_reasoning):
|
||||
delta_r = reasoning[len(self._accumulated_reasoning):]
|
||||
self._accumulated_reasoning = reasoning
|
||||
else:
|
||||
delta_r = reasoning
|
||||
self._accumulated_reasoning += reasoning
|
||||
if delta_r:
|
||||
result["reasoning_content"] = delta_r
|
||||
|
||||
result["done"] = is_done
|
||||
|
||||
except json.JSONDecodeError as e:
|
||||
logger.debug(f"[AIStudioParser] JSON 解析失败: {e}")
|
||||
result["error"] = str(e)
|
||||
except Exception as e:
|
||||
logger.debug(f"[AIStudioParser] 解析异常: {e}")
|
||||
result["error"] = str(e)
|
||||
|
||||
return result
|
||||
|
||||
def reset(self):
|
||||
"""重置状态"""
|
||||
self._accumulated_content = ""
|
||||
self._accumulated_reasoning = ""
|
||||
self._is_done = False
|
||||
self._last_raw_length = 0
|
||||
self._last_raw_response = ""
|
||||
self._pending_raw = ""
|
||||
self._visible_text_scan_pending = ""
|
||||
self._visible_text_fallback_content = ""
|
||||
|
||||
@staticmethod
|
||||
def _normalize_raw_text(raw_response: Any) -> str:
|
||||
if isinstance(raw_response, bytes):
|
||||
raw_response = raw_response.decode("utf-8", errors="ignore")
|
||||
elif not isinstance(raw_response, str):
|
||||
raw_response = str(raw_response)
|
||||
|
||||
text = raw_response.lstrip("\ufeff")
|
||||
stripped = text.lstrip()
|
||||
if stripped.startswith(")]}'"):
|
||||
stripped = stripped[4:].lstrip("\r\n")
|
||||
return stripped
|
||||
|
||||
def _decode_payloads(self, raw_response: Any, incremental: bool = False) -> List[Any]:
|
||||
if isinstance(raw_response, (list, dict)):
|
||||
return [raw_response]
|
||||
|
||||
normalized = self._normalize_raw_text(raw_response)
|
||||
if not normalized:
|
||||
return []
|
||||
|
||||
if incremental:
|
||||
normalized = f"{self._pending_raw}{normalized}"
|
||||
|
||||
try:
|
||||
payload = json.loads(normalized)
|
||||
if incremental:
|
||||
self._pending_raw = ""
|
||||
return [payload]
|
||||
except json.JSONDecodeError as exc:
|
||||
payloads, remainder = self._decode_json_prefix_values(normalized)
|
||||
if incremental:
|
||||
self._pending_raw = remainder
|
||||
if payloads:
|
||||
return payloads
|
||||
|
||||
repaired = self._try_repair_unclosed_json(normalized)
|
||||
if repaired is not None:
|
||||
return [repaired]
|
||||
|
||||
logger.debug_throttled(
|
||||
f"aistudio.incomplete_json.{id(self)}",
|
||||
"[AIStudioParser] JSON 未完整,继续等待流增长 "
|
||||
f"(body_len={len(normalized)}, error={exc})",
|
||||
interval_sec=3.0,
|
||||
)
|
||||
return []
|
||||
|
||||
@staticmethod
|
||||
def _try_repair_unclosed_json(text: str) -> Any:
|
||||
stripped = text.strip()
|
||||
if not stripped:
|
||||
return None
|
||||
for pad in range(0, 30):
|
||||
try:
|
||||
attempt = stripped + (']' * pad)
|
||||
return json.loads(attempt)
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
attempt = stripped + '"]' + (']' * pad)
|
||||
return json.loads(attempt)
|
||||
except Exception:
|
||||
pass
|
||||
return None
|
||||
|
||||
@classmethod
|
||||
def _extract_visible_text_from_raw_body(cls, raw_response: Any) -> str:
|
||||
raw_text = cls._normalize_raw_text(raw_response)
|
||||
if not raw_text:
|
||||
return ""
|
||||
|
||||
pieces, _last_match_end = cls._scan_visible_text_matches(raw_text)
|
||||
return "".join(pieces)
|
||||
|
||||
@classmethod
|
||||
def _scan_visible_text_matches(cls, raw_text: str) -> Tuple[List[str], int]:
|
||||
pieces: List[str] = []
|
||||
last_match_end = 0
|
||||
for match in cls._VISIBLE_TEXT_BLOCK_RE.finditer(raw_text):
|
||||
last_match_end = match.end()
|
||||
piece = cls._decode_json_fragment(match.group(1))
|
||||
if piece and cls._looks_like_visible_text(piece):
|
||||
pieces.append(piece)
|
||||
|
||||
return pieces, last_match_end
|
||||
|
||||
@classmethod
|
||||
def _build_visible_text_scan_pending(cls, scan_text: str, last_match_end: int) -> str:
|
||||
remainder = scan_text[last_match_end:] if last_match_end > 0 else scan_text
|
||||
marker_index = remainder.rfind(cls._VISIBLE_TEXT_BLOCK_PREFIX)
|
||||
if marker_index >= 0:
|
||||
return remainder[marker_index:]
|
||||
|
||||
keep_chars = max(0, len(cls._VISIBLE_TEXT_BLOCK_PREFIX) - 1)
|
||||
return remainder[-keep_chars:] if keep_chars else ""
|
||||
|
||||
def _extract_visible_text_from_increment(self, new_data: str) -> str:
|
||||
if not new_data and not self._visible_text_scan_pending:
|
||||
return self._visible_text_fallback_content
|
||||
|
||||
normalized_new_data = self._normalize_raw_text(new_data)
|
||||
scan_text = f"{self._visible_text_scan_pending}{normalized_new_data}"
|
||||
if not scan_text:
|
||||
return self._visible_text_fallback_content
|
||||
|
||||
pieces, last_match_end = self._scan_visible_text_matches(scan_text)
|
||||
if pieces:
|
||||
self._visible_text_fallback_content += "".join(pieces)
|
||||
self._visible_text_scan_pending = self._build_visible_text_scan_pending(
|
||||
scan_text,
|
||||
last_match_end,
|
||||
)
|
||||
return self._visible_text_fallback_content
|
||||
|
||||
@classmethod
|
||||
def _looks_like_obfuscated_payload(cls, value: Any) -> bool:
|
||||
text = str(value or "").strip()
|
||||
if len(text) < 96:
|
||||
return False
|
||||
if any(ch.isspace() for ch in text):
|
||||
return False
|
||||
if cls._CJK_RE.search(text):
|
||||
return False
|
||||
if not cls._OBFUSCATED_PAYLOAD_RE.fullmatch(text):
|
||||
return False
|
||||
if not any(ch in "+/=_-" for ch in text):
|
||||
return False
|
||||
|
||||
distinct_chars = len(set(text))
|
||||
if distinct_chars < 20:
|
||||
return False
|
||||
|
||||
alpha_num_ratio = sum(ch.isalnum() for ch in text) / len(text)
|
||||
return alpha_num_ratio >= 0.85
|
||||
|
||||
@classmethod
|
||||
def _looks_like_visible_text(cls, value: Any) -> bool:
|
||||
text = str(value or "")
|
||||
stripped = text.strip()
|
||||
if not stripped:
|
||||
return False
|
||||
if cls._looks_like_obfuscated_payload(stripped):
|
||||
return False
|
||||
if cls._CJK_RE.search(stripped):
|
||||
return True
|
||||
if any(ch.isspace() for ch in stripped):
|
||||
return True
|
||||
if stripped.startswith(("{", "[", "<", "```", "http://", "https://")):
|
||||
return True
|
||||
|
||||
punctuation_count = sum(
|
||||
ch in '.,!?;:()[]{}<>"\'`~@#$%^&*\\|'
|
||||
for ch in stripped
|
||||
)
|
||||
if punctuation_count > 0:
|
||||
return True
|
||||
|
||||
return len(stripped) <= 80
|
||||
|
||||
@classmethod
|
||||
def _pick_visible_text_candidate(cls, values: List[Any]) -> str:
|
||||
for item in values:
|
||||
if not isinstance(item, str):
|
||||
continue
|
||||
if cls._looks_like_visible_text(item):
|
||||
return item
|
||||
return ""
|
||||
|
||||
@staticmethod
|
||||
def _decode_json_fragment(value: str) -> str:
|
||||
text = str(value or "")
|
||||
if not text:
|
||||
return ""
|
||||
|
||||
try:
|
||||
decoded = json.loads(f'"{text}"')
|
||||
return decoded if isinstance(decoded, str) else ""
|
||||
except json.JSONDecodeError:
|
||||
return (
|
||||
text.replace("\\n", "\n")
|
||||
.replace("\\r", "\r")
|
||||
.replace("\\t", "\t")
|
||||
.replace('\\"', '"')
|
||||
.replace("\\\\", "\\")
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _decode_json_prefix_values(raw_text: str) -> Tuple[List[Any], str]:
|
||||
decoder = json.JSONDecoder()
|
||||
values: List[Any] = []
|
||||
index = 0
|
||||
length = len(raw_text)
|
||||
|
||||
while index < length:
|
||||
while index < length and raw_text[index].isspace():
|
||||
index += 1
|
||||
if index >= length:
|
||||
break
|
||||
|
||||
try:
|
||||
value, next_index = decoder.raw_decode(raw_text, index)
|
||||
except json.JSONDecodeError:
|
||||
return values, raw_text[index:]
|
||||
|
||||
values.append(value)
|
||||
index = next_index
|
||||
|
||||
return values, ""
|
||||
|
||||
@staticmethod
|
||||
def _normalize_outer_blocks(data: Any) -> List[Any]:
|
||||
if not isinstance(data, list) or not data:
|
||||
return []
|
||||
|
||||
if any(isinstance(item, list) for item in data):
|
||||
if (
|
||||
len(data) == 1
|
||||
and isinstance(data[0], list)
|
||||
and any(isinstance(item, list) for item in data[0])
|
||||
):
|
||||
return data[0]
|
||||
return data
|
||||
|
||||
return []
|
||||
|
||||
def _extract_content(self, data: Any) -> Tuple[str, str, bool]:
|
||||
"""
|
||||
从响应数据中提取文本内容与思考过程
|
||||
|
||||
Returns:
|
||||
(accumulated_text, accumulated_reasoning, is_done)
|
||||
"""
|
||||
try:
|
||||
outer = self._normalize_outer_blocks(data)
|
||||
if not outer:
|
||||
return "", "", False
|
||||
|
||||
accumulated = ""
|
||||
accumulated_reasoning = ""
|
||||
is_done = False
|
||||
|
||||
for block in outer:
|
||||
if not isinstance(block, list):
|
||||
continue
|
||||
|
||||
# 检查是否是统计块(结束标志之一)
|
||||
if self._is_stats_block(block):
|
||||
is_done = True
|
||||
continue
|
||||
|
||||
# 提取文本
|
||||
text, block_done, is_thinking = self._extract_block_content(block)
|
||||
|
||||
if is_thinking:
|
||||
if text:
|
||||
accumulated_reasoning += text
|
||||
else:
|
||||
if text:
|
||||
accumulated += text
|
||||
|
||||
if block_done:
|
||||
is_done = True
|
||||
|
||||
return accumulated, accumulated_reasoning, is_done
|
||||
|
||||
except Exception as e:
|
||||
logger.debug(f"[AIStudioParser] _extract_content 异常: {e}")
|
||||
return "", "", False
|
||||
|
||||
def _is_stats_block(self, block: list) -> bool:
|
||||
"""检查是否是统计块(响应结束)"""
|
||||
try:
|
||||
# 格式: [null, null, null, [timestamp, ...]]
|
||||
if len(block) >= 4:
|
||||
if block[0] is None and block[1] is None and block[2] is None:
|
||||
if isinstance(block[3], list):
|
||||
return True
|
||||
return False
|
||||
except:
|
||||
return False
|
||||
|
||||
def _extract_block_content(self, block: list) -> tuple:
|
||||
"""
|
||||
从单个块提取内容
|
||||
|
||||
Returns:
|
||||
(text, is_done, is_thinking)
|
||||
"""
|
||||
try:
|
||||
# 路径: block[0][0][0][0][0]
|
||||
# thinking: [13 items, None, "**Thinking...", ..., 1]
|
||||
# 正常内容: [2 items, None, "文本内容"]
|
||||
|
||||
if not isinstance(block, list) or len(block) == 0:
|
||||
return "", False, False
|
||||
|
||||
level1 = block[0]
|
||||
if not isinstance(level1, list) or len(level1) == 0:
|
||||
return "", False, False
|
||||
|
||||
level2 = level1[0]
|
||||
if not isinstance(level2, list) or len(level2) == 0:
|
||||
return "", False, False
|
||||
|
||||
level3 = level2[0]
|
||||
if not isinstance(level3, list) or len(level3) == 0:
|
||||
return "", False, False
|
||||
|
||||
level4 = level3[0]
|
||||
if not isinstance(level4, list) or len(level4) == 0:
|
||||
return "", False, False
|
||||
|
||||
# 这里就是 content_arr
|
||||
content_arr = level4[0]
|
||||
if not isinstance(content_arr, list):
|
||||
return "", False, False
|
||||
|
||||
# 提取文本:只接受像“可见正文”的字符串,避免把内部高熵 payload 误当回复。
|
||||
text = ""
|
||||
primary_text = content_arr[1] if len(content_arr) > 1 else ""
|
||||
if self._looks_like_visible_text(primary_text):
|
||||
text = primary_text
|
||||
else:
|
||||
text = self._pick_visible_text_candidate(content_arr[2:])
|
||||
if not text:
|
||||
suppressed_payload = next(
|
||||
(
|
||||
item for item in content_arr[2:]
|
||||
if isinstance(item, str)
|
||||
and self._looks_like_obfuscated_payload(item)
|
||||
),
|
||||
"",
|
||||
)
|
||||
if suppressed_payload:
|
||||
logger.debug_throttled(
|
||||
f"aistudio.hidden_payload.{id(self)}",
|
||||
"[AIStudioParser] 忽略疑似内部高熵 payload "
|
||||
f"(len={len(str(suppressed_payload).strip())})",
|
||||
interval_sec=3.0,
|
||||
)
|
||||
|
||||
# 判断是否是 thinking
|
||||
# thinking 块特征:len >= 13 且索引 12 == 1
|
||||
is_thinking = False
|
||||
if len(content_arr) >= 13:
|
||||
if len(content_arr) > 12 and content_arr[12] == 1:
|
||||
is_thinking = True
|
||||
|
||||
# 检查是否结束
|
||||
# 1. level2 的第二个元素是 1
|
||||
is_done = False
|
||||
if len(level2) > 1 and level2[1] == 1:
|
||||
is_done = True
|
||||
|
||||
# 2. 有长 token 字符串(索引 13 或 14)
|
||||
if len(content_arr) > 13 and isinstance(content_arr[13], str):
|
||||
if len(content_arr[13]) > 50:
|
||||
is_done = True
|
||||
|
||||
return text, is_done, is_thinking
|
||||
|
||||
except Exception as e:
|
||||
logger.debug(f"[AIStudioParser] _extract_block_content 异常: {e}")
|
||||
return "", False, False
|
||||
|
||||
# ============ 元数据 ============
|
||||
|
||||
@classmethod
|
||||
def get_id(cls) -> str:
|
||||
return "aistudio"
|
||||
|
||||
@classmethod
|
||||
def get_name(cls) -> str:
|
||||
return "Google AI Studio"
|
||||
|
||||
@classmethod
|
||||
def get_description(cls) -> str:
|
||||
return "解析 Google AI Studio (MakerSuite) 的 GenerateContent 响应"
|
||||
|
||||
@classmethod
|
||||
def get_supported_patterns(cls) -> List[str]:
|
||||
return ["MakerSuiteService/GenerateContent", "GenerateContent"]
|
||||
|
||||
def should_fallback_to_dom_when_no_visible_content(self) -> bool:
|
||||
return True
|
||||
|
||||
|
||||
__all__ = ['AIStudioParser']
|
||||
Reference in new issue
Block a user