Initial commit

This commit is contained in:
Alex committed 2026-11-02 22:54:14 +08:00
commit 57fc3525eb
7311 files changed
+1291398

No files matched your search

+70
View File
@@ -0,0 +1,70 @@
"""
app/core/page_capture - page-side provider helpers.
This package groups provider-specific page runtime capabilities while keeping
their dispatch APIs small and generic.
"""
from .base import PageFetchCapture
from .registry import (
create_page_fetch_capture as _create_page_fetch_capture,
register_page_fetch_capture,
)
from .request_transport import (
execute_page_request_transport as _execute_page_request_transport,
get_page_request_transport_profile as _get_page_request_transport_profile,
get_page_request_transport_profiles as _get_page_request_transport_profiles,
register_page_request_transport,
)
_FETCH_CAPTURE_BUILTINS_LOADED = False
_REQUEST_TRANSPORT_BUILTINS_LOADED = False
def _load_builtin_fetch_captures() -> None:
global _FETCH_CAPTURE_BUILTINS_LOADED
if _FETCH_CAPTURE_BUILTINS_LOADED:
return
from . import kimi_fetch_capture as _kimi_fetch_capture # noqa: F401
_FETCH_CAPTURE_BUILTINS_LOADED = True
def _load_builtin_request_transports() -> None:
global _REQUEST_TRANSPORT_BUILTINS_LOADED
if _REQUEST_TRANSPORT_BUILTINS_LOADED:
return
from . import deepseek_request_transport as _deepseek_request_transport # noqa: F401
_REQUEST_TRANSPORT_BUILTINS_LOADED = True
def create_page_fetch_capture(*args, **kwargs):
_load_builtin_fetch_captures()
return _create_page_fetch_capture(*args, **kwargs)
def get_page_request_transport_profiles():
_load_builtin_request_transports()
return _get_page_request_transport_profiles()
def get_page_request_transport_profile(profile_id):
_load_builtin_request_transports()
return _get_page_request_transport_profile(profile_id)
def execute_page_request_transport(*args, **kwargs):
_load_builtin_request_transports()
return _execute_page_request_transport(*args, **kwargs)
__all__ = [
"PageFetchCapture",
"create_page_fetch_capture",
"execute_page_request_transport",
"get_page_request_transport_profile",
"get_page_request_transport_profiles",
"register_page_fetch_capture",
"register_page_request_transport",
]
+71
View File
@@ -0,0 +1,71 @@
"""
app/core/page_capture/base.py - shared page-side fetch capture contract.
"""
from __future__ import annotations
from contextlib import nullcontext
from typing import Any, Callable, ContextManager, Dict, Generator, Optional
from app.core.config import SSEFormatter
InteractionSlot = Callable[[str, str], ContextManager[bool]]
class PageFetchCapture:
"""Base class for page-runtime fetch capture implementations."""
parser_id = ""
monitor_id = "page_fetch"
mode_name = "页面抓流"
def __init__(
self,
*,
tab,
formatter: SSEFormatter,
stream_config: Optional[Dict[str, Any]] = None,
network_config: Optional[Dict[str, Any]] = None,
stop_checker: Optional[Callable[[], bool]] = None,
interaction_slot: Optional[InteractionSlot] = None,
) -> None:
self.tab = tab
self.formatter = formatter
self._stream_config = stream_config or {}
self._network_config = network_config or {}
self._should_stop = stop_checker or (lambda: False)
self._interaction_slot = interaction_slot
self._sent_content_length = 0
@classmethod
def get_parser_id(cls) -> str:
return cls.parser_id
@classmethod
def get_monitor_id(cls) -> str:
return cls.monitor_id
@classmethod
def get_mode_name(cls) -> str:
return cls.mode_name
def prepare(self) -> None:
raise NotImplementedError
def monitor(self, completion_id: str) -> Generator[str, None, None]:
raise NotImplementedError
def get_sent_content_length(self) -> int:
try:
return max(0, int(self._sent_content_length or 0))
except Exception:
return 0
def _check_cancelled(self) -> bool:
return bool(self._should_stop())
def _page_interaction_slot(self, action: str, target_key: str = ""):
if callable(self._interaction_slot):
return self._interaction_slot(action, target_key)
return nullcontext(True)
@@ -0,0 +1,425 @@
"""
app/core/page_capture/deepseek_request_transport.py - DeepSeek page-side completion transport.
"""
from __future__ import annotations
import json
from typing import Any, Dict
from app.core.config import logger
from .request_transport import register_page_request_transport
from app.core.driver import driver_for_tab
DEEPSEEK_COMPLETION_PROFILE: Dict[str, Any] = {
"id": "deepseek_completion",
"name": "DeepSeek 直发",
"description": "在当前 DeepSeek 页面中直接新建会话并发起 completion 请求。",
"transport_mode": "page_fetch",
"supported_domains": ["chat.deepseek.com"],
"supported_parsers": ["deepseek"],
"supports_images": False,
"supports_file_inputs": False,
"options": [
{
"key": "model_type",
"label": "模型类型",
"type": "enum",
"default": "auto",
"description": "auto 跟随 profile 配置或页面状态,也可固定为 default / expert / vision。",
"choices": [
{"value": "auto", "label": "自动"},
{"value": "default", "label": "快速模式"},
{"value": "expert", "label": "专家模式"},
{"value": "vision", "label": "识图模式"},
],
},
{
"key": "context_mode",
"label": "上下文模式",
"type": "enum",
"default": "full_prompt",
"description": "v1 固定使用主链已构造好的完整 prompt 字符串。",
"choices": [
{"value": "full_prompt", "label": "完整上下文 prompt"},
],
},
{
"key": "search_enabled",
"label": "联网搜索",
"type": "enum",
"default": "auto",
"description": "auto 跟随页面当前开关,也可强制开/关。",
"choices": [
{"value": "auto", "label": "自动"},
{"value": "on", "label": "开启"},
{"value": "off", "label": "关闭"},
],
},
{
"key": "thinking_enabled",
"label": "深度思考",
"type": "enum",
"default": "auto",
"description": "auto 跟随页面当前开关,也可强制开/关。",
"choices": [
{"value": "auto", "label": "自动"},
{"value": "on", "label": "开启"},
{"value": "off", "label": "关闭"},
],
},
{
"key": "fallback_mode",
"label": "失败回退",
"type": "enum",
"default": "workflow",
"description": "workflow 表示直发失败时回退原工作流。",
"choices": [
{"value": "workflow", "label": "回退工作流"},
{"value": "error", "label": "直接报错"},
],
},
{
"key": "client_version",
"label": "客户端版本",
"type": "string",
"default": "2.0.0",
"description": "附加到 x-client-version 请求头。",
},
{
"key": "app_version",
"label": "应用版本",
"type": "string",
"default": "2.0.0",
"description": "附加到 x-app-version 请求头。",
},
],
}
def _build_deepseek_completion_script(
prompt: str,
options: Dict[str, Any],
*,
consume_response: bool,
) -> str:
explicit_model_type = str(options.get("model_type") or "").strip().lower()
if explicit_model_type not in {"", "auto", "default", "expert", "vision"}:
explicit_model_type = ""
context_mode = str(options.get("context_mode") or "full_prompt").strip().lower() or "full_prompt"
if context_mode != "full_prompt":
context_mode = "full_prompt"
search_enabled = str(options.get("search_enabled") or "auto").strip().lower() or "auto"
thinking_enabled = str(options.get("thinking_enabled") or "auto").strip().lower() or "auto"
client_version = str(options.get("client_version") or "2.0.0").strip() or "2.0.0"
app_version = str(options.get("app_version") or client_version).strip() or client_version
script = f"""
return (async () => {{
const prompt = {json.dumps(prompt, ensure_ascii=False)};
const preferredModelType = {json.dumps(explicit_model_type)};
const contextMode = {json.dumps(context_mode)};
const searchEnabledMode = {json.dumps(search_enabled)};
const thinkingEnabledMode = {json.dumps(thinking_enabled)};
const clientVersion = {json.dumps(client_version)};
const appVersion = {json.dumps(app_version)};
const consumeResponse = {str(bool(consume_response)).lower()};
const chunk = window.webpackChunk_deepseek_chat;
if (!chunk || typeof chunk.push !== 'function') {{
return {{ ok: false, error: 'deepseek_webpack_missing' }};
}}
let __req = null;
chunk.push([[Symbol('request_transport_deepseek_completion')], {{}}, (req) => {{ __req = req; }}]);
if (!__req) {{
return {{ ok: false, error: 'deepseek_require_missing' }};
}}
const app = __req('86389').y;
const createPowManager = __req('87109').Qc;
const buildPowHeader = __req('42587').Bx;
const base64Encode = (text) => btoa(unescape(encodeURIComponent(text)));
const tracker = {{
info() {{}},
error() {{}},
withError: (error, payload) => ({{
...(payload || {{}}),
error: String(error && error.message ? error.message : error),
}}),
}};
if (contextMode !== 'full_prompt') {{
return {{ ok: false, error: 'deepseek_context_mode_unsupported' }};
}}
const readStoredBool = (key, fallbackValue) => {{
try {{
const raw = localStorage.getItem(key);
if (!raw) return fallbackValue;
const parsed = JSON.parse(raw);
if (typeof parsed === 'boolean') return parsed;
if (parsed && typeof parsed === 'object' && 'value' in parsed) {{
return !!parsed.value;
}}
return !!parsed;
}} catch (error) {{
return fallbackValue;
}}
}};
const resolveToggleMode = (mode, storageKey, fallbackValue) => {{
const text = String(mode || '').trim().toLowerCase();
if (text === 'on' || text === 'true' || text === '1') return true;
if (text === 'off' || text === 'false' || text === '0') return false;
return readStoredBool(storageKey, fallbackValue);
}};
const getToken = () => {{
try {{
if (app && typeof app.getStorageUserToken === 'function') {{
const tokenValue = app.getStorageUserToken();
if (typeof tokenValue === 'string' && tokenValue) return tokenValue;
if (tokenValue && typeof tokenValue === 'object' && tokenValue.value) {{
return String(tokenValue.value);
}}
}}
}} catch (error) {{}}
try {{
const raw = localStorage.getItem('userToken');
if (!raw) return '';
const parsed = JSON.parse(raw);
if (typeof parsed === 'string' && parsed) return parsed;
if (parsed && typeof parsed === 'object' && parsed.value) {{
return String(parsed.value);
}}
}} catch (error) {{}}
return '';
}};
const getLocale = () => {{
try {{
if (app && typeof app.getLocale === 'function') {{
const locale = app.getLocale();
if (locale) return String(locale).replace('-', '_');
}}
}} catch (error) {{}}
return String(document.documentElement.lang || navigator.language || 'zh-CN').replace('-', '_');
}};
const inferModelType = (rawValue) => {{
const text = String(rawValue || '').trim().toLowerCase();
if (!text) return '';
if (text.includes('expert') || text.includes('专家')) return 'expert';
if (text.includes('vision') || text.includes('识图')) return 'vision';
if (text.includes('default') || text.includes('快速')) return 'default';
return '';
}};
const getSelectedModelType = () => {{
const radios = Array.from(document.querySelectorAll('[role="radio"][aria-checked="true"], input[type="radio"]:checked'));
for (const radio of radios) {{
const samples = [
radio.innerText,
radio.textContent,
radio.getAttribute && radio.getAttribute('aria-label'),
radio.value,
radio.parentElement && radio.parentElement.innerText,
];
for (const sample of samples) {{
const parsed = inferModelType(sample);
if (parsed) return parsed;
}}
}}
return inferModelType(preferredModelType) || 'default';
}};
const token = getToken();
if (!token) {{
return {{ ok: false, error: 'deepseek_user_token_missing' }};
}}
const baseHeaders = {{
authorization: `Bearer ${{token}}`,
'x-client-locale': getLocale(),
'x-client-timezone-offset': String(-new Date().getTimezoneOffset() * 60),
'x-client-platform': 'web',
'x-client-version': clientVersion,
'x-app-version': appVersion,
'content-type': 'application/json',
}};
let latestChallenge = null;
const powManager = createPowManager({{
getTracker: () => tracker,
getChallengeByScene: async (scene) => {{
const targetPath = scene === 'upload_file'
? '/api/v0/file/upload_file'
: '/api/v0/chat/completion';
const challengeResp = await fetch('/api/v0/chat/create_pow_challenge', {{
method: 'POST',
headers: baseHeaders,
credentials: 'include',
body: JSON.stringify({{ target_path: targetPath }}),
}});
const challengeJson = await challengeResp.json();
const challengeData = challengeJson && challengeJson.data && challengeJson.data.biz_data
? challengeJson.data.biz_data
: null;
const challenge = challengeData ? challengeData.challenge : null;
if (!challenge) {{
return Promise.reject(new Error('deepseek_pow_challenge_missing'));
}}
latestChallenge = {{
...challenge,
expireAt: challenge.expire_at,
expireAfter: challenge.expire_after,
targetPath,
}};
return latestChallenge;
}},
}});
const createSessionResp = await fetch('/api/v0/chat_session/create', {{
method: 'POST',
headers: baseHeaders,
credentials: 'include',
body: '{{}}',
}});
const createSessionText = await createSessionResp.text();
let createSessionJson = null;
try {{
createSessionJson = JSON.parse(createSessionText);
}} catch (error) {{}}
const bizData = createSessionJson && createSessionJson.data ? createSessionJson.data.biz_data || {{}} : {{}};
const sessionId = (bizData.chat_session && bizData.chat_session.id) || bizData.id || '';
if (!sessionId) {{
return {{
ok: false,
error: 'deepseek_create_session_failed',
status: createSessionResp.status,
responsePreview: createSessionText.slice(0, 1200),
}};
}}
const powAnswer = await powManager.retrieveAnswer('completion_like');
if (!powAnswer || !powAnswer.res || !latestChallenge) {{
return {{ ok: false, error: 'deepseek_pow_answer_missing', sessionId }};
}}
const [powHeaderName, powHeaderValue] = buildPowHeader(
{{ ...latestChallenge, ...powAnswer.res }},
'/api/v0/chat/completion',
base64Encode,
);
const resolvedModelType = preferredModelType && preferredModelType !== 'auto'
? preferredModelType
: getSelectedModelType();
const searchEnabled = resolveToggleMode(searchEnabledMode, 'searchEnabled', true);
const thinkingEnabled = resolveToggleMode(thinkingEnabledMode, 'thinkingEnabled', false);
const completionResp = await fetch('/api/v0/chat/completion', {{
method: 'POST',
headers: {{
...baseHeaders,
[powHeaderName]: powHeaderValue,
}},
credentials: 'include',
body: JSON.stringify({{
chat_session_id: sessionId,
parent_message_id: null,
model_type: resolvedModelType,
prompt,
ref_file_ids: [],
thinking_enabled: thinkingEnabled,
search_enabled: searchEnabled,
preempt: false,
}}),
}});
const contentType = completionResp.headers.get('content-type') || '';
if (!completionResp.ok) {{
let errorText = '';
try {{
errorText = await completionResp.text();
}} catch (error) {{}}
return {{
ok: false,
error: 'deepseek_completion_http_error',
status: completionResp.status,
content_type: contentType,
responsePreview: errorText.slice(0, 1200),
session_id: sessionId,
model_type: resolvedModelType,
}};
}}
if (!consumeResponse) {{
return {{
ok: true,
status: completionResp.status,
url: completionResp.url || '/api/v0/chat/completion',
content_type: contentType,
session_id: sessionId,
model_type: resolvedModelType,
raw_text: '',
}};
}}
const rawText = await completionResp.text();
return {{
ok: true,
status: completionResp.status,
url: completionResp.url || '/api/v0/chat/completion',
content_type: contentType,
session_id: sessionId,
model_type: resolvedModelType,
raw_text: rawText,
}};
}})()
"""
return script
def execute_deepseek_completion_transport(
*,
tab: Any,
options: Dict[str, Any],
prompt: str,
consume_response: bool,
) -> Dict[str, Any]:
script = _build_deepseek_completion_script(
prompt=str(prompt or ""),
options=options if isinstance(options, dict) else {},
consume_response=consume_response,
)
try:
result = driver_for_tab(tab).run_js(script) or {}
except Exception as e:
logger.warning(f"[REQUEST_TRANSPORT] 页面直发执行异常: {e}")
return {"ok": False, "error": str(e)}
if not isinstance(result, dict):
return {"ok": False, "error": "invalid_request_transport_result"}
return result
register_page_request_transport(
DEEPSEEK_COMPLETION_PROFILE,
execute_deepseek_completion_transport,
)
__all__ = [
"DEEPSEEK_COMPLETION_PROFILE",
"execute_deepseek_completion_transport",
]
+379
View File
@@ -0,0 +1,379 @@
"""
app/core/page_capture/kimi_fetch_capture.py - Kimi page-side fetch stream capture.
"""
from __future__ import annotations
import time
import uuid
from typing import Any, Dict, Generator
from app.core.cdp_hygiene import decode_js_json
from app.core.config import logger
from app.core.network_monitor import NetworkMonitorError, NetworkMonitorTimeout
from app.core.parsers import ParserRegistry
from .base import PageFetchCapture
from .registry import register_page_fetch_capture
from app.core.driver import driver_for_tab
class KimiPageFetchCapture(PageFetchCapture):
"""Capture Kimi connect+json streams from the page fetch runtime."""
parser_id = "kimi"
monitor_id = "kimi_page"
mode_name = "Kimi 页面抓流"
_BOOTSTRAP_JS = r"""
(() => {
const W = window;
const KEY = "__KIMI_CAPTURE__";
const TARGET = "/apiv2/kimi.gateway.chat.v1.ChatService/Chat";
// 修复(7b):逐字节 += 字符串拼接是 O(n^2),改为数组 map + join 一次性拼接
const toEscapedBytes = (chunk) => {
return Array.from(chunk, (b) => "\\u00" + b.toString(16).padStart(2, "0")).join("");
};
const cap = W[KEY] = W[KEY] || {
installed: false,
seq: 0,
requests: [],
currentToken: null,
maxRequests: 12
};
if (cap.installed) {
return { installed: true, patched: false, requests: cap.requests.length };
}
if (typeof W.fetch !== "function") {
return { installed: false, reason: "fetch_missing" };
}
const originalFetch = W.fetch.bind(W);
cap.installed = true;
cap.installedAt = Date.now();
W.fetch = async function(input, init) {
const response = await originalFetch(input, init);
try {
const url = input && typeof input === "object" && "url" in input
? String(input.url || "")
: String(input || "");
if (!url.includes(TARGET)) {
return response;
}
const request = {
id: "kimi_" + (++cap.seq),
url,
token: cap.currentToken || null,
startedAt: Date.now(),
lastChunkAt: 0,
chunkCount: 0,
escapedFullText: "",
complete: false,
error: null,
contentType: response.headers ? (response.headers.get("content-type") || "") : ""
};
cap.requests.push(request);
while (cap.requests.length > (cap.maxRequests || 12)) {
cap.requests.shift();
}
const cloned = response.clone();
if (cloned.body && typeof cloned.body.getReader === "function") {
const reader = cloned.body.getReader();
(async () => {
try {
while (true) {
const { done, value } = await reader.read();
if (done) {
request.complete = true;
request.endedAt = Date.now();
break;
}
if (!value) {
continue;
}
request.chunkCount += 1;
request.lastChunkAt = Date.now();
request.escapedFullText += toEscapedBytes(value);
}
} catch (error) {
request.error = String(error && error.message ? error.message : error);
request.complete = true;
request.endedAt = Date.now();
}
})();
} else {
request.complete = true;
request.endedAt = Date.now();
}
} catch (error) {
cap.lastHookError = String(error && error.message ? error.message : error);
}
return response;
};
return { installed: true, patched: true, requests: cap.requests.length };
})();
"""
def __init__(self, **kwargs: Any) -> None:
super().__init__(**kwargs)
self._capture_token = ""
self._init_js_id = None
self._parser = ParserRegistry.get("kimi")
self.ensure_init_js()
def prepare(self) -> None:
self.ensure_init_js()
token = f"kimi_{uuid.uuid4().hex[:12]}"
with self._page_interaction_slot("JS_EXEC", "kimi_capture_prepare") as acquired:
if not acquired or self._check_cancelled():
return
install_result = driver_for_tab(self.tab).run_js(
f"return {self._BOOTSTRAP_JS.strip()}"
)
driver_for_tab(self.tab).run_js(
"""
return (function(token) {
const cap = window.__KIMI_CAPTURE__ = window.__KIMI_CAPTURE__ || {};
cap.currentToken = token;
cap.requests = [];
cap.lastResetAt = Date.now();
return { ok: true, token: cap.currentToken };
})(arguments[0]);
""",
token,
)
self._capture_token = token
self._sent_content_length = 0
if install_result is not None:
logger.debug(f"[Executor] Kimi 页面抓流已准备: {install_result}")
def ensure_init_js(self) -> None:
if self._init_js_id:
return
# 修复(8):去重标记挂在 tab 对象属性上做 tab 级去重——
# 每请求新建实例时,实例级 _init_js_id 无法阻止向同一 tab 重复注册 init script。
existing_id = getattr(self.tab, "_kimi_fetch_init_js_id", None)
if existing_id:
self._init_js_id = existing_id
return
try:
self._init_js_id = self.tab.add_init_js(
self._BOOTSTRAP_JS.strip()
)
try:
setattr(self.tab, "_kimi_fetch_init_js_id", self._init_js_id)
except Exception:
pass
logger.debug(
f"[Executor] Kimi 页面抓流已注册 document-start 注入: {self._init_js_id}"
)
except Exception as e:
logger.debug(f"[Executor] Kimi document-start 注入失败: {e}")
def get_state(self, since_length: int = 0) -> Dict[str, Any]:
# 修复(7a):增加 sinceLength 增量参数,页面侧只返回 escapedFullText.slice(sinceLength)
# 与 totalLength,避免每 0.3s 轮询全量回传导致 O(n^2) 传输;
# 修复(7c):requestId 作为请求标识返回,Python 侧据此检测页面自动重试换流。
try:
since = max(0, int(since_length or 0))
except Exception:
since = 0
# P0-2:结果在页面内 JSON.stringify,只回传字符串,不留 RemoteObject
state = driver_for_tab(self.tab).run_js(
"""
return JSON.stringify((function(token, sinceLength) {
const cap = window.__KIMI_CAPTURE__;
if (!cap) {
return { installed: false, found: false };
}
const requests = Array.isArray(cap.requests) ? cap.requests : [];
let target = null;
for (let i = requests.length - 1; i >= 0; i -= 1) {
const item = requests[i];
if (!token || item.token === token) {
target = item;
break;
}
}
const fullText = target ? (target.escapedFullText || "") : "";
const totalLength = fullText.length;
const since = Math.max(0, Math.min(Number(sinceLength) || 0, totalLength));
return {
installed: true,
currentToken: cap.currentToken || null,
found: !!target,
requestId: target ? (target.id || "") : "",
escapedDelta: fullText.slice(since),
totalLength: totalLength,
complete: !!(target && target.complete),
error: target ? (target.error || null) : null,
chunkCount: target ? (target.chunkCount || 0) : 0,
startedAt: target ? (target.startedAt || 0) : 0,
lastChunkAt: target ? (target.lastChunkAt || 0) : 0
};
})(arguments[0], arguments[1]));
""",
self._capture_token or "",
since,
)
state = decode_js_json(state, {})
return state if isinstance(state, dict) else {}
def _clear_page_capture_buffer(self) -> None:
# 修复(7d):monitor 结束时尽力清空当前 token 对应请求的页面侧缓冲,失败忽略
try:
driver_for_tab(self.tab).run_js(
"""
return (function(token) {
const cap = window.__KIMI_CAPTURE__;
if (!cap || !Array.isArray(cap.requests)) {
return { ok: false };
}
let cleared = 0;
for (const item of cap.requests) {
if ((!token || item.token === token) && item.escapedFullText) {
item.escapedFullText = "";
cleared += 1;
}
}
return { ok: true, cleared: cleared };
})(arguments[0]);
""",
self._capture_token or "",
)
except Exception as e:
logger.debug(f"[Executor] Kimi 页面抓流缓冲清理失败(忽略): {e}")
def monitor(self, completion_id: str) -> Generator[str, None, None]:
parser = self._parser
parser.reset()
hard_timeout = float(
self._stream_config.get("hard_timeout", 300) or 300
)
first_response_timeout = float(
self._network_config.get("first_response_timeout", hard_timeout) or hard_timeout
)
response_interval = float(
self._network_config.get("response_interval", 0.3) or 0.3
)
silence_threshold = float(
self._network_config.get("silence_threshold", 3) or 3
)
phase_start = time.time()
last_activity = phase_start
seen_request = False
# 修复(7c):锁定首个命中的请求标识;标识变化说明页面自动重试换了流
locked_request_id = ""
# 修复(7a):Python 侧按偏移量增量取回并累积转义文本,避免全量回传
escaped_buffer = ""
try:
while True:
if self._check_cancelled():
logger.debug("[Executor] Kimi 页面抓流被取消")
break
now = time.time()
if now - phase_start > hard_timeout:
raise NetworkMonitorError(f"kimi_page_capture_hard_timeout:{hard_timeout:.1f}s")
state = self.get_state(len(escaped_buffer))
if not state.get("installed"):
raise NetworkMonitorError("kimi_page_capture_not_installed")
if state.get("error"):
raise NetworkMonitorError(f"kimi_page_capture_error:{state.get('error')}")
request_id = str(state.get("requestId", "") or "")
if state.get("found"):
if not seen_request:
logger.debug(
"[Executor] Kimi 页面抓流已命中请求 "
f"(request_id={state.get('requestId')}, token={self._capture_token})"
)
seen_request = True
locked_request_id = request_id
elif locked_request_id and request_id and request_id != locked_request_id:
# 修复(7c):页面自动重试换了流——重置解析状态与偏移,
# 从新流头部重新取数(本次按旧偏移取到的增量作废)
logger.warning(
"[Executor] Kimi 页面抓流请求标识变化,重置解析状态 "
f"({locked_request_id} -> {request_id})"
)
parser.reset()
escaped_buffer = ""
locked_request_id = request_id
state = self.get_state(0)
if state.get("error"):
raise NetworkMonitorError(f"kimi_page_capture_error:{state.get('error')}")
delta = str(state.get("escapedDelta", "") or "")
if delta:
escaped_buffer += delta
last_activity = now
if escaped_buffer:
parse_result = parser.parse_chunk(escaped_buffer)
if parse_result.get("error"):
raise NetworkMonitorError(f"kimi_page_capture_parse_error:{parse_result['error']}")
content = parse_result.get("content", "")
done = bool(parse_result.get("done")) or bool(state.get("complete"))
if content:
logger.debug(f"[Executor] Kimi 页面抓流产出: {repr(content)[:240]}")
self._sent_content_length += len(content)
yield self.formatter.pack_chunk(content, completion_id=completion_id)
if done:
logger.debug("[Executor] Kimi 页面抓流完成")
break
elif seen_request and state.get("complete"):
logger.debug("[Executor] Kimi 页面抓流请求已结束但无有效内容")
break
if not seen_request and (now - phase_start) > first_response_timeout:
raise NetworkMonitorTimeout(f"kimi_page_capture_first_response_timeout:{first_response_timeout:.1f}s")
if seen_request and (now - last_activity) > silence_threshold:
logger.warning(
"[Executor] Kimi 页面抓流静默超时 "
f"({now - last_activity:.1f}s)"
)
raise NetworkMonitorTimeout(
f"kimi_page_capture_silence_timeout:{silence_threshold:.1f}s"
)
time.sleep(max(0.05, response_interval))
finally:
# 修复(7d):monitor 结束(正常/异常/取消)时清空页面侧缓冲,尽力而为
self._clear_page_capture_buffer()
register_page_fetch_capture(KimiPageFetchCapture)
__all__ = ["KimiPageFetchCapture"]
+53
View File
@@ -0,0 +1,53 @@
"""
app/core/page_capture/registry.py - registry for provider-specific page captures.
"""
from __future__ import annotations
from typing import Any, Callable, Dict, Optional, Type
from app.core.config import SSEFormatter, logger
from .base import InteractionSlot, PageFetchCapture
_CAPTURE_CLASSES: Dict[str, Type[PageFetchCapture]] = {}
def register_page_fetch_capture(capture_class: Type[PageFetchCapture]) -> None:
parser_id = str(capture_class.get_parser_id() or "").strip().lower()
if not parser_id:
raise ValueError("page fetch capture parser_id is required")
_CAPTURE_CLASSES[parser_id] = capture_class
logger.info(f"已注册页面抓流实现: {parser_id} -> {capture_class.__name__}")
def create_page_fetch_capture(
*,
tab,
formatter: SSEFormatter,
stream_config: Optional[Dict[str, Any]] = None,
stop_checker: Optional[Callable[[], bool]] = None,
interaction_slot: Optional[InteractionSlot] = None,
) -> Optional[PageFetchCapture]:
stream_config = stream_config or {}
if stream_config.get("mode", "dom") != "network":
return None
network_config = stream_config.get("network", {}) or {}
parser_id = str(network_config.get("parser", "") or "").strip().lower()
if not parser_id:
return None
capture_class = _CAPTURE_CLASSES.get(parser_id)
if capture_class is None:
return None
return capture_class(
tab=tab,
formatter=formatter,
stream_config=stream_config,
network_config=network_config,
stop_checker=stop_checker,
interaction_slot=interaction_slot,
)
@@ -0,0 +1,67 @@
"""
app/core/page_capture/request_transport.py - page-side request transport registry.
"""
from __future__ import annotations
import copy
from typing import Any, Callable, Dict, List
PageRequestTransportExecutor = Callable[..., Dict[str, Any]]
_PROFILES: Dict[str, Dict[str, Any]] = {}
_EXECUTORS: Dict[str, PageRequestTransportExecutor] = {}
def register_page_request_transport(
profile: Dict[str, Any],
executor: PageRequestTransportExecutor,
) -> None:
profile_id = str((profile or {}).get("id") or "").strip()
if not profile_id:
raise ValueError("page request transport profile id is required")
if not callable(executor):
raise ValueError(f"page request transport executor is not callable: {profile_id}")
_PROFILES[profile_id] = copy.deepcopy(profile)
_EXECUTORS[profile_id] = executor
def get_page_request_transport_profiles() -> List[Dict[str, Any]]:
return [copy.deepcopy(profile) for profile in _PROFILES.values()]
def get_page_request_transport_profile(profile_id: Any) -> Dict[str, Any] | None:
pid = str(profile_id or "").strip()
profile = _PROFILES.get(pid)
return copy.deepcopy(profile) if profile else None
def execute_page_request_transport(
tab: Any,
profile_id: Any,
*,
options: Dict[str, Any],
prompt: str,
consume_response: bool,
) -> Dict[str, Any]:
pid = str(profile_id or "").strip()
executor = _EXECUTORS.get(pid)
if executor is None:
return {"ok": False, "error": f"unsupported_request_transport_profile:{pid}"}
return executor(
tab=tab,
options=options if isinstance(options, dict) else {},
prompt=str(prompt or ""),
consume_response=consume_response,
)
__all__ = [
"execute_page_request_transport",
"get_page_request_transport_profile",
"get_page_request_transport_profiles",
"register_page_request_transport",
]