Initial commit
This commit is contained in:
commit
57fc3525eb
7311 files changed
+1291398
No files matched your search
@@ -0,0 +1,70 @@
|
||||
"""
|
||||
app/core/page_capture - page-side provider helpers.
|
||||
|
||||
This package groups provider-specific page runtime capabilities while keeping
|
||||
their dispatch APIs small and generic.
|
||||
"""
|
||||
|
||||
from .base import PageFetchCapture
|
||||
from .registry import (
|
||||
create_page_fetch_capture as _create_page_fetch_capture,
|
||||
register_page_fetch_capture,
|
||||
)
|
||||
from .request_transport import (
|
||||
execute_page_request_transport as _execute_page_request_transport,
|
||||
get_page_request_transport_profile as _get_page_request_transport_profile,
|
||||
get_page_request_transport_profiles as _get_page_request_transport_profiles,
|
||||
register_page_request_transport,
|
||||
)
|
||||
|
||||
_FETCH_CAPTURE_BUILTINS_LOADED = False
|
||||
_REQUEST_TRANSPORT_BUILTINS_LOADED = False
|
||||
|
||||
|
||||
def _load_builtin_fetch_captures() -> None:
|
||||
global _FETCH_CAPTURE_BUILTINS_LOADED
|
||||
if _FETCH_CAPTURE_BUILTINS_LOADED:
|
||||
return
|
||||
from . import kimi_fetch_capture as _kimi_fetch_capture # noqa: F401
|
||||
|
||||
_FETCH_CAPTURE_BUILTINS_LOADED = True
|
||||
|
||||
|
||||
def _load_builtin_request_transports() -> None:
|
||||
global _REQUEST_TRANSPORT_BUILTINS_LOADED
|
||||
if _REQUEST_TRANSPORT_BUILTINS_LOADED:
|
||||
return
|
||||
from . import deepseek_request_transport as _deepseek_request_transport # noqa: F401
|
||||
|
||||
_REQUEST_TRANSPORT_BUILTINS_LOADED = True
|
||||
|
||||
|
||||
def create_page_fetch_capture(*args, **kwargs):
|
||||
_load_builtin_fetch_captures()
|
||||
return _create_page_fetch_capture(*args, **kwargs)
|
||||
|
||||
|
||||
def get_page_request_transport_profiles():
|
||||
_load_builtin_request_transports()
|
||||
return _get_page_request_transport_profiles()
|
||||
|
||||
|
||||
def get_page_request_transport_profile(profile_id):
|
||||
_load_builtin_request_transports()
|
||||
return _get_page_request_transport_profile(profile_id)
|
||||
|
||||
|
||||
def execute_page_request_transport(*args, **kwargs):
|
||||
_load_builtin_request_transports()
|
||||
return _execute_page_request_transport(*args, **kwargs)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"PageFetchCapture",
|
||||
"create_page_fetch_capture",
|
||||
"execute_page_request_transport",
|
||||
"get_page_request_transport_profile",
|
||||
"get_page_request_transport_profiles",
|
||||
"register_page_fetch_capture",
|
||||
"register_page_request_transport",
|
||||
]
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -0,0 +1,71 @@
|
||||
"""
|
||||
app/core/page_capture/base.py - shared page-side fetch capture contract.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from contextlib import nullcontext
|
||||
from typing import Any, Callable, ContextManager, Dict, Generator, Optional
|
||||
|
||||
from app.core.config import SSEFormatter
|
||||
|
||||
|
||||
InteractionSlot = Callable[[str, str], ContextManager[bool]]
|
||||
|
||||
|
||||
class PageFetchCapture:
|
||||
"""Base class for page-runtime fetch capture implementations."""
|
||||
|
||||
parser_id = ""
|
||||
monitor_id = "page_fetch"
|
||||
mode_name = "页面抓流"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
tab,
|
||||
formatter: SSEFormatter,
|
||||
stream_config: Optional[Dict[str, Any]] = None,
|
||||
network_config: Optional[Dict[str, Any]] = None,
|
||||
stop_checker: Optional[Callable[[], bool]] = None,
|
||||
interaction_slot: Optional[InteractionSlot] = None,
|
||||
) -> None:
|
||||
self.tab = tab
|
||||
self.formatter = formatter
|
||||
self._stream_config = stream_config or {}
|
||||
self._network_config = network_config or {}
|
||||
self._should_stop = stop_checker or (lambda: False)
|
||||
self._interaction_slot = interaction_slot
|
||||
self._sent_content_length = 0
|
||||
|
||||
@classmethod
|
||||
def get_parser_id(cls) -> str:
|
||||
return cls.parser_id
|
||||
|
||||
@classmethod
|
||||
def get_monitor_id(cls) -> str:
|
||||
return cls.monitor_id
|
||||
|
||||
@classmethod
|
||||
def get_mode_name(cls) -> str:
|
||||
return cls.mode_name
|
||||
|
||||
def prepare(self) -> None:
|
||||
raise NotImplementedError
|
||||
|
||||
def monitor(self, completion_id: str) -> Generator[str, None, None]:
|
||||
raise NotImplementedError
|
||||
|
||||
def get_sent_content_length(self) -> int:
|
||||
try:
|
||||
return max(0, int(self._sent_content_length or 0))
|
||||
except Exception:
|
||||
return 0
|
||||
|
||||
def _check_cancelled(self) -> bool:
|
||||
return bool(self._should_stop())
|
||||
|
||||
def _page_interaction_slot(self, action: str, target_key: str = ""):
|
||||
if callable(self._interaction_slot):
|
||||
return self._interaction_slot(action, target_key)
|
||||
return nullcontext(True)
|
||||
@@ -0,0 +1,425 @@
|
||||
"""
|
||||
app/core/page_capture/deepseek_request_transport.py - DeepSeek page-side completion transport.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from typing import Any, Dict
|
||||
|
||||
from app.core.config import logger
|
||||
|
||||
from .request_transport import register_page_request_transport
|
||||
from app.core.driver import driver_for_tab
|
||||
|
||||
|
||||
DEEPSEEK_COMPLETION_PROFILE: Dict[str, Any] = {
|
||||
"id": "deepseek_completion",
|
||||
"name": "DeepSeek 直发",
|
||||
"description": "在当前 DeepSeek 页面中直接新建会话并发起 completion 请求。",
|
||||
"transport_mode": "page_fetch",
|
||||
"supported_domains": ["chat.deepseek.com"],
|
||||
"supported_parsers": ["deepseek"],
|
||||
"supports_images": False,
|
||||
"supports_file_inputs": False,
|
||||
"options": [
|
||||
{
|
||||
"key": "model_type",
|
||||
"label": "模型类型",
|
||||
"type": "enum",
|
||||
"default": "auto",
|
||||
"description": "auto 跟随 profile 配置或页面状态,也可固定为 default / expert / vision。",
|
||||
"choices": [
|
||||
{"value": "auto", "label": "自动"},
|
||||
{"value": "default", "label": "快速模式"},
|
||||
{"value": "expert", "label": "专家模式"},
|
||||
{"value": "vision", "label": "识图模式"},
|
||||
],
|
||||
},
|
||||
{
|
||||
"key": "context_mode",
|
||||
"label": "上下文模式",
|
||||
"type": "enum",
|
||||
"default": "full_prompt",
|
||||
"description": "v1 固定使用主链已构造好的完整 prompt 字符串。",
|
||||
"choices": [
|
||||
{"value": "full_prompt", "label": "完整上下文 prompt"},
|
||||
],
|
||||
},
|
||||
{
|
||||
"key": "search_enabled",
|
||||
"label": "联网搜索",
|
||||
"type": "enum",
|
||||
"default": "auto",
|
||||
"description": "auto 跟随页面当前开关,也可强制开/关。",
|
||||
"choices": [
|
||||
{"value": "auto", "label": "自动"},
|
||||
{"value": "on", "label": "开启"},
|
||||
{"value": "off", "label": "关闭"},
|
||||
],
|
||||
},
|
||||
{
|
||||
"key": "thinking_enabled",
|
||||
"label": "深度思考",
|
||||
"type": "enum",
|
||||
"default": "auto",
|
||||
"description": "auto 跟随页面当前开关,也可强制开/关。",
|
||||
"choices": [
|
||||
{"value": "auto", "label": "自动"},
|
||||
{"value": "on", "label": "开启"},
|
||||
{"value": "off", "label": "关闭"},
|
||||
],
|
||||
},
|
||||
{
|
||||
"key": "fallback_mode",
|
||||
"label": "失败回退",
|
||||
"type": "enum",
|
||||
"default": "workflow",
|
||||
"description": "workflow 表示直发失败时回退原工作流。",
|
||||
"choices": [
|
||||
{"value": "workflow", "label": "回退工作流"},
|
||||
{"value": "error", "label": "直接报错"},
|
||||
],
|
||||
},
|
||||
{
|
||||
"key": "client_version",
|
||||
"label": "客户端版本",
|
||||
"type": "string",
|
||||
"default": "2.0.0",
|
||||
"description": "附加到 x-client-version 请求头。",
|
||||
},
|
||||
{
|
||||
"key": "app_version",
|
||||
"label": "应用版本",
|
||||
"type": "string",
|
||||
"default": "2.0.0",
|
||||
"description": "附加到 x-app-version 请求头。",
|
||||
},
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def _build_deepseek_completion_script(
|
||||
prompt: str,
|
||||
options: Dict[str, Any],
|
||||
*,
|
||||
consume_response: bool,
|
||||
) -> str:
|
||||
explicit_model_type = str(options.get("model_type") or "").strip().lower()
|
||||
if explicit_model_type not in {"", "auto", "default", "expert", "vision"}:
|
||||
explicit_model_type = ""
|
||||
|
||||
context_mode = str(options.get("context_mode") or "full_prompt").strip().lower() or "full_prompt"
|
||||
if context_mode != "full_prompt":
|
||||
context_mode = "full_prompt"
|
||||
|
||||
search_enabled = str(options.get("search_enabled") or "auto").strip().lower() or "auto"
|
||||
thinking_enabled = str(options.get("thinking_enabled") or "auto").strip().lower() or "auto"
|
||||
client_version = str(options.get("client_version") or "2.0.0").strip() or "2.0.0"
|
||||
app_version = str(options.get("app_version") or client_version).strip() or client_version
|
||||
|
||||
script = f"""
|
||||
return (async () => {{
|
||||
const prompt = {json.dumps(prompt, ensure_ascii=False)};
|
||||
const preferredModelType = {json.dumps(explicit_model_type)};
|
||||
const contextMode = {json.dumps(context_mode)};
|
||||
const searchEnabledMode = {json.dumps(search_enabled)};
|
||||
const thinkingEnabledMode = {json.dumps(thinking_enabled)};
|
||||
const clientVersion = {json.dumps(client_version)};
|
||||
const appVersion = {json.dumps(app_version)};
|
||||
const consumeResponse = {str(bool(consume_response)).lower()};
|
||||
|
||||
const chunk = window.webpackChunk_deepseek_chat;
|
||||
if (!chunk || typeof chunk.push !== 'function') {{
|
||||
return {{ ok: false, error: 'deepseek_webpack_missing' }};
|
||||
}}
|
||||
|
||||
let __req = null;
|
||||
chunk.push([[Symbol('request_transport_deepseek_completion')], {{}}, (req) => {{ __req = req; }}]);
|
||||
if (!__req) {{
|
||||
return {{ ok: false, error: 'deepseek_require_missing' }};
|
||||
}}
|
||||
|
||||
const app = __req('86389').y;
|
||||
const createPowManager = __req('87109').Qc;
|
||||
const buildPowHeader = __req('42587').Bx;
|
||||
const base64Encode = (text) => btoa(unescape(encodeURIComponent(text)));
|
||||
const tracker = {{
|
||||
info() {{}},
|
||||
error() {{}},
|
||||
withError: (error, payload) => ({{
|
||||
...(payload || {{}}),
|
||||
error: String(error && error.message ? error.message : error),
|
||||
}}),
|
||||
}};
|
||||
|
||||
if (contextMode !== 'full_prompt') {{
|
||||
return {{ ok: false, error: 'deepseek_context_mode_unsupported' }};
|
||||
}}
|
||||
|
||||
const readStoredBool = (key, fallbackValue) => {{
|
||||
try {{
|
||||
const raw = localStorage.getItem(key);
|
||||
if (!raw) return fallbackValue;
|
||||
const parsed = JSON.parse(raw);
|
||||
if (typeof parsed === 'boolean') return parsed;
|
||||
if (parsed && typeof parsed === 'object' && 'value' in parsed) {{
|
||||
return !!parsed.value;
|
||||
}}
|
||||
return !!parsed;
|
||||
}} catch (error) {{
|
||||
return fallbackValue;
|
||||
}}
|
||||
}};
|
||||
|
||||
const resolveToggleMode = (mode, storageKey, fallbackValue) => {{
|
||||
const text = String(mode || '').trim().toLowerCase();
|
||||
if (text === 'on' || text === 'true' || text === '1') return true;
|
||||
if (text === 'off' || text === 'false' || text === '0') return false;
|
||||
return readStoredBool(storageKey, fallbackValue);
|
||||
}};
|
||||
|
||||
const getToken = () => {{
|
||||
try {{
|
||||
if (app && typeof app.getStorageUserToken === 'function') {{
|
||||
const tokenValue = app.getStorageUserToken();
|
||||
if (typeof tokenValue === 'string' && tokenValue) return tokenValue;
|
||||
if (tokenValue && typeof tokenValue === 'object' && tokenValue.value) {{
|
||||
return String(tokenValue.value);
|
||||
}}
|
||||
}}
|
||||
}} catch (error) {{}}
|
||||
|
||||
try {{
|
||||
const raw = localStorage.getItem('userToken');
|
||||
if (!raw) return '';
|
||||
const parsed = JSON.parse(raw);
|
||||
if (typeof parsed === 'string' && parsed) return parsed;
|
||||
if (parsed && typeof parsed === 'object' && parsed.value) {{
|
||||
return String(parsed.value);
|
||||
}}
|
||||
}} catch (error) {{}}
|
||||
|
||||
return '';
|
||||
}};
|
||||
|
||||
const getLocale = () => {{
|
||||
try {{
|
||||
if (app && typeof app.getLocale === 'function') {{
|
||||
const locale = app.getLocale();
|
||||
if (locale) return String(locale).replace('-', '_');
|
||||
}}
|
||||
}} catch (error) {{}}
|
||||
return String(document.documentElement.lang || navigator.language || 'zh-CN').replace('-', '_');
|
||||
}};
|
||||
|
||||
const inferModelType = (rawValue) => {{
|
||||
const text = String(rawValue || '').trim().toLowerCase();
|
||||
if (!text) return '';
|
||||
if (text.includes('expert') || text.includes('专家')) return 'expert';
|
||||
if (text.includes('vision') || text.includes('识图')) return 'vision';
|
||||
if (text.includes('default') || text.includes('快速')) return 'default';
|
||||
return '';
|
||||
}};
|
||||
|
||||
const getSelectedModelType = () => {{
|
||||
const radios = Array.from(document.querySelectorAll('[role="radio"][aria-checked="true"], input[type="radio"]:checked'));
|
||||
for (const radio of radios) {{
|
||||
const samples = [
|
||||
radio.innerText,
|
||||
radio.textContent,
|
||||
radio.getAttribute && radio.getAttribute('aria-label'),
|
||||
radio.value,
|
||||
radio.parentElement && radio.parentElement.innerText,
|
||||
];
|
||||
for (const sample of samples) {{
|
||||
const parsed = inferModelType(sample);
|
||||
if (parsed) return parsed;
|
||||
}}
|
||||
}}
|
||||
return inferModelType(preferredModelType) || 'default';
|
||||
}};
|
||||
|
||||
const token = getToken();
|
||||
if (!token) {{
|
||||
return {{ ok: false, error: 'deepseek_user_token_missing' }};
|
||||
}}
|
||||
|
||||
const baseHeaders = {{
|
||||
authorization: `Bearer ${{token}}`,
|
||||
'x-client-locale': getLocale(),
|
||||
'x-client-timezone-offset': String(-new Date().getTimezoneOffset() * 60),
|
||||
'x-client-platform': 'web',
|
||||
'x-client-version': clientVersion,
|
||||
'x-app-version': appVersion,
|
||||
'content-type': 'application/json',
|
||||
}};
|
||||
|
||||
let latestChallenge = null;
|
||||
const powManager = createPowManager({{
|
||||
getTracker: () => tracker,
|
||||
getChallengeByScene: async (scene) => {{
|
||||
const targetPath = scene === 'upload_file'
|
||||
? '/api/v0/file/upload_file'
|
||||
: '/api/v0/chat/completion';
|
||||
const challengeResp = await fetch('/api/v0/chat/create_pow_challenge', {{
|
||||
method: 'POST',
|
||||
headers: baseHeaders,
|
||||
credentials: 'include',
|
||||
body: JSON.stringify({{ target_path: targetPath }}),
|
||||
}});
|
||||
const challengeJson = await challengeResp.json();
|
||||
const challengeData = challengeJson && challengeJson.data && challengeJson.data.biz_data
|
||||
? challengeJson.data.biz_data
|
||||
: null;
|
||||
const challenge = challengeData ? challengeData.challenge : null;
|
||||
if (!challenge) {{
|
||||
return Promise.reject(new Error('deepseek_pow_challenge_missing'));
|
||||
}}
|
||||
latestChallenge = {{
|
||||
...challenge,
|
||||
expireAt: challenge.expire_at,
|
||||
expireAfter: challenge.expire_after,
|
||||
targetPath,
|
||||
}};
|
||||
return latestChallenge;
|
||||
}},
|
||||
}});
|
||||
|
||||
const createSessionResp = await fetch('/api/v0/chat_session/create', {{
|
||||
method: 'POST',
|
||||
headers: baseHeaders,
|
||||
credentials: 'include',
|
||||
body: '{{}}',
|
||||
}});
|
||||
const createSessionText = await createSessionResp.text();
|
||||
let createSessionJson = null;
|
||||
try {{
|
||||
createSessionJson = JSON.parse(createSessionText);
|
||||
}} catch (error) {{}}
|
||||
const bizData = createSessionJson && createSessionJson.data ? createSessionJson.data.biz_data || {{}} : {{}};
|
||||
const sessionId = (bizData.chat_session && bizData.chat_session.id) || bizData.id || '';
|
||||
if (!sessionId) {{
|
||||
return {{
|
||||
ok: false,
|
||||
error: 'deepseek_create_session_failed',
|
||||
status: createSessionResp.status,
|
||||
responsePreview: createSessionText.slice(0, 1200),
|
||||
}};
|
||||
}}
|
||||
|
||||
const powAnswer = await powManager.retrieveAnswer('completion_like');
|
||||
if (!powAnswer || !powAnswer.res || !latestChallenge) {{
|
||||
return {{ ok: false, error: 'deepseek_pow_answer_missing', sessionId }};
|
||||
}}
|
||||
|
||||
const [powHeaderName, powHeaderValue] = buildPowHeader(
|
||||
{{ ...latestChallenge, ...powAnswer.res }},
|
||||
'/api/v0/chat/completion',
|
||||
base64Encode,
|
||||
);
|
||||
|
||||
const resolvedModelType = preferredModelType && preferredModelType !== 'auto'
|
||||
? preferredModelType
|
||||
: getSelectedModelType();
|
||||
const searchEnabled = resolveToggleMode(searchEnabledMode, 'searchEnabled', true);
|
||||
const thinkingEnabled = resolveToggleMode(thinkingEnabledMode, 'thinkingEnabled', false);
|
||||
|
||||
const completionResp = await fetch('/api/v0/chat/completion', {{
|
||||
method: 'POST',
|
||||
headers: {{
|
||||
...baseHeaders,
|
||||
[powHeaderName]: powHeaderValue,
|
||||
}},
|
||||
credentials: 'include',
|
||||
body: JSON.stringify({{
|
||||
chat_session_id: sessionId,
|
||||
parent_message_id: null,
|
||||
model_type: resolvedModelType,
|
||||
prompt,
|
||||
ref_file_ids: [],
|
||||
thinking_enabled: thinkingEnabled,
|
||||
search_enabled: searchEnabled,
|
||||
preempt: false,
|
||||
}}),
|
||||
}});
|
||||
|
||||
const contentType = completionResp.headers.get('content-type') || '';
|
||||
if (!completionResp.ok) {{
|
||||
let errorText = '';
|
||||
try {{
|
||||
errorText = await completionResp.text();
|
||||
}} catch (error) {{}}
|
||||
return {{
|
||||
ok: false,
|
||||
error: 'deepseek_completion_http_error',
|
||||
status: completionResp.status,
|
||||
content_type: contentType,
|
||||
responsePreview: errorText.slice(0, 1200),
|
||||
session_id: sessionId,
|
||||
model_type: resolvedModelType,
|
||||
}};
|
||||
}}
|
||||
|
||||
if (!consumeResponse) {{
|
||||
return {{
|
||||
ok: true,
|
||||
status: completionResp.status,
|
||||
url: completionResp.url || '/api/v0/chat/completion',
|
||||
content_type: contentType,
|
||||
session_id: sessionId,
|
||||
model_type: resolvedModelType,
|
||||
raw_text: '',
|
||||
}};
|
||||
}}
|
||||
|
||||
const rawText = await completionResp.text();
|
||||
return {{
|
||||
ok: true,
|
||||
status: completionResp.status,
|
||||
url: completionResp.url || '/api/v0/chat/completion',
|
||||
content_type: contentType,
|
||||
session_id: sessionId,
|
||||
model_type: resolvedModelType,
|
||||
raw_text: rawText,
|
||||
}};
|
||||
}})()
|
||||
"""
|
||||
return script
|
||||
|
||||
|
||||
def execute_deepseek_completion_transport(
|
||||
*,
|
||||
tab: Any,
|
||||
options: Dict[str, Any],
|
||||
prompt: str,
|
||||
consume_response: bool,
|
||||
) -> Dict[str, Any]:
|
||||
script = _build_deepseek_completion_script(
|
||||
prompt=str(prompt or ""),
|
||||
options=options if isinstance(options, dict) else {},
|
||||
consume_response=consume_response,
|
||||
)
|
||||
|
||||
try:
|
||||
result = driver_for_tab(tab).run_js(script) or {}
|
||||
except Exception as e:
|
||||
logger.warning(f"[REQUEST_TRANSPORT] 页面直发执行异常: {e}")
|
||||
return {"ok": False, "error": str(e)}
|
||||
|
||||
if not isinstance(result, dict):
|
||||
return {"ok": False, "error": "invalid_request_transport_result"}
|
||||
|
||||
return result
|
||||
|
||||
|
||||
register_page_request_transport(
|
||||
DEEPSEEK_COMPLETION_PROFILE,
|
||||
execute_deepseek_completion_transport,
|
||||
)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"DEEPSEEK_COMPLETION_PROFILE",
|
||||
"execute_deepseek_completion_transport",
|
||||
]
|
||||
@@ -0,0 +1,379 @@
|
||||
"""
|
||||
app/core/page_capture/kimi_fetch_capture.py - Kimi page-side fetch stream capture.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
import uuid
|
||||
from typing import Any, Dict, Generator
|
||||
|
||||
from app.core.cdp_hygiene import decode_js_json
|
||||
from app.core.config import logger
|
||||
from app.core.network_monitor import NetworkMonitorError, NetworkMonitorTimeout
|
||||
from app.core.parsers import ParserRegistry
|
||||
|
||||
from .base import PageFetchCapture
|
||||
from .registry import register_page_fetch_capture
|
||||
from app.core.driver import driver_for_tab
|
||||
|
||||
|
||||
class KimiPageFetchCapture(PageFetchCapture):
|
||||
"""Capture Kimi connect+json streams from the page fetch runtime."""
|
||||
|
||||
parser_id = "kimi"
|
||||
monitor_id = "kimi_page"
|
||||
mode_name = "Kimi 页面抓流"
|
||||
|
||||
_BOOTSTRAP_JS = r"""
|
||||
(() => {
|
||||
const W = window;
|
||||
const KEY = "__KIMI_CAPTURE__";
|
||||
const TARGET = "/apiv2/kimi.gateway.chat.v1.ChatService/Chat";
|
||||
|
||||
// 修复(7b):逐字节 += 字符串拼接是 O(n^2),改为数组 map + join 一次性拼接
|
||||
const toEscapedBytes = (chunk) => {
|
||||
return Array.from(chunk, (b) => "\\u00" + b.toString(16).padStart(2, "0")).join("");
|
||||
};
|
||||
|
||||
const cap = W[KEY] = W[KEY] || {
|
||||
installed: false,
|
||||
seq: 0,
|
||||
requests: [],
|
||||
currentToken: null,
|
||||
maxRequests: 12
|
||||
};
|
||||
|
||||
if (cap.installed) {
|
||||
return { installed: true, patched: false, requests: cap.requests.length };
|
||||
}
|
||||
|
||||
if (typeof W.fetch !== "function") {
|
||||
return { installed: false, reason: "fetch_missing" };
|
||||
}
|
||||
|
||||
const originalFetch = W.fetch.bind(W);
|
||||
cap.installed = true;
|
||||
cap.installedAt = Date.now();
|
||||
|
||||
W.fetch = async function(input, init) {
|
||||
const response = await originalFetch(input, init);
|
||||
|
||||
try {
|
||||
const url = input && typeof input === "object" && "url" in input
|
||||
? String(input.url || "")
|
||||
: String(input || "");
|
||||
|
||||
if (!url.includes(TARGET)) {
|
||||
return response;
|
||||
}
|
||||
|
||||
const request = {
|
||||
id: "kimi_" + (++cap.seq),
|
||||
url,
|
||||
token: cap.currentToken || null,
|
||||
startedAt: Date.now(),
|
||||
lastChunkAt: 0,
|
||||
chunkCount: 0,
|
||||
escapedFullText: "",
|
||||
complete: false,
|
||||
error: null,
|
||||
contentType: response.headers ? (response.headers.get("content-type") || "") : ""
|
||||
};
|
||||
|
||||
cap.requests.push(request);
|
||||
while (cap.requests.length > (cap.maxRequests || 12)) {
|
||||
cap.requests.shift();
|
||||
}
|
||||
|
||||
const cloned = response.clone();
|
||||
if (cloned.body && typeof cloned.body.getReader === "function") {
|
||||
const reader = cloned.body.getReader();
|
||||
(async () => {
|
||||
try {
|
||||
while (true) {
|
||||
const { done, value } = await reader.read();
|
||||
if (done) {
|
||||
request.complete = true;
|
||||
request.endedAt = Date.now();
|
||||
break;
|
||||
}
|
||||
if (!value) {
|
||||
continue;
|
||||
}
|
||||
request.chunkCount += 1;
|
||||
request.lastChunkAt = Date.now();
|
||||
request.escapedFullText += toEscapedBytes(value);
|
||||
}
|
||||
} catch (error) {
|
||||
request.error = String(error && error.message ? error.message : error);
|
||||
request.complete = true;
|
||||
request.endedAt = Date.now();
|
||||
}
|
||||
})();
|
||||
} else {
|
||||
request.complete = true;
|
||||
request.endedAt = Date.now();
|
||||
}
|
||||
} catch (error) {
|
||||
cap.lastHookError = String(error && error.message ? error.message : error);
|
||||
}
|
||||
|
||||
return response;
|
||||
};
|
||||
|
||||
return { installed: true, patched: true, requests: cap.requests.length };
|
||||
})();
|
||||
"""
|
||||
|
||||
def __init__(self, **kwargs: Any) -> None:
|
||||
super().__init__(**kwargs)
|
||||
self._capture_token = ""
|
||||
self._init_js_id = None
|
||||
self._parser = ParserRegistry.get("kimi")
|
||||
self.ensure_init_js()
|
||||
|
||||
def prepare(self) -> None:
|
||||
self.ensure_init_js()
|
||||
token = f"kimi_{uuid.uuid4().hex[:12]}"
|
||||
|
||||
with self._page_interaction_slot("JS_EXEC", "kimi_capture_prepare") as acquired:
|
||||
if not acquired or self._check_cancelled():
|
||||
return
|
||||
install_result = driver_for_tab(self.tab).run_js(
|
||||
f"return {self._BOOTSTRAP_JS.strip()}"
|
||||
)
|
||||
driver_for_tab(self.tab).run_js(
|
||||
"""
|
||||
return (function(token) {
|
||||
const cap = window.__KIMI_CAPTURE__ = window.__KIMI_CAPTURE__ || {};
|
||||
cap.currentToken = token;
|
||||
cap.requests = [];
|
||||
cap.lastResetAt = Date.now();
|
||||
return { ok: true, token: cap.currentToken };
|
||||
})(arguments[0]);
|
||||
""",
|
||||
token,
|
||||
)
|
||||
|
||||
self._capture_token = token
|
||||
self._sent_content_length = 0
|
||||
if install_result is not None:
|
||||
logger.debug(f"[Executor] Kimi 页面抓流已准备: {install_result}")
|
||||
|
||||
def ensure_init_js(self) -> None:
|
||||
if self._init_js_id:
|
||||
return
|
||||
|
||||
# 修复(8):去重标记挂在 tab 对象属性上做 tab 级去重——
|
||||
# 每请求新建实例时,实例级 _init_js_id 无法阻止向同一 tab 重复注册 init script。
|
||||
existing_id = getattr(self.tab, "_kimi_fetch_init_js_id", None)
|
||||
if existing_id:
|
||||
self._init_js_id = existing_id
|
||||
return
|
||||
|
||||
try:
|
||||
self._init_js_id = self.tab.add_init_js(
|
||||
self._BOOTSTRAP_JS.strip()
|
||||
)
|
||||
try:
|
||||
setattr(self.tab, "_kimi_fetch_init_js_id", self._init_js_id)
|
||||
except Exception:
|
||||
pass
|
||||
logger.debug(
|
||||
f"[Executor] Kimi 页面抓流已注册 document-start 注入: {self._init_js_id}"
|
||||
)
|
||||
except Exception as e:
|
||||
logger.debug(f"[Executor] Kimi document-start 注入失败: {e}")
|
||||
|
||||
def get_state(self, since_length: int = 0) -> Dict[str, Any]:
|
||||
# 修复(7a):增加 sinceLength 增量参数,页面侧只返回 escapedFullText.slice(sinceLength)
|
||||
# 与 totalLength,避免每 0.3s 轮询全量回传导致 O(n^2) 传输;
|
||||
# 修复(7c):requestId 作为请求标识返回,Python 侧据此检测页面自动重试换流。
|
||||
try:
|
||||
since = max(0, int(since_length or 0))
|
||||
except Exception:
|
||||
since = 0
|
||||
# P0-2:结果在页面内 JSON.stringify,只回传字符串,不留 RemoteObject
|
||||
state = driver_for_tab(self.tab).run_js(
|
||||
"""
|
||||
return JSON.stringify((function(token, sinceLength) {
|
||||
const cap = window.__KIMI_CAPTURE__;
|
||||
if (!cap) {
|
||||
return { installed: false, found: false };
|
||||
}
|
||||
|
||||
const requests = Array.isArray(cap.requests) ? cap.requests : [];
|
||||
let target = null;
|
||||
|
||||
for (let i = requests.length - 1; i >= 0; i -= 1) {
|
||||
const item = requests[i];
|
||||
if (!token || item.token === token) {
|
||||
target = item;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
const fullText = target ? (target.escapedFullText || "") : "";
|
||||
const totalLength = fullText.length;
|
||||
const since = Math.max(0, Math.min(Number(sinceLength) || 0, totalLength));
|
||||
|
||||
return {
|
||||
installed: true,
|
||||
currentToken: cap.currentToken || null,
|
||||
found: !!target,
|
||||
requestId: target ? (target.id || "") : "",
|
||||
escapedDelta: fullText.slice(since),
|
||||
totalLength: totalLength,
|
||||
complete: !!(target && target.complete),
|
||||
error: target ? (target.error || null) : null,
|
||||
chunkCount: target ? (target.chunkCount || 0) : 0,
|
||||
startedAt: target ? (target.startedAt || 0) : 0,
|
||||
lastChunkAt: target ? (target.lastChunkAt || 0) : 0
|
||||
};
|
||||
})(arguments[0], arguments[1]));
|
||||
""",
|
||||
self._capture_token or "",
|
||||
since,
|
||||
)
|
||||
state = decode_js_json(state, {})
|
||||
return state if isinstance(state, dict) else {}
|
||||
|
||||
def _clear_page_capture_buffer(self) -> None:
|
||||
# 修复(7d):monitor 结束时尽力清空当前 token 对应请求的页面侧缓冲,失败忽略
|
||||
try:
|
||||
driver_for_tab(self.tab).run_js(
|
||||
"""
|
||||
return (function(token) {
|
||||
const cap = window.__KIMI_CAPTURE__;
|
||||
if (!cap || !Array.isArray(cap.requests)) {
|
||||
return { ok: false };
|
||||
}
|
||||
let cleared = 0;
|
||||
for (const item of cap.requests) {
|
||||
if ((!token || item.token === token) && item.escapedFullText) {
|
||||
item.escapedFullText = "";
|
||||
cleared += 1;
|
||||
}
|
||||
}
|
||||
return { ok: true, cleared: cleared };
|
||||
})(arguments[0]);
|
||||
""",
|
||||
self._capture_token or "",
|
||||
)
|
||||
except Exception as e:
|
||||
logger.debug(f"[Executor] Kimi 页面抓流缓冲清理失败(忽略): {e}")
|
||||
|
||||
def monitor(self, completion_id: str) -> Generator[str, None, None]:
|
||||
parser = self._parser
|
||||
parser.reset()
|
||||
|
||||
hard_timeout = float(
|
||||
self._stream_config.get("hard_timeout", 300) or 300
|
||||
)
|
||||
first_response_timeout = float(
|
||||
self._network_config.get("first_response_timeout", hard_timeout) or hard_timeout
|
||||
)
|
||||
response_interval = float(
|
||||
self._network_config.get("response_interval", 0.3) or 0.3
|
||||
)
|
||||
silence_threshold = float(
|
||||
self._network_config.get("silence_threshold", 3) or 3
|
||||
)
|
||||
|
||||
phase_start = time.time()
|
||||
last_activity = phase_start
|
||||
seen_request = False
|
||||
# 修复(7c):锁定首个命中的请求标识;标识变化说明页面自动重试换了流
|
||||
locked_request_id = ""
|
||||
# 修复(7a):Python 侧按偏移量增量取回并累积转义文本,避免全量回传
|
||||
escaped_buffer = ""
|
||||
|
||||
try:
|
||||
while True:
|
||||
if self._check_cancelled():
|
||||
logger.debug("[Executor] Kimi 页面抓流被取消")
|
||||
break
|
||||
|
||||
now = time.time()
|
||||
if now - phase_start > hard_timeout:
|
||||
raise NetworkMonitorError(f"kimi_page_capture_hard_timeout:{hard_timeout:.1f}s")
|
||||
|
||||
state = self.get_state(len(escaped_buffer))
|
||||
if not state.get("installed"):
|
||||
raise NetworkMonitorError("kimi_page_capture_not_installed")
|
||||
|
||||
if state.get("error"):
|
||||
raise NetworkMonitorError(f"kimi_page_capture_error:{state.get('error')}")
|
||||
|
||||
request_id = str(state.get("requestId", "") or "")
|
||||
if state.get("found"):
|
||||
if not seen_request:
|
||||
logger.debug(
|
||||
"[Executor] Kimi 页面抓流已命中请求 "
|
||||
f"(request_id={state.get('requestId')}, token={self._capture_token})"
|
||||
)
|
||||
seen_request = True
|
||||
locked_request_id = request_id
|
||||
elif locked_request_id and request_id and request_id != locked_request_id:
|
||||
# 修复(7c):页面自动重试换了流——重置解析状态与偏移,
|
||||
# 从新流头部重新取数(本次按旧偏移取到的增量作废)
|
||||
logger.warning(
|
||||
"[Executor] Kimi 页面抓流请求标识变化,重置解析状态 "
|
||||
f"({locked_request_id} -> {request_id})"
|
||||
)
|
||||
parser.reset()
|
||||
escaped_buffer = ""
|
||||
locked_request_id = request_id
|
||||
state = self.get_state(0)
|
||||
if state.get("error"):
|
||||
raise NetworkMonitorError(f"kimi_page_capture_error:{state.get('error')}")
|
||||
|
||||
delta = str(state.get("escapedDelta", "") or "")
|
||||
if delta:
|
||||
escaped_buffer += delta
|
||||
last_activity = now
|
||||
|
||||
if escaped_buffer:
|
||||
parse_result = parser.parse_chunk(escaped_buffer)
|
||||
if parse_result.get("error"):
|
||||
raise NetworkMonitorError(f"kimi_page_capture_parse_error:{parse_result['error']}")
|
||||
|
||||
content = parse_result.get("content", "")
|
||||
done = bool(parse_result.get("done")) or bool(state.get("complete"))
|
||||
|
||||
if content:
|
||||
logger.debug(f"[Executor] Kimi 页面抓流产出: {repr(content)[:240]}")
|
||||
self._sent_content_length += len(content)
|
||||
yield self.formatter.pack_chunk(content, completion_id=completion_id)
|
||||
|
||||
if done:
|
||||
logger.debug("[Executor] Kimi 页面抓流完成")
|
||||
break
|
||||
|
||||
elif seen_request and state.get("complete"):
|
||||
logger.debug("[Executor] Kimi 页面抓流请求已结束但无有效内容")
|
||||
break
|
||||
|
||||
if not seen_request and (now - phase_start) > first_response_timeout:
|
||||
raise NetworkMonitorTimeout(f"kimi_page_capture_first_response_timeout:{first_response_timeout:.1f}s")
|
||||
|
||||
if seen_request and (now - last_activity) > silence_threshold:
|
||||
logger.warning(
|
||||
"[Executor] Kimi 页面抓流静默超时 "
|
||||
f"({now - last_activity:.1f}s)"
|
||||
)
|
||||
raise NetworkMonitorTimeout(
|
||||
f"kimi_page_capture_silence_timeout:{silence_threshold:.1f}s"
|
||||
)
|
||||
|
||||
time.sleep(max(0.05, response_interval))
|
||||
finally:
|
||||
# 修复(7d):monitor 结束(正常/异常/取消)时清空页面侧缓冲,尽力而为
|
||||
self._clear_page_capture_buffer()
|
||||
|
||||
|
||||
register_page_fetch_capture(KimiPageFetchCapture)
|
||||
|
||||
|
||||
__all__ = ["KimiPageFetchCapture"]
|
||||
@@ -0,0 +1,53 @@
|
||||
"""
|
||||
app/core/page_capture/registry.py - registry for provider-specific page captures.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Callable, Dict, Optional, Type
|
||||
|
||||
from app.core.config import SSEFormatter, logger
|
||||
|
||||
from .base import InteractionSlot, PageFetchCapture
|
||||
|
||||
|
||||
_CAPTURE_CLASSES: Dict[str, Type[PageFetchCapture]] = {}
|
||||
|
||||
|
||||
def register_page_fetch_capture(capture_class: Type[PageFetchCapture]) -> None:
|
||||
parser_id = str(capture_class.get_parser_id() or "").strip().lower()
|
||||
if not parser_id:
|
||||
raise ValueError("page fetch capture parser_id is required")
|
||||
_CAPTURE_CLASSES[parser_id] = capture_class
|
||||
logger.info(f"已注册页面抓流实现: {parser_id} -> {capture_class.__name__}")
|
||||
|
||||
|
||||
def create_page_fetch_capture(
|
||||
*,
|
||||
tab,
|
||||
formatter: SSEFormatter,
|
||||
stream_config: Optional[Dict[str, Any]] = None,
|
||||
stop_checker: Optional[Callable[[], bool]] = None,
|
||||
interaction_slot: Optional[InteractionSlot] = None,
|
||||
) -> Optional[PageFetchCapture]:
|
||||
stream_config = stream_config or {}
|
||||
if stream_config.get("mode", "dom") != "network":
|
||||
return None
|
||||
|
||||
network_config = stream_config.get("network", {}) or {}
|
||||
parser_id = str(network_config.get("parser", "") or "").strip().lower()
|
||||
if not parser_id:
|
||||
return None
|
||||
|
||||
capture_class = _CAPTURE_CLASSES.get(parser_id)
|
||||
if capture_class is None:
|
||||
return None
|
||||
|
||||
return capture_class(
|
||||
tab=tab,
|
||||
formatter=formatter,
|
||||
stream_config=stream_config,
|
||||
network_config=network_config,
|
||||
stop_checker=stop_checker,
|
||||
interaction_slot=interaction_slot,
|
||||
)
|
||||
@@ -0,0 +1,67 @@
|
||||
"""
|
||||
app/core/page_capture/request_transport.py - page-side request transport registry.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import copy
|
||||
from typing import Any, Callable, Dict, List
|
||||
|
||||
|
||||
PageRequestTransportExecutor = Callable[..., Dict[str, Any]]
|
||||
|
||||
_PROFILES: Dict[str, Dict[str, Any]] = {}
|
||||
_EXECUTORS: Dict[str, PageRequestTransportExecutor] = {}
|
||||
|
||||
|
||||
def register_page_request_transport(
|
||||
profile: Dict[str, Any],
|
||||
executor: PageRequestTransportExecutor,
|
||||
) -> None:
|
||||
profile_id = str((profile or {}).get("id") or "").strip()
|
||||
if not profile_id:
|
||||
raise ValueError("page request transport profile id is required")
|
||||
if not callable(executor):
|
||||
raise ValueError(f"page request transport executor is not callable: {profile_id}")
|
||||
|
||||
_PROFILES[profile_id] = copy.deepcopy(profile)
|
||||
_EXECUTORS[profile_id] = executor
|
||||
|
||||
|
||||
def get_page_request_transport_profiles() -> List[Dict[str, Any]]:
|
||||
return [copy.deepcopy(profile) for profile in _PROFILES.values()]
|
||||
|
||||
|
||||
def get_page_request_transport_profile(profile_id: Any) -> Dict[str, Any] | None:
|
||||
pid = str(profile_id or "").strip()
|
||||
profile = _PROFILES.get(pid)
|
||||
return copy.deepcopy(profile) if profile else None
|
||||
|
||||
|
||||
def execute_page_request_transport(
|
||||
tab: Any,
|
||||
profile_id: Any,
|
||||
*,
|
||||
options: Dict[str, Any],
|
||||
prompt: str,
|
||||
consume_response: bool,
|
||||
) -> Dict[str, Any]:
|
||||
pid = str(profile_id or "").strip()
|
||||
executor = _EXECUTORS.get(pid)
|
||||
if executor is None:
|
||||
return {"ok": False, "error": f"unsupported_request_transport_profile:{pid}"}
|
||||
|
||||
return executor(
|
||||
tab=tab,
|
||||
options=options if isinstance(options, dict) else {},
|
||||
prompt=str(prompt or ""),
|
||||
consume_response=consume_response,
|
||||
)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"execute_page_request_transport",
|
||||
"get_page_request_transport_profile",
|
||||
"get_page_request_transport_profiles",
|
||||
"register_page_request_transport",
|
||||
]
|
||||
Reference in new issue
Block a user