Initial commit
This commit is contained in:
commit
57fc3525eb
7311 files changed
+1291398
No files matched your search
@@ -0,0 +1,10 @@
|
||||
from .clipboard import get_clipboard_lock
|
||||
from .manager import TabPoolManager
|
||||
from .session import TabSession, TabStatus
|
||||
|
||||
__all__ = [
|
||||
"TabStatus",
|
||||
"TabSession",
|
||||
"TabPoolManager",
|
||||
"get_clipboard_lock",
|
||||
]
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -0,0 +1,264 @@
|
||||
"""
|
||||
app/core/tab_pool_parts/_arena_snapshot.py
|
||||
|
||||
无外部依赖的纯 JavaScript 常量定义模块。
|
||||
供 network, arena_tab_listener 以及命令引擎沙箱安全引用。
|
||||
"""
|
||||
|
||||
_ARENA_STORE_SNAPSHOT_JS = r"""
|
||||
return (() => {
|
||||
function safe(fn, fallback) {
|
||||
try { return fn(); } catch (error) { return fallback; }
|
||||
}
|
||||
function isExplicitDirectUrl(url) {
|
||||
try {
|
||||
const u = new URL(url || location.href);
|
||||
const p = String(u.pathname || '').toLowerCase();
|
||||
if (
|
||||
p === '/direct' ||
|
||||
p.startsWith('/direct/') ||
|
||||
p === '/text/direct' ||
|
||||
p.startsWith('/text/direct/') ||
|
||||
p === '/image/direct' ||
|
||||
p.startsWith('/image/direct/') ||
|
||||
p === '/code/direct' ||
|
||||
p.startsWith('/code/direct/') ||
|
||||
p === '/search/direct' ||
|
||||
p.startsWith('/search/direct/') ||
|
||||
p.endsWith('/direct') ||
|
||||
u.searchParams.get('mode') === 'direct'
|
||||
) return true;
|
||||
return false;
|
||||
} catch (e) {
|
||||
const s = String(url || location.href || '').toLowerCase();
|
||||
return s.includes('/direct') || s.includes('mode=direct');
|
||||
}
|
||||
}
|
||||
function textOf(value) {
|
||||
if (typeof value === 'string') return value;
|
||||
if (Array.isArray(value)) {
|
||||
return value.map(item => {
|
||||
if (typeof item === 'string') return item;
|
||||
if (item && typeof item === 'object') return item.text || item.content || '';
|
||||
return '';
|
||||
}).filter(Boolean).join('\n');
|
||||
}
|
||||
return '';
|
||||
}
|
||||
function preferredModelName(value) {
|
||||
if (!value || typeof value !== 'object') return '';
|
||||
return String(value.displayName || value.publicName || value.name || value.modelName || value.slug || '').trim();
|
||||
}
|
||||
function modelIdOf(message) {
|
||||
if (!message || typeof message !== 'object') return '';
|
||||
const model = message.model;
|
||||
if (model && typeof model === 'object') {
|
||||
return String(model.id || model.modelId || '').trim();
|
||||
}
|
||||
return String(message.modelId || message.model || message.modelName || message.modelSlug || '').trim();
|
||||
}
|
||||
function getPageModelMap() {
|
||||
const html = safe(() => document.documentElement.outerHTML || '', '');
|
||||
const cache = window.__arenaDetectorModelMap;
|
||||
if (cache && cache.htmlLength === html.length && cache.map) return cache.map;
|
||||
|
||||
const map = {};
|
||||
const re = /\{\\"id\\":\\"[a-f0-9-]+\\"/g;
|
||||
let match;
|
||||
while ((match = re.exec(html)) && Object.keys(map).length < 2000) {
|
||||
const start = match.index;
|
||||
let openBraces = 0;
|
||||
let end = -1;
|
||||
const limit = Math.min(html.length, start + 20000);
|
||||
for (let i = start; i < limit; i += 1) {
|
||||
if (html[i] === '{') openBraces += 1;
|
||||
else if (html[i] === '}') {
|
||||
openBraces -= 1;
|
||||
if (openBraces === 0) {
|
||||
end = i + 1;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (end < 0) continue;
|
||||
const raw = html.slice(start, end).replace(/\\"/g, '"').replace(/\\\\/g, '\\');
|
||||
const item = safe(() => JSON.parse(raw), null);
|
||||
if (!item || typeof item !== 'object' || !item.id) continue;
|
||||
const name = preferredModelName(item);
|
||||
if (name) map[String(item.id).trim()] = name;
|
||||
}
|
||||
window.__arenaDetectorModelMap = { htmlLength: html.length, map };
|
||||
return map;
|
||||
}
|
||||
function modelNameOf(message) {
|
||||
if (!message || typeof message !== 'object') return '';
|
||||
const direct = preferredModelName(message.model) || preferredModelName(message);
|
||||
if (direct) return direct;
|
||||
const modelId = modelIdOf(message);
|
||||
if (!modelId) return '';
|
||||
return getPageModelMap()[modelId] || modelId;
|
||||
}
|
||||
function findReactFiber(el) {
|
||||
if (!el) return null;
|
||||
const key = Object.keys(el).find(k => k.startsWith('__reactFiber$') || k.startsWith('__reactInternalInstance$'));
|
||||
return key ? el[key] : null;
|
||||
}
|
||||
function looksLikeArenaStore(value) {
|
||||
if (!value || typeof value !== 'object') return false;
|
||||
if (typeof value.getState !== 'function') return false;
|
||||
const state = safe(() => value.getState(), null);
|
||||
return !!(state && typeof state === 'object' && Array.isArray(state.messages) && typeof state.id === 'string');
|
||||
}
|
||||
function findArenaStoreIn(value, depth, seen) {
|
||||
if (!value || typeof value !== 'object' || depth < 0 || seen.has(value)) return null;
|
||||
seen.add(value);
|
||||
if (looksLikeArenaStore(value)) return value;
|
||||
const keys = safe(() => Object.keys(value), []);
|
||||
for (const key of keys.slice(0, 100)) {
|
||||
if (['_owner', 'return', 'child', 'sibling', 'alternate'].includes(key)) continue;
|
||||
const found = findArenaStoreIn(value[key], depth - 1, seen);
|
||||
if (found) return found;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
function findStoreFromFiber() {
|
||||
const roots = [
|
||||
document.querySelector('main'),
|
||||
document.querySelector('form'),
|
||||
document.body,
|
||||
].filter(Boolean);
|
||||
for (const root of roots) {
|
||||
const fiber = findReactFiber(root);
|
||||
for (let cur = fiber, depth = 0; cur && depth < 100; depth += 1, cur = cur.return) {
|
||||
const found = findArenaStoreIn(cur.memoizedProps, 5, new WeakSet())
|
||||
|| findArenaStoreIn(cur.memoizedState, 5, new WeakSet());
|
||||
if (found) return found;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
const store = findStoreFromFiber();
|
||||
const state = store && safe(() => store.getState(), null);
|
||||
const stateMode = String((state && (state.evaluationMode || state.mode || state.sessionMode || state.type)) || '').toLowerCase();
|
||||
const isDirect = stateMode.includes('direct') || stateMode.includes('single') || isExplicitDirectUrl(location.href);
|
||||
|
||||
if (isDirect) {
|
||||
return {
|
||||
is_direct: true,
|
||||
mode: 'direct',
|
||||
url: location.href,
|
||||
conversation_id: String(state && state.id || ''),
|
||||
message_id_a: '',
|
||||
message_id_b: '',
|
||||
status_a: '',
|
||||
status_b: '',
|
||||
model_a: '',
|
||||
model_b: '',
|
||||
model_id_a: '',
|
||||
model_id_b: '',
|
||||
response_a: '',
|
||||
response_b: '',
|
||||
};
|
||||
}
|
||||
|
||||
const messages = state && Array.isArray(state.messages) ? state.messages : [];
|
||||
const byId = new Map(messages.map(message => [String(message && message.id || ''), message]));
|
||||
let assistantIds = Array.isArray(state && state.lastMessageIds)
|
||||
? state.lastMessageIds.map(id => String(id || '')).filter(id => byId.get(id) && byId.get(id).role === 'assistant')
|
||||
: [];
|
||||
|
||||
// 严格校验双侧对战:必须有成对的 2 个 assistant 消息
|
||||
if (assistantIds.length < 2) {
|
||||
const lastAssistants = messages.filter(m => m && m.role === 'assistant');
|
||||
if (lastAssistants.length >= 2) {
|
||||
const cand1 = lastAssistants[lastAssistants.length - 1];
|
||||
const cand2 = lastAssistants[lastAssistants.length - 2];
|
||||
const p1 = String((Array.isArray(cand1.parentMessageIds) ? cand1.parentMessageIds[0] : cand1.parentId) || '');
|
||||
const p2 = String((Array.isArray(cand2.parentMessageIds) ? cand2.parentMessageIds[0] : cand2.parentId) || '');
|
||||
const s1 = String(cand1.side || cand1.modelSide || '').toLowerCase();
|
||||
const s2 = String(cand2.side || cand2.modelSide || '').toLowerCase();
|
||||
const m1 = modelIdOf(cand1);
|
||||
const m2 = modelIdOf(cand2);
|
||||
const isBattlePair = (p1 && p2 && p1 === p2) && (
|
||||
(s1 && s2 && s1 !== s2) ||
|
||||
(m1 && m2 && m1 !== m2)
|
||||
);
|
||||
if (isBattlePair) {
|
||||
assistantIds = [String(cand2.id || ''), String(cand1.id || '')];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (assistantIds.length < 2) {
|
||||
return {
|
||||
is_direct: false,
|
||||
mode: stateMode || 'arena',
|
||||
url: location.href,
|
||||
conversation_id: String(state && state.id || ''),
|
||||
message_id_a: '',
|
||||
message_id_b: '',
|
||||
status_a: '',
|
||||
status_b: '',
|
||||
model_a: '',
|
||||
model_b: '',
|
||||
model_id_a: '',
|
||||
model_id_b: '',
|
||||
response_a: '',
|
||||
response_b: '',
|
||||
};
|
||||
}
|
||||
|
||||
const a = byId.get(assistantIds[0]) || null;
|
||||
const b = byId.get(assistantIds[1]) || null;
|
||||
const parentIds = []
|
||||
.concat(Array.isArray(a && a.parentMessageIds) ? a.parentMessageIds : [])
|
||||
.concat(Array.isArray(b && b.parentMessageIds) ? b.parentMessageIds : []);
|
||||
let userMessage = null;
|
||||
for (const parentId of parentIds) {
|
||||
const candidate = byId.get(String(parentId || ''));
|
||||
if (candidate && candidate.role === 'user') {
|
||||
userMessage = candidate;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!userMessage) {
|
||||
for (let i = messages.length - 1; i >= 0; i -= 1) {
|
||||
if (messages[i] && messages[i].role === 'user') {
|
||||
userMessage = messages[i];
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
return {
|
||||
is_direct: false,
|
||||
mode: stateMode || 'arena',
|
||||
url: location.href,
|
||||
conversation_id: String(state && state.id || ''),
|
||||
prompt: textOf(userMessage && userMessage.content),
|
||||
message_id_a: String(a && a.id || ''),
|
||||
message_id_b: String(b && b.id || ''),
|
||||
status_a: String(a && a.status || ''),
|
||||
status_b: String(b && b.status || ''),
|
||||
model_a: modelNameOf(a),
|
||||
model_b: modelNameOf(b),
|
||||
model_id_a: modelIdOf(a),
|
||||
model_id_b: modelIdOf(b),
|
||||
response_a: textOf(a && a.content),
|
||||
response_b: textOf(b && b.content),
|
||||
};
|
||||
})()
|
||||
""".strip()
|
||||
|
||||
# P0-2:轮询用的 JSON 版本。页面内 JSON.stringify 后只回传一个字符串,
|
||||
# 不会在渲染进程里留下 RemoteObject,也省掉 DrissionPage 额外的一次序列化往返。
|
||||
# 原常量保留(命令沙箱脚本可能依赖它返回 dict)。调用方用 json.loads 解码。
|
||||
_ARENA_STORE_SNAPSHOT_JSON_JS = (
|
||||
"function(){"
|
||||
"const __uwapiResult = (function(){\n"
|
||||
+ _ARENA_STORE_SNAPSHOT_JS
|
||||
+ "\n}).apply(this, arguments);"
|
||||
"return JSON.stringify(__uwapiResult === undefined ? null : __uwapiResult);"
|
||||
"}"
|
||||
)
|
||||
|
||||
__all__ = ["_ARENA_STORE_SNAPSHOT_JS", "_ARENA_STORE_SNAPSHOT_JSON_JS"]
|
||||
@@ -0,0 +1,52 @@
|
||||
from typing import Any
|
||||
|
||||
from app.utils.site_url import is_remote_site_url
|
||||
|
||||
|
||||
_POOL_SKIP_URL_PREFIXES = (
|
||||
"about:",
|
||||
"chrome://",
|
||||
"chrome-devtools://",
|
||||
"devtools://",
|
||||
"edge://",
|
||||
"brave://",
|
||||
"javascript:",
|
||||
"data:",
|
||||
"blob:",
|
||||
)
|
||||
|
||||
_POOL_SKIP_URL_CONTAINS = (
|
||||
"chrome-error://",
|
||||
"about:neterror",
|
||||
)
|
||||
|
||||
_TAB_HEALTH_CACHE_TTL_SEC = 5.0
|
||||
|
||||
|
||||
def _looks_like_transient_local_debug_error(error: Any) -> bool:
|
||||
text = str(error or "").strip().lower()
|
||||
if not text:
|
||||
return False
|
||||
if "winerror 10048" in text:
|
||||
return True
|
||||
if "max retries exceeded" in text and "127.0.0.1" in text and "/json" in text:
|
||||
return True
|
||||
if "failed to establish a new connection" in text and "127.0.0.1" in text:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _should_skip_pool_url(url: str) -> bool:
|
||||
raw = str(url or "").strip()
|
||||
if not raw:
|
||||
return True
|
||||
|
||||
lowered = raw.lower()
|
||||
if lowered.startswith(_POOL_SKIP_URL_PREFIXES):
|
||||
return True
|
||||
if any(marker in lowered for marker in _POOL_SKIP_URL_CONTAINS):
|
||||
return True
|
||||
|
||||
return not is_remote_site_url(lowered)
|
||||
|
||||
|
||||
@@ -0,0 +1,8 @@
|
||||
import threading
|
||||
|
||||
|
||||
_clipboard_lock = threading.Lock()
|
||||
|
||||
|
||||
def get_clipboard_lock() -> threading.Lock:
|
||||
return _clipboard_lock
|
||||
@@ -0,0 +1,904 @@
|
||||
"""
|
||||
app/core/tab_pool_parts/idle_maintenance.py - 空闲标签页维护(P0-3 / P0-6)
|
||||
|
||||
P0-3 CDP 会话回收(recycle)
|
||||
DrissionPage 执行 run_js / 元素查找时产生的 RemoteObject、DOM agent 的
|
||||
nodeId 映射都挂在 CDP 会话上,只有会话断开才会整体释放;它们会把早已从
|
||||
页面移除的旧对话 DOM 钉在渲染进程里。空闲标签页满足触发条件时,断开并
|
||||
重新建立该标签页的 CDP 连接(不刷新页面、不影响登录态),释放这些引用。
|
||||
|
||||
顺序:maint 占用会话 → 停止全局网络监听 → tab.disconnect() →
|
||||
tab._driver_init(target_id) → tab._get_document() →
|
||||
重置 listener 驱动状态 / 会话级注入脚本标记 → 释放会话(释放流程会恢复
|
||||
可见性模拟状态,全局监听随后自动重启)。
|
||||
|
||||
触发(任一满足,且自上次回收后标签页确有使用):
|
||||
- 自上次回收后已处理 BROWSER_CDP_RECYCLE_AFTER_REQUESTS 个请求(默认 20)
|
||||
- 距上次回收超过 BROWSER_CDP_RECYCLE_INTERVAL_SEC(默认 1800s)
|
||||
- Memory.getDOMCounters 节点数 ≥ BROWSER_CDP_RECYCLE_DOM_NODES(默认 150000)
|
||||
|
||||
P0-6 空闲冻结(freeze)
|
||||
空闲 ≥ BROWSER_IDLE_FREEZE_AFTER_SEC(默认 60s)、且真实 visibilityState 为
|
||||
hidden 的标签页执行 Page.setWebLifecycleState=frozen(计时器 / rAF / 页面任务
|
||||
全部暂停)。被占用(acquire / acquire_for_command)或 page_check 唤醒时
|
||||
先恢复为 active。可见(用户正在看)的标签页永远不会被冻结。
|
||||
|
||||
环境变量:
|
||||
BROWSER_IDLE_FREEZE_ENABLED 默认 true
|
||||
BROWSER_IDLE_FREEZE_AFTER_SEC 默认 60(最小 15)
|
||||
BROWSER_IDLE_FREEZE_REQUIRE_HIDDEN 默认 true(仅冻结后台/被遮挡的标签页)
|
||||
BROWSER_CDP_RECYCLE_ENABLED 默认 true
|
||||
BROWSER_CDP_RECYCLE_IDLE_SEC 默认 30(最小 10)
|
||||
BROWSER_CDP_RECYCLE_AFTER_REQUESTS 默认 20(0 = 不按请求数触发)
|
||||
BROWSER_CDP_RECYCLE_INTERVAL_SEC 默认 1800(0 = 不按时间触发)
|
||||
BROWSER_CDP_RECYCLE_DOM_NODES 默认 150000(0 = 不按节点数触发)
|
||||
BROWSER_CDP_RECYCLE_DOM_CHECK_SEC 默认 300
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import threading
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Optional
|
||||
|
||||
from app.core.config import logger
|
||||
from app.core.driver import browser_driver, driver_for_tab
|
||||
|
||||
|
||||
# 真实可见性:可见性模拟在 document 实例上定义了 own property,
|
||||
# 这里直接调用 Document.prototype 上的原生 getter,得到浏览器真实状态。
|
||||
REAL_VISIBILITY_JS = (
|
||||
"return (function(){try{"
|
||||
"var d=Object.getOwnPropertyDescriptor(Document.prototype,'visibilityState');"
|
||||
"return String(d&&d.get?d.get.call(document):document.visibilityState);"
|
||||
"}catch(e){return 'unknown';}})();"
|
||||
)
|
||||
|
||||
_LOCK_FACTORY_GUARD = threading.Lock()
|
||||
|
||||
|
||||
def _env_bool(name: str, default: bool) -> bool:
|
||||
raw = os.getenv(name)
|
||||
if raw is None or not str(raw).strip():
|
||||
return default
|
||||
return str(raw).strip().lower() in {"1", "true", "yes", "y", "on"}
|
||||
|
||||
|
||||
def _env_float(name: str, default: float, minimum: float = 0.0) -> float:
|
||||
raw = os.getenv(name)
|
||||
try:
|
||||
value = float(str(raw).strip()) if raw not in (None, "") else float(default)
|
||||
except (TypeError, ValueError):
|
||||
value = float(default)
|
||||
if not math.isfinite(value):
|
||||
# B6:inf / nan 不是有效配置(int(inf) 会 OverflowError,nan 让所有比较为假)。
|
||||
# 想要「禁用」请写 0。
|
||||
logger.warning(f"[IdleMaintenance] {name}={raw!r} 不是有限数值,使用默认值 {default}(禁用请设为 0)")
|
||||
value = float(default)
|
||||
if value <= 0:
|
||||
return 0.0
|
||||
return max(minimum, value)
|
||||
|
||||
|
||||
def _env_int(name: str, default: int, minimum: int = 0) -> int:
|
||||
return int(_env_float(name, float(default), float(minimum)))
|
||||
|
||||
|
||||
@dataclass
|
||||
class IdleMaintenanceConfig:
|
||||
freeze_enabled: bool = True
|
||||
freeze_after_sec: float = 60.0
|
||||
freeze_require_hidden: bool = True
|
||||
recycle_enabled: bool = True
|
||||
recycle_idle_sec: float = 30.0
|
||||
recycle_after_requests: int = 20
|
||||
recycle_interval_sec: float = 1800.0
|
||||
recycle_dom_nodes: int = 150000
|
||||
dom_check_interval_sec: float = 300.0
|
||||
|
||||
@classmethod
|
||||
def from_env(cls) -> "IdleMaintenanceConfig":
|
||||
return cls(
|
||||
freeze_enabled=_env_bool("BROWSER_IDLE_FREEZE_ENABLED", True),
|
||||
freeze_after_sec=_env_float("BROWSER_IDLE_FREEZE_AFTER_SEC", 60.0, 15.0) or 60.0,
|
||||
freeze_require_hidden=_env_bool("BROWSER_IDLE_FREEZE_REQUIRE_HIDDEN", True),
|
||||
recycle_enabled=_env_bool("BROWSER_CDP_RECYCLE_ENABLED", True),
|
||||
recycle_idle_sec=_env_float("BROWSER_CDP_RECYCLE_IDLE_SEC", 30.0, 10.0) or 30.0,
|
||||
recycle_after_requests=_env_int("BROWSER_CDP_RECYCLE_AFTER_REQUESTS", 20, 1),
|
||||
recycle_interval_sec=_env_float("BROWSER_CDP_RECYCLE_INTERVAL_SEC", 1800.0, 60.0),
|
||||
recycle_dom_nodes=_env_int("BROWSER_CDP_RECYCLE_DOM_NODES", 150000, 1000),
|
||||
dom_check_interval_sec=_env_float("BROWSER_CDP_RECYCLE_DOM_CHECK_SEC", 300.0, 30.0) or 300.0,
|
||||
)
|
||||
|
||||
|
||||
def is_idle_freeze_enabled() -> bool:
|
||||
return _env_bool("BROWSER_IDLE_FREEZE_ENABLED", True)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# helpers
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
def _run_cdp(tab: Any, method: str, *, timeout: float = 2.0, **params):
|
||||
runner = getattr(tab, "run_cdp", None)
|
||||
if not callable(runner):
|
||||
runner = getattr(tab, "_run_cdp", None)
|
||||
if not callable(runner):
|
||||
raise AttributeError(f"CDP runner unavailable for {method}")
|
||||
try:
|
||||
return runner(method, _timeout=max(0.1, float(timeout)), **params)
|
||||
except TypeError as exc:
|
||||
if "timeout" not in str(exc).lower():
|
||||
raise
|
||||
return runner(method, **params)
|
||||
|
||||
|
||||
def lifecycle_lock(session: Any) -> threading.Lock:
|
||||
lock = getattr(session, "_uwapi_lifecycle_lock", None)
|
||||
if lock is None:
|
||||
with _LOCK_FACTORY_GUARD:
|
||||
lock = getattr(session, "_uwapi_lifecycle_lock", None)
|
||||
if lock is None:
|
||||
lock = threading.Lock()
|
||||
setattr(session, "_uwapi_lifecycle_lock", lock)
|
||||
return lock
|
||||
|
||||
|
||||
def _status_name(session: Any) -> str:
|
||||
status = getattr(session, "status", None)
|
||||
return str(getattr(status, "value", status) or "").lower()
|
||||
|
||||
|
||||
def _is_idle(session: Any) -> bool:
|
||||
return _status_name(session) == "idle" and not bool(getattr(session, "_termination_in_progress", False))
|
||||
|
||||
|
||||
def last_activity_at(session: Any) -> float:
|
||||
"""Most recent sign of life: request/command use, page_check wake, network event."""
|
||||
values = [
|
||||
getattr(session, "last_used_at", 0.0),
|
||||
getattr(session, "_pc_last_wake_at", 0.0),
|
||||
getattr(session, "_uwapi_last_network_event_at", 0.0),
|
||||
getattr(session, "_uwapi_last_resumed_at", 0.0),
|
||||
]
|
||||
best = 0.0
|
||||
for value in values:
|
||||
try:
|
||||
best = max(best, float(value or 0.0))
|
||||
except (TypeError, ValueError):
|
||||
continue
|
||||
return best
|
||||
|
||||
|
||||
def note_network_activity(session: Any) -> None:
|
||||
try:
|
||||
setattr(session, "_uwapi_last_network_event_at", time.time())
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# freeze / resume (P0-6)
|
||||
# --------------------------------------------------------------------------- #
|
||||
#
|
||||
# 注意:DrissionPage 在 _driver_init 中对每个标签页调用
|
||||
# Emulation.setFocusEmulationEnabled(true)。Chromium 用“增加 capturer 计数”
|
||||
# 实现该仿真,结果是被控制的标签页永远被视为 visible —— 后台标签页既不会被
|
||||
# 浏览器降频,Page.setWebLifecycleState=frozen 也会被静默忽略(实测:冻结后
|
||||
# 计时器照常运行)。因此冻结前必须先在本会话上关闭焦点仿真,读取真实可见性;
|
||||
# 恢复时重新打开。
|
||||
|
||||
|
||||
def _set_focus_emulation(tab: Any, enabled: bool) -> bool:
|
||||
try:
|
||||
_run_cdp(tab, "Emulation.setFocusEmulationEnabled", enabled=bool(enabled), timeout=1.5)
|
||||
return True
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def _read_real_visibility(tab: Any) -> str:
|
||||
try:
|
||||
return str(driver_for_tab(tab).run_js(REAL_VISIBILITY_JS, timeout=1.0) or "").strip().lower()
|
||||
except Exception:
|
||||
return "unknown"
|
||||
|
||||
|
||||
def _wait_real_hidden(tab: Any, timeout_sec: float = 0.6, should_abort=None) -> str:
|
||||
"""关闭焦点仿真后,可见性变化是异步传到渲染进程的,短暂轮询。"""
|
||||
deadline = time.time() + max(0.0, timeout_sec)
|
||||
state = _read_real_visibility(tab)
|
||||
while state != "hidden" and time.time() < deadline:
|
||||
if should_abort is not None and should_abort():
|
||||
return "aborted"
|
||||
time.sleep(0.05)
|
||||
state = _read_real_visibility(tab)
|
||||
return state
|
||||
|
||||
|
||||
# 冻结核验:CDP 冻结可能被 Chrome 静默忽略或在我们背后解除(发起冻结的 DevTools 会话断开、
|
||||
# 用户切回标签页、与其它会话附加并发等,均已实测)。冻结前在页面挂一次性的标准生命周期事件
|
||||
# freeze / resume 监听(不可枚举属性,不影响页面逻辑),用同步 run_js 读取——冻结的页面也能执行。
|
||||
_FREEZE_MARK_INSTALL_JS = (
|
||||
"return (function(t){try{var st={t:t,f:0,r:0};"
|
||||
"Object.defineProperty(window,'__uwapiFz',{value:st,configurable:true,enumerable:false,writable:true});"
|
||||
"document.addEventListener('freeze',function(){st.f=Date.now();},{once:true});"
|
||||
"document.addEventListener('resume',function(){st.r=Date.now();},{once:true});"
|
||||
"return 1;}catch(e){return 0;}})(arguments[0]);"
|
||||
)
|
||||
_FREEZE_MARK_READ_JS = (
|
||||
"return (function(){try{var s=window.__uwapiFz;"
|
||||
"return s?JSON.stringify({t:s.t,f:s.f,r:s.r}):'';}catch(e){return '';}})();"
|
||||
)
|
||||
|
||||
|
||||
def _read_freeze_mark(tab: Any) -> Optional[dict]:
|
||||
"""``None`` = could not read; ``{}`` = marker missing (document replaced)."""
|
||||
try:
|
||||
raw = driver_for_tab(tab).run_js(_FREEZE_MARK_READ_JS, timeout=1.0)
|
||||
except Exception:
|
||||
return None
|
||||
if not raw:
|
||||
return {}
|
||||
try:
|
||||
data = json.loads(raw)
|
||||
return data if isinstance(data, dict) else {}
|
||||
except Exception:
|
||||
return {}
|
||||
|
||||
|
||||
def _freeze_took_effect(tab: Any, token: str, timeout_sec: float = 0.4, should_abort=None) -> bool:
|
||||
deadline = time.time() + max(0.0, timeout_sec)
|
||||
while True:
|
||||
mark = _read_freeze_mark(tab)
|
||||
if mark is None:
|
||||
return True # 读不到(非真实页面对象等)时不否定冻结,交给后续巡检
|
||||
if mark.get("t") == token and mark.get("f") and not mark.get("r"):
|
||||
return True
|
||||
if should_abort is not None and should_abort():
|
||||
return True # 已被占用:不再核验,按已冻结登记,由等锁的占用方立刻解冻
|
||||
if time.time() >= deadline:
|
||||
return False
|
||||
time.sleep(0.03)
|
||||
|
||||
|
||||
def freeze_was_lifted(session: Any) -> bool:
|
||||
"""True when the page is known to be running again although we think it is frozen."""
|
||||
tab = getattr(session, "tab", None)
|
||||
token = str(getattr(session, "_uwapi_freeze_token", "") or "")
|
||||
if tab is None or not token:
|
||||
return False
|
||||
mark = _read_freeze_mark(tab)
|
||||
if mark is None:
|
||||
return False
|
||||
return mark.get("t") != token or bool(mark.get("r"))
|
||||
|
||||
|
||||
def _front_tab_hint(tab: Any) -> str:
|
||||
"""Guess whether the user can see ``tab`` WITHOUT touching focus emulation.
|
||||
|
||||
Toggling focus emulation off/on (the authoritative probe) fires ``blur`` /
|
||||
``focus`` on a tab the user is looking at, and sites with refetch-on-focus
|
||||
react to that. So first use browser-side facts that have no page-visible
|
||||
side effects:
|
||||
|
||||
* ``"minimized"`` – the tab's window is minimized (hidden for sure)
|
||||
* ``"front"`` – the tab is the most recently activated page of its
|
||||
(non-minimized) window, i.e. the tab currently shown there. The DevTools
|
||||
``/json`` list is MRU-ordered (DrissionPage's ``latest_tab`` relies on it).
|
||||
* ``"background"`` – another page is in front of it in the same window
|
||||
* ``"unknown"`` – cannot tell (ws-only mode, errors, …) → caller probes
|
||||
"""
|
||||
try:
|
||||
info = _run_cdp(tab, "Browser.getWindowForTarget", timeout=1.0) or {}
|
||||
window_id = info.get("windowId")
|
||||
window_state = str((info.get("bounds") or {}).get("windowState") or "").lower()
|
||||
except Exception:
|
||||
return "unknown"
|
||||
if window_state == "minimized":
|
||||
return "minimized"
|
||||
browser = getattr(tab, "browser", None)
|
||||
my_id = str(getattr(tab, "tab_id", "") or "")
|
||||
if browser is None or not my_id or window_id is None or getattr(browser, "_ws_only", False):
|
||||
return "unknown"
|
||||
try:
|
||||
mru_ids = list(browser.tab_ids)
|
||||
except Exception:
|
||||
return "unknown"
|
||||
for target_id in mru_ids[:32]:
|
||||
if str(target_id) == my_id:
|
||||
return "front"
|
||||
try:
|
||||
other = browser_driver(browser).run_cdp("Browser.getWindowForTarget", targetId=target_id) or {}
|
||||
except Exception:
|
||||
continue
|
||||
if other.get("windowId") == window_id:
|
||||
return "background"
|
||||
return "unknown"
|
||||
|
||||
|
||||
_VISIBLE_BACKOFF_MAX_SEC = 900.0
|
||||
|
||||
|
||||
def _schedule_visible_retry(session: Any, config: "IdleMaintenanceConfig", *, probed: bool) -> None:
|
||||
"""Back off after finding the tab visible.
|
||||
|
||||
A side-effect-free ``front`` hint retries at the normal cadence; an actual
|
||||
probe (which toggles focus emulation) backs off exponentially up to 15 min
|
||||
so a tab the user keeps looking at sees at most a handful of blur/focus
|
||||
pairs. Any real activity resets the streak (see schedule_idle_maintenance).
|
||||
"""
|
||||
base = max(30.0, float(config.freeze_after_sec))
|
||||
if not probed:
|
||||
setattr(session, "_uwapi_freeze_retry_at", time.time() + base)
|
||||
return
|
||||
activity = last_activity_at(session)
|
||||
streak = int(getattr(session, "_uwapi_visible_probe_streak", 0) or 0)
|
||||
if activity > float(getattr(session, "_uwapi_visible_probe_mark", 0.0) or 0.0):
|
||||
streak = 0
|
||||
streak += 1
|
||||
setattr(session, "_uwapi_visible_probe_streak", streak)
|
||||
setattr(session, "_uwapi_visible_probe_mark", activity)
|
||||
delay = min(_VISIBLE_BACKOFF_MAX_SEC, base * (2 ** (streak - 1)))
|
||||
setattr(session, "_uwapi_freeze_retry_at", time.time() + delay)
|
||||
|
||||
|
||||
def _restore_focus_emulation(session: Any, tab: Any) -> None:
|
||||
if getattr(session, "_uwapi_focus_emulation_off", False):
|
||||
_set_focus_emulation(tab, True)
|
||||
setattr(session, "_uwapi_focus_emulation_off", False)
|
||||
|
||||
|
||||
def resume_if_frozen(session: Any, reason: str = "") -> bool:
|
||||
"""Set a frozen tab back to ``active``. Cheap no-op when not frozen.
|
||||
|
||||
Always goes through the lifecycle lock so a concurrent freeze cannot land
|
||||
after the session has been acquired.
|
||||
"""
|
||||
if session is None:
|
||||
return False
|
||||
if (
|
||||
not getattr(session, "_uwapi_frozen", False)
|
||||
and not getattr(session, "_uwapi_freeze_in_progress", False)
|
||||
and not getattr(session, "_uwapi_focus_emulation_off", False)
|
||||
):
|
||||
return False
|
||||
with lifecycle_lock(session):
|
||||
tab = getattr(session, "tab", None)
|
||||
if not getattr(session, "_uwapi_frozen", False):
|
||||
_restore_focus_emulation(session, tab)
|
||||
return False
|
||||
ok = False
|
||||
try:
|
||||
_run_cdp(tab, "Page.setWebLifecycleState", state="active", timeout=2.0)
|
||||
ok = True
|
||||
except Exception as exc:
|
||||
logger.warning(
|
||||
f"[{getattr(session, 'id', '?')}] 解除标签页冻结失败(reason={reason or '-'}): {exc}"
|
||||
)
|
||||
finally:
|
||||
_restore_focus_emulation(session, tab)
|
||||
if ok:
|
||||
setattr(session, "_uwapi_frozen", False)
|
||||
setattr(session, "_uwapi_freeze_token", "")
|
||||
setattr(session, "_uwapi_last_resumed_at", time.time())
|
||||
setattr(session, "_uwapi_resume_failed_at", 0.0)
|
||||
else:
|
||||
# B5:解冻失败时保留「待确认冻结」状态,调用方据此拒绝交付该会话;
|
||||
# 旧实现无条件清除冻结标志,acquire() 会把仍冻结的页面当作可用标签页交出去。
|
||||
setattr(session, "_uwapi_resume_failed_at", time.time())
|
||||
frozen_at = float(getattr(session, "_uwapi_frozen_at", 0.0) or 0.0)
|
||||
if ok:
|
||||
logger.debug(
|
||||
f"[{getattr(session, 'id', '?')}] 标签页已解除冻结 "
|
||||
f"(reason={reason or '-'}, frozen_for={max(0.0, time.time() - frozen_at):.0f}s)"
|
||||
)
|
||||
return ok
|
||||
|
||||
|
||||
def freeze_session(manager: Any, session: Any, config: IdleMaintenanceConfig) -> bool:
|
||||
"""Freeze one idle, hidden tab. Returns True when the tab was frozen."""
|
||||
if session is None or getattr(manager, "_shutdown", False):
|
||||
return False
|
||||
lock = lifecycle_lock(session)
|
||||
if not lock.acquire(timeout=0.5):
|
||||
return False
|
||||
try:
|
||||
setattr(session, "_uwapi_freeze_in_progress", True)
|
||||
session_lock = getattr(session, "_lock", None)
|
||||
if session_lock is not None:
|
||||
with session_lock:
|
||||
eligible = _is_idle(session)
|
||||
else:
|
||||
eligible = _is_idle(session)
|
||||
if not eligible or getattr(session, "_uwapi_frozen", False):
|
||||
return False
|
||||
if getattr(session, "_cdp_recycle_in_progress", False):
|
||||
return False
|
||||
if (time.time() - last_activity_at(session)) < config.freeze_after_sec:
|
||||
return False
|
||||
|
||||
tab = getattr(session, "tab", None)
|
||||
if tab is None:
|
||||
return False
|
||||
|
||||
def claimed() -> bool:
|
||||
# 冻结给占用让路:acquire 先把状态置为 BUSY 再在 lifecycle 锁上等待解冻,
|
||||
# 这里每一步都检查,发现被占用立刻收手,不让请求等冻结流程走完
|
||||
return (not _is_idle(session)) or bool(getattr(manager, "_shutdown", False))
|
||||
|
||||
if config.freeze_require_hidden and _front_tab_hint(tab) == "front":
|
||||
# 用户很可能正看着它:不切换焦点仿真(避免给页面发 blur/focus),直接跳过
|
||||
_schedule_visible_retry(session, config, probed=False)
|
||||
return False
|
||||
if claimed():
|
||||
return False
|
||||
# 先关闭焦点仿真,否则 Chromium 视其为 visible,冻结无效(见上方说明)
|
||||
setattr(session, "_uwapi_focus_emulation_off", True)
|
||||
_set_focus_emulation(tab, False)
|
||||
if config.freeze_require_hidden:
|
||||
state = _wait_real_hidden(tab, should_abort=claimed)
|
||||
if state == "aborted" or claimed():
|
||||
_restore_focus_emulation(session, tab)
|
||||
return False
|
||||
if state != "hidden":
|
||||
# 用户正在看这个标签页(前台 / 独立窗口未最小化)或无法判断:不冻结
|
||||
_restore_focus_emulation(session, tab)
|
||||
_schedule_visible_retry(session, config, probed=True)
|
||||
return False
|
||||
|
||||
token = f"{getattr(session, 'id', '?')}:{time.time_ns()}"
|
||||
mark_installed = False
|
||||
try:
|
||||
mark_installed = bool(driver_for_tab(tab).run_js(_FREEZE_MARK_INSTALL_JS, token, timeout=1.0))
|
||||
except Exception:
|
||||
mark_installed = False
|
||||
|
||||
if claimed():
|
||||
_restore_focus_emulation(session, tab)
|
||||
return False
|
||||
# 此后若被占用,占用方会在 lifecycle 锁上等待并解冻
|
||||
try:
|
||||
_run_cdp(tab, "Page.setWebLifecycleState", state="frozen", timeout=1.5)
|
||||
except Exception:
|
||||
_restore_focus_emulation(session, tab)
|
||||
raise
|
||||
if mark_installed and not _freeze_took_effect(tab, token, should_abort=claimed):
|
||||
# Chrome 没有真正冻结(页面仍在跑):撤销,稍后重试
|
||||
try:
|
||||
_run_cdp(tab, "Page.setWebLifecycleState", state="active", timeout=1.5)
|
||||
except Exception:
|
||||
pass
|
||||
_restore_focus_emulation(session, tab)
|
||||
setattr(session, "_uwapi_freeze_retry_at", time.time() + 30.0)
|
||||
setattr(session, "_uwapi_freeze_ineffective_count",
|
||||
int(getattr(session, "_uwapi_freeze_ineffective_count", 0) or 0) + 1)
|
||||
logger.debug(f"[{getattr(session, 'id', '?')}] 冻结未生效(页面仍在运行),30s 后重试")
|
||||
return False
|
||||
setattr(session, "_uwapi_freeze_token", token if mark_installed else "")
|
||||
setattr(session, "_uwapi_frozen", True)
|
||||
setattr(session, "_uwapi_frozen_at", time.time())
|
||||
setattr(session, "_uwapi_frozen_check_at", time.time())
|
||||
setattr(session, "_uwapi_visible_probe_streak", 0)
|
||||
logger.debug(f"[{getattr(session, 'id', '?')}] 空闲标签页已冻结 (Page.setWebLifecycleState=frozen)")
|
||||
return True
|
||||
except Exception as exc:
|
||||
logger.debug_throttled(
|
||||
f"tab_pool.freeze_error.{getattr(session, 'id', '?')}",
|
||||
f"[{getattr(session, 'id', '?')}] 空闲标签页冻结失败(忽略): {exc}",
|
||||
interval_sec=300.0,
|
||||
)
|
||||
setattr(session, "_uwapi_freeze_retry_at", time.time() + 300.0)
|
||||
return False
|
||||
finally:
|
||||
setattr(session, "_uwapi_freeze_in_progress", False)
|
||||
lock.release()
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# CDP session recycle (P0-3)
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
def _supports_recycle(tab: Any) -> bool:
|
||||
return all(callable(getattr(tab, name, None)) for name in ("disconnect", "_driver_init", "_get_document"))
|
||||
|
||||
|
||||
def recycle_tab_connection(tab: Any) -> None:
|
||||
"""Drop and re-create the tab's CDP websocket without touching the page.
|
||||
|
||||
Chrome detaches the old DevTools session, which releases every RemoteObject
|
||||
and DOM-agent node mapping held for it.
|
||||
"""
|
||||
target_id = getattr(tab, "_target_id", None) or getattr(tab, "tab_id", None)
|
||||
if not target_id:
|
||||
raise RuntimeError("tab target id unavailable")
|
||||
listener = getattr(tab, "_listener", None)
|
||||
# 每次尝试都从 disconnect 开始:_driver_init 第一步就取新驱动,中途失败会留下一个
|
||||
# 半初始化的驱动(_is_loading 卡住、load 回调未注册、文档根未获取)。实测半初始化 / 取驱动失败 /
|
||||
# _get_document 失败三种情况下,标签页都会在后续请求里报错或空等 10s+,必须完整重建。
|
||||
# 另外 DrissionPage 的 _get_document() 失败时不抛异常:读不到返回 False;
|
||||
# 另一线程(页面加载事件)正在读时直接返回 None;读的过程中抛异常则会让 _is_reading 标志一直卡住,
|
||||
# 之后每次都“直接返回 None”。只看异常会把这些情况当成功、留下失效的文档根(run_js 报 ElementLostError)。
|
||||
last_exc: Optional[BaseException] = None
|
||||
for attempt in range(RECYCLE_REBUILD_ATTEMPTS):
|
||||
try:
|
||||
tab.disconnect()
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
if getattr(tab, "_is_reading", False):
|
||||
tab._is_reading = False # 旧会话上中断的读取留下的标志,新驱动上重新读取
|
||||
tab._driver_init(target_id)
|
||||
doc = tab._get_document()
|
||||
if doc is None:
|
||||
# 新驱动的页面加载事件正在读取文档:等它读完
|
||||
deadline = time.monotonic() + 5.0
|
||||
while getattr(tab, "_is_reading", False) and time.monotonic() < deadline:
|
||||
time.sleep(0.05)
|
||||
elif doc is False:
|
||||
raise RuntimeError("document root unavailable after reconnect")
|
||||
if not _tab_connection_healthy(tab):
|
||||
raise RuntimeError("tab does not respond after reconnect")
|
||||
last_exc = None
|
||||
break
|
||||
except Exception as exc:
|
||||
last_exc = exc
|
||||
if attempt + 1 < RECYCLE_REBUILD_ATTEMPTS:
|
||||
time.sleep(0.5 * (attempt + 1))
|
||||
if last_exc is not None:
|
||||
raise last_exc
|
||||
|
||||
# 监听器(补丁版 _reuse_driver 模式)引用的是旧驱动
|
||||
if listener is not None:
|
||||
for attr, value in (("_driver", None), ("_network_enabled", False), ("listening", False)):
|
||||
try:
|
||||
if hasattr(listener, attr):
|
||||
setattr(listener, attr, value)
|
||||
except Exception:
|
||||
pass
|
||||
# 会话级注入脚本随旧会话一起失效
|
||||
try:
|
||||
init_jss = getattr(tab, "_init_jss", None)
|
||||
if isinstance(init_jss, list):
|
||||
init_jss.clear()
|
||||
except Exception:
|
||||
pass
|
||||
for attr in ("_kimi_fetch_init_js_id",):
|
||||
try:
|
||||
if hasattr(tab, attr):
|
||||
delattr(tab, attr)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def _reinstall_command_init_scripts(session: Any, tab: Any) -> int:
|
||||
"""Re-register command ``run_js_file`` new-document scripts on the fresh CDP session.
|
||||
|
||||
``Page.addScriptToEvaluateOnNewDocument`` registrations belong to the CDP session and die
|
||||
with it, while the command engine's per-session registry (and the bootstrap mtime cache)
|
||||
still say "installed" -- so page-resilience / monitoring scripts would silently stop being
|
||||
injected after the next reload. Re-register the same source and update the identifier.
|
||||
The page itself was not reloaded, so the script is NOT executed again in the current document.
|
||||
"""
|
||||
registry = getattr(session, "_command_init_js_registry", None)
|
||||
if not isinstance(registry, dict) or not registry:
|
||||
return 0
|
||||
restored = 0
|
||||
lost = False
|
||||
for key, entry in list(registry.items()):
|
||||
source = str(entry.get("source", "") or "") if isinstance(entry, dict) else ""
|
||||
if not source:
|
||||
continue
|
||||
try:
|
||||
result = _run_cdp(tab, "Page.addScriptToEvaluateOnNewDocument", timeout=5.0, source=source)
|
||||
identifier = ""
|
||||
if isinstance(result, dict):
|
||||
identifier = str(result.get("identifier") or result.get("scriptId") or "").strip()
|
||||
entry["identifier"] = identifier
|
||||
restored += 1
|
||||
except Exception as exc:
|
||||
# 让命令引擎下一轮重新同步(启动脚本会重新注册)
|
||||
registry.pop(key, None)
|
||||
lost = True
|
||||
logger.warning(
|
||||
f"[{getattr(session, 'id', '?')}] CDP 回收后重新注册命令预注入脚本失败: "
|
||||
f"{entry.get('path', key) if isinstance(entry, dict) else key}: {exc}"
|
||||
)
|
||||
if lost:
|
||||
state = getattr(session, "_command_bootstrap_js_files", None)
|
||||
if isinstance(state, dict):
|
||||
state.clear()
|
||||
return restored
|
||||
|
||||
|
||||
RECYCLE_REBUILD_ATTEMPTS = 3
|
||||
|
||||
|
||||
def _tab_connection_healthy(tab: Any) -> bool:
|
||||
try:
|
||||
return driver_for_tab(tab).run_js("return 1", timeout=2.0) == 1
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def _quarantine_broken_tab(session: Any, tab: Any, reason: str) -> None:
|
||||
"""Recycle could not rebuild the connection: never hand this tab out again as-is.
|
||||
|
||||
Stops whatever half-initialised driver is left, drops the DrissionPage singleton so a
|
||||
later pool rescan builds a fresh tab object (instead of getting the same broken one back),
|
||||
and marks the session ERROR so the pool's existing cleanup / quarantine takes over.
|
||||
"""
|
||||
try:
|
||||
tab.disconnect()
|
||||
except Exception:
|
||||
pass
|
||||
from app.core.driver.drission_internals import forget_tab_object
|
||||
|
||||
forget_tab_object(tab)
|
||||
try:
|
||||
session.mark_error(reason)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def _dom_node_count(tab: Any) -> int:
|
||||
try:
|
||||
result = _run_cdp(tab, "Memory.getDOMCounters", timeout=1.0)
|
||||
except Exception:
|
||||
return 0
|
||||
if isinstance(result, dict):
|
||||
try:
|
||||
return int(result.get("nodes") or 0)
|
||||
except (TypeError, ValueError):
|
||||
return 0
|
||||
return 0
|
||||
|
||||
|
||||
def _recycle_reason(session: Any, config: IdleMaintenanceConfig, now: float) -> str:
|
||||
if not config.recycle_enabled:
|
||||
return ""
|
||||
tab = getattr(session, "tab", None)
|
||||
if tab is None or not _supports_recycle(tab):
|
||||
return ""
|
||||
if now < float(getattr(session, "_uwapi_recycle_retry_at", 0.0) or 0.0):
|
||||
return ""
|
||||
last_recycle = getattr(session, "_uwapi_last_recycle_at", None)
|
||||
if last_recycle is None:
|
||||
# 从未回收:以会话创建为起点,创建以来处理过的请求都计数
|
||||
setattr(session, "_uwapi_last_recycle_at", float(getattr(session, "created_at", now) or now))
|
||||
setattr(session, "_uwapi_recycle_request_mark", 0)
|
||||
last_recycle = getattr(session, "_uwapi_last_recycle_at")
|
||||
last_recycle = float(last_recycle or 0.0)
|
||||
|
||||
# 回收只对“上次回收后有过使用”的标签页有意义
|
||||
used_since = float(getattr(session, "last_used_at", 0.0) or 0.0) > last_recycle + 1.0 or float(
|
||||
getattr(session, "_pc_last_wake_at", 0.0) or 0.0
|
||||
) > last_recycle + 1.0
|
||||
if not used_since:
|
||||
return ""
|
||||
if (now - float(getattr(session, "last_used_at", 0.0) or 0.0)) < config.recycle_idle_sec:
|
||||
return ""
|
||||
|
||||
if config.recycle_after_requests > 0:
|
||||
mark = int(getattr(session, "_uwapi_recycle_request_mark", 0) or 0)
|
||||
served = int(getattr(session, "request_count", 0) or 0) - mark
|
||||
if served >= config.recycle_after_requests:
|
||||
return f"requests={served}"
|
||||
if config.recycle_interval_sec > 0 and (now - last_recycle) >= config.recycle_interval_sec:
|
||||
return f"interval={now - last_recycle:.0f}s"
|
||||
if config.recycle_dom_nodes > 0:
|
||||
last_check = float(getattr(session, "_uwapi_dom_check_at", 0.0) or 0.0)
|
||||
if (now - last_check) >= config.dom_check_interval_sec:
|
||||
setattr(session, "_uwapi_dom_check_at", now)
|
||||
nodes = _dom_node_count(tab)
|
||||
if nodes >= config.recycle_dom_nodes:
|
||||
return f"dom_nodes={nodes}"
|
||||
return ""
|
||||
|
||||
|
||||
def recycle_session(manager: Any, session: Any, reason: str) -> bool:
|
||||
"""Recycle one idle session's CDP connection under a maintenance lease."""
|
||||
if session is None or getattr(manager, "_shutdown", False):
|
||||
return False
|
||||
tabs = getattr(manager, "_tabs", {}) or {}
|
||||
if tabs.get(getattr(session, "id", None)) is not session:
|
||||
return False
|
||||
tab = getattr(session, "tab", None)
|
||||
if tab is None or not _supports_recycle(tab):
|
||||
return False
|
||||
|
||||
task_id = f"maint:cdp_recycle:{int(time.time() * 1000)}"
|
||||
saved_last_used = float(getattr(session, "last_used_at", 0.0) or 0.0)
|
||||
# 本次若因条件不满足而跳过,60s 内不再尝试回收(否则每个 tick 都选中回收,冻结永远轮不到)
|
||||
setattr(session, "_uwapi_recycle_retry_at", time.time() + 60.0)
|
||||
if not session.acquire_for_command(task_id):
|
||||
return False
|
||||
|
||||
started = time.perf_counter()
|
||||
ok = False
|
||||
setattr(session, "_cdp_recycle_in_progress", True)
|
||||
try:
|
||||
stop = getattr(manager, "_stop_global_monitor_for_session", None)
|
||||
if callable(stop) and not stop(session.id, reason="cdp_recycle", wait=True):
|
||||
logger.debug(f"[{session.id}] 全局监听未能及时停止,跳过本次 CDP 回收")
|
||||
return False
|
||||
listener = getattr(tab, "_listener", None)
|
||||
if listener is not None and getattr(listener, "listening", False):
|
||||
logger.debug(f"[{session.id}] 标签页仍在监听网络,跳过本次 CDP 回收")
|
||||
return False
|
||||
|
||||
recycle_tab_connection(tab)
|
||||
_reinstall_command_init_scripts(session, tab)
|
||||
# 旧会话注册的可见性模拟脚本已随会话失效;标签页释放时本来就会恢复页面状态
|
||||
try:
|
||||
from app.core.page_lifecycle import _clear_visibility_emulation_attrs
|
||||
|
||||
_clear_visibility_emulation_attrs(tab)
|
||||
_clear_visibility_emulation_attrs(session)
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
setattr(session, "_audio_capture_init_script_source", None)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
setattr(session, "_uwapi_last_recycle_at", time.time())
|
||||
setattr(session, "_uwapi_recycle_request_mark", int(getattr(session, "request_count", 0) or 0))
|
||||
setattr(session, "_uwapi_recycle_count", int(getattr(session, "_uwapi_recycle_count", 0) or 0) + 1)
|
||||
setattr(session, "_uwapi_recycle_retry_at", 0.0)
|
||||
ok = True
|
||||
logger.debug(
|
||||
f"[{session.id}] 空闲标签页 CDP 会话已回收({reason},"
|
||||
f"耗时 {time.perf_counter() - started:.2f}s)"
|
||||
)
|
||||
return True
|
||||
except Exception as exc:
|
||||
# 避免失败后每个 tick 重试
|
||||
setattr(session, "_uwapi_last_recycle_at", time.time())
|
||||
setattr(session, "_uwapi_recycle_failures", int(getattr(session, "_uwapi_recycle_failures", 0) or 0) + 1)
|
||||
if _tab_connection_healthy(tab):
|
||||
logger.warning(f"[{session.id}] 空闲标签页 CDP 会话回收失败,但连接仍可用,继续使用: {exc}")
|
||||
else:
|
||||
logger.warning(f"[{session.id}] 空闲标签页 CDP 会话回收失败且连接不可用,标记为错误交给池清理: {exc}")
|
||||
_quarantine_broken_tab(session, tab, "cdp_recycle_failed")
|
||||
return False
|
||||
finally:
|
||||
setattr(session, "_cdp_recycle_in_progress", False)
|
||||
try:
|
||||
manager.release(session.id, check_triggers=False, expected_task_id=task_id)
|
||||
except Exception as exc:
|
||||
logger.warning(f"[{session.id}] CDP 回收后释放标签页失败: {exc}")
|
||||
# 维护占用不算“使用”:恢复空闲计时,冻结计时不被推迟
|
||||
try:
|
||||
session_lock = getattr(session, "_lock", None)
|
||||
if session_lock is not None:
|
||||
with session_lock:
|
||||
if _is_idle(session):
|
||||
session.last_used_at = saved_last_used
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# scheduling (called from TabPoolManager.run_watchdog_tick)
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
FROZEN_VISIBILITY_CHECK_SEC = 10.0
|
||||
|
||||
|
||||
def sync_frozen_visibility(session: Any) -> bool:
|
||||
"""Chrome lifts a CDP freeze by itself when the user switches to the tab or
|
||||
restores the window (measured). Bring our bookkeeping in line: mark the tab
|
||||
resumed and turn focus emulation back on, so it can be re-frozen later once
|
||||
it is hidden and idle again. Returns True when the tab was found visible.
|
||||
"""
|
||||
if session is None or not getattr(session, "_uwapi_frozen", False):
|
||||
return False
|
||||
tab = getattr(session, "tab", None)
|
||||
if tab is None:
|
||||
return False
|
||||
# 冻结状态下同步 run_js 仍可执行;此时焦点仿真已关闭,读到的是真实可见性
|
||||
if _read_real_visibility(tab) == "visible":
|
||||
resume_if_frozen(session, reason="user_visible")
|
||||
return True
|
||||
# 页面已在我们背后恢复运行(会话断开 / 用户切回又切走 / 导航):记账改回,之后可重新冻结
|
||||
if freeze_was_lifted(session):
|
||||
resume_if_frozen(session, reason="lifted_externally")
|
||||
setattr(session, "_uwapi_freeze_retry_at", 0.0)
|
||||
return False
|
||||
|
||||
|
||||
def _run_maintenance(manager: Any, session: Any, config: IdleMaintenanceConfig, action: str, reason: str) -> None:
|
||||
try:
|
||||
if action == "recycle":
|
||||
recycle_session(manager, session, reason)
|
||||
elif action == "freeze":
|
||||
freeze_session(manager, session, config)
|
||||
elif action == "sync":
|
||||
sync_frozen_visibility(session)
|
||||
finally:
|
||||
setattr(session, "_uwapi_maint_pending", False)
|
||||
|
||||
|
||||
def schedule_idle_maintenance(manager: Any, config: Optional[IdleMaintenanceConfig] = None) -> int:
|
||||
"""Pick idle sessions that need recycle/freeze and submit them to the maintenance pool."""
|
||||
config = config or getattr(manager, "_idle_maintenance_config", None) or IdleMaintenanceConfig.from_env()
|
||||
if not (config.freeze_enabled or config.recycle_enabled):
|
||||
return 0
|
||||
if getattr(manager, "_shutdown", False):
|
||||
return 0
|
||||
executor = getattr(manager, "_maintenance_executor", None)
|
||||
if executor is None:
|
||||
return 0
|
||||
lock = getattr(manager, "_lock", None)
|
||||
if lock is not None:
|
||||
with lock:
|
||||
sessions = list((getattr(manager, "_tabs", {}) or {}).values())
|
||||
else:
|
||||
sessions = list((getattr(manager, "_tabs", {}) or {}).values())
|
||||
|
||||
now = time.time()
|
||||
scheduled = 0
|
||||
for session in sessions:
|
||||
try:
|
||||
if getattr(session, "_uwapi_maint_pending", False):
|
||||
continue
|
||||
if not _is_idle(session) or getattr(session, "_cdp_recycle_in_progress", False):
|
||||
continue
|
||||
action = ""
|
||||
reason = ""
|
||||
frozen = bool(getattr(session, "_uwapi_frozen", False))
|
||||
if frozen:
|
||||
last_check = float(getattr(session, "_uwapi_frozen_check_at", 0.0) or 0.0)
|
||||
if now - last_check >= FROZEN_VISIBILITY_CHECK_SEC:
|
||||
setattr(session, "_uwapi_frozen_check_at", now)
|
||||
action = "sync"
|
||||
else:
|
||||
reason = _recycle_reason(session, config, now)
|
||||
if reason:
|
||||
action = "recycle"
|
||||
if not action and config.freeze_enabled and not frozen:
|
||||
retry_at = float(getattr(session, "_uwapi_freeze_retry_at", 0.0) or 0.0)
|
||||
activity = last_activity_at(session)
|
||||
# 被“可见”退避推迟时,一旦有新活动就不再等退避期满
|
||||
if int(getattr(session, "_uwapi_visible_probe_streak", 0) or 0) > 0 and activity > float(
|
||||
getattr(session, "_uwapi_visible_probe_mark", 0.0) or 0.0
|
||||
):
|
||||
retry_at = min(retry_at, activity + config.freeze_after_sec)
|
||||
if now >= retry_at and (now - activity) >= config.freeze_after_sec:
|
||||
action = "freeze"
|
||||
if not action:
|
||||
continue
|
||||
setattr(session, "_uwapi_maint_pending", True)
|
||||
try:
|
||||
executor.submit(_run_maintenance, manager, session, config, action, reason)
|
||||
scheduled += 1
|
||||
except RuntimeError:
|
||||
setattr(session, "_uwapi_maint_pending", False)
|
||||
break
|
||||
except Exception as exc:
|
||||
logger.debug(f"[TabPool] 空闲维护调度异常(忽略): {exc}")
|
||||
return scheduled
|
||||
|
||||
|
||||
__all__ = [
|
||||
"IdleMaintenanceConfig",
|
||||
"REAL_VISIBILITY_JS",
|
||||
"freeze_session",
|
||||
"is_idle_freeze_enabled",
|
||||
"last_activity_at",
|
||||
"lifecycle_lock",
|
||||
"note_network_activity",
|
||||
"recycle_session",
|
||||
"recycle_tab_connection",
|
||||
"resume_if_frozen",
|
||||
"schedule_idle_maintenance",
|
||||
"sync_frozen_visibility",
|
||||
]
|
||||
@@ -0,0 +1,273 @@
|
||||
"""标签页池管理器。
|
||||
|
||||
R2-3:原 4500 行的单个类按职责拆成 mixin(pool_*.py),这里只保留类属性、__init__ 与 shutdown;
|
||||
对外接口(TabPoolManager 的全部方法与属性)保持不变。
|
||||
"""
|
||||
|
||||
import os
|
||||
import threading
|
||||
from collections import OrderedDict, deque
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from app.core.config import BrowserConstants, logger
|
||||
from app.utils.tab_route_groups import normalize_route_groups
|
||||
|
||||
from .idle_maintenance import IdleMaintenanceConfig
|
||||
from .network import _GlobalNetworkInterceptionManager
|
||||
from .recovery import TabQuarantineEntry, TabRecoveryService
|
||||
from .session import TabSession
|
||||
from .pool_config import TabPoolConfigMixin
|
||||
from .pool_contexts import TabPoolContextMixin
|
||||
from .pool_tabs import TabPoolTabsMixin
|
||||
from .pool_sessions import TabPoolSessionsMixin
|
||||
from .pool_allocation import TabPoolAllocationMixin
|
||||
from .pool_maintenance import TabPoolMaintenanceMixin
|
||||
from .pool_acquire import TabPoolAcquireMixin
|
||||
from .pool_routing import TabPoolRoutingMixin
|
||||
from .pool_status import TabPoolStatusMixin
|
||||
|
||||
|
||||
class TabPoolManager(
|
||||
TabPoolConfigMixin,
|
||||
TabPoolContextMixin,
|
||||
TabPoolTabsMixin,
|
||||
TabPoolSessionsMixin,
|
||||
TabPoolAllocationMixin,
|
||||
TabPoolMaintenanceMixin,
|
||||
TabPoolAcquireMixin,
|
||||
TabPoolRoutingMixin,
|
||||
TabPoolStatusMixin,
|
||||
):
|
||||
"""标签页池管理器"""
|
||||
DOMAIN_ABBR_MAP = {
|
||||
"chatgpt": "gpt",
|
||||
"openai": "gpt",
|
||||
"gemini": "gemini",
|
||||
"aistudio": "aistudio",
|
||||
"claude": "claude",
|
||||
"anthropic": "claude",
|
||||
"poe": "poe",
|
||||
"bing": "bing",
|
||||
"copilot": "copilot",
|
||||
"perplexity": "pplx",
|
||||
"lmarena": "lmarena",
|
||||
"chat": "chat",
|
||||
}
|
||||
# 卡死超时时间(秒)
|
||||
STUCK_TIMEOUT = 180
|
||||
# 新标签页扫描间隔(秒)
|
||||
SCAN_INTERVAL = 10
|
||||
QUERY_SCAN_MIN_INTERVAL_SEC = 1.0
|
||||
ISOLATED_CONTEXT_ORPHAN_GRACE_SEC = 3.0
|
||||
ISOLATED_CONTEXT_REBIND_GRACE_SEC = 20.0
|
||||
GET_TABS_FAILURE_COOLDOWN_SEC = 5.0
|
||||
GET_TABS_WARNING_INTERVAL_SEC = 10.0
|
||||
ROUTE_CURSOR_LIMIT = 1000
|
||||
MAINTENANCE_WORKER_LIMIT = 4
|
||||
TERMINATION_RELEASE_WAIT_SEC = 5.0
|
||||
IDLE_MEMORY_PURGE_MIN_SEC = 60.0
|
||||
CONTEXT_RECOVERY_CDP_TIMEOUT_SEC = 2.0
|
||||
CONTEXT_RECOVERY_COOLDOWN_SEC = 30.0
|
||||
CONTEXT_RECOVERY_CACHE_LIMIT = 2048
|
||||
# A cancelled HTTP request must not leave its worker blocked in a 60s
|
||||
# acquire. Polling this frequently keeps cancellation cleanup bounded
|
||||
# without adding a second notification channel from RequestManager.
|
||||
ACQUIRE_CANCEL_POLL_SEC = 0.25
|
||||
def __init__(
|
||||
self,
|
||||
browser_page,
|
||||
max_tabs: int = 5,
|
||||
min_tabs: int = 1,
|
||||
idle_timeout: float = 300,
|
||||
acquire_timeout: float = 60,
|
||||
stuck_timeout: float = STUCK_TIMEOUT,
|
||||
allocation_mode: str = "first_idle",
|
||||
excluded_urls: Optional[List[str]] = None,
|
||||
preserve_error_tabs: bool = False,
|
||||
auto_remember_url_presets: bool = False,
|
||||
model_name_overrides: Optional[Dict[str, Any]] = None,
|
||||
preset_overrides: Optional[Dict[str, Any]] = None,
|
||||
route_groups: Optional[List[Dict[str, Any]]] = None,
|
||||
):
|
||||
self.page = browser_page
|
||||
self.max_tabs = max(1, int(max_tabs))
|
||||
self.min_tabs = min(self.max_tabs, max(1, int(min_tabs)))
|
||||
self.idle_timeout = max(1.0, float(idle_timeout))
|
||||
self.acquire_timeout = max(1.0, float(acquire_timeout))
|
||||
self.stuck_timeout = max(1.0, float(stuck_timeout))
|
||||
self.allocation_mode = self._normalize_allocation_mode(allocation_mode)
|
||||
self.excluded_urls = self._normalize_excluded_urls(excluded_urls)
|
||||
self.preserve_error_tabs = self._to_bool(preserve_error_tabs, False)
|
||||
self.auto_remember_url_presets = self._to_bool(auto_remember_url_presets, False)
|
||||
self.model_name_overrides = self._normalize_model_name_overrides(model_name_overrides)
|
||||
self.preset_overrides = self._normalize_preset_overrides(preset_overrides)
|
||||
self.route_groups = normalize_route_groups(route_groups)
|
||||
|
||||
self._tabs: Dict[str, TabSession] = {}
|
||||
self._lock = threading.RLock()
|
||||
self._condition = threading.Condition(self._lock)
|
||||
self._scan_snapshot_lock = threading.Lock()
|
||||
self._global_monitor_transition_lock = threading.RLock()
|
||||
|
||||
self._initialized = False
|
||||
self._shutdown = False
|
||||
self._tab_counter = 0
|
||||
|
||||
self._last_scan_time: float = 0
|
||||
self._get_tabs_retry_after: float = 0.0
|
||||
self._last_get_tabs_warning_at: float = 0.0
|
||||
self._context_recovery_lock = threading.Lock()
|
||||
self._context_recovery_pending: set[str] = set()
|
||||
self._context_recovery_requested_at: Dict[str, float] = {}
|
||||
|
||||
# 记录已知的标签页底层 ID(用于检测新标签页)
|
||||
self._known_tab_ids: set = set()
|
||||
# 🆕 记录当前活动的标签页 ID(避免重复激活)
|
||||
self._active_session_id: Optional[str] = None
|
||||
self._auto_activate_on_acquire = self._to_bool(
|
||||
os.getenv("TAB_AUTO_ACTIVATE_ON_ACQUIRE"), False
|
||||
)
|
||||
self._acquire_waiters: deque[str] = deque()
|
||||
self._index_waiters: Dict[int, deque[str]] = {}
|
||||
self._route_waiters: Dict[str, deque[str]] = {}
|
||||
self._group_waiters: Dict[str, deque[str]] = {}
|
||||
self._route_group_bindings: Dict[str, Dict[str, str]] = {}
|
||||
self._waiter_counter = 0
|
||||
# H9:所有 acquire 等待者(通用 / 按编号 / 按域名 / 按路由组)的总量上限,0 = 不限
|
||||
try:
|
||||
self._max_acquire_waiters = max(0, int(os.getenv("TAB_ACQUIRE_MAX_WAITERS", "128") or 128))
|
||||
except ValueError:
|
||||
self._max_acquire_waiters = 128
|
||||
self._queue_full_rejections: "OrderedDict[str, float]" = OrderedDict()
|
||||
|
||||
# 🆕 持久化编号系统
|
||||
self._next_persistent_index: int = 1 # 下一个可分配的编号
|
||||
self._raw_id_to_persistent: Dict[str, int] = {} # raw_tab_id → persistent_index
|
||||
self._persistent_to_session_id: Dict[int, str] = {} # persistent_index → session.id
|
||||
self._isolated_context_by_raw_id: Dict[str, str] = {}
|
||||
self._orphaned_isolated_contexts: Dict[str, float] = {}
|
||||
self._round_robin_cursor: int = 0
|
||||
self._route_round_robin_cursor: OrderedDict[str, int] = OrderedDict()
|
||||
self._preserved_error_session_ids = set()
|
||||
self._maintenance_executor: Optional[ThreadPoolExecutor] = ThreadPoolExecutor(
|
||||
max_workers=self.MAINTENANCE_WORKER_LIMIT,
|
||||
thread_name_prefix="tab-maint",
|
||||
)
|
||||
# acquire 专用线程池(懒创建):阻塞式 acquire 可占线程最长 acquire_timeout,
|
||||
# 若复用 asyncio 默认 executor,池满时一批被取消/慢的 acquire 会占满默认
|
||||
# executor,拖垮事件循环上所有 to_thread 调用。创建用独立小锁保护,
|
||||
# 避免事件循环线程为建池去竞争可能被长持有的池锁。
|
||||
self._acquire_executor: Optional[ThreadPoolExecutor] = None
|
||||
self._acquire_executor_lock = threading.Lock()
|
||||
|
||||
# 错误标签页隔离集合(池锁 self._condition 保护下读写)。
|
||||
# ERROR 会话被移出池但浏览器标签页保留时,其 raw id 先进隔离集合,
|
||||
# 阻止 _scan_new_tabs 立即把同一 tab 当"新标签页"重新入池——
|
||||
# 旧请求的 worker 线程可能仍阻塞在对该 tab 的同步 DrissionPage 调用里,
|
||||
# 直接重入池会造成新旧工作流并发驱动同一页面。
|
||||
self._quarantined_raw_ids: Dict[str, TabQuarantineEntry] = {}
|
||||
# 恢复服务懒创建:创建仅涉及线程池构造(非阻塞),
|
||||
# 用独立小锁保护,不与池锁竞争。
|
||||
self._recovery_service: Optional[TabRecoveryService] = None
|
||||
self._recovery_service_lock = threading.Lock()
|
||||
|
||||
# 全局常驻网络监听(可配置)
|
||||
self._global_network_enabled = self._to_bool(
|
||||
BrowserConstants.get("GLOBAL_NETWORK_INTERCEPTION_ENABLED"), False
|
||||
)
|
||||
self._global_network_listen_pattern = str(
|
||||
BrowserConstants.get("GLOBAL_NETWORK_INTERCEPTION_LISTEN_PATTERN") or "http"
|
||||
).strip() or "http"
|
||||
self._global_network_wait_timeout = max(
|
||||
0.1,
|
||||
self._to_float(BrowserConstants.get("GLOBAL_NETWORK_INTERCEPTION_WAIT_TIMEOUT"), 0.5),
|
||||
)
|
||||
self._global_network_retry_delay = max(
|
||||
0.2,
|
||||
self._to_float(BrowserConstants.get("GLOBAL_NETWORK_INTERCEPTION_RETRY_DELAY"), 1.0),
|
||||
)
|
||||
# A JavaScript heap purge does not freeze or discard a page. Run it
|
||||
# only for sessions that have been idle for a while, so DOM clicks and
|
||||
# active workflows keep their existing timing behavior.
|
||||
self._idle_memory_purge_enabled = self._to_bool(
|
||||
os.getenv("BROWSER_IDLE_MEMORY_PURGE_ENABLED"),
|
||||
True,
|
||||
)
|
||||
self._idle_memory_purge_after_sec = max(
|
||||
self.IDLE_MEMORY_PURGE_MIN_SEC,
|
||||
self._to_float(os.getenv("BROWSER_IDLE_MEMORY_PURGE_AFTER_SEC"), 300.0),
|
||||
)
|
||||
self._idle_memory_purge_interval_sec = max(
|
||||
self.IDLE_MEMORY_PURGE_MIN_SEC,
|
||||
self._to_float(os.getenv("BROWSER_IDLE_MEMORY_PURGE_INTERVAL_SEC"), 180.0),
|
||||
)
|
||||
self._idle_maintenance_config = IdleMaintenanceConfig.from_env()
|
||||
self._global_network_monitor: Optional[_GlobalNetworkInterceptionManager] = None
|
||||
if self._global_network_enabled:
|
||||
self._global_network_monitor = _GlobalNetworkInterceptionManager(
|
||||
get_session_fn=self._get_session_for_monitor_snapshot,
|
||||
is_shutdown_fn=lambda: self._shutdown,
|
||||
listen_pattern=self._global_network_listen_pattern,
|
||||
wait_timeout=self._global_network_wait_timeout,
|
||||
retry_delay=self._global_network_retry_delay,
|
||||
res_types=BrowserConstants.get("GLOBAL_NETWORK_INTERCEPTION_RES_TYPES"),
|
||||
)
|
||||
try:
|
||||
from app.services.arena_tab_listener import register_arena_tab_listener
|
||||
register_arena_tab_listener(self._global_network_monitor)
|
||||
except Exception as e:
|
||||
logger.debug(f"[TabPool] 注册 Arena 扩展监听器失败(忽略): {e}")
|
||||
|
||||
logger.debug(
|
||||
f"TabPoolManager 初始化 (max={max_tabs}, stuck_timeout={self.stuck_timeout}s, "
|
||||
f"allocation_mode={self.allocation_mode}, preserve_error_tabs={self.preserve_error_tabs})"
|
||||
)
|
||||
def shutdown(self):
|
||||
monitor = None
|
||||
maintenance_executor = None
|
||||
with self._lock:
|
||||
self._shutdown = True
|
||||
self._condition.notify_all()
|
||||
monitor = self._global_network_monitor
|
||||
self._global_network_monitor = None
|
||||
maintenance_executor = self._maintenance_executor
|
||||
self._maintenance_executor = None
|
||||
# acquire 专用线程池随池一起关闭(不等待:阻塞中的 acquire 线程会因
|
||||
# _shutdown + notify_all 自行退出)。
|
||||
with self._acquire_executor_lock:
|
||||
acquire_executor = self._acquire_executor
|
||||
self._acquire_executor = None
|
||||
with self._recovery_service_lock:
|
||||
recovery_service = self._recovery_service
|
||||
self._recovery_service = None
|
||||
|
||||
if monitor:
|
||||
monitor.shutdown()
|
||||
if maintenance_executor:
|
||||
maintenance_executor.shutdown(wait=False, cancel_futures=True)
|
||||
if acquire_executor:
|
||||
acquire_executor.shutdown(wait=False, cancel_futures=True)
|
||||
if recovery_service:
|
||||
# 内部同样以 wait=False, cancel_futures=True 关闭其线程池
|
||||
recovery_service.shutdown()
|
||||
|
||||
with self._lock:
|
||||
self._tabs.clear()
|
||||
self._known_tab_ids.clear()
|
||||
self._quarantined_raw_ids.clear()
|
||||
self._active_session_id = None # 🆕 重置活动标签页记录
|
||||
recovery_lock = getattr(self, "_context_recovery_lock", None)
|
||||
if recovery_lock is not None:
|
||||
with recovery_lock:
|
||||
getattr(self, "_context_recovery_pending", set()).clear()
|
||||
getattr(self, "_context_recovery_requested_at", {}).clear()
|
||||
# 🆕 清理编号映射
|
||||
self._raw_id_to_persistent.clear()
|
||||
self._persistent_to_session_id.clear()
|
||||
self._isolated_context_by_raw_id.clear()
|
||||
self._orphaned_isolated_contexts.clear()
|
||||
self._round_robin_cursor = 0
|
||||
self._route_round_robin_cursor.clear()
|
||||
self._next_persistent_index = 1
|
||||
logger.info("TabPoolManager 已关闭")
|
||||
@@ -0,0 +1,619 @@
|
||||
import threading
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Callable, Dict, List, Optional, Tuple, Union
|
||||
|
||||
from app.core.config import logger
|
||||
|
||||
from ._arena_snapshot import _ARENA_STORE_SNAPSHOT_JS
|
||||
from .idle_maintenance import note_network_activity
|
||||
from .session import TabSession
|
||||
from app.core.driver import driver_for_tab
|
||||
|
||||
|
||||
ResponseListenerCallback = Callable[[TabSession, Any, Dict[str, Any], threading.Event], None]
|
||||
ResponseFilter = Union[str, Callable[[Dict[str, Any]], bool], None]
|
||||
|
||||
|
||||
@dataclass
|
||||
class _GlobalNetworkWorker:
|
||||
"""单个标签页的全局网络监听工作线程。"""
|
||||
session_id: str
|
||||
thread: threading.Thread
|
||||
stop_event: threading.Event
|
||||
|
||||
|
||||
class _GlobalNetworkInterceptionManager:
|
||||
"""
|
||||
全局常驻网络事件监听。
|
||||
|
||||
设计要点:
|
||||
- 仅在标签页空闲时运行;
|
||||
- 标签页被任务占用时暂停,让位给工作流内监听器;
|
||||
- 命令触发逻辑仍由 CommandEngine 决定;
|
||||
- 通过 register_response_listener 机制支持插件化挂载各种响应监听器(例如 Arena 翻牌探测等)。
|
||||
"""
|
||||
|
||||
LISTENER_STOP_TIMEOUT_SEC = 2.0
|
||||
LISTENER_CLEAR_INTERVAL_SEC = 60.0
|
||||
LISTENER_CLEAR_EVENT_INTERVAL = 200
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
get_session_fn,
|
||||
is_shutdown_fn,
|
||||
listen_pattern: str = "http",
|
||||
wait_timeout: float = 0.5,
|
||||
retry_delay: float = 1.0,
|
||||
res_types: Any = None,
|
||||
):
|
||||
self._get_session = get_session_fn
|
||||
self._is_shutdown = is_shutdown_fn
|
||||
self._listen_pattern = str(listen_pattern or "http").strip() or "http"
|
||||
# P0-4:只捕获 API 类资源。补丁版 DrissionPage 会对每个命中的包附加
|
||||
# streamResourceContent 并缓存响应体,放开到脚本/图片/样式会显著增加内存与 CPU。
|
||||
self._res_types = self.parse_res_types(res_types)
|
||||
self._wait_timeout = max(0.1, float(wait_timeout or 0.5))
|
||||
self._retry_delay = max(0.2, float(retry_delay or 1.0))
|
||||
self._workers: Dict[str, _GlobalNetworkWorker] = {}
|
||||
self._lock = threading.RLock()
|
||||
self._stop_join_timeout = max(2.0, self._wait_timeout + self._retry_delay + 0.2)
|
||||
self._response_listeners: List[Tuple[ResponseFilter, ResponseListenerCallback]] = []
|
||||
self._listeners_lock = threading.RLock()
|
||||
|
||||
def register_response_listener(
|
||||
self,
|
||||
pattern_or_filter: ResponseFilter,
|
||||
callback: ResponseListenerCallback,
|
||||
) -> Callable[[], None]:
|
||||
"""
|
||||
注册全局网络响应观察者。
|
||||
|
||||
Args:
|
||||
pattern_or_filter: 匹配规则。可为 URL 匹配子串、Callable[[event_dict], bool] 或 None(匹配全部)。
|
||||
callback: 响应处理回调,签名 (session, response, event, stop_event) -> None。
|
||||
|
||||
Returns:
|
||||
注销函数。
|
||||
"""
|
||||
with self._listeners_lock:
|
||||
entry = (pattern_or_filter, callback)
|
||||
self._response_listeners.append(entry)
|
||||
|
||||
def _unregister():
|
||||
self.unregister_response_listener(callback)
|
||||
|
||||
return _unregister
|
||||
|
||||
def unregister_response_listener(self, callback: ResponseListenerCallback) -> bool:
|
||||
"""注销全局网络响应观察者。"""
|
||||
with self._listeners_lock:
|
||||
original_len = len(self._response_listeners)
|
||||
self._response_listeners = [
|
||||
item for item in self._response_listeners if item[1] != callback
|
||||
]
|
||||
return len(self._response_listeners) < original_len
|
||||
|
||||
@staticmethod
|
||||
def _extract_event(response: Any) -> Dict[str, Any]:
|
||||
req = getattr(response, "request", None)
|
||||
resp = getattr(response, "response", None)
|
||||
|
||||
url = (
|
||||
getattr(req, "url", None)
|
||||
or getattr(resp, "url", None)
|
||||
or getattr(response, "url", None)
|
||||
or ""
|
||||
)
|
||||
method = (
|
||||
getattr(req, "method", None)
|
||||
or getattr(response, "method", None)
|
||||
or ""
|
||||
)
|
||||
status = (
|
||||
getattr(resp, "status", None)
|
||||
or getattr(resp, "status_code", None)
|
||||
or getattr(response, "status", None)
|
||||
or 0
|
||||
)
|
||||
|
||||
try:
|
||||
status = int(status)
|
||||
except (TypeError, ValueError, OverflowError):
|
||||
status = 0
|
||||
|
||||
return {
|
||||
"url": str(url or ""),
|
||||
"method": str(method or "").upper(),
|
||||
"status": status,
|
||||
"timestamp": time.time(),
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _is_expected_stop_error(error: Any) -> bool:
|
||||
text = str(error or "").strip().lower()
|
||||
if not text:
|
||||
return False
|
||||
expected_markers = (
|
||||
"监听未启动或已停止",
|
||||
"target closed",
|
||||
"invalid session",
|
||||
"no such window",
|
||||
"not connected",
|
||||
"connection refused",
|
||||
"disconnected",
|
||||
)
|
||||
if "nonetype" in text and "is_running" in text:
|
||||
return True
|
||||
return any(marker in text for marker in expected_markers)
|
||||
|
||||
@staticmethod
|
||||
def _force_reset_listen_state(tab: Any) -> bool:
|
||||
listener = getattr(tab, "listen", None)
|
||||
if listener is None:
|
||||
return True
|
||||
|
||||
ok = True
|
||||
for attr, value in (
|
||||
("listening", False),
|
||||
("_network_enabled", False),
|
||||
("_driver", None),
|
||||
):
|
||||
try:
|
||||
if hasattr(listener, attr):
|
||||
setattr(listener, attr, value)
|
||||
except Exception:
|
||||
ok = False
|
||||
|
||||
try:
|
||||
clear = getattr(listener, "clear", None)
|
||||
if callable(clear):
|
||||
clear()
|
||||
except Exception:
|
||||
ok = False
|
||||
|
||||
return ok
|
||||
|
||||
@staticmethod
|
||||
def _force_stop_listener_driver(listener: Any) -> bool:
|
||||
driver = getattr(listener, "_driver", None)
|
||||
if driver is None:
|
||||
return False
|
||||
|
||||
stopped = False
|
||||
for method_name in ("stop", "close", "disconnect"):
|
||||
method = getattr(driver, method_name, None)
|
||||
if not callable(method):
|
||||
continue
|
||||
try:
|
||||
method()
|
||||
stopped = True
|
||||
break
|
||||
except Exception:
|
||||
pass
|
||||
return stopped
|
||||
|
||||
@staticmethod
|
||||
def _listener_is_marked_active(listener: Any) -> bool:
|
||||
try:
|
||||
return bool(getattr(listener, "listening", False))
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
@classmethod
|
||||
def _safe_stop_listen(cls, tab: Any) -> bool:
|
||||
listener = getattr(tab, "listen", None)
|
||||
if listener is None:
|
||||
return True
|
||||
|
||||
try:
|
||||
if cls._listener_is_marked_active(listener):
|
||||
stop_result: Dict[str, Any] = {"error": None}
|
||||
|
||||
def _stop_listener():
|
||||
try:
|
||||
listener.stop()
|
||||
except Exception as e:
|
||||
stop_result["error"] = e
|
||||
|
||||
stop_thread = threading.Thread(
|
||||
target=_stop_listener,
|
||||
daemon=True,
|
||||
name="global-net-listen-stop",
|
||||
)
|
||||
stop_thread.start()
|
||||
stop_thread.join(timeout=cls.LISTENER_STOP_TIMEOUT_SEC)
|
||||
if stop_thread.is_alive():
|
||||
logger.warning(
|
||||
f"[GlobalNet] listen.stop timed out after "
|
||||
f"{cls.LISTENER_STOP_TIMEOUT_SEC:.1f}s; "
|
||||
"listener remains unavailable"
|
||||
)
|
||||
# The stop thread can still mutate the listener when it eventually
|
||||
# returns. Never report success or reset its private fields here,
|
||||
# otherwise a new owner could start listening before that late write.
|
||||
return False
|
||||
if stop_result["error"] is not None:
|
||||
raise stop_result["error"]
|
||||
except Exception as e:
|
||||
reset_ok = cls._force_reset_listen_state(tab)
|
||||
log = logger.debug if cls._is_expected_stop_error(e) else logger.warning
|
||||
log(f"[GlobalNet] listen.stop failed; forced_reset={reset_ok}: {e}")
|
||||
return reset_ok and not cls._listener_is_marked_active(listener)
|
||||
|
||||
try:
|
||||
if cls._listener_is_marked_active(listener):
|
||||
reset_ok = cls._force_reset_listen_state(tab)
|
||||
if cls._listener_is_marked_active(listener):
|
||||
logger.warning(
|
||||
f"[GlobalNet] listen.stop returned but listener is still active "
|
||||
f"(forced_reset={reset_ok})"
|
||||
)
|
||||
return False
|
||||
logger.debug("[GlobalNet] listener state reset after stop")
|
||||
except Exception as e:
|
||||
reset_ok = cls._force_reset_listen_state(tab)
|
||||
log = logger.debug if cls._is_expected_stop_error(e) else logger.warning
|
||||
log(f"[GlobalNet] listen state check failed; forced_reset={reset_ok}: {e}")
|
||||
return reset_ok
|
||||
|
||||
try:
|
||||
clear = getattr(listener, "clear", None)
|
||||
if callable(clear):
|
||||
clear()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
cls._reset_listener_res_type(listener)
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
def _reset_listener_res_type(listener: Any) -> None:
|
||||
"""tab.listen 是共享对象:让出时把资源类型过滤恢复为“全部”,
|
||||
避免工作流/命令脚本里不带 res_type 的 listen.start() 继承全局监听的过滤。"""
|
||||
try:
|
||||
if getattr(listener, "_res_type", True) is not True:
|
||||
listener._res_type = True
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
@staticmethod
|
||||
def _safe_clear_listener(tab: Any, session_id: str, reason: str) -> bool:
|
||||
listener = getattr(tab, "listen", None)
|
||||
if listener is None:
|
||||
return True
|
||||
|
||||
try:
|
||||
clear = getattr(listener, "clear", None)
|
||||
if callable(clear):
|
||||
clear()
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.debug(f"[GlobalNet] 清理监听残留失败: {session_id}, reason={reason}, err={e}")
|
||||
return False
|
||||
|
||||
return False
|
||||
|
||||
def _should_cleanup_worker_listener(
|
||||
self,
|
||||
session_id: str,
|
||||
stop_event: threading.Event,
|
||||
tab: Any = None,
|
||||
) -> bool:
|
||||
with self._lock:
|
||||
current = self._workers.get(session_id)
|
||||
if current is None or current.stop_event is stop_event:
|
||||
return True
|
||||
if tab is None:
|
||||
return False
|
||||
try:
|
||||
session = self._get_session(session_id)
|
||||
return session is None or getattr(session, "tab", None) is not tab
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
def _dispatch_event(self, session: TabSession, event: Dict[str, Any]):
|
||||
try:
|
||||
from app.services.command_engine import command_engine
|
||||
command_engine.handle_network_event(session, event)
|
||||
except Exception as e:
|
||||
logger.debug(f"[GlobalNet] 事件上报失败(忽略): {e}")
|
||||
|
||||
@classmethod
|
||||
def extract_request_post_data(cls, response: Any) -> Any:
|
||||
req = getattr(response, "request", None)
|
||||
if req is None:
|
||||
return None
|
||||
for attr in ("postData", "post_data", "body"):
|
||||
try:
|
||||
value = getattr(req, attr, None)
|
||||
except Exception:
|
||||
value = None
|
||||
if value:
|
||||
return value
|
||||
return None
|
||||
|
||||
_extract_request_post_data = extract_request_post_data
|
||||
|
||||
@classmethod
|
||||
def read_response_body(cls, response: Any, stop_event: Optional[threading.Event] = None) -> tuple[Any, str]:
|
||||
if getattr(response, "_stream_enabled", False):
|
||||
deadline = time.time() + 120.0
|
||||
while time.time() < deadline and (stop_event is None or not stop_event.is_set()):
|
||||
stream = getattr(response, "_stream", None)
|
||||
if isinstance(stream, dict) and stream.get("complete"):
|
||||
break
|
||||
time.sleep(0.1)
|
||||
stream = getattr(response, "_stream", None)
|
||||
if isinstance(stream, dict):
|
||||
return stream.get("fullText") or "", "drission_stream"
|
||||
return "", "drission_stream"
|
||||
|
||||
resp = getattr(response, "response", None)
|
||||
if resp is None:
|
||||
return "", "response_missing"
|
||||
|
||||
try:
|
||||
body = getattr(resp, "body", "")
|
||||
return body or "", "response_body"
|
||||
except Exception as e:
|
||||
logger.debug(f"[GlobalNet] 读取候选响应体失败: {e}")
|
||||
return "", "response_body_error"
|
||||
|
||||
_read_response_body = read_response_body
|
||||
|
||||
def _dispatch_response_listeners(
|
||||
self,
|
||||
session: TabSession,
|
||||
response: Any,
|
||||
event: Dict[str, Any],
|
||||
stop_event: threading.Event,
|
||||
) -> None:
|
||||
with self._listeners_lock:
|
||||
listeners = list(self._response_listeners)
|
||||
|
||||
for pattern_or_filter, callback in listeners:
|
||||
if stop_event.is_set() or self._is_shutdown():
|
||||
break
|
||||
try:
|
||||
matched = False
|
||||
if pattern_or_filter is None or pattern_or_filter == "*":
|
||||
matched = True
|
||||
elif isinstance(pattern_or_filter, str):
|
||||
matched = pattern_or_filter.lower() in str(event.get("url", "")).lower()
|
||||
elif callable(pattern_or_filter):
|
||||
matched = bool(pattern_or_filter(event))
|
||||
|
||||
if matched and callable(callback):
|
||||
callback(session, response, event, stop_event)
|
||||
except Exception as e:
|
||||
logger.debug(f"[GlobalNet] 响应监听器回调执行异常(忽略): {e}")
|
||||
|
||||
def _forget_worker_if_current(self, worker: _GlobalNetworkWorker) -> None:
|
||||
with self._lock:
|
||||
current = self._workers.get(worker.session_id)
|
||||
if current is worker:
|
||||
self._workers.pop(worker.session_id, None)
|
||||
|
||||
def _join_stopping_worker(self, worker: _GlobalNetworkWorker, reason: str) -> bool:
|
||||
if not worker.thread.is_alive():
|
||||
self._forget_worker_if_current(worker)
|
||||
return True
|
||||
if worker.thread is threading.current_thread():
|
||||
return False
|
||||
|
||||
worker.thread.join(timeout=self._stop_join_timeout)
|
||||
if worker.thread.is_alive():
|
||||
logger.warning(
|
||||
f"[GlobalNet] previous worker still stopping: {worker.session_id} "
|
||||
f"(reason={reason or '-'})"
|
||||
)
|
||||
return False
|
||||
|
||||
self._forget_worker_if_current(worker)
|
||||
return True
|
||||
|
||||
DEFAULT_RES_TYPES = ("XHR", "Fetch", "EventSource", "Document")
|
||||
|
||||
@classmethod
|
||||
def parse_res_types(cls, value: Any) -> Any:
|
||||
"""``None`` → default API types; ``*``/``all``/``True`` → no filter."""
|
||||
if value is None:
|
||||
return tuple(cls.DEFAULT_RES_TYPES)
|
||||
if value is True:
|
||||
return True
|
||||
if isinstance(value, str):
|
||||
text = value.strip()
|
||||
if not text:
|
||||
return tuple(cls.DEFAULT_RES_TYPES)
|
||||
if text.lower() in {"*", "all", "true", "any"}:
|
||||
return True
|
||||
items = [part.strip() for part in text.replace(";", ",").split(",")]
|
||||
elif isinstance(value, (list, tuple, set)):
|
||||
items = [str(part).strip() for part in value]
|
||||
else:
|
||||
return tuple(cls.DEFAULT_RES_TYPES)
|
||||
items = [item for item in items if item]
|
||||
return tuple(items) if items else tuple(cls.DEFAULT_RES_TYPES)
|
||||
|
||||
def _start_listen(self, tab: Any) -> None:
|
||||
try:
|
||||
driver_for_tab(tab).listener.start(self._listen_pattern, res_type=self._res_types)
|
||||
except TypeError:
|
||||
# 简化的测试替身 / 旧版 Listener 不支持 res_type
|
||||
driver_for_tab(tab).listener.start(self._listen_pattern)
|
||||
|
||||
def start_for_session(self, session: TabSession) -> bool:
|
||||
if not session:
|
||||
return False
|
||||
|
||||
while True:
|
||||
with self._lock:
|
||||
existing = self._workers.get(session.id)
|
||||
if existing is None:
|
||||
break
|
||||
if not existing.thread.is_alive():
|
||||
self._workers.pop(session.id, None)
|
||||
break
|
||||
if not existing.stop_event.is_set():
|
||||
return True
|
||||
|
||||
if not self._join_stopping_worker(existing, "start"):
|
||||
return False
|
||||
|
||||
with self._lock:
|
||||
existing = self._workers.get(session.id)
|
||||
if existing and existing.thread.is_alive():
|
||||
return True
|
||||
stop_event = threading.Event()
|
||||
thread = threading.Thread(
|
||||
target=self._worker_loop,
|
||||
args=(session.id, stop_event),
|
||||
daemon=True,
|
||||
name=f"global-net-{session.id}",
|
||||
)
|
||||
self._workers[session.id] = _GlobalNetworkWorker(
|
||||
session_id=session.id,
|
||||
thread=thread,
|
||||
stop_event=stop_event,
|
||||
)
|
||||
thread.start()
|
||||
# logger.debug(f"[GlobalNet] 启动监听: {session.id} pattern={self._listen_pattern!r}")
|
||||
return True
|
||||
|
||||
def stop_for_session(self, session_id: str, reason: str = "", join: bool = False) -> bool:
|
||||
if not session_id:
|
||||
return True
|
||||
|
||||
with self._lock:
|
||||
worker = self._workers.get(session_id)
|
||||
if not worker:
|
||||
return True
|
||||
|
||||
worker.stop_event.set()
|
||||
|
||||
session = self._get_session(session_id)
|
||||
stop_ok = True
|
||||
if session is not None:
|
||||
stop_ok = self._safe_stop_listen(session.tab)
|
||||
|
||||
should_join = bool(join or not stop_ok)
|
||||
if should_join and worker.thread.is_alive() and worker.thread is not threading.current_thread():
|
||||
worker.thread.join(timeout=self._stop_join_timeout)
|
||||
if worker.thread.is_alive():
|
||||
logger.warning(
|
||||
f"[GlobalNet] worker did not stop promptly: {session_id} "
|
||||
f"(reason={reason or '-'}, stop_listen_ok={stop_ok}, "
|
||||
f"requested_join={join})"
|
||||
)
|
||||
return False
|
||||
|
||||
if not worker.thread.is_alive():
|
||||
self._forget_worker_if_current(worker)
|
||||
|
||||
return True
|
||||
|
||||
def request_stop_for_session(
|
||||
self,
|
||||
session_id: str,
|
||||
reason: str = "",
|
||||
*,
|
||||
detach: bool = False,
|
||||
) -> bool:
|
||||
if not session_id:
|
||||
return True
|
||||
|
||||
with self._lock:
|
||||
worker = self._workers.get(session_id)
|
||||
if not worker:
|
||||
return True
|
||||
|
||||
worker.stop_event.set()
|
||||
return True
|
||||
|
||||
def shutdown(self):
|
||||
with self._lock:
|
||||
session_ids = list(self._workers.keys())
|
||||
for session_id in session_ids:
|
||||
self.stop_for_session(session_id, reason="shutdown", join=True)
|
||||
logger.info("[GlobalNet] 全局网络监听已关闭")
|
||||
|
||||
def _worker_loop(self, session_id: str, stop_event: threading.Event):
|
||||
tab = None
|
||||
listening = False
|
||||
last_listener_clear_at = time.monotonic()
|
||||
events_since_listener_clear = 0
|
||||
|
||||
try:
|
||||
while not stop_event.is_set():
|
||||
if self._is_shutdown():
|
||||
break
|
||||
|
||||
session = self._get_session(session_id)
|
||||
if session is None:
|
||||
break
|
||||
|
||||
tab = session.tab
|
||||
|
||||
if not listening:
|
||||
try:
|
||||
# 复用连接,降低对 CDP session 的额外占用
|
||||
driver_for_tab(tab).listener.reuse_driver = True
|
||||
self._start_listen(tab)
|
||||
listening = True
|
||||
last_listener_clear_at = time.monotonic()
|
||||
events_since_listener_clear = 0
|
||||
except Exception as e:
|
||||
logger.debug(f"[GlobalNet] 启动监听失败: {session_id}, err={e}")
|
||||
stop_event.wait(self._retry_delay)
|
||||
continue
|
||||
|
||||
now = time.monotonic()
|
||||
if listening and now - last_listener_clear_at >= self.LISTENER_CLEAR_INTERVAL_SEC:
|
||||
if self._safe_clear_listener(tab, session_id, "interval"):
|
||||
last_listener_clear_at = now
|
||||
events_since_listener_clear = 0
|
||||
|
||||
try:
|
||||
response = driver_for_tab(tab).listener.wait(timeout=self._wait_timeout)
|
||||
except Exception as e:
|
||||
if stop_event.is_set() or self._is_shutdown():
|
||||
break
|
||||
err_text = str(e)
|
||||
if "NoneType" in err_text and "is_running" in err_text:
|
||||
logger.debug(f"[GlobalNet] 监听状态失效,准备重启: {session_id}")
|
||||
else:
|
||||
logger.debug(f"[GlobalNet] wait 异常: {session_id}, err={e}")
|
||||
listening = False
|
||||
if not self._safe_stop_listen(tab):
|
||||
logger.warning(
|
||||
f"[GlobalNet] listener could not stop safely; "
|
||||
f"ending worker: {session_id}"
|
||||
)
|
||||
break
|
||||
stop_event.wait(self._retry_delay)
|
||||
continue
|
||||
|
||||
if response is None or response is False:
|
||||
continue
|
||||
|
||||
if stop_event.is_set() or self._is_shutdown():
|
||||
break
|
||||
|
||||
note_network_activity(session)
|
||||
event = self._extract_event(response)
|
||||
self._dispatch_event(session, event)
|
||||
self._dispatch_response_listeners(session, response, event, stop_event)
|
||||
events_since_listener_clear += 1
|
||||
if events_since_listener_clear >= self.LISTENER_CLEAR_EVENT_INTERVAL:
|
||||
if self._safe_clear_listener(tab, session_id, "event_budget"):
|
||||
last_listener_clear_at = time.monotonic()
|
||||
events_since_listener_clear = 0
|
||||
|
||||
finally:
|
||||
if tab is not None and self._should_cleanup_worker_listener(session_id, stop_event, tab):
|
||||
self._safe_stop_listen(tab)
|
||||
with self._lock:
|
||||
current = self._workers.get(session_id)
|
||||
if current is not None and current.stop_event is stop_event:
|
||||
self._workers.pop(session_id, None)
|
||||
@@ -0,0 +1,730 @@
|
||||
"""TabPoolManager 的一部分(R2-3 拆分):借用与归还:等待队列、通用/按编号/按原始 ID 借用、释放。
|
||||
|
||||
方法原样从 manager.py 迁出,行为不变;通过 mixin 组合回 TabPoolManager,
|
||||
类属性与实例状态都在 TabPoolManager(manager.py)里定义。
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import threading
|
||||
import time
|
||||
from collections import OrderedDict, deque
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from contextlib import asynccontextmanager
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from app.core.config import logger
|
||||
|
||||
from .session import TabSession, TabStatus
|
||||
|
||||
|
||||
class TabPoolAcquireMixin:
|
||||
"""借用与归还:等待队列、通用/按编号/按原始 ID 借用、释放"""
|
||||
|
||||
def _normalize_acquire_timeout(self, timeout: Optional[float]) -> float:
|
||||
"""统一各 acquire 路径的 timeout 语义。
|
||||
|
||||
原先大多数路径写 `timeout = timeout or self.acquire_timeout`,于是
|
||||
timeout=0(调用方想表达"不等待,立刻失败")会被当成"未传"而退化成默认
|
||||
60 秒阻塞,只有 acquire_by_route_group 一条路径按显式值处理。
|
||||
统一成:None -> 池默认;其余按显式值处理,负数收敛到 0。
|
||||
"""
|
||||
if timeout is None:
|
||||
return float(self.acquire_timeout)
|
||||
try:
|
||||
return max(0.0, float(timeout))
|
||||
except (TypeError, ValueError):
|
||||
return float(self.acquire_timeout)
|
||||
@staticmethod
|
||||
def _request_should_stop(task_id: str) -> bool:
|
||||
"""Return whether the request owning a blocking acquire was cancelled."""
|
||||
request_id = str(task_id or "").strip()
|
||||
if not request_id:
|
||||
return False
|
||||
try:
|
||||
from app.services.request_manager import request_manager
|
||||
|
||||
context = request_manager.get_request(request_id)
|
||||
return bool(context is not None and context.should_stop())
|
||||
except Exception:
|
||||
# Pool acquisition is also used by command/internal callers that do
|
||||
# not have a RequestContext; cancellation lookup must stay best effort.
|
||||
return False
|
||||
def _total_acquire_waiters(self) -> int:
|
||||
total = len(self._acquire_waiters)
|
||||
for group in (self._index_waiters, self._route_waiters, self._group_waiters):
|
||||
for waiters in group.values():
|
||||
total += len(waiters)
|
||||
return total
|
||||
def _acquire_queue_full(self, task_id: str) -> bool:
|
||||
"""H9:所有 acquire 等待队列的总量上限(调用方持有 self._condition)。
|
||||
|
||||
满了立即返回 True:调用方直接返回 None,并记下该 task 是「队列满」而不是
|
||||
「没有标签页」,工作流层据此给出明确的 503(可重试)而不是再排 60 秒。
|
||||
"""
|
||||
limit = int(getattr(self, "_max_acquire_waiters", 0) or 0)
|
||||
if limit <= 0:
|
||||
return False
|
||||
total = self._total_acquire_waiters()
|
||||
if total < limit:
|
||||
return False
|
||||
rejections = getattr(self, "_queue_full_rejections", None)
|
||||
if rejections is None:
|
||||
rejections = OrderedDict()
|
||||
self._queue_full_rejections = rejections
|
||||
rejections[str(task_id or "")] = time.time()
|
||||
while len(rejections) > 256:
|
||||
rejections.popitem(last=False)
|
||||
logger.warning(
|
||||
f"[TabPool] acquire 等待队列已满({total}/{limit}),立即拒绝 task={task_id}"
|
||||
)
|
||||
return True
|
||||
def consume_queue_full_rejection(self, task_id: str) -> bool:
|
||||
"""工作流层查询:该 task 的 acquire 是否因等待队列已满被拒(查询即清除)。"""
|
||||
rejections = getattr(self, "_queue_full_rejections", None)
|
||||
if not rejections:
|
||||
return False
|
||||
with self._condition:
|
||||
return rejections.pop(str(task_id or ""), None) is not None
|
||||
def _next_waiter_token(self, task_id: str) -> str:
|
||||
self._waiter_counter += 1
|
||||
base = str(task_id or "task").strip() or "task"
|
||||
return f"{base}#{self._waiter_counter}"
|
||||
@staticmethod
|
||||
def _is_waiter_turn(waiters: deque[str], waiter_token: str) -> bool:
|
||||
return not waiters or waiters[0] == waiter_token
|
||||
@staticmethod
|
||||
def _count_waiters_ahead(waiters: deque[str], waiter_token: str) -> int:
|
||||
try:
|
||||
return waiters.index(waiter_token)
|
||||
except ValueError:
|
||||
return 0
|
||||
def _unregister_waiter(
|
||||
self,
|
||||
waiters: deque[str],
|
||||
waiter_token: str,
|
||||
*,
|
||||
owner_map: Optional[Dict[Any, deque[str]]] = None,
|
||||
owner_key: Optional[Any] = None,
|
||||
) -> bool:
|
||||
removed = False
|
||||
try:
|
||||
waiters.remove(waiter_token)
|
||||
removed = True
|
||||
except ValueError:
|
||||
removed = False
|
||||
|
||||
# 只有当 owner_map 里登记的仍是同一个队列对象时才摘除。
|
||||
# 成功路径会先显式 _unregister_waiter 一次(此时队列可能已空并被摘除),
|
||||
# 随后 finally 再调一次;期间池锁可能被 _complete_acquired_session_for_return
|
||||
# 短暂释放,其他线程会用同一个 key 建出**新的**队列。若这里无条件 pop,
|
||||
# 就会把新队列连同里面排队的等待者一起丢掉,破坏 FIFO 公平性。
|
||||
if (
|
||||
owner_map is not None
|
||||
and owner_key is not None
|
||||
and not waiters
|
||||
and owner_map.get(owner_key) is waiters
|
||||
):
|
||||
owner_map.pop(owner_key, None)
|
||||
|
||||
if removed:
|
||||
self._condition.notify_all()
|
||||
return removed
|
||||
@staticmethod
|
||||
def _describe_session(session: Optional[TabSession]) -> str:
|
||||
if session is None:
|
||||
return "session=-"
|
||||
try:
|
||||
return session.debug_summary()
|
||||
except Exception as e:
|
||||
return f"session={getattr(session, 'id', '-')}, snapshot_error={e}"
|
||||
def _describe_index_wait_state(
|
||||
self,
|
||||
persistent_index: int,
|
||||
waiters: deque[str],
|
||||
waiter_token: str,
|
||||
) -> str:
|
||||
session_id = self._persistent_to_session_id.get(persistent_index)
|
||||
session = self._tabs.get(session_id) if session_id else None
|
||||
ahead = self._count_waiters_ahead(waiters, waiter_token)
|
||||
busy_for = "-"
|
||||
if session is not None:
|
||||
try:
|
||||
if getattr(session, "status", None) == TabStatus.BUSY:
|
||||
busy_for = (
|
||||
f"{max(0.0, time.time() - float(getattr(session, 'last_used_at', 0.0) or 0.0)):.1f}s"
|
||||
)
|
||||
except Exception:
|
||||
busy_for = "?"
|
||||
return (
|
||||
f"idx=#{persistent_index}, ahead={ahead}, waiters={len(waiters)}, "
|
||||
f"busy_for={busy_for}, holder={self._describe_session(session)}"
|
||||
)
|
||||
async def acquire_async(self, task_id: str, timeout: float = None) -> Optional[TabSession]:
|
||||
return await self._run_cancellable_acquire(
|
||||
lambda: self.acquire(task_id, timeout),
|
||||
task_id,
|
||||
)
|
||||
def _ensure_acquire_executor(self) -> ThreadPoolExecutor:
|
||||
# 双重检查懒创建:读属性无锁,创建走专用小锁(不碰池锁,事件循环线程不会
|
||||
# 被池维护路径阻塞)。max_workers=32 覆盖"池满 + 一批客户端断连"的堆积场景。
|
||||
executor = self._acquire_executor
|
||||
if executor is not None:
|
||||
return executor
|
||||
with self._acquire_executor_lock:
|
||||
if self._acquire_executor is None:
|
||||
self._acquire_executor = ThreadPoolExecutor(
|
||||
max_workers=32,
|
||||
thread_name_prefix="tab-acquire",
|
||||
)
|
||||
return self._acquire_executor
|
||||
async def _run_cancellable_acquire(self, acquire_fn, task_id: str) -> Optional[TabSession]:
|
||||
"""Release a session acquired after the awaiting task was cancelled."""
|
||||
# 改用专用 executor:默认 executor 线程数有限(约 min(32, cpu+4)),被取消的
|
||||
# acquire 仍会占线程直到超时,占满后所有 to_thread 排不上队。
|
||||
# run_in_executor 返回的 Future 同样兼容 shield 与 add_done_callback,
|
||||
# shield 语义不变:外层取消不打断底层 acquire 线程,结果由回调兜底释放。
|
||||
loop = asyncio.get_running_loop()
|
||||
acquire_task = loop.run_in_executor(self._ensure_acquire_executor(), acquire_fn)
|
||||
try:
|
||||
return await asyncio.shield(acquire_task)
|
||||
except asyncio.CancelledError:
|
||||
def _release_late_session(completed_task) -> None:
|
||||
try:
|
||||
session = completed_task.result()
|
||||
except BaseException:
|
||||
return
|
||||
if session is None:
|
||||
return
|
||||
|
||||
def _do_release() -> None:
|
||||
try:
|
||||
self.release(
|
||||
session.id,
|
||||
check_triggers=False,
|
||||
rollback_request_count=True,
|
||||
expected_task_id=task_id,
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.warning(
|
||||
f"Cancelled acquire cleanup failed (task={task_id}, "
|
||||
f"tab={getattr(session, 'id', '-')}, error={exc})"
|
||||
)
|
||||
|
||||
# 锁序考量:done callback 在事件循环线程执行,release 含池锁竞争与
|
||||
# CDP 调用(visibility 恢复各 0.5s 超时),不能同步跑在事件循环里。
|
||||
# 投递到 maintenance executor;submit 失败(已关闭)回退 daemon 线程。
|
||||
executor = self._maintenance_executor
|
||||
submitted = False
|
||||
if executor is not None:
|
||||
try:
|
||||
executor.submit(_do_release)
|
||||
submitted = True
|
||||
except RuntimeError as exc:
|
||||
logger.debug(
|
||||
f"[TabPool] maintenance submit failed "
|
||||
f"(late-release:{getattr(session, 'id', '-')}): {exc}"
|
||||
)
|
||||
if not submitted:
|
||||
threading.Thread(
|
||||
target=_do_release,
|
||||
name="tab-late-release",
|
||||
daemon=True,
|
||||
).start()
|
||||
|
||||
acquire_task.add_done_callback(_release_late_session)
|
||||
raise
|
||||
def release(
|
||||
self,
|
||||
tab_id: str,
|
||||
clear_page: bool = False,
|
||||
check_triggers: bool = True,
|
||||
rollback_request_count: bool = False,
|
||||
expected_task_id: str = "",
|
||||
) -> bool:
|
||||
"""释放标签页"""
|
||||
with self._condition:
|
||||
session = self._tabs.get(tab_id)
|
||||
if not session:
|
||||
return False
|
||||
|
||||
before_snapshot = self._describe_session(session)
|
||||
expected_task = str(expected_task_id or "").strip()
|
||||
current_task = str(getattr(session, "current_task_id", "") or "").strip()
|
||||
session_status = getattr(getattr(session, "status", None), "value", "")
|
||||
if current_task and not expected_task:
|
||||
logger.warning(
|
||||
f"[{tab_id}] 跳过无所有权释放 "
|
||||
f"(current_task={current_task}, snapshot={before_snapshot})"
|
||||
)
|
||||
return False
|
||||
if expected_task:
|
||||
if current_task and current_task != expected_task:
|
||||
logger.warning(
|
||||
f"[{tab_id}] 跳过释放:标签页已被其他任务接管 "
|
||||
f"(expected_task={expected_task}, current_task={current_task}, "
|
||||
f"snapshot={before_snapshot})"
|
||||
)
|
||||
return False
|
||||
if not current_task:
|
||||
logger.warning(
|
||||
f"[{tab_id}] 跳过释放:标签页 task_id 已丢失 "
|
||||
f"(expected_task={expected_task}, status={session_status or 'unknown'}, "
|
||||
f"snapshot={before_snapshot})"
|
||||
)
|
||||
return False
|
||||
|
||||
release_state = session._begin_release_state(
|
||||
clear_page=clear_page,
|
||||
rollback_request_count=rollback_request_count,
|
||||
force=False,
|
||||
expected_task_id=expected_task,
|
||||
reject_ownerless=True,
|
||||
)
|
||||
if release_state is None:
|
||||
self._condition.notify_all()
|
||||
return False
|
||||
|
||||
session._run_release_from_state(
|
||||
release_state,
|
||||
clear_page=clear_page,
|
||||
check_triggers=check_triggers,
|
||||
rollback_request_count=rollback_request_count,
|
||||
)
|
||||
|
||||
with self._condition:
|
||||
if self._tabs.get(tab_id) is session:
|
||||
# 锁序考量:不在持池锁时同步启动 monitor——start_for_session 可能
|
||||
# join 停止中的 worker 最长 2s(本文件他处已注明持池锁禁止 join worker)。
|
||||
# 复用 _restart_global_monitor_for_session_async 投递到 maintenance
|
||||
# executor:它会在池锁外复核会话仍存在且 IDLE 再启动,start_for_session
|
||||
# 本身幂等(已有存活 worker 时直接返回);若会话已被再次 acquire,
|
||||
# 复核发现非 IDLE 即放弃,acquire 侧的 handoff stop 亦会兜底摘除。
|
||||
self._restart_global_monitor_for_session_async(tab_id, "release")
|
||||
self._condition.notify_all()
|
||||
logger.debug(
|
||||
f"[{tab_id}] 已释放 "
|
||||
f"(expected_task={expected_task or '-'}, clear_page={clear_page}, "
|
||||
f"check_triggers={check_triggers}, rollback_request_count={rollback_request_count}, "
|
||||
f"before={before_snapshot}, after={self._describe_session(session)})"
|
||||
)
|
||||
return session.status == TabStatus.IDLE
|
||||
return False
|
||||
def force_release_all(self):
|
||||
"""强制释放所有标签页(调试用)"""
|
||||
with self._condition:
|
||||
pending: List[tuple[TabSession, Dict[str, Any]]] = []
|
||||
for session in self._tabs.values():
|
||||
if session.status == TabStatus.BUSY:
|
||||
# 先请求取消旧工作流再强制释放:否则 BUSY 会话被洗成 IDLE 后,
|
||||
# 旧工作流尚未退出,同一 tab 可能立刻被再分配,新旧任务并发操作
|
||||
# 同一页面。锁序安全:_cancel_active_request_for_session 只向
|
||||
# maintenance executor 投递,持池锁时不外呼。
|
||||
busy_task = str(session.current_task_id or "").strip()
|
||||
cancel_submitted = self._cancel_active_request_for_session(
|
||||
session,
|
||||
"force_release_all",
|
||||
detail=f"task={busy_task or '-'}",
|
||||
)
|
||||
logger.warning(
|
||||
f"[{session.id}] force_release_all 强制释放忙碌会话 "
|
||||
f"(task={busy_task or '-'}, cancel_submitted={cancel_submitted})"
|
||||
)
|
||||
release_state = session._begin_release_state(
|
||||
clear_page=False,
|
||||
force=True,
|
||||
)
|
||||
if release_state is not None:
|
||||
pending.append((session, release_state))
|
||||
|
||||
for session, release_state in pending:
|
||||
session._run_force_release_from_state(
|
||||
release_state,
|
||||
clear_page=False,
|
||||
check_triggers=False,
|
||||
)
|
||||
|
||||
with self._condition:
|
||||
for session, _release_state in pending:
|
||||
if self._tabs.get(session.id) is session and session.status == TabStatus.IDLE:
|
||||
self._start_global_monitor_for_session(session)
|
||||
self._condition.notify_all()
|
||||
count = len(pending)
|
||||
logger.info(f"强制释放 {count} 个标签页")
|
||||
return count
|
||||
@asynccontextmanager
|
||||
async def get_tab(self, task_id: str, timeout: float = None):
|
||||
session = await self.acquire_async(task_id, timeout)
|
||||
if session is None:
|
||||
raise TimeoutError(f"获取标签页超时 (task: {task_id})")
|
||||
|
||||
try:
|
||||
yield session
|
||||
except Exception as e:
|
||||
session.mark_error(str(e))
|
||||
raise
|
||||
finally:
|
||||
# 锁序考量:release 含池锁竞争与 CDP 调用(visibility 恢复各 0.5s 超时),
|
||||
# 移出事件循环线程执行。shield 保证 __aexit__ 期间再次被取消时,
|
||||
# release 仍在后台任务/线程中完成,不会泄漏标签页占用。
|
||||
await asyncio.shield(
|
||||
asyncio.to_thread(self.release, session.id, expected_task_id=task_id)
|
||||
)
|
||||
def acquire_by_raw_tab_id(
|
||||
self,
|
||||
raw_tab_id: str,
|
||||
task_id: str,
|
||||
timeout: float = None,
|
||||
count_request: bool = True,
|
||||
activate: Optional[bool] = None,
|
||||
) -> Optional[TabSession]:
|
||||
"""
|
||||
根据底层浏览器标签页 ID 获取指定标签页会话。
|
||||
|
||||
这用于外部已经明确知道目标标签页时,复用 TabPool 的运行时上下文,
|
||||
保持与正常工作流执行一致的监听、占用与释放行为。
|
||||
"""
|
||||
raw_tab_id = str(raw_tab_id or "").strip()
|
||||
if not raw_tab_id:
|
||||
return None
|
||||
|
||||
timeout = self._normalize_acquire_timeout(timeout)
|
||||
deadline = time.time() + timeout
|
||||
|
||||
with self._condition:
|
||||
while True:
|
||||
if self._shutdown:
|
||||
return None
|
||||
if self._request_should_stop(task_id):
|
||||
logger.debug(f"Raw tab acquire cancelled (task={task_id})")
|
||||
return None
|
||||
|
||||
if self._should_scan():
|
||||
self._scan_new_tabs()
|
||||
|
||||
self._check_stuck_tabs()
|
||||
self._cleanup_unhealthy_tabs()
|
||||
|
||||
persistent_index = self._raw_id_to_persistent.get(raw_tab_id)
|
||||
if not persistent_index:
|
||||
logger.warning(f"底层标签页 ID 不存在: {raw_tab_id}")
|
||||
return None
|
||||
|
||||
session_id = self._persistent_to_session_id.get(persistent_index)
|
||||
if not session_id:
|
||||
logger.warning(f"底层标签页 {raw_tab_id} 未绑定会话")
|
||||
return None
|
||||
|
||||
session = self._tabs.get(session_id)
|
||||
if not session:
|
||||
logger.warning(f"底层标签页 {raw_tab_id} 对应会话已移除")
|
||||
return None
|
||||
|
||||
if not session.is_healthy(allow_live_check=False):
|
||||
logger.warning(f"[{session.id}] 标签页不健康")
|
||||
return None
|
||||
|
||||
if self._should_defer_to_command(session, task_id):
|
||||
logger.debug(f"[{session.id}] defer raw-tab acquire to high-priority command")
|
||||
remaining = deadline - time.time()
|
||||
if remaining <= 0:
|
||||
return None
|
||||
self._condition.wait(timeout=min(remaining, 0.5))
|
||||
continue
|
||||
|
||||
if session.status == TabStatus.IDLE:
|
||||
acquired = (
|
||||
session.acquire(task_id)
|
||||
if count_request
|
||||
else session.acquire_for_command(task_id)
|
||||
)
|
||||
if acquired:
|
||||
if not self._complete_acquired_session_for_return(
|
||||
session,
|
||||
"acquire_by_raw_tab_id",
|
||||
task_id,
|
||||
rollback_request_count=count_request,
|
||||
activate=(
|
||||
self._auto_activate_on_acquire and session.id != self._active_session_id
|
||||
if activate is None
|
||||
else bool(activate)
|
||||
),
|
||||
):
|
||||
continue
|
||||
logger.debug(
|
||||
f"TabPool → {session.id} "
|
||||
f"(task={task_id}, raw_tab_id={raw_tab_id}, idx=#{persistent_index}, "
|
||||
f"count_request={count_request}, snapshot={self._describe_session(session)})"
|
||||
)
|
||||
return session
|
||||
|
||||
remaining = deadline - time.time()
|
||||
if remaining <= 0:
|
||||
logger.warning(
|
||||
f"获取底层标签页 {raw_tab_id} 超时 "
|
||||
f"(task={task_id}, 当前状态: {session.status.value}, "
|
||||
f"snapshot={self._describe_session(session)})"
|
||||
)
|
||||
return None
|
||||
|
||||
logger.debug_throttled(
|
||||
f"tab_pool.wait_raw.{raw_tab_id}",
|
||||
f"等待底层标签页 {raw_tab_id} 释放...",
|
||||
interval_sec=5.0,
|
||||
)
|
||||
self._condition.wait(
|
||||
timeout=min(remaining, self.ACQUIRE_CANCEL_POLL_SEC)
|
||||
)
|
||||
async def acquire_by_index_async(self, persistent_index: int, task_id: str, timeout: float = None) -> Optional[TabSession]:
|
||||
"""异步版本的按编号获取"""
|
||||
return await self._run_cancellable_acquire(
|
||||
lambda: self.acquire_by_index(persistent_index, task_id, timeout),
|
||||
task_id,
|
||||
)
|
||||
def acquire(self, task_id: str, timeout: float = None) -> Optional[TabSession]:
|
||||
"""ASCII-safe fair acquire for generic request routing."""
|
||||
timeout = self._normalize_acquire_timeout(timeout)
|
||||
deadline = time.time() + timeout
|
||||
logged_waiting = False
|
||||
first_iteration = True
|
||||
|
||||
with self._condition:
|
||||
if self._acquire_queue_full(task_id):
|
||||
return None
|
||||
waiter_token = self._next_waiter_token(task_id)
|
||||
self._acquire_waiters.append(waiter_token)
|
||||
try:
|
||||
while True:
|
||||
if self._shutdown:
|
||||
return None
|
||||
if self._request_should_stop(task_id):
|
||||
logger.debug(f"Acquire cancelled (task={task_id})")
|
||||
return None
|
||||
|
||||
if first_iteration or self._should_scan():
|
||||
self._scan_new_tabs()
|
||||
first_iteration = False
|
||||
|
||||
self._check_stuck_tabs()
|
||||
self._cleanup_unhealthy_tabs()
|
||||
|
||||
if not self._is_waiter_turn(self._acquire_waiters, waiter_token):
|
||||
remaining = deadline - time.time()
|
||||
if remaining <= 0:
|
||||
busy_info = [
|
||||
f"{s.id}({s.current_task_id})"
|
||||
for s in self._tabs.values()
|
||||
if s.status == TabStatus.BUSY
|
||||
]
|
||||
unhealthy_count = sum(
|
||||
1 for s in self._tabs.values()
|
||||
if s.status == TabStatus.IDLE and not s.is_healthy(allow_live_check=False)
|
||||
)
|
||||
logger.warning(
|
||||
f"Acquire tab timed out (task={task_id}, busy: {', '.join(busy_info) or 'none'}, "
|
||||
f"unhealthy: {unhealthy_count})"
|
||||
)
|
||||
return None
|
||||
|
||||
if not logged_waiting:
|
||||
busy_tabs = [s.id for s in self._tabs.values() if s.status == TabStatus.BUSY]
|
||||
ahead = self._count_waiters_ahead(self._acquire_waiters, waiter_token)
|
||||
if busy_tabs:
|
||||
logger.debug(f"Waiting for tab (busy: {', '.join(busy_tabs)})")
|
||||
elif ahead > 0:
|
||||
logger.debug(f"Waiting in queue (ahead: {ahead})")
|
||||
logged_waiting = True
|
||||
|
||||
self._condition.wait(
|
||||
timeout=min(remaining, self.ACQUIRE_CANCEL_POLL_SEC)
|
||||
)
|
||||
continue
|
||||
|
||||
dynamic_sessions = self._get_sessions_for_dynamic_routing()
|
||||
if not dynamic_sessions and self._tabs:
|
||||
logger.warning(
|
||||
f"Acquire tab skipped: all open tabs are excluded from dynamic routing "
|
||||
f"(task={task_id})"
|
||||
)
|
||||
return None
|
||||
|
||||
session = self._try_acquire_session_for_request(
|
||||
dynamic_sessions,
|
||||
task_id,
|
||||
)
|
||||
if session is not None:
|
||||
self._unregister_waiter(self._acquire_waiters, waiter_token)
|
||||
if not self._complete_acquired_session_for_return(
|
||||
session,
|
||||
"acquire",
|
||||
task_id,
|
||||
activate=self._auto_activate_on_acquire and session.id != self._active_session_id,
|
||||
):
|
||||
return None
|
||||
self._mark_allocation_cursor(session)
|
||||
|
||||
if logged_waiting:
|
||||
logger.debug(
|
||||
f"Acquire finished -> {session.id} "
|
||||
f"(task={task_id}, snapshot={self._describe_session(session)})"
|
||||
)
|
||||
else:
|
||||
logger.debug(
|
||||
f"TabPool -> {session.id} "
|
||||
f"(task={task_id}, snapshot={self._describe_session(session)})"
|
||||
)
|
||||
return session
|
||||
|
||||
remaining = deadline - time.time()
|
||||
if remaining <= 0:
|
||||
busy_info = [
|
||||
f"{s.id}({s.current_task_id})"
|
||||
for s in self._tabs.values()
|
||||
if s.status == TabStatus.BUSY
|
||||
]
|
||||
unhealthy_count = sum(
|
||||
1 for s in self._tabs.values()
|
||||
if s.status == TabStatus.IDLE and not s.is_healthy(allow_live_check=False)
|
||||
)
|
||||
logger.warning(
|
||||
f"Acquire tab timed out (task={task_id}, busy: {', '.join(busy_info) or 'none'}, "
|
||||
f"unhealthy: {unhealthy_count})"
|
||||
)
|
||||
return None
|
||||
|
||||
if not logged_waiting:
|
||||
busy_tabs = [s.id for s in self._tabs.values() if s.status == TabStatus.BUSY]
|
||||
if busy_tabs:
|
||||
logger.debug(f"Waiting for tab (busy: {', '.join(busy_tabs)})")
|
||||
logged_waiting = True
|
||||
|
||||
self._condition.wait(
|
||||
timeout=min(remaining, self.ACQUIRE_CANCEL_POLL_SEC)
|
||||
)
|
||||
finally:
|
||||
self._unregister_waiter(self._acquire_waiters, waiter_token)
|
||||
def acquire_by_index(self, persistent_index: int, task_id: str, timeout: float = None) -> Optional[TabSession]:
|
||||
"""ASCII-safe fair acquire for a fixed persistent tab index."""
|
||||
timeout = self._normalize_acquire_timeout(timeout)
|
||||
deadline = time.time() + timeout
|
||||
|
||||
with self._condition:
|
||||
if self._acquire_queue_full(task_id):
|
||||
return None
|
||||
waiters = self._index_waiters.setdefault(persistent_index, deque())
|
||||
waiter_token = self._next_waiter_token(task_id)
|
||||
waiters.append(waiter_token)
|
||||
try:
|
||||
while True:
|
||||
if self._shutdown:
|
||||
return None
|
||||
if self._request_should_stop(task_id):
|
||||
logger.debug(f"Index acquire cancelled (task={task_id})")
|
||||
return None
|
||||
|
||||
if self._should_scan():
|
||||
self._scan_new_tabs()
|
||||
|
||||
# Keep fixed-index acquisition aligned with other wait paths so a wedged
|
||||
# holder can be cancelled and released instead of blocking the queue.
|
||||
released_stuck = self._check_stuck_tabs()
|
||||
self._cleanup_unhealthy_tabs()
|
||||
if released_stuck:
|
||||
logger.debug(
|
||||
f"Rechecking tab #{persistent_index} after stuck-tab maintenance "
|
||||
f"(task={task_id}, wait_state={self._describe_index_wait_state(persistent_index, waiters, waiter_token)})"
|
||||
)
|
||||
|
||||
if not self._is_waiter_turn(waiters, waiter_token):
|
||||
remaining = deadline - time.time()
|
||||
if remaining <= 0:
|
||||
logger.warning(
|
||||
f"Timed out waiting for tab #{persistent_index} "
|
||||
f"(task={task_id}, wait_state={self._describe_index_wait_state(persistent_index, waiters, waiter_token)})"
|
||||
)
|
||||
return None
|
||||
logger.debug_throttled(
|
||||
f"tab_pool.wait_index.{persistent_index}",
|
||||
f"Waiting for tab #{persistent_index} release... "
|
||||
f"({self._describe_index_wait_state(persistent_index, waiters, waiter_token)})",
|
||||
interval_sec=5.0,
|
||||
)
|
||||
self._condition.wait(
|
||||
timeout=min(remaining, self.ACQUIRE_CANCEL_POLL_SEC)
|
||||
)
|
||||
continue
|
||||
|
||||
session_id = self._persistent_to_session_id.get(persistent_index)
|
||||
if not session_id:
|
||||
logger.warning(f"Persistent tab #{persistent_index} does not exist")
|
||||
return None
|
||||
|
||||
session = self._tabs.get(session_id)
|
||||
if not session:
|
||||
logger.warning(f"Tab {session_id} (#{persistent_index}) was removed")
|
||||
return None
|
||||
|
||||
if not session.is_healthy(allow_live_check=False):
|
||||
logger.warning(
|
||||
f"[{session.id}] tab is unhealthy "
|
||||
f"(task={task_id}, snapshot={self._describe_session(session)})"
|
||||
)
|
||||
return None
|
||||
|
||||
if self._should_defer_to_command(session, task_id):
|
||||
logger.debug(
|
||||
f"[{session.id}] defer by index acquire to high-priority command "
|
||||
f"(task={task_id}, snapshot={self._describe_session(session)})"
|
||||
)
|
||||
remaining = deadline - time.time()
|
||||
if remaining <= 0:
|
||||
return None
|
||||
self._condition.wait(timeout=min(remaining, 0.5))
|
||||
continue
|
||||
|
||||
if session.status == TabStatus.IDLE and session.acquire(task_id):
|
||||
self._unregister_waiter(
|
||||
waiters,
|
||||
waiter_token,
|
||||
owner_map=self._index_waiters,
|
||||
owner_key=persistent_index,
|
||||
)
|
||||
if not self._complete_acquired_session_for_return(
|
||||
session,
|
||||
"acquire_by_index",
|
||||
task_id,
|
||||
activate=self._auto_activate_on_acquire and session.id != self._active_session_id,
|
||||
):
|
||||
return None
|
||||
logger.debug(
|
||||
f"TabPool -> {session.id} "
|
||||
f"(task={task_id}, idx=#{persistent_index}, "
|
||||
f"snapshot={self._describe_session(session)})"
|
||||
)
|
||||
return session
|
||||
|
||||
remaining = deadline - time.time()
|
||||
if remaining <= 0:
|
||||
logger.warning(
|
||||
f"Timed out waiting for tab #{persistent_index} "
|
||||
f"(task={task_id}, status: {session.status.value}, "
|
||||
f"wait_state={self._describe_index_wait_state(persistent_index, waiters, waiter_token)}, "
|
||||
f"snapshot={self._describe_session(session)})"
|
||||
)
|
||||
return None
|
||||
|
||||
logger.debug_throttled(
|
||||
f"tab_pool.wait_index.{persistent_index}",
|
||||
f"Waiting for tab #{persistent_index} release... "
|
||||
f"({self._describe_index_wait_state(persistent_index, waiters, waiter_token)})",
|
||||
interval_sec=5.0,
|
||||
)
|
||||
self._condition.wait(
|
||||
timeout=min(remaining, self.ACQUIRE_CANCEL_POLL_SEC)
|
||||
)
|
||||
finally:
|
||||
self._unregister_waiter(
|
||||
waiters,
|
||||
waiter_token,
|
||||
owner_map=self._index_waiters,
|
||||
owner_key=persistent_index,
|
||||
)
|
||||
@@ -0,0 +1,366 @@
|
||||
"""TabPoolManager 的一部分(R2-3 拆分):分配顺序与游标、全局网络监听的交接(借出/归还时的启停)。
|
||||
|
||||
方法原样从 manager.py 迁出,行为不变;通过 mixin 组合回 TabPoolManager,
|
||||
类属性与实例状态都在 TabPoolManager(manager.py)里定义。
|
||||
"""
|
||||
|
||||
import random
|
||||
import threading
|
||||
import time
|
||||
from typing import Any, List, Optional
|
||||
|
||||
from app.core.config import logger
|
||||
|
||||
from .session import TabSession, TabStatus
|
||||
|
||||
|
||||
class TabPoolAllocationMixin:
|
||||
"""分配顺序与游标、全局网络监听的交接(借出/归还时的启停)"""
|
||||
|
||||
def _order_sessions_for_allocation(
|
||||
self,
|
||||
sessions: List[TabSession],
|
||||
*,
|
||||
route_domain: Optional[str] = None,
|
||||
allocation_mode: Optional[str] = None,
|
||||
) -> List[TabSession]:
|
||||
ordered = sorted(list(sessions or []), key=self._session_allocation_key)
|
||||
mode = self._normalize_allocation_mode(allocation_mode or self.allocation_mode)
|
||||
if mode == "random" and len(ordered) > 1:
|
||||
randomized = ordered[:]
|
||||
random.shuffle(randomized)
|
||||
return randomized
|
||||
|
||||
if mode != "round_robin" or len(ordered) <= 1:
|
||||
return ordered
|
||||
|
||||
cursor = (
|
||||
self._route_round_robin_cursor.get(route_domain, 0)
|
||||
if route_domain
|
||||
else self._round_robin_cursor
|
||||
)
|
||||
cursor_index = self._persistent_index_value(cursor)
|
||||
next_items = [
|
||||
item for item in ordered
|
||||
if self._persistent_index_value(item.persistent_index) > cursor_index
|
||||
]
|
||||
return next_items + [
|
||||
item for item in ordered
|
||||
if self._persistent_index_value(item.persistent_index) <= cursor_index
|
||||
]
|
||||
@staticmethod
|
||||
def _persistent_index_value(value: Any) -> int:
|
||||
try:
|
||||
return int(value or 0)
|
||||
except (TypeError, ValueError):
|
||||
return 0
|
||||
@classmethod
|
||||
def _session_allocation_key(cls, session: TabSession):
|
||||
return (
|
||||
cls._persistent_index_value(getattr(session, "persistent_index", 0)),
|
||||
str(getattr(session, "id", "") or ""),
|
||||
)
|
||||
def _select_first_idle_acquire_candidate(
|
||||
self,
|
||||
sessions,
|
||||
task_id: str,
|
||||
attempted_ids: set,
|
||||
*,
|
||||
defer_context: str = "acquire",
|
||||
) -> Optional[TabSession]:
|
||||
selected = None
|
||||
selected_key = None
|
||||
for session in sessions or []:
|
||||
session_id = str(getattr(session, "id", "") or "")
|
||||
if session_id in attempted_ids:
|
||||
continue
|
||||
if session.status != TabStatus.IDLE:
|
||||
continue
|
||||
if not session.is_healthy(allow_live_check=False):
|
||||
logger.warning(f"[{session.id}] skip unhealthy tab")
|
||||
continue
|
||||
if self._should_defer_to_command(session, task_id):
|
||||
logger.debug(f"[{session.id}] defer {defer_context} to high-priority command")
|
||||
continue
|
||||
|
||||
candidate_key = self._session_allocation_key(session)
|
||||
if selected is None or candidate_key < selected_key:
|
||||
selected = session
|
||||
selected_key = candidate_key
|
||||
|
||||
return selected
|
||||
def _try_acquire_session_for_request(
|
||||
self,
|
||||
sessions,
|
||||
task_id: str,
|
||||
*,
|
||||
route_domain: Optional[str] = None,
|
||||
allocation_mode: Optional[str] = None,
|
||||
defer_context: str = "acquire",
|
||||
) -> Optional[TabSession]:
|
||||
mode = self._normalize_allocation_mode(allocation_mode or self.allocation_mode)
|
||||
if mode == "first_idle":
|
||||
attempted_ids = set()
|
||||
while True:
|
||||
session = self._select_first_idle_acquire_candidate(
|
||||
sessions,
|
||||
task_id,
|
||||
attempted_ids,
|
||||
defer_context=defer_context,
|
||||
)
|
||||
if session is None:
|
||||
return None
|
||||
attempted_ids.add(str(session.id or ""))
|
||||
if session.acquire(task_id):
|
||||
return session
|
||||
|
||||
for session in self._order_sessions_for_allocation(
|
||||
sessions,
|
||||
route_domain=route_domain,
|
||||
allocation_mode=mode,
|
||||
):
|
||||
if session.status != TabStatus.IDLE:
|
||||
continue
|
||||
if not session.is_healthy(allow_live_check=False):
|
||||
logger.warning(f"[{session.id}] skip unhealthy tab")
|
||||
continue
|
||||
if self._should_defer_to_command(session, task_id):
|
||||
logger.debug(f"[{session.id}] defer {defer_context} to high-priority command")
|
||||
continue
|
||||
if session.acquire(task_id):
|
||||
return session
|
||||
return None
|
||||
def _mark_allocation_cursor(self, session: TabSession, route_domain: Optional[str] = None) -> None:
|
||||
current_index = int(session.persistent_index or 0)
|
||||
if route_domain:
|
||||
self._route_round_robin_cursor[route_domain] = current_index
|
||||
self._route_round_robin_cursor.move_to_end(route_domain)
|
||||
while len(self._route_round_robin_cursor) > self.ROUTE_CURSOR_LIMIT:
|
||||
self._route_round_robin_cursor.popitem(last=False)
|
||||
else:
|
||||
self._round_robin_cursor = current_index
|
||||
def _get_session_for_monitor(self, session_id: str) -> Optional[TabSession]:
|
||||
with self._lock:
|
||||
return self._tabs.get(session_id)
|
||||
def _get_session_for_monitor_snapshot(self, session_id: str) -> Optional[TabSession]:
|
||||
# Global monitor workers may be joined while the pool lock is held elsewhere.
|
||||
# Use a lock-free snapshot lookup here so the worker can observe removal and exit
|
||||
# instead of blocking behind the pool lock during shutdown/rebind paths.
|
||||
return self._tabs.get(session_id)
|
||||
def _start_global_monitor_for_session(self, session: Optional[TabSession]) -> bool:
|
||||
if not session or not self._global_network_monitor:
|
||||
return False
|
||||
transition_lock = getattr(self, "_global_monitor_transition_lock", None)
|
||||
if transition_lock is None:
|
||||
transition_lock = self._global_monitor_transition_lock = threading.RLock()
|
||||
# Start and stop use one transition lock. If acquisition changes the
|
||||
# session to BUSY during a delayed start, its handoff stop waits here and
|
||||
# removes the newly registered worker before returning the session.
|
||||
with transition_lock:
|
||||
if self._shutdown:
|
||||
return False
|
||||
if self._tabs.get(session.id) is not session:
|
||||
return False
|
||||
# 仅在空闲标签页常驻监听,任务执行时让位
|
||||
if session.status != TabStatus.IDLE:
|
||||
return False
|
||||
return bool(self._global_network_monitor.start_for_session(session))
|
||||
def _restart_global_monitor_for_session_async(self, session_id: str, reason: str) -> None:
|
||||
"""Restart an idle session monitor outside the pool lock after a target rebind."""
|
||||
session_key = str(session_id or "").strip()
|
||||
executor = self._maintenance_executor
|
||||
if not session_key or executor is None or not self._global_network_monitor:
|
||||
return
|
||||
|
||||
def _restart() -> None:
|
||||
deadline = time.monotonic() + 5.0
|
||||
while True:
|
||||
with self._lock:
|
||||
if self._shutdown:
|
||||
return
|
||||
session = self._tabs.get(session_key)
|
||||
if session is None or session.status != TabStatus.IDLE:
|
||||
return
|
||||
if self._start_global_monitor_for_session(session):
|
||||
# logger.debug(f"[GlobalNet] rebound listener restarted: {session_key} ({reason})")
|
||||
return
|
||||
if time.monotonic() >= deadline:
|
||||
logger.warning(f"[GlobalNet] rebound listener restart failed: {session_key} ({reason})")
|
||||
return
|
||||
time.sleep(0.1)
|
||||
|
||||
try:
|
||||
executor.submit(_restart)
|
||||
except RuntimeError as e:
|
||||
logger.debug(f"[GlobalNet] restart submit failed ({session_key}): {e}")
|
||||
def _stop_global_monitor_for_session(self, session_id: str, reason: str = "", wait: bool = False) -> bool:
|
||||
if not self._global_network_monitor:
|
||||
return True
|
||||
transition_lock = getattr(self, "_global_monitor_transition_lock", None)
|
||||
if transition_lock is None:
|
||||
transition_lock = self._global_monitor_transition_lock = threading.RLock()
|
||||
# Do not hold the pool lock while joining a worker: command callbacks can
|
||||
# legitimately need that lock while the worker is winding down.
|
||||
with transition_lock:
|
||||
monitor = self._global_network_monitor
|
||||
if not monitor:
|
||||
return True
|
||||
return bool(monitor.stop_for_session(session_id, reason=reason, join=wait))
|
||||
def _request_global_monitor_stop_for_session(
|
||||
self,
|
||||
session_id: str,
|
||||
reason: str = "",
|
||||
*,
|
||||
detach: bool = False,
|
||||
) -> bool:
|
||||
if not self._global_network_monitor:
|
||||
return True
|
||||
request_stop = getattr(self._global_network_monitor, "request_stop_for_session", None)
|
||||
if callable(request_stop):
|
||||
return bool(request_stop(session_id, reason=reason, detach=detach))
|
||||
return self._stop_global_monitor_for_session(session_id, reason=reason, wait=False)
|
||||
def _detach_global_monitor_for_session(
|
||||
self,
|
||||
session_id: str,
|
||||
reason: str = "",
|
||||
) -> bool:
|
||||
session_id = str(session_id or "").strip()
|
||||
if not session_id:
|
||||
return True
|
||||
if not self._global_network_monitor:
|
||||
return True
|
||||
return self._request_global_monitor_stop_for_session(
|
||||
session_id,
|
||||
reason=reason,
|
||||
detach=True,
|
||||
)
|
||||
def _prepare_acquired_session_for_handoff(
|
||||
self,
|
||||
session: TabSession,
|
||||
reason: str,
|
||||
*,
|
||||
rollback_request_count: bool = True,
|
||||
) -> bool:
|
||||
self._request_global_monitor_stop_for_session(session.id, reason=reason)
|
||||
return True
|
||||
def _finish_acquired_session_handoff(
|
||||
self,
|
||||
session: TabSession,
|
||||
reason: str,
|
||||
task_id: str,
|
||||
*,
|
||||
rollback_request_count: bool = True,
|
||||
) -> bool:
|
||||
if self._stop_global_monitor_for_session(session.id, reason=reason, wait=True):
|
||||
return True
|
||||
|
||||
logger.warning(
|
||||
f"[{session.id}] global monitor did not stop before handoff "
|
||||
f"(reason={reason}); marking session unhealthy"
|
||||
)
|
||||
expected_task = str(task_id or "").strip()
|
||||
release_state = None
|
||||
with self._condition:
|
||||
current = self._tabs.get(session.id)
|
||||
current_task = str(getattr(session, "current_task_id", "") or "").strip()
|
||||
if (
|
||||
current is session
|
||||
and session.status == TabStatus.BUSY
|
||||
and (not expected_task or current_task == expected_task)
|
||||
):
|
||||
release_state = session._begin_release_state(
|
||||
clear_page=False,
|
||||
rollback_request_count=rollback_request_count,
|
||||
force=False,
|
||||
)
|
||||
|
||||
if release_state is not None:
|
||||
session._run_release_from_state(
|
||||
release_state,
|
||||
clear_page=False,
|
||||
check_triggers=False,
|
||||
rollback_request_count=rollback_request_count,
|
||||
)
|
||||
|
||||
with self._condition:
|
||||
current = self._tabs.get(session.id)
|
||||
current_task = str(getattr(session, "current_task_id", "") or "").strip()
|
||||
if current is session and (not expected_task or not current_task or current_task == expected_task):
|
||||
session.mark_error("global_monitor_stop_timeout")
|
||||
self._condition.notify_all()
|
||||
return False
|
||||
def _complete_acquired_session_for_return(
|
||||
self,
|
||||
session: TabSession,
|
||||
reason: str,
|
||||
task_id: str,
|
||||
*,
|
||||
rollback_request_count: bool = True,
|
||||
activate: bool = False,
|
||||
) -> bool:
|
||||
self._prepare_acquired_session_for_handoff(
|
||||
session,
|
||||
reason,
|
||||
rollback_request_count=rollback_request_count,
|
||||
)
|
||||
|
||||
self._condition.release()
|
||||
try:
|
||||
handoff_ok = self._finish_acquired_session_handoff(
|
||||
session,
|
||||
reason,
|
||||
task_id,
|
||||
rollback_request_count=rollback_request_count,
|
||||
)
|
||||
finally:
|
||||
self._condition.acquire()
|
||||
|
||||
if not handoff_ok:
|
||||
return False
|
||||
|
||||
current_task = str(getattr(session, "current_task_id", "") or "").strip()
|
||||
expected_task = str(task_id or "").strip()
|
||||
if (
|
||||
self._tabs.get(session.id) is not session
|
||||
or session.status != TabStatus.BUSY
|
||||
or (expected_task and current_task != expected_task)
|
||||
):
|
||||
logger.warning(
|
||||
f"[{session.id}] acquire handoff lost ownership "
|
||||
f"(reason={reason}, expected_task={expected_task or '-'}, "
|
||||
f"current_task={current_task or '-'}, status={session.status.value})"
|
||||
)
|
||||
return False
|
||||
|
||||
if activate:
|
||||
self._condition.release()
|
||||
try:
|
||||
session.activate()
|
||||
finally:
|
||||
self._condition.acquire()
|
||||
|
||||
current_task = str(getattr(session, "current_task_id", "") or "").strip()
|
||||
if (
|
||||
self._tabs.get(session.id) is not session
|
||||
or session.status != TabStatus.BUSY
|
||||
or (expected_task and current_task != expected_task)
|
||||
):
|
||||
logger.warning(
|
||||
f"[{session.id}] acquire activation lost ownership "
|
||||
f"(reason={reason}, expected_task={expected_task or '-'}, "
|
||||
f"current_task={current_task or '-'}, status={session.status.value})"
|
||||
)
|
||||
return False
|
||||
self._active_session_id = session.id
|
||||
return True
|
||||
def suspend_global_network_monitor(self, tab_id: str, reason: str = "manual"):
|
||||
self._stop_global_monitor_for_session(tab_id, reason=reason)
|
||||
def resume_global_network_monitor(self, tab_id: str, reason: str = "manual"):
|
||||
with self._lock:
|
||||
session = self._tabs.get(tab_id)
|
||||
if not session:
|
||||
return
|
||||
if session.status != TabStatus.IDLE or not session.is_healthy():
|
||||
return
|
||||
self._start_global_monitor_for_session(session)
|
||||
logger.debug(f"[GlobalNet] 恢复监听: {tab_id} ({reason})")
|
||||
@@ -0,0 +1,289 @@
|
||||
"""TabPoolManager 的一部分(R2-3 拆分):配置解析与运行时配置(排除 URL、模型名/预设覆盖、动态路由候选)。
|
||||
|
||||
方法原样从 manager.py 迁出,行为不变;通过 mixin 组合回 TabPoolManager,
|
||||
类属性与实例状态都在 TabPoolManager(manager.py)里定义。
|
||||
"""
|
||||
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from app.core.config import logger
|
||||
from app.utils.site_url import normalize_exact_tab_url, normalize_route_domain, tab_url_matches
|
||||
from app.utils.tab_route_groups import normalize_route_groups, route_groups_by_id
|
||||
|
||||
from .session import TabSession
|
||||
|
||||
|
||||
class TabPoolConfigMixin:
|
||||
"""配置解析与运行时配置(排除 URL、模型名/预设覆盖、动态路由候选)"""
|
||||
|
||||
@staticmethod
|
||||
def _to_bool(value: Any, default: bool = False) -> bool:
|
||||
if isinstance(value, bool):
|
||||
return value
|
||||
if value is None:
|
||||
return default
|
||||
return str(value).strip().lower() in {"1", "true", "yes", "y", "on"}
|
||||
@staticmethod
|
||||
def _to_float(value: Any, default: float) -> float:
|
||||
try:
|
||||
return float(value)
|
||||
except (TypeError, ValueError, OverflowError):
|
||||
return default
|
||||
@staticmethod
|
||||
def _normalize_allocation_mode(value: Any) -> str:
|
||||
normalized = str(value or "").strip().lower()
|
||||
if normalized in {"round_robin", "random"}:
|
||||
return normalized
|
||||
return "first_idle"
|
||||
@staticmethod
|
||||
def _normalize_excluded_urls(value: Any) -> List[str]:
|
||||
if not isinstance(value, list):
|
||||
return []
|
||||
|
||||
normalized: List[str] = []
|
||||
seen = set()
|
||||
for item in value:
|
||||
text = str(item or "").strip()
|
||||
normalized_text = normalize_exact_tab_url(text) or text
|
||||
if not normalized_text or normalized_text in seen:
|
||||
continue
|
||||
seen.add(normalized_text)
|
||||
normalized.append(normalized_text)
|
||||
return normalized
|
||||
@staticmethod
|
||||
def _normalize_model_name(value: Any) -> str:
|
||||
return str(value or "").strip()
|
||||
@classmethod
|
||||
def _normalize_model_name_overrides(cls, value: Any) -> Dict[str, Dict[str, str]]:
|
||||
payload = value if isinstance(value, dict) else {}
|
||||
normalized = {"sites": {}, "urls": {}}
|
||||
|
||||
sites = payload.get("sites") if isinstance(payload, dict) else {}
|
||||
if isinstance(sites, dict):
|
||||
for key, model_name in sites.items():
|
||||
route_key = normalize_route_domain(key)
|
||||
display_name = cls._normalize_model_name(model_name)
|
||||
if route_key and display_name:
|
||||
normalized["sites"][route_key] = display_name
|
||||
|
||||
urls = payload.get("urls") if isinstance(payload, dict) else {}
|
||||
if isinstance(urls, dict):
|
||||
for key, model_name in urls.items():
|
||||
url_key = normalize_exact_tab_url(str(key or "").strip())
|
||||
display_name = cls._normalize_model_name(model_name)
|
||||
if url_key and display_name:
|
||||
normalized["urls"][url_key] = display_name
|
||||
|
||||
return normalized
|
||||
@classmethod
|
||||
def _normalize_preset_overrides(cls, value: Any) -> Dict[str, Dict[str, str]]:
|
||||
payload = value if isinstance(value, dict) else {}
|
||||
normalized = {"urls": {}}
|
||||
|
||||
urls = payload.get("urls") if isinstance(payload, dict) else {}
|
||||
if isinstance(urls, dict):
|
||||
for key, preset_name in urls.items():
|
||||
url_key = normalize_exact_tab_url(str(key or "").strip())
|
||||
display_preset = str(preset_name or "").strip()
|
||||
if url_key and display_preset:
|
||||
normalized["urls"][url_key] = display_preset
|
||||
|
||||
return normalized
|
||||
def _apply_preset_overrides(self, session: TabSession, info: Optional[Dict[str, Any]] = None) -> None:
|
||||
"""如果开启了自动按 URL 记忆预设,自动为标签页恢复该 URL 上次保存的预设"""
|
||||
if not getattr(self, "auto_remember_url_presets", False):
|
||||
return
|
||||
|
||||
current_url = str((info.get("url") if info else None) or session.last_known_url or "").strip()
|
||||
if not current_url:
|
||||
try:
|
||||
current_url = str(session.tab.url or "").strip()
|
||||
except Exception:
|
||||
pass
|
||||
normalized_url = normalize_exact_tab_url(current_url)
|
||||
if not normalized_url:
|
||||
return
|
||||
|
||||
overrides = self._normalize_preset_overrides(
|
||||
getattr(self, "preset_overrides", None)
|
||||
)
|
||||
saved_preset = overrides["urls"].get(normalized_url)
|
||||
if saved_preset:
|
||||
applied = False
|
||||
with session._lock:
|
||||
if session.preset_name is None or session.preset_name == saved_preset:
|
||||
session.set_preset(saved_preset, source="url")
|
||||
applied = True
|
||||
if applied and info is not None:
|
||||
info["preset_name"] = saved_preset
|
||||
info["preset_override_source"] = "url"
|
||||
@staticmethod
|
||||
def _default_model_name_for_info(info: Dict[str, Any]) -> str:
|
||||
route_domain = str(info.get("route_domain") or "").strip()
|
||||
current_domain = str(info.get("current_domain") or "").strip()
|
||||
tab_id = str(info.get("id") or "").strip()
|
||||
return route_domain or current_domain or tab_id or "web-browser"
|
||||
def _apply_exposed_model_name(self, info: Dict[str, Any]) -> None:
|
||||
default_name = self._default_model_name_for_info(info)
|
||||
override_source = ""
|
||||
override_name = self._normalize_model_name(info.get("model_name_override"))
|
||||
|
||||
if override_name:
|
||||
override_source = "tab"
|
||||
else:
|
||||
overrides = self._normalize_model_name_overrides(
|
||||
getattr(self, "model_name_overrides", None)
|
||||
)
|
||||
current_url = str(info.get("url") or "").strip()
|
||||
normalized_url = normalize_exact_tab_url(current_url)
|
||||
if normalized_url:
|
||||
override_name = overrides["urls"].get(normalized_url, "")
|
||||
if override_name:
|
||||
override_source = "url"
|
||||
|
||||
if not override_name:
|
||||
route_domain = normalize_route_domain(
|
||||
info.get("route_domain") or info.get("current_domain") or ""
|
||||
)
|
||||
if route_domain:
|
||||
override_name = overrides["sites"].get(route_domain, "")
|
||||
if override_name:
|
||||
override_source = "site"
|
||||
|
||||
info["default_model_name"] = default_name
|
||||
info["exposed_model_name"] = override_name or default_name
|
||||
info["model_name_override_source"] = override_source
|
||||
def _get_session_override_model_name(self, session: TabSession) -> str:
|
||||
"""获取 session 当前被用户显式覆盖的重命名模型名(若未重命名则返回空串)。"""
|
||||
override_name = self._normalize_model_name(getattr(session, "model_name_override", None))
|
||||
if override_name:
|
||||
return override_name
|
||||
try:
|
||||
current_url, route_domain = session.get_cached_route_snapshot()
|
||||
except Exception:
|
||||
current_url, route_domain = "", ""
|
||||
overrides = self._normalize_model_name_overrides(
|
||||
getattr(self, "model_name_overrides", None)
|
||||
)
|
||||
normalized_url = normalize_exact_tab_url(current_url)
|
||||
if normalized_url and normalized_url in overrides.get("urls", {}):
|
||||
return str(overrides["urls"][normalized_url] or "").strip()
|
||||
norm_domain = normalize_route_domain(route_domain)
|
||||
if norm_domain and norm_domain in overrides.get("sites", {}):
|
||||
return str(overrides["sites"][norm_domain] or "").strip()
|
||||
return ""
|
||||
def is_url_excluded(self, url: str) -> bool:
|
||||
current_url = str(url or "").strip()
|
||||
if not current_url:
|
||||
return False
|
||||
|
||||
with self._lock:
|
||||
excluded_urls = list(self.excluded_urls)
|
||||
|
||||
return any(tab_url_matches(excluded_url, current_url) for excluded_url in excluded_urls)
|
||||
def _is_session_excluded_from_dynamic_routing(
|
||||
self,
|
||||
session: TabSession,
|
||||
current_url: Optional[str] = None,
|
||||
) -> bool:
|
||||
if current_url is None:
|
||||
try:
|
||||
current_url, _actual_domain = session.get_cached_route_snapshot()
|
||||
except Exception:
|
||||
current_url = ""
|
||||
return self.is_url_excluded(current_url)
|
||||
def _get_sessions_for_dynamic_routing(self) -> List[TabSession]:
|
||||
return [
|
||||
session
|
||||
for session in self._tabs.values()
|
||||
if not self._is_session_excluded_from_dynamic_routing(session)
|
||||
]
|
||||
def apply_runtime_config(
|
||||
self,
|
||||
*,
|
||||
max_tabs: Optional[int] = None,
|
||||
min_tabs: Optional[int] = None,
|
||||
idle_timeout: Optional[float] = None,
|
||||
acquire_timeout: Optional[float] = None,
|
||||
stuck_timeout: Optional[float] = None,
|
||||
allocation_mode: Optional[str] = None,
|
||||
excluded_urls: Optional[List[str]] = None,
|
||||
preserve_error_tabs: Optional[bool] = None,
|
||||
auto_remember_url_presets: Optional[bool] = None,
|
||||
model_name_overrides: Optional[Dict[str, Any]] = None,
|
||||
preset_overrides: Optional[Dict[str, Any]] = None,
|
||||
route_groups: Optional[List[Dict[str, Any]]] = None,
|
||||
) -> Dict[str, Any]:
|
||||
"""同步更新运行中的标签页池参数。"""
|
||||
with self._lock:
|
||||
new_max_tabs = self.max_tabs if max_tabs is None else max(1, int(max_tabs))
|
||||
new_min_tabs = self.min_tabs if min_tabs is None else max(1, int(min_tabs))
|
||||
if new_min_tabs > new_max_tabs:
|
||||
new_min_tabs = new_max_tabs
|
||||
|
||||
self.max_tabs = new_max_tabs
|
||||
self.min_tabs = new_min_tabs
|
||||
|
||||
if idle_timeout is not None:
|
||||
self.idle_timeout = max(1.0, float(idle_timeout))
|
||||
if acquire_timeout is not None:
|
||||
self.acquire_timeout = max(1.0, float(acquire_timeout))
|
||||
if stuck_timeout is not None:
|
||||
self.stuck_timeout = max(1.0, float(stuck_timeout))
|
||||
if allocation_mode is not None:
|
||||
self.allocation_mode = self._normalize_allocation_mode(allocation_mode)
|
||||
if excluded_urls is not None:
|
||||
self.excluded_urls = self._normalize_excluded_urls(excluded_urls)
|
||||
if preserve_error_tabs is not None:
|
||||
self.preserve_error_tabs = self._to_bool(preserve_error_tabs, False)
|
||||
if auto_remember_url_presets is not None:
|
||||
self.auto_remember_url_presets = self._to_bool(auto_remember_url_presets, False)
|
||||
if model_name_overrides is not None:
|
||||
self.model_name_overrides = self._normalize_model_name_overrides(model_name_overrides)
|
||||
if preset_overrides is not None:
|
||||
self.preset_overrides = self._normalize_preset_overrides(preset_overrides)
|
||||
if route_groups is not None:
|
||||
self.route_groups = normalize_route_groups(route_groups)
|
||||
valid_groups = route_groups_by_id(self.route_groups)
|
||||
self._route_group_bindings = {
|
||||
group_id: bindings
|
||||
for group_id, bindings in self._route_group_bindings.items()
|
||||
if group_id in valid_groups
|
||||
}
|
||||
self._condition.notify_all()
|
||||
|
||||
updated = {
|
||||
"max_tabs": self.max_tabs,
|
||||
"min_tabs": self.min_tabs,
|
||||
"idle_timeout": self.idle_timeout,
|
||||
"acquire_timeout": self.acquire_timeout,
|
||||
"stuck_timeout": self.stuck_timeout,
|
||||
"allocation_mode": self.allocation_mode,
|
||||
"excluded_urls": list(self.excluded_urls),
|
||||
"preserve_error_tabs": self.preserve_error_tabs,
|
||||
"auto_remember_url_presets": self.auto_remember_url_presets,
|
||||
"model_name_overrides": {
|
||||
"sites": dict(getattr(self, "model_name_overrides", {}).get("sites", {})),
|
||||
"urls": dict(getattr(self, "model_name_overrides", {}).get("urls", {})),
|
||||
},
|
||||
"preset_overrides": {
|
||||
"urls": dict(getattr(self, "preset_overrides", {}).get("urls", {})),
|
||||
},
|
||||
"route_groups": normalize_route_groups(self.route_groups),
|
||||
}
|
||||
|
||||
logger.info(
|
||||
"[TabPool] 运行时配置已更新: "
|
||||
f"max_tabs={self.max_tabs}, min_tabs={self.min_tabs}, "
|
||||
f"idle_timeout={self.idle_timeout}, acquire_timeout={self.acquire_timeout}, "
|
||||
f"stuck_timeout={self.stuck_timeout}, allocation_mode={self.allocation_mode}, "
|
||||
f"excluded_urls={len(self.excluded_urls)}, "
|
||||
f"preserve_error_tabs={self.preserve_error_tabs}, "
|
||||
f"auto_remember_url_presets={self.auto_remember_url_presets}, "
|
||||
f"model_name_overrides="
|
||||
f"{len(updated['model_name_overrides']['sites'])}/"
|
||||
f"{len(updated['model_name_overrides']['urls'])}, "
|
||||
f"preset_overrides={len(updated['preset_overrides']['urls'])}"
|
||||
f", route_groups={len(updated['route_groups'])}"
|
||||
)
|
||||
return updated
|
||||
@@ -0,0 +1,574 @@
|
||||
"""TabPoolManager 的一部分(R2-3 拆分):浏览器上下文与 CDP:独立 Cookie 上下文、目标快照、强制重载、上下文恢复。
|
||||
|
||||
方法原样从 manager.py 迁出,行为不变;通过 mixin 组合回 TabPoolManager,
|
||||
类属性与实例状态都在 TabPoolManager(manager.py)里定义。
|
||||
"""
|
||||
|
||||
import json
|
||||
import threading
|
||||
import time
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from app.core.config import logger
|
||||
|
||||
from ._utils import _looks_like_transient_local_debug_error
|
||||
from .session import TabSession, TabStatus
|
||||
from app.core.driver import browser_driver
|
||||
|
||||
|
||||
class TabPoolContextMixin:
|
||||
"""浏览器上下文与 CDP:独立 Cookie 上下文、目标快照、强制重载、上下文恢复"""
|
||||
|
||||
def _get_target_info(self, raw_tab_id: str) -> Dict[str, Any]:
|
||||
if not raw_tab_id:
|
||||
return {}
|
||||
|
||||
try:
|
||||
browser = self._get_browser_handle()
|
||||
if browser is None or not hasattr(browser, "_run_cdp"):
|
||||
return {}
|
||||
|
||||
result = browser_driver(browser).run_cdp("Target.getTargetInfo", targetId=raw_tab_id) or {}
|
||||
info = result.get("targetInfo") or {}
|
||||
return info if isinstance(info, dict) else {}
|
||||
except Exception as e:
|
||||
logger.debug(f"[TabPool] get target info failed ({raw_tab_id}): {e}")
|
||||
return {}
|
||||
def _get_browser_context_id(self, raw_tab_id: str) -> Optional[str]:
|
||||
if raw_tab_id in self._isolated_context_by_raw_id:
|
||||
return self._isolated_context_by_raw_id.get(raw_tab_id)
|
||||
|
||||
info = self._get_target_info(raw_tab_id)
|
||||
browser_context_id = str(info.get("browserContextId") or "").strip()
|
||||
return browser_context_id or None
|
||||
def _snapshot_current_tab_targets(
|
||||
self,
|
||||
) -> tuple[List[Any], List[str], Dict[str, str], Dict[str, str], set[str]]:
|
||||
current_tabs = self._list_current_tab_refs()
|
||||
current_tab_ids = [
|
||||
raw_id for raw_id in
|
||||
(self._get_tab_ref_id(tab_ref) for tab_ref in current_tabs)
|
||||
if raw_id
|
||||
]
|
||||
|
||||
current_context_by_raw: Dict[str, str] = {}
|
||||
current_url_by_raw: Dict[str, str] = {}
|
||||
current_url_known_raw: set[str] = set()
|
||||
for tab_ref in current_tabs:
|
||||
raw_id = self._get_tab_ref_id(tab_ref)
|
||||
if not raw_id:
|
||||
continue
|
||||
if self._tab_ref_has_url_field(tab_ref):
|
||||
current_url_known_raw.add(raw_id)
|
||||
url = self._get_tab_ref_url(tab_ref)
|
||||
if url:
|
||||
current_url_by_raw[raw_id] = url
|
||||
|
||||
target_info_by_raw: Dict[str, Dict[str, Any]] = {}
|
||||
for tab_ref in current_tabs:
|
||||
raw_id = self._get_tab_ref_id(tab_ref)
|
||||
if not raw_id:
|
||||
continue
|
||||
if isinstance(tab_ref, tuple) and len(tab_ref) >= 2 and isinstance(tab_ref[1], dict):
|
||||
target_info_by_raw[raw_id] = tab_ref[1]
|
||||
elif isinstance(tab_ref, dict):
|
||||
target_info_by_raw[raw_id] = tab_ref
|
||||
|
||||
with self._lock:
|
||||
cached_contexts = dict(self._isolated_context_by_raw_id)
|
||||
for raw_id in current_tab_ids:
|
||||
cached_context = str(cached_contexts.get(raw_id) or "").strip()
|
||||
if cached_context:
|
||||
current_context_by_raw[raw_id] = cached_context
|
||||
continue
|
||||
target_info = target_info_by_raw.get(raw_id) or {}
|
||||
context_id = str(target_info.get("browserContextId") or "").strip()
|
||||
if context_id:
|
||||
current_context_by_raw[raw_id] = context_id
|
||||
if raw_id in current_context_by_raw:
|
||||
continue
|
||||
try:
|
||||
info = self._get_target_info(raw_id)
|
||||
context_id = str(info.get("browserContextId") or "").strip()
|
||||
if context_id:
|
||||
current_context_by_raw[raw_id] = context_id
|
||||
except Exception as e:
|
||||
logger.debug(f"[TabPool] skip context lookup for {raw_id}: {e}")
|
||||
|
||||
return current_tabs, current_tab_ids, current_context_by_raw, current_url_by_raw, current_url_known_raw
|
||||
def _is_site_independent_cookie_enabled(self, domain: str) -> bool:
|
||||
normalized_domain = str(domain or "").strip()
|
||||
if not normalized_domain:
|
||||
return False
|
||||
|
||||
try:
|
||||
from app.services.config_engine import config_engine
|
||||
|
||||
advanced = config_engine.get_site_advanced_config(normalized_domain)
|
||||
return bool(advanced.get("independent_cookies", False))
|
||||
except Exception as e:
|
||||
logger.debug(f"[TabPool] load site advanced config failed ({normalized_domain}): {e}")
|
||||
return False
|
||||
def _is_site_independent_cookie_auto_takeover_enabled(self, domain: str) -> bool:
|
||||
normalized_domain = str(domain or "").strip()
|
||||
if not normalized_domain:
|
||||
return False
|
||||
|
||||
try:
|
||||
from app.services.config_engine import config_engine
|
||||
|
||||
advanced = config_engine.get_site_advanced_config(normalized_domain)
|
||||
return bool(advanced.get("independent_cookies_auto_takeover", False))
|
||||
except Exception as e:
|
||||
logger.debug(f"[TabPool] load site advanced config failed ({normalized_domain}): {e}")
|
||||
return False
|
||||
def _register_isolated_context(self, raw_tab_id: str, browser_context_id: Optional[str]) -> None:
|
||||
context_id = str(browser_context_id or "").strip()
|
||||
if raw_tab_id and context_id:
|
||||
self._isolated_context_by_raw_id[raw_tab_id] = context_id
|
||||
self._orphaned_isolated_contexts.pop(context_id, None)
|
||||
def _mark_orphaned_isolated_context(self, browser_context_id: Optional[str]) -> None:
|
||||
context_id = str(browser_context_id or "").strip()
|
||||
if context_id:
|
||||
self._orphaned_isolated_contexts[context_id] = (
|
||||
time.time() + self.ISOLATED_CONTEXT_ORPHAN_GRACE_SEC
|
||||
)
|
||||
def _cleanup_orphaned_isolated_contexts(
|
||||
self,
|
||||
current_context_ids: Optional[List[str]] = None,
|
||||
) -> None:
|
||||
if not self._orphaned_isolated_contexts:
|
||||
return
|
||||
|
||||
now = time.time()
|
||||
active_contexts = {
|
||||
str(context_id).strip()
|
||||
for context_id in (current_context_ids or [])
|
||||
if str(context_id).strip()
|
||||
}
|
||||
active_contexts.update(
|
||||
str(context_id).strip()
|
||||
for context_id in self._isolated_context_by_raw_id.values()
|
||||
if str(context_id).strip()
|
||||
)
|
||||
|
||||
for context_id, expire_at in list(self._orphaned_isolated_contexts.items()):
|
||||
if context_id in active_contexts:
|
||||
self._orphaned_isolated_contexts.pop(context_id, None)
|
||||
continue
|
||||
if now < expire_at:
|
||||
continue
|
||||
self._orphaned_isolated_contexts.pop(context_id, None)
|
||||
self._dispose_browser_context_async(context_id)
|
||||
logger.info(
|
||||
f"[TabPool] disposed orphaned isolated context after grace: {context_id}"
|
||||
)
|
||||
def _dispose_browser_context(self, browser_context_id: Optional[str]) -> None:
|
||||
context_id = str(browser_context_id or "").strip()
|
||||
if not context_id:
|
||||
return
|
||||
|
||||
try:
|
||||
browser = self._get_browser_handle()
|
||||
if browser is None or not hasattr(browser, "_run_cdp"):
|
||||
return
|
||||
browser_driver(browser).run_cdp("Target.disposeBrowserContext", browserContextId=context_id)
|
||||
except Exception as e:
|
||||
logger.debug(f"[TabPool] dispose browser context failed ({context_id}): {e}")
|
||||
def _dispose_browser_context_async(self, browser_context_id: Optional[str]) -> None:
|
||||
context_id = str(browser_context_id or "").strip()
|
||||
if not context_id:
|
||||
return
|
||||
|
||||
executor = self._maintenance_executor
|
||||
if executor is None:
|
||||
return
|
||||
|
||||
try:
|
||||
executor.submit(self._dispose_browser_context, context_id)
|
||||
except RuntimeError as e:
|
||||
logger.debug(f"[TabPool] maintenance submit failed (dispose:{context_id}): {e}")
|
||||
def _arm_isolated_rebind_grace(self, session: TabSession, reason: str, detail: str) -> bool:
|
||||
now = time.time()
|
||||
same_reason = session.transient_disconnect_reason == reason
|
||||
if same_reason and session.transient_disconnect_until > now:
|
||||
return True
|
||||
if same_reason and session.transient_disconnect_until > 0:
|
||||
logger.warning(
|
||||
f"[{session.id}] isolated session grace expired ({detail}), removing from pool"
|
||||
)
|
||||
session.clear_transient_disconnect()
|
||||
return False
|
||||
session.mark_transient_disconnect(self.ISOLATED_CONTEXT_REBIND_GRACE_SEC, reason=reason)
|
||||
logger.warning(
|
||||
f"[{session.id}] isolated session entered {self.ISOLATED_CONTEXT_REBIND_GRACE_SEC:.0f}s "
|
||||
f"rebind grace ({detail})"
|
||||
)
|
||||
return True
|
||||
def _refresh_isolated_session_binding(self, session: TabSession, raw_tab_id: str) -> bool:
|
||||
if not session.is_isolated_context or not raw_tab_id:
|
||||
return False
|
||||
|
||||
replacement_tab = self._resolve_tab_from_ref(raw_tab_id)
|
||||
if not replacement_tab:
|
||||
return False
|
||||
|
||||
self._detach_global_monitor_for_session(session.id, reason="target_rebind")
|
||||
session.tab = replacement_tab
|
||||
session.clear_transient_disconnect()
|
||||
try:
|
||||
_cached_url, cached_domain = session.get_cached_route_snapshot()
|
||||
if cached_domain:
|
||||
session.current_domain = cached_domain
|
||||
except Exception:
|
||||
pass
|
||||
if session.status == TabStatus.IDLE:
|
||||
self._restart_global_monitor_for_session_async(session.id, "target_rebind")
|
||||
return True
|
||||
def _get_browser_handle(self):
|
||||
browser = getattr(self.page, "browser", None)
|
||||
if browser is not None:
|
||||
return browser
|
||||
return self.page
|
||||
@staticmethod
|
||||
def _run_cdp_compat(target: Any, method: str, *, timeout: float = 2.0, **params):
|
||||
"""Call DrissionPage CDP APIs across minor 4.x signature differences."""
|
||||
runner = getattr(target, "run_cdp", None)
|
||||
if not callable(runner):
|
||||
runner = getattr(target, "_run_cdp", None)
|
||||
if not callable(runner):
|
||||
raise AttributeError(f"CDP runner unavailable for {method}")
|
||||
|
||||
timeout_value = max(0.1, float(timeout))
|
||||
try:
|
||||
return runner(method, _timeout=timeout_value, **params)
|
||||
except TypeError as exc:
|
||||
# A few older wrappers do not expose DrissionPage's private timeout
|
||||
# keyword. Retry only when the exception clearly concerns that kwarg.
|
||||
error_text = str(exc).lower()
|
||||
if "timeout" not in error_text:
|
||||
raise
|
||||
return runner(method, **params)
|
||||
@staticmethod
|
||||
def _is_execution_context_error(error: Any) -> bool:
|
||||
text = str(error or "").strip().lower()
|
||||
if not text:
|
||||
return False
|
||||
return any(
|
||||
marker in text
|
||||
for marker in (
|
||||
"cannot find default execution context",
|
||||
"execution context was destroyed",
|
||||
"execution context is not available",
|
||||
"context was destroyed",
|
||||
)
|
||||
)
|
||||
@staticmethod
|
||||
def _is_unsupported_cdp_method(error: Any) -> bool:
|
||||
text = str(error or "").strip().lower()
|
||||
return any(
|
||||
marker in text
|
||||
for marker in (
|
||||
"unknown method",
|
||||
"method not found",
|
||||
"not supported",
|
||||
"unsupported command",
|
||||
)
|
||||
)
|
||||
def _find_idle_tab_by_raw_id(self, raw_tab_id: str) -> Optional[Any]:
|
||||
"""Find an existing idle wrapper before opening another CDP session."""
|
||||
raw_id = str(raw_tab_id or "").strip()
|
||||
if not raw_id:
|
||||
return None
|
||||
|
||||
with self._lock:
|
||||
sessions = list(getattr(self, "_tabs", {}).values())
|
||||
for session in sessions:
|
||||
try:
|
||||
with session._lock:
|
||||
if session.status != TabStatus.IDLE or session._termination_in_progress:
|
||||
continue
|
||||
tab = getattr(session, "tab", None)
|
||||
if self._get_tab_ref_id(tab) == raw_id:
|
||||
return tab
|
||||
except Exception:
|
||||
continue
|
||||
return None
|
||||
def _force_reload_flat_via_cdp(self, browser: Any, target_id: str) -> bool:
|
||||
"""Fallback for protocol versions where nested Target messaging is unavailable."""
|
||||
driver = getattr(browser, "_driver", None)
|
||||
send = getattr(driver, "_send", None)
|
||||
if not callable(send):
|
||||
return False
|
||||
|
||||
session_id = None
|
||||
try:
|
||||
result = self._run_cdp_compat(
|
||||
browser,
|
||||
"Target.attachToTarget",
|
||||
timeout=self.CONTEXT_RECOVERY_CDP_TIMEOUT_SEC,
|
||||
targetId=target_id,
|
||||
flatten=True,
|
||||
) or {}
|
||||
session_id = result.get("sessionId") if isinstance(result, dict) else None
|
||||
if not session_id:
|
||||
return False
|
||||
|
||||
message = {
|
||||
"method": "Page.reload",
|
||||
"params": {},
|
||||
"sessionId": session_id,
|
||||
}
|
||||
try:
|
||||
response = send(message, timeout=self.CONTEXT_RECOVERY_CDP_TIMEOUT_SEC)
|
||||
except TypeError as exc:
|
||||
if "timeout" not in str(exc).lower():
|
||||
raise
|
||||
response = send(message)
|
||||
return isinstance(response, dict) and "error" not in response
|
||||
except Exception as exc:
|
||||
logger.debug(f"[TabPool] flat CDP 刷新唤醒失败 (target={target_id}): {exc}")
|
||||
return False
|
||||
finally:
|
||||
if session_id:
|
||||
try:
|
||||
self._run_cdp_compat(
|
||||
browser,
|
||||
"Target.detachFromTarget",
|
||||
timeout=self.CONTEXT_RECOVERY_CDP_TIMEOUT_SEC,
|
||||
sessionId=session_id,
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.debug(
|
||||
f"[TabPool] flat CDP session detach 失败 "
|
||||
f"(target={target_id}, session={session_id}): {exc}"
|
||||
)
|
||||
def _force_reload_tab_via_cdp(self, target_id: str) -> bool:
|
||||
"""Best-effort reload that works with old nested and newer flat CDP sessions."""
|
||||
raw_id = str(target_id or "").strip()
|
||||
if not raw_id or self._shutdown:
|
||||
return False
|
||||
|
||||
existing_tab = self._find_idle_tab_by_raw_id(raw_id)
|
||||
if existing_tab is not None:
|
||||
try:
|
||||
reload_result = self._run_cdp_compat(
|
||||
existing_tab,
|
||||
"Page.reload",
|
||||
timeout=self.CONTEXT_RECOVERY_CDP_TIMEOUT_SEC,
|
||||
)
|
||||
if isinstance(reload_result, dict) and reload_result.get("error"):
|
||||
raise RuntimeError(str(reload_result.get("error")))
|
||||
return True
|
||||
except Exception as exc:
|
||||
logger.debug(
|
||||
f"[TabPool] 复用现有 tab wrapper 刷新失败,改用 browser CDP "
|
||||
f"(target={raw_id}): {exc}"
|
||||
)
|
||||
|
||||
try:
|
||||
browser = self._get_browser_handle()
|
||||
if browser is None:
|
||||
return False
|
||||
run_cdp = getattr(browser, "_run_cdp", None)
|
||||
if not callable(run_cdp):
|
||||
# Some older/public wrappers expose only run_cdp().
|
||||
run_cdp = getattr(browser, "run_cdp", None)
|
||||
if not callable(run_cdp):
|
||||
return False
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
session_id = None
|
||||
use_flat_fallback = False
|
||||
try:
|
||||
# Omitting flatten keeps the legacy nested-session path compatible
|
||||
# with older Chromium versions. Current Chrome accepts this too.
|
||||
result = self._run_cdp_compat(
|
||||
browser,
|
||||
"Target.attachToTarget",
|
||||
timeout=self.CONTEXT_RECOVERY_CDP_TIMEOUT_SEC,
|
||||
targetId=raw_id,
|
||||
) or {}
|
||||
session_id = result.get("sessionId") if isinstance(result, dict) else None
|
||||
if not session_id:
|
||||
return False
|
||||
|
||||
message = json.dumps(
|
||||
{"id": 1001, "method": "Page.reload", "params": {}},
|
||||
separators=(",", ":"),
|
||||
)
|
||||
try:
|
||||
nested_result = self._run_cdp_compat(
|
||||
browser,
|
||||
"Target.sendMessageToTarget",
|
||||
timeout=self.CONTEXT_RECOVERY_CDP_TIMEOUT_SEC,
|
||||
message=message,
|
||||
sessionId=session_id,
|
||||
)
|
||||
if isinstance(nested_result, dict) and nested_result.get("error"):
|
||||
raise RuntimeError(str(nested_result.get("error")))
|
||||
return True
|
||||
except Exception as exc:
|
||||
use_flat_fallback = self._is_unsupported_cdp_method(exc)
|
||||
if not use_flat_fallback:
|
||||
logger.debug(
|
||||
f"[TabPool] 嵌套 CDP 刷新唤醒失败 "
|
||||
f"(target={raw_id}): {exc}"
|
||||
)
|
||||
return False
|
||||
except Exception as exc:
|
||||
logger.debug(f"[TabPool] CDP attach 刷新唤醒失败 (target={raw_id}): {exc}")
|
||||
return False
|
||||
finally:
|
||||
if session_id:
|
||||
try:
|
||||
self._run_cdp_compat(
|
||||
browser,
|
||||
"Target.detachFromTarget",
|
||||
timeout=self.CONTEXT_RECOVERY_CDP_TIMEOUT_SEC,
|
||||
sessionId=session_id,
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.debug(
|
||||
f"[TabPool] CDP session detach 失败 "
|
||||
f"(target={raw_id}, session={session_id}): {exc}"
|
||||
)
|
||||
|
||||
return use_flat_fallback and self._force_reload_flat_via_cdp(browser, raw_id)
|
||||
def _run_context_recovery(self, raw_tab_id: str) -> None:
|
||||
try:
|
||||
self._force_reload_tab_via_cdp(raw_tab_id)
|
||||
except Exception as exc:
|
||||
logger.debug(f"[TabPool] 上下文恢复任务失败 (target={raw_tab_id}): {exc}")
|
||||
finally:
|
||||
recovery_lock = getattr(self, "_context_recovery_lock", None)
|
||||
if recovery_lock is not None:
|
||||
with recovery_lock:
|
||||
getattr(self, "_context_recovery_pending", set()).discard(raw_tab_id)
|
||||
def _schedule_context_recovery(self, raw_tab_id: str) -> bool:
|
||||
"""Schedule one reload per target cooldown without holding the pool lock."""
|
||||
raw_id = str(raw_tab_id or "").strip()
|
||||
if not raw_id or self._shutdown:
|
||||
return False
|
||||
|
||||
recovery_lock = getattr(self, "_context_recovery_lock", None)
|
||||
if recovery_lock is None:
|
||||
recovery_lock = threading.Lock()
|
||||
self._context_recovery_lock = recovery_lock
|
||||
pending = getattr(self, "_context_recovery_pending", None)
|
||||
requested_at = getattr(self, "_context_recovery_requested_at", None)
|
||||
if pending is None:
|
||||
pending = set()
|
||||
self._context_recovery_pending = pending
|
||||
if requested_at is None:
|
||||
requested_at = {}
|
||||
self._context_recovery_requested_at = requested_at
|
||||
|
||||
now = time.monotonic()
|
||||
with recovery_lock:
|
||||
last_requested = float(requested_at.get(raw_id, 0.0) or 0.0)
|
||||
if raw_id in pending or (
|
||||
last_requested
|
||||
and now - last_requested < self.CONTEXT_RECOVERY_COOLDOWN_SEC
|
||||
):
|
||||
return False
|
||||
|
||||
requested_at[raw_id] = now
|
||||
pending.add(raw_id)
|
||||
if len(requested_at) > self.CONTEXT_RECOVERY_CACHE_LIMIT:
|
||||
oldest_id = min(requested_at, key=requested_at.get)
|
||||
requested_at.pop(oldest_id, None)
|
||||
|
||||
executor = getattr(self, "_maintenance_executor", None)
|
||||
if executor is None:
|
||||
with recovery_lock:
|
||||
pending.discard(raw_id)
|
||||
return False
|
||||
try:
|
||||
executor.submit(self._run_context_recovery, raw_id)
|
||||
return True
|
||||
except RuntimeError as exc:
|
||||
with recovery_lock:
|
||||
pending.discard(raw_id)
|
||||
logger.debug(f"[TabPool] 上下文恢复任务提交失败 (target={raw_id}): {exc}")
|
||||
return False
|
||||
def _list_current_tab_ids_via_cdp(self) -> List[str]:
|
||||
return [
|
||||
item[0]
|
||||
for item in (self._list_current_tab_targets_via_cdp() or [])
|
||||
if item and item[0]
|
||||
]
|
||||
def _list_current_tab_targets_via_cdp(self) -> Optional[List[tuple[str, Dict[str, Any]]]]:
|
||||
browser = self._get_browser_handle()
|
||||
if browser is None or not hasattr(browser, "_run_cdp"):
|
||||
return None
|
||||
|
||||
try:
|
||||
result = browser_driver(browser).run_cdp("Target.getTargets")
|
||||
if not isinstance(result, dict) or not isinstance(result.get("targetInfos"), list):
|
||||
logger.debug("[TabPool] Target.getTargets returned an incomplete snapshot")
|
||||
return None
|
||||
target_infos = result["targetInfos"]
|
||||
except Exception as e:
|
||||
logger.debug(f"[TabPool] Target.getTargets 失败: {e}")
|
||||
return None
|
||||
|
||||
targets: List[tuple[str, Dict[str, Any]]] = []
|
||||
for info in target_infos:
|
||||
if not isinstance(info, dict):
|
||||
continue
|
||||
if str(info.get("type") or "").strip().lower() != "page":
|
||||
continue
|
||||
target_id = str(info.get("targetId") or "").strip()
|
||||
if target_id:
|
||||
targets.append((target_id, info))
|
||||
return targets
|
||||
def _log_get_tabs_warning(self, message: str) -> None:
|
||||
now = time.time()
|
||||
if now - self._last_get_tabs_warning_at < self.GET_TABS_WARNING_INTERVAL_SEC:
|
||||
return
|
||||
self._last_get_tabs_warning_at = now
|
||||
logger.warning(message)
|
||||
def _list_current_tab_refs(self) -> List[Any]:
|
||||
browser = self._get_browser_handle()
|
||||
if browser is None:
|
||||
raise RuntimeError("browser_handle_unavailable")
|
||||
|
||||
cdp_targets = self._list_current_tab_targets_via_cdp()
|
||||
if cdp_targets is not None:
|
||||
self._get_tabs_retry_after = 0.0
|
||||
return cdp_targets
|
||||
|
||||
now = time.time()
|
||||
if now < self._get_tabs_retry_after:
|
||||
raise RuntimeError("tab_snapshot_temporarily_unavailable")
|
||||
|
||||
try:
|
||||
tabs = browser.get_tabs()
|
||||
if tabs is None:
|
||||
raise RuntimeError("browser.get_tabs returned no snapshot")
|
||||
self._get_tabs_retry_after = 0.0
|
||||
return list(tabs)
|
||||
except Exception as e:
|
||||
self._get_tabs_retry_after = max(
|
||||
self._get_tabs_retry_after,
|
||||
time.time() + self.GET_TABS_FAILURE_COOLDOWN_SEC,
|
||||
)
|
||||
cdp_targets = self._list_current_tab_targets_via_cdp()
|
||||
if cdp_targets is not None:
|
||||
message = f"[TabPool] browser.get_tabs() 失败,回退到 Target.getTargets 扫描: {e}"
|
||||
if _looks_like_transient_local_debug_error(e):
|
||||
logger.debug(message)
|
||||
else:
|
||||
self._log_get_tabs_warning(message)
|
||||
return cdp_targets
|
||||
fallback_ids = list(getattr(browser, "tab_ids", []) or [])
|
||||
if fallback_ids:
|
||||
message = f"[TabPool] browser.get_tabs() 失败,回退到 tab_ids 扫描: {e}"
|
||||
if _looks_like_transient_local_debug_error(e):
|
||||
logger.debug(message)
|
||||
else:
|
||||
self._log_get_tabs_warning(message)
|
||||
return fallback_ids
|
||||
raise
|
||||
@@ -0,0 +1,381 @@
|
||||
"""TabPoolManager 的一部分(R2-3 拆分):看门狗、卡死检测、空闲维护与内存清理、不健康标签页清理。
|
||||
|
||||
方法原样从 manager.py 迁出,行为不变;通过 mixin 组合回 TabPoolManager,
|
||||
类属性与实例状态都在 TabPoolManager(manager.py)里定义。
|
||||
"""
|
||||
|
||||
import time
|
||||
from typing import Dict, List, Optional
|
||||
|
||||
from app.core.config import logger
|
||||
|
||||
from .idle_maintenance import schedule_idle_maintenance
|
||||
from .recovery import TabQuarantineEntry
|
||||
from .session import TabSession, TabStatus
|
||||
|
||||
|
||||
class TabPoolMaintenanceMixin:
|
||||
"""看门狗、卡死检测、空闲维护与内存清理、不健康标签页清理"""
|
||||
|
||||
def _check_stuck_tabs(self):
|
||||
"""检查并释放卡死的标签页"""
|
||||
now = time.time()
|
||||
released_any = False
|
||||
|
||||
for session in self._tabs.values():
|
||||
if session.status == TabStatus.BUSY:
|
||||
try:
|
||||
timeout_info = session.get_busy_timeout_info(now=now)
|
||||
except Exception:
|
||||
timeout_info = {
|
||||
"basis": "session",
|
||||
"duration": now - session.last_used_at,
|
||||
"command_loop": {},
|
||||
}
|
||||
busy_duration = float(timeout_info.get("duration") or 0.0)
|
||||
timeout_basis = str(timeout_info.get("basis") or "session")
|
||||
command_loop = timeout_info.get("command_loop") if isinstance(timeout_info, dict) else {}
|
||||
|
||||
if busy_duration > self.stuck_timeout:
|
||||
task_id = session.current_task_id or ""
|
||||
detail_parts = [
|
||||
f"busy_duration={busy_duration:.0f}s",
|
||||
f"basis={timeout_basis}",
|
||||
]
|
||||
if isinstance(command_loop, dict) and command_loop.get("active"):
|
||||
loop_iteration = command_loop.get("iteration")
|
||||
loop_total = command_loop.get("total")
|
||||
loop_label = str(command_loop.get("label") or "").strip()
|
||||
loop_text = f"loop={loop_iteration or '-'}"
|
||||
if loop_total:
|
||||
loop_text = f"{loop_text}/{loop_total}"
|
||||
if loop_label:
|
||||
loop_text = f"{loop_label}:{loop_text}"
|
||||
detail_parts.append(loop_text)
|
||||
cancel_submitted = self._cancel_active_request_for_session(
|
||||
session,
|
||||
"stuck_timeout",
|
||||
detail=", ".join(detail_parts),
|
||||
)
|
||||
snapshot = self._describe_session(session)
|
||||
action_label = "record stuck session" if self.preserve_error_tabs else "retire session"
|
||||
logger.warning(
|
||||
f"[{session.id}] stuck for {busy_duration:.0f}s "
|
||||
f"(basis={timeout_basis}), {action_label} "
|
||||
f"(task={task_id or '-'}, cancel_submitted={cancel_submitted}, "
|
||||
f"snapshot={snapshot})"
|
||||
)
|
||||
session.mark_error("stuck_timeout")
|
||||
released_any = True
|
||||
|
||||
if released_any:
|
||||
self._condition.notify_all()
|
||||
return released_any
|
||||
def run_watchdog_tick(self) -> bool:
|
||||
"""Run periodic stuck-tab maintenance from an external watchdog thread."""
|
||||
with self._condition:
|
||||
if self._shutdown:
|
||||
return False
|
||||
changed = bool(self._check_stuck_tabs())
|
||||
self._cleanup_unhealthy_tabs()
|
||||
|
||||
self._schedule_idle_memory_purge()
|
||||
self._schedule_idle_maintenance()
|
||||
return changed
|
||||
def _schedule_idle_maintenance(self) -> int:
|
||||
"""P0-3 / P0-6:空闲标签页 CDP 会话回收与冻结(在 maintenance 线程池执行)。"""
|
||||
try:
|
||||
return schedule_idle_maintenance(self, getattr(self, "_idle_maintenance_config", None))
|
||||
except Exception as e:
|
||||
logger.debug(f"[TabPool] idle maintenance scheduling failed: {e}")
|
||||
return 0
|
||||
def _schedule_idle_memory_purge(self) -> bool:
|
||||
"""Request best-effort JS GC for long-idle tabs without holding pool locks."""
|
||||
if not self._idle_memory_purge_enabled:
|
||||
return False
|
||||
|
||||
now_wall = time.time()
|
||||
now_mono = time.monotonic()
|
||||
with self._lock:
|
||||
if self._shutdown:
|
||||
return False
|
||||
sessions = list(self._tabs.values())
|
||||
|
||||
candidates: List[TabSession] = []
|
||||
for session in sessions:
|
||||
try:
|
||||
with session._lock:
|
||||
if session.status != TabStatus.IDLE or session._termination_in_progress:
|
||||
continue
|
||||
idle_for = now_wall - float(session.last_used_at or now_wall)
|
||||
last_requested = float(
|
||||
getattr(session, "_last_memory_purge_requested_at", 0.0) or 0.0
|
||||
)
|
||||
if idle_for < self._idle_memory_purge_after_sec:
|
||||
continue
|
||||
if last_requested and (now_mono - last_requested) < self._idle_memory_purge_interval_sec:
|
||||
continue
|
||||
setattr(session, "_last_memory_purge_requested_at", now_mono)
|
||||
candidates.append(session)
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
if not candidates:
|
||||
return False
|
||||
|
||||
executor = self._maintenance_executor
|
||||
if executor is None:
|
||||
return False
|
||||
|
||||
scheduled = False
|
||||
for session in candidates:
|
||||
try:
|
||||
executor.submit(self._purge_idle_session_memory, session)
|
||||
scheduled = True
|
||||
except RuntimeError:
|
||||
break
|
||||
return scheduled
|
||||
def _purge_idle_session_memory(self, session: TabSession) -> None:
|
||||
"""Force renderer GC for one idle session; never changes page lifecycle."""
|
||||
if self._shutdown:
|
||||
return
|
||||
heap_profiler_enabled = False
|
||||
try:
|
||||
with session._lock:
|
||||
if session.status != TabStatus.IDLE or session._termination_in_progress:
|
||||
return
|
||||
|
||||
try:
|
||||
self._run_cdp_compat(
|
||||
session.tab,
|
||||
"HeapProfiler.enable",
|
||||
timeout=1.0,
|
||||
)
|
||||
heap_profiler_enabled = True
|
||||
except Exception:
|
||||
# collectGarbage is available without enabling the domain on
|
||||
# some older Chromium/DrissionPage combinations.
|
||||
pass
|
||||
|
||||
self._run_cdp_compat(
|
||||
session.tab,
|
||||
"HeapProfiler.collectGarbage",
|
||||
timeout=1.0,
|
||||
)
|
||||
logger.debug_throttled(
|
||||
f"tab_pool.memory_purge.{session.id}",
|
||||
f"[{session.id}] 空闲标签页已执行 JavaScript 内存回收",
|
||||
interval_sec=self._idle_memory_purge_interval_sec,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.debug_throttled(
|
||||
f"tab_pool.memory_purge_error.{session.id}",
|
||||
f"[{session.id}] 空闲标签页内存回收失败(忽略): {e}",
|
||||
interval_sec=self._idle_memory_purge_interval_sec,
|
||||
)
|
||||
finally:
|
||||
if heap_profiler_enabled:
|
||||
try:
|
||||
self._run_cdp_compat(
|
||||
session.tab,
|
||||
"HeapProfiler.disable",
|
||||
timeout=1.0,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.debug_throttled(
|
||||
f"tab_pool.memory_purge_disable_error.{session.id}",
|
||||
f"[{session.id}] HeapProfiler 关闭失败(忽略): {e}",
|
||||
interval_sec=self._idle_memory_purge_interval_sec,
|
||||
)
|
||||
def _cancel_active_request_for_session(
|
||||
self,
|
||||
session: Optional[TabSession],
|
||||
reason: str,
|
||||
*,
|
||||
detail: str = "",
|
||||
) -> bool:
|
||||
"""Request the bound workflow to stop when a session becomes unusable."""
|
||||
if session is None:
|
||||
return False
|
||||
|
||||
request_ids = []
|
||||
for attr_name in ("current_task_id", "_command_request_id", "_bound_request_id", "_command_loop_request_id"):
|
||||
try:
|
||||
request_id = str(getattr(session, attr_name, "") or "").strip()
|
||||
except Exception:
|
||||
request_id = ""
|
||||
if request_id and request_id not in request_ids:
|
||||
request_ids.append(request_id)
|
||||
if not request_ids:
|
||||
return False
|
||||
|
||||
task_id = request_ids[0]
|
||||
cancel_key = "|".join(request_ids)
|
||||
if getattr(session, "_last_cancel_request_task_id", None) == cancel_key:
|
||||
logger.debug(
|
||||
f"[{session.id}] duplicate cancel skipped "
|
||||
f"(task={cancel_key}, reason={reason}, "
|
||||
f"previous_reason={getattr(session, '_last_cancel_request_reason', '-')}, "
|
||||
f"detail={detail or '-'})"
|
||||
)
|
||||
return False
|
||||
|
||||
try:
|
||||
setattr(session, "_workflow_stop_reason", reason)
|
||||
setattr(session, "_last_cancel_request_task_id", cancel_key)
|
||||
setattr(session, "_last_cancel_request_reason", reason)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
cancel_submitted = False
|
||||
for request_id in request_ids:
|
||||
cancel_submitted = bool(self._submit_request_cancel(
|
||||
request_id,
|
||||
reason,
|
||||
session_id=session.id,
|
||||
detail=detail,
|
||||
)) or cancel_submitted
|
||||
|
||||
logger.warning(
|
||||
f"[{session.id}] 会话失效,已请求取消任务 "
|
||||
f"(task={task_id}, requests={','.join(request_ids)}, reason={reason}, "
|
||||
f"cancel_submitted={cancel_submitted}, detail={detail or '-'})"
|
||||
)
|
||||
return cancel_submitted
|
||||
def _cleanup_unhealthy_tabs(self):
|
||||
"""清理不健康的空闲标签页、关闭状态和错误状态的标签页"""
|
||||
to_remove: Dict[str, str] = {}
|
||||
|
||||
for tab_id, session in list(self._tabs.items()):
|
||||
try:
|
||||
if session.is_isolated_context and session.is_healthy(allow_live_check=False):
|
||||
session.clear_transient_disconnect()
|
||||
if session.status != TabStatus.ERROR:
|
||||
self._preserved_error_session_ids.discard(tab_id)
|
||||
|
||||
if session.status == TabStatus.CLOSED:
|
||||
to_remove[tab_id] = "closed"
|
||||
# 清理 ERROR 状态的标签页(包括强制释放失败的)
|
||||
elif session.status == TabStatus.ERROR:
|
||||
if self.preserve_error_tabs:
|
||||
if tab_id not in self._preserved_error_session_ids:
|
||||
logger.warning(
|
||||
f"[{tab_id}] 错误状态已记录,按配置保留标签页 "
|
||||
"(preserve_error_tabs=True)"
|
||||
)
|
||||
self._preserved_error_session_ids.add(tab_id)
|
||||
continue
|
||||
to_remove[tab_id] = "error"
|
||||
# 清理空闲但不健康的标签页
|
||||
elif session.status == TabStatus.IDLE and not session.is_healthy(allow_live_check=False):
|
||||
if session.is_isolated_context:
|
||||
if session.is_in_transient_disconnect():
|
||||
continue
|
||||
if self._arm_isolated_rebind_grace(
|
||||
session,
|
||||
reason="health_probe_failed",
|
||||
detail="health probe failed",
|
||||
):
|
||||
continue
|
||||
to_remove[tab_id] = "unhealthy"
|
||||
except Exception as e:
|
||||
logger.warning(f"[TabPool] cleanup check failed for tab {tab_id}: {e}")
|
||||
to_remove[tab_id] = "health_check_failed"
|
||||
|
||||
newly_quarantined: List[TabQuarantineEntry] = []
|
||||
for tab_id, removal_reason in list(to_remove.items()):
|
||||
try:
|
||||
session = self._tabs.get(tab_id)
|
||||
if not session:
|
||||
continue
|
||||
# mark_closed 会清空 current_task_id,先捕获用于隔离条目
|
||||
quarantine_task_id = str(
|
||||
getattr(session, "current_task_id", "") or ""
|
||||
).strip()
|
||||
self._cancel_active_request_for_session(
|
||||
session,
|
||||
removal_reason,
|
||||
detail=(
|
||||
f"status={getattr(getattr(session, 'status', None), 'value', 'unknown')}, "
|
||||
f"reason={removal_reason}"
|
||||
),
|
||||
)
|
||||
logger.warning(
|
||||
f"[{tab_id}] 不健康或错误状态,从池中移除但保留浏览器标签页 "
|
||||
f"({removal_reason})"
|
||||
)
|
||||
session.mark_closed(removal_reason)
|
||||
self._detach_global_monitor_for_session(tab_id, reason=removal_reason)
|
||||
|
||||
# 清理映射表,允许相同 raw_tab_id 被重新扫描
|
||||
raw_ids_to_remove = [
|
||||
raw_id for raw_id, p_idx in list(self._raw_id_to_persistent.items())
|
||||
if self._persistent_to_session_id.get(p_idx) == tab_id
|
||||
]
|
||||
for raw_id in raw_ids_to_remove:
|
||||
self._known_tab_ids.discard(raw_id)
|
||||
self._raw_id_to_persistent.pop(raw_id, None)
|
||||
browser_context_id = self._isolated_context_by_raw_id.pop(raw_id, None)
|
||||
if browser_context_id:
|
||||
self._mark_orphaned_isolated_context(browser_context_id)
|
||||
if removal_reason == "error":
|
||||
# ERROR 会话移出池但浏览器标签页保留:旧 worker 可能仍
|
||||
# 阻塞在该 tab 的同步调用里。先隔离 raw id,阻止下次扫描
|
||||
# 立即重新入池(双工作流串扰的根源);由 TabRecoveryService
|
||||
# 在锁外确认旧 worker 退出后决定解除或永久隔离。
|
||||
entry = TabQuarantineEntry(
|
||||
raw_tab_id=raw_id,
|
||||
persistent_index=int(
|
||||
getattr(session, "persistent_index", 0) or 0
|
||||
),
|
||||
task_id=quarantine_task_id,
|
||||
tab=getattr(session, "tab", None),
|
||||
session_id=tab_id,
|
||||
reason=str(
|
||||
getattr(session, "_workflow_stop_reason", "") or ""
|
||||
).strip() or removal_reason,
|
||||
url=str(getattr(session, "last_known_url", "") or ""),
|
||||
)
|
||||
self._quarantined_raw_ids[raw_id] = entry
|
||||
newly_quarantined.append(entry)
|
||||
logger.info(
|
||||
f"[TabRecovery] 已隔离错误标签页 "
|
||||
f"(raw={raw_id}, tab={tab_id}, "
|
||||
f"task={quarantine_task_id or '-'}, "
|
||||
f"reason={entry.reason})"
|
||||
)
|
||||
|
||||
# 清理持久编号映射
|
||||
p_idx = session.persistent_index
|
||||
if p_idx and self._persistent_to_session_id.get(p_idx) == tab_id:
|
||||
self._persistent_to_session_id.pop(p_idx, None)
|
||||
|
||||
# 清理活动标签页记录
|
||||
if self._active_session_id == tab_id:
|
||||
self._active_session_id = None
|
||||
|
||||
self._tabs.pop(tab_id, None)
|
||||
self._preserved_error_session_ids.discard(tab_id)
|
||||
self._on_session_removed(tab_id)
|
||||
except Exception as e:
|
||||
logger.warning(f"[TabPool] cleanup remove failed for tab {tab_id}: {e}")
|
||||
|
||||
# 投递到 maintenance executor 由恢复服务处理(submit 仅入队,
|
||||
# 持池锁调用不会阻塞,也不触发任何 CDP/网络操作)。
|
||||
for entry in newly_quarantined:
|
||||
self._submit_recovery(entry)
|
||||
|
||||
if to_remove:
|
||||
self._condition.notify_all()
|
||||
def _should_defer_to_command(self, session: TabSession, task_id: str) -> bool:
|
||||
"""Whether request acquisition should defer to high-priority pending/running commands."""
|
||||
task = str(task_id or "").strip().lower()
|
||||
if task.startswith("cmd_") or task.startswith("group_"):
|
||||
return False
|
||||
try:
|
||||
from app.services.command_engine import command_engine
|
||||
if hasattr(command_engine, "should_block_request_for_session"):
|
||||
return bool(command_engine.should_block_request_for_session(session, task_id=task_id))
|
||||
except Exception:
|
||||
return False
|
||||
return False
|
||||
@@ -0,0 +1,603 @@
|
||||
"""TabPoolManager 的一部分(R2-3 拆分):按域名、路由组、精确 URL 借用标签页。
|
||||
|
||||
方法原样从 manager.py 迁出,行为不变;通过 mixin 组合回 TabPoolManager,
|
||||
类属性与实例状态都在 TabPoolManager(manager.py)里定义。
|
||||
"""
|
||||
|
||||
import time
|
||||
from collections import deque
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from app.core.config import logger
|
||||
from app.utils.site_url import (
|
||||
encode_tab_url_route_token,
|
||||
normalize_exact_tab_url,
|
||||
normalize_route_domain,
|
||||
route_domain_matches,
|
||||
tab_url_matches,
|
||||
)
|
||||
from app.utils.tab_route_groups import (
|
||||
normalize_route_group_id,
|
||||
normalize_route_groups,
|
||||
route_group_member_key,
|
||||
route_groups_by_id,
|
||||
)
|
||||
|
||||
from .session import TabSession, TabStatus
|
||||
|
||||
|
||||
class TabPoolRoutingMixin:
|
||||
"""按域名、路由组、精确 URL 借用标签页"""
|
||||
|
||||
def _get_sessions_for_route_domain(self, route_domain: str) -> List[TabSession]:
|
||||
target = normalize_route_domain(route_domain)
|
||||
if not target:
|
||||
return []
|
||||
|
||||
matches: List[TabSession] = []
|
||||
for session in self._tabs.values():
|
||||
current_url, actual_domain = session.get_cached_route_snapshot()
|
||||
if route_domain_matches(target, actual_domain):
|
||||
if self._is_session_excluded_from_dynamic_routing(session, current_url):
|
||||
continue
|
||||
matches.append(session)
|
||||
|
||||
return matches
|
||||
def _get_route_group_config(self, group_id: str) -> Optional[Dict[str, Any]]:
|
||||
normalized_id = normalize_route_group_id(group_id)
|
||||
if not normalized_id:
|
||||
return None
|
||||
return route_groups_by_id(getattr(self, "route_groups", [])).get(normalized_id)
|
||||
@staticmethod
|
||||
def _session_matches_route_group_member(
|
||||
session: TabSession,
|
||||
member: Dict[str, Any],
|
||||
) -> bool:
|
||||
current_url, _actual_domain = session.get_cached_route_snapshot()
|
||||
member_url = str(member.get("url") or "").strip()
|
||||
member_token = str(member.get("url_token") or "").strip().lower()
|
||||
if member_url and tab_url_matches(member_url, current_url):
|
||||
return True
|
||||
if member_token and encode_tab_url_route_token(current_url) == member_token:
|
||||
return True
|
||||
return False
|
||||
def _get_sessions_for_route_group(
|
||||
self,
|
||||
group_id: str,
|
||||
*,
|
||||
bind: bool = True,
|
||||
) -> List[TabSession]:
|
||||
"""解析路由组当前的成员标签页。
|
||||
|
||||
bind=False 供只读查询(/api/tabs 面板轮询、组快照)使用:只做匹配,
|
||||
不写 _route_group_bindings。否则一次与请求无关的面板轮询就会替真实请求
|
||||
提前把成员钉死在某个标签页上(绑定是粘滞的),后续请求再也轮不到同 URL
|
||||
的其他标签页。
|
||||
"""
|
||||
group = self._get_route_group_config(group_id)
|
||||
if not group:
|
||||
return []
|
||||
|
||||
normalized_id = group["id"]
|
||||
members = group.get("members") or []
|
||||
valid_member_keys = {route_group_member_key(member) for member in members}
|
||||
if bind:
|
||||
bindings = self._route_group_bindings.setdefault(normalized_id, {})
|
||||
for member_key in list(bindings):
|
||||
session_id = bindings.get(member_key)
|
||||
if member_key not in valid_member_keys or session_id not in self._tabs:
|
||||
bindings.pop(member_key, None)
|
||||
else:
|
||||
bindings = {
|
||||
member_key: session_id
|
||||
for member_key, session_id in self._route_group_bindings.get(normalized_id, {}).items()
|
||||
if member_key in valid_member_keys and session_id in self._tabs
|
||||
}
|
||||
|
||||
sessions: List[TabSession] = []
|
||||
used_session_ids = set()
|
||||
ordered_sessions = sorted(self._tabs.values(), key=self._session_allocation_key)
|
||||
for member in members:
|
||||
member_key = route_group_member_key(member)
|
||||
bound_session = self._tabs.get(bindings.get(member_key, ""))
|
||||
if bound_session is not None and bound_session.id not in used_session_ids:
|
||||
sessions.append(bound_session)
|
||||
used_session_ids.add(bound_session.id)
|
||||
continue
|
||||
|
||||
matching = [
|
||||
session
|
||||
for session in ordered_sessions
|
||||
if session.id not in used_session_ids
|
||||
and self._session_matches_route_group_member(session, member)
|
||||
]
|
||||
try:
|
||||
preferred_index = int(member.get("tab_index") or 0)
|
||||
except (TypeError, ValueError):
|
||||
preferred_index = 0
|
||||
if preferred_index > 0:
|
||||
matching.sort(
|
||||
key=lambda session: (
|
||||
int(session.persistent_index or 0) != preferred_index,
|
||||
self._session_allocation_key(session),
|
||||
)
|
||||
)
|
||||
if not matching:
|
||||
bindings.pop(member_key, None)
|
||||
continue
|
||||
|
||||
selected = matching[0]
|
||||
bindings[member_key] = selected.id
|
||||
sessions.append(selected)
|
||||
used_session_ids.add(selected.id)
|
||||
|
||||
return sessions
|
||||
def get_route_groups_snapshot(self) -> List[Dict[str, Any]]:
|
||||
with self._lock:
|
||||
groups = normalize_route_groups(getattr(self, "route_groups", []))
|
||||
result = []
|
||||
for group in groups:
|
||||
sessions = self._get_sessions_for_route_group(group["id"], bind=False)
|
||||
item = dict(group)
|
||||
item["live_member_count"] = len(sessions)
|
||||
item["idle_member_count"] = sum(
|
||||
1 for session in sessions if session.status == TabStatus.IDLE
|
||||
)
|
||||
item["busy_member_count"] = sum(
|
||||
1 for session in sessions if session.status == TabStatus.BUSY
|
||||
)
|
||||
item["live_tab_indices"] = [
|
||||
int(session.persistent_index or 0) for session in sessions
|
||||
]
|
||||
result.append(item)
|
||||
return result
|
||||
def _refresh_route_snapshots_for_sessions(self, sessions: List[TabSession]) -> None:
|
||||
"""Refresh live tab URL/domain snapshots; callers must not hold self._condition."""
|
||||
for session in sessions:
|
||||
try:
|
||||
# BUSY 标签页跳过实时读取:
|
||||
# 1) 该 tab 正被别的工作流线程用同步 DrissionPage 调用驱动,
|
||||
# 这里再发一次 CDP 是并发访问同一 target(_scan_new_tabs 同样避开 BUSY);
|
||||
# 2) 工作流中途导航时 tab.url 会瞬时变成 about:blank,
|
||||
# _refresh_current_domain 会据此把 current_domain 清空,
|
||||
# 使并发的 /url/<domain> 请求匹配不到这个标签页而误报 404。
|
||||
# BUSY 标签页的 URL 已由 _scan_new_tabs 从 Target.getTargets 快照刷新。
|
||||
if session.status == TabStatus.BUSY:
|
||||
continue
|
||||
current_url = session._safe_get_url(allow_live_when_busy=True)
|
||||
if current_url:
|
||||
session._refresh_current_domain(current_url)
|
||||
except Exception as e:
|
||||
logger.debug(f"[{session.id}] failed to refresh route snapshot: {e}")
|
||||
def _refresh_route_snapshots_unlocked_once(self) -> None:
|
||||
"""Refresh route snapshots once without holding the pool condition."""
|
||||
sessions_to_refresh = list(self._tabs.values())
|
||||
self._condition.release()
|
||||
try:
|
||||
self._refresh_route_snapshots_for_sessions(sessions_to_refresh)
|
||||
finally:
|
||||
self._condition.acquire()
|
||||
async def acquire_by_route_domain_async(
|
||||
self,
|
||||
route_domain: str,
|
||||
task_id: str,
|
||||
timeout: float = None,
|
||||
allocation_mode: Optional[str] = None,
|
||||
) -> Optional[TabSession]:
|
||||
"""异步版本的按域名路由获取。"""
|
||||
return await self._run_cancellable_acquire(
|
||||
lambda: self.acquire_by_route_domain(
|
||||
route_domain,
|
||||
task_id,
|
||||
timeout,
|
||||
allocation_mode,
|
||||
),
|
||||
task_id,
|
||||
)
|
||||
async def acquire_by_route_group_async(
|
||||
self,
|
||||
group_id: str,
|
||||
task_id: str,
|
||||
timeout: float = None,
|
||||
allocation_mode: Optional[str] = None,
|
||||
requested_model: Optional[str] = None,
|
||||
) -> Optional[TabSession]:
|
||||
return await self._run_cancellable_acquire(
|
||||
lambda: self.acquire_by_route_group(
|
||||
group_id,
|
||||
task_id,
|
||||
timeout,
|
||||
allocation_mode,
|
||||
requested_model=requested_model,
|
||||
),
|
||||
task_id,
|
||||
)
|
||||
def _get_sessions_for_exact_url(self, exact_url: str) -> List[TabSession]:
|
||||
target = str(exact_url or "").strip()
|
||||
if not target:
|
||||
return []
|
||||
|
||||
matches: List[TabSession] = []
|
||||
for session in self._tabs.values():
|
||||
current_url, _actual_domain = session.get_cached_route_snapshot()
|
||||
if tab_url_matches(target, current_url):
|
||||
matches.append(session)
|
||||
|
||||
return matches
|
||||
def acquire_by_exact_url(self, exact_url: str, task_id: str, timeout: float = None) -> Optional[TabSession]:
|
||||
"""Acquire a tab by strict full-URL match, round-robin within identical URLs."""
|
||||
raw_target = str(exact_url or "").strip()
|
||||
if not raw_target:
|
||||
logger.warning("Exact tab URL is empty; cannot acquire a tab")
|
||||
return None
|
||||
|
||||
# 队列 key / 轮询游标 key 必须用归一化后的 URL:匹配侧 tab_url_matches 会剥掉
|
||||
# 默认端口等差异,若这里仍用原始串,"https://x/a" 与 "https://x:443/a"
|
||||
# 会落进两条互不相干的等待队列,各自都是队首,FIFO 互斥形同虚设。
|
||||
target = normalize_exact_tab_url(raw_target) or raw_target
|
||||
waiter_key = f"url::{target}"
|
||||
|
||||
timeout = self._normalize_acquire_timeout(timeout)
|
||||
deadline = time.time() + timeout
|
||||
|
||||
with self._condition:
|
||||
if self._acquire_queue_full(task_id):
|
||||
return None
|
||||
waiters = self._route_waiters.setdefault(waiter_key, deque())
|
||||
waiter_token = self._next_waiter_token(task_id)
|
||||
waiters.append(waiter_token)
|
||||
try:
|
||||
while True:
|
||||
if self._shutdown:
|
||||
return None
|
||||
if self._request_should_stop(task_id):
|
||||
logger.debug(f"Exact URL acquire cancelled (task={task_id})")
|
||||
return None
|
||||
|
||||
if self._should_scan():
|
||||
self._scan_new_tabs()
|
||||
|
||||
self._check_stuck_tabs()
|
||||
self._cleanup_unhealthy_tabs()
|
||||
|
||||
if not self._is_waiter_turn(waiters, waiter_token):
|
||||
remaining = deadline - time.time()
|
||||
if remaining <= 0:
|
||||
logger.warning(
|
||||
f"Timed out waiting for exact URL '{target}' (task={task_id})"
|
||||
)
|
||||
return None
|
||||
logger.debug_throttled(
|
||||
f"tab_pool.wait_exact_url.{target}",
|
||||
f"Waiting for exact URL '{target}' release...",
|
||||
interval_sec=5.0,
|
||||
)
|
||||
self._condition.wait(
|
||||
timeout=min(remaining, self.ACQUIRE_CANCEL_POLL_SEC)
|
||||
)
|
||||
continue
|
||||
|
||||
self._refresh_route_snapshots_unlocked_once()
|
||||
matching_sessions = self._get_sessions_for_exact_url(target)
|
||||
if not matching_sessions:
|
||||
logger.warning(f"No tab matches exact URL '{target}'")
|
||||
return None
|
||||
session = self._try_acquire_session_for_request(
|
||||
matching_sessions,
|
||||
task_id,
|
||||
route_domain=waiter_key,
|
||||
defer_context="exact-url acquire",
|
||||
)
|
||||
if session is not None:
|
||||
self._unregister_waiter(
|
||||
waiters,
|
||||
waiter_token,
|
||||
owner_map=self._route_waiters,
|
||||
owner_key=waiter_key,
|
||||
)
|
||||
if not self._complete_acquired_session_for_return(
|
||||
session,
|
||||
"acquire_by_exact_url",
|
||||
task_id,
|
||||
activate=self._auto_activate_on_acquire and session.id != self._active_session_id,
|
||||
):
|
||||
return None
|
||||
self._mark_allocation_cursor(session, route_domain=waiter_key)
|
||||
|
||||
logger.debug(
|
||||
f"TabPool -> {session.id} "
|
||||
f"(task={task_id}, exact_url={target}, "
|
||||
f"idx=#{session.persistent_index}, snapshot={self._describe_session(session)})"
|
||||
)
|
||||
return session
|
||||
|
||||
remaining = deadline - time.time()
|
||||
if remaining <= 0:
|
||||
logger.warning(
|
||||
f"Timed out waiting for exact URL '{target}' "
|
||||
f"(task={task_id}, matching tabs: "
|
||||
f"{', '.join(f'{s.id}(#{s.persistent_index}:{s.status.value})' for s in matching_sessions) or 'none'})"
|
||||
)
|
||||
return None
|
||||
|
||||
logger.debug_throttled(
|
||||
f"tab_pool.wait_exact_url.{target}",
|
||||
f"Waiting for exact URL '{target}' release...",
|
||||
interval_sec=5.0,
|
||||
)
|
||||
self._condition.wait(
|
||||
timeout=min(remaining, self.ACQUIRE_CANCEL_POLL_SEC)
|
||||
)
|
||||
finally:
|
||||
self._unregister_waiter(
|
||||
waiters,
|
||||
waiter_token,
|
||||
owner_map=self._route_waiters,
|
||||
owner_key=waiter_key,
|
||||
)
|
||||
def acquire_by_route_domain(
|
||||
self,
|
||||
route_domain: str,
|
||||
task_id: str,
|
||||
timeout: float = None,
|
||||
allocation_mode: Optional[str] = None,
|
||||
) -> Optional[TabSession]:
|
||||
"""ASCII-safe fair acquire for tabs matching the same route domain."""
|
||||
target = normalize_route_domain(route_domain)
|
||||
if not target:
|
||||
logger.warning("Route domain is empty; cannot acquire a tab")
|
||||
return None
|
||||
|
||||
timeout = self._normalize_acquire_timeout(timeout)
|
||||
deadline = time.time() + timeout
|
||||
|
||||
with self._condition:
|
||||
if self._acquire_queue_full(task_id):
|
||||
return None
|
||||
waiters = self._route_waiters.setdefault(target, deque())
|
||||
waiter_token = self._next_waiter_token(task_id)
|
||||
waiters.append(waiter_token)
|
||||
try:
|
||||
while True:
|
||||
if self._shutdown:
|
||||
return None
|
||||
if self._request_should_stop(task_id):
|
||||
logger.debug(f"Route domain acquire cancelled (task={task_id})")
|
||||
return None
|
||||
|
||||
if self._should_scan():
|
||||
self._scan_new_tabs()
|
||||
|
||||
self._check_stuck_tabs()
|
||||
self._cleanup_unhealthy_tabs()
|
||||
|
||||
if not self._is_waiter_turn(waiters, waiter_token):
|
||||
remaining = deadline - time.time()
|
||||
if remaining <= 0:
|
||||
logger.warning(
|
||||
f"Timed out waiting for route domain '{target}' (task={task_id})"
|
||||
)
|
||||
return None
|
||||
logger.debug_throttled(
|
||||
f"tab_pool.wait_route.{target}",
|
||||
f"Waiting for route domain '{target}' release...",
|
||||
interval_sec=5.0,
|
||||
)
|
||||
self._condition.wait(
|
||||
timeout=min(remaining, self.ACQUIRE_CANCEL_POLL_SEC)
|
||||
)
|
||||
continue
|
||||
|
||||
self._refresh_route_snapshots_unlocked_once()
|
||||
matching_sessions = self._get_sessions_for_route_domain(target)
|
||||
if not matching_sessions:
|
||||
logger.warning(f"No tab matches route domain '{target}'")
|
||||
return None
|
||||
|
||||
session = self._try_acquire_session_for_request(
|
||||
matching_sessions,
|
||||
task_id,
|
||||
route_domain=target,
|
||||
allocation_mode=allocation_mode,
|
||||
defer_context="route-domain acquire",
|
||||
)
|
||||
if session is not None:
|
||||
self._unregister_waiter(
|
||||
waiters,
|
||||
waiter_token,
|
||||
owner_map=self._route_waiters,
|
||||
owner_key=target,
|
||||
)
|
||||
if not self._complete_acquired_session_for_return(
|
||||
session,
|
||||
"acquire_by_route_domain",
|
||||
task_id,
|
||||
activate=self._auto_activate_on_acquire and session.id != self._active_session_id,
|
||||
):
|
||||
return None
|
||||
self._mark_allocation_cursor(session, route_domain=target)
|
||||
|
||||
logger.debug(
|
||||
f"TabPool -> {session.id} "
|
||||
f"(task={task_id}, route_domain={target}, "
|
||||
f"idx=#{session.persistent_index}, snapshot={self._describe_session(session)})"
|
||||
)
|
||||
return session
|
||||
|
||||
remaining = deadline - time.time()
|
||||
if remaining <= 0:
|
||||
busy_info = [
|
||||
f"{session.id}(#{session.persistent_index}:{session.status.value})"
|
||||
for session in matching_sessions
|
||||
]
|
||||
logger.warning(
|
||||
f"Timed out waiting for route domain '{target}' "
|
||||
f"(task={task_id}, matching tabs: {', '.join(busy_info) or 'none'})"
|
||||
)
|
||||
return None
|
||||
|
||||
logger.debug_throttled(
|
||||
f"tab_pool.wait_route.{target}",
|
||||
f"Waiting for route domain '{target}' release...",
|
||||
interval_sec=5.0,
|
||||
)
|
||||
self._condition.wait(
|
||||
timeout=min(remaining, self.ACQUIRE_CANCEL_POLL_SEC)
|
||||
)
|
||||
finally:
|
||||
self._unregister_waiter(
|
||||
waiters,
|
||||
waiter_token,
|
||||
owner_map=self._route_waiters,
|
||||
owner_key=target,
|
||||
)
|
||||
def acquire_by_route_group(
|
||||
self,
|
||||
group_id: str,
|
||||
task_id: str,
|
||||
timeout: float = None,
|
||||
allocation_mode: Optional[str] = None,
|
||||
requested_model: Optional[str] = None,
|
||||
) -> Optional[TabSession]:
|
||||
target = normalize_route_group_id(group_id)
|
||||
if not target:
|
||||
logger.warning("Route group ID is invalid")
|
||||
return None
|
||||
|
||||
timeout = self._normalize_acquire_timeout(timeout)
|
||||
deadline = time.time() + timeout
|
||||
first_iteration = True
|
||||
|
||||
with self._condition:
|
||||
group = self._get_route_group_config(target)
|
||||
if not group:
|
||||
logger.warning(f"Route group '{target}' does not exist")
|
||||
return None
|
||||
|
||||
if self._acquire_queue_full(task_id):
|
||||
return None
|
||||
waiters = self._group_waiters.setdefault(target, deque())
|
||||
waiter_token = self._next_waiter_token(task_id)
|
||||
waiters.append(waiter_token)
|
||||
try:
|
||||
while True:
|
||||
if self._shutdown:
|
||||
return None
|
||||
if self._request_should_stop(task_id):
|
||||
logger.debug(f"Route group acquire cancelled (task={task_id})")
|
||||
return None
|
||||
|
||||
# 组配置每轮重新读取:apply_runtime_config 可能在等待期间
|
||||
# 删除/改名该组或改掉分配模式。用循环外的旧快照会让等待者既不
|
||||
# 报"组不存在"也匹配不到成员,只能空转到超时。
|
||||
group = self._get_route_group_config(target)
|
||||
if not group:
|
||||
logger.warning(
|
||||
f"Route group '{target}' no longer exists (task={task_id})"
|
||||
)
|
||||
return None
|
||||
|
||||
if first_iteration or self._should_scan():
|
||||
self._scan_new_tabs()
|
||||
first_iteration = False
|
||||
|
||||
self._check_stuck_tabs()
|
||||
self._cleanup_unhealthy_tabs()
|
||||
|
||||
if not self._is_waiter_turn(waiters, waiter_token):
|
||||
remaining = deadline - time.time()
|
||||
if remaining <= 0:
|
||||
logger.warning(
|
||||
f"Timed out waiting in route group '{target}' (task={task_id})"
|
||||
)
|
||||
return None
|
||||
logger.debug_throttled(
|
||||
f"tab_pool.wait_group.{target}",
|
||||
f"Waiting in route group '{target}' queue...",
|
||||
interval_sec=5.0,
|
||||
)
|
||||
self._condition.wait(
|
||||
timeout=min(remaining, self.ACQUIRE_CANCEL_POLL_SEC)
|
||||
)
|
||||
continue
|
||||
|
||||
matching_sessions = self._get_sessions_for_route_group(target)
|
||||
# 空组没有任何可能在后续释放的成员。立即返回,让上游的
|
||||
# 多 URL 轮询跳到下一个 URL,而不是按默认 60 秒超时等待。
|
||||
if not matching_sessions:
|
||||
logger.info(
|
||||
f"Route group '{target}' has no live members; skip immediately "
|
||||
f"(task={task_id})"
|
||||
)
|
||||
return None
|
||||
|
||||
if requested_model:
|
||||
clean_req_model = str(requested_model).strip().lower()
|
||||
model_specific = [
|
||||
s for s in matching_sessions
|
||||
if self._get_session_override_model_name(s).strip().lower() == clean_req_model
|
||||
]
|
||||
if model_specific:
|
||||
matching_sessions = model_specific
|
||||
|
||||
mode = allocation_mode or group.get("allocation_mode") or self.allocation_mode
|
||||
session = self._try_acquire_session_for_request(
|
||||
matching_sessions,
|
||||
task_id,
|
||||
route_domain=f"group::{target}",
|
||||
allocation_mode=mode,
|
||||
defer_context=f"route-group {target} acquire",
|
||||
)
|
||||
if session is not None:
|
||||
self._unregister_waiter(
|
||||
waiters,
|
||||
waiter_token,
|
||||
owner_map=self._group_waiters,
|
||||
owner_key=target,
|
||||
)
|
||||
if not self._complete_acquired_session_for_return(
|
||||
session,
|
||||
"acquire_by_route_group",
|
||||
task_id,
|
||||
activate=self._auto_activate_on_acquire and session.id != self._active_session_id,
|
||||
):
|
||||
# 与 acquire/acquire_by_route_domain 等路径保持一致:
|
||||
# handoff 失败直接返回 None。此处 waiter 已出队,若 continue
|
||||
# 会因 _is_waiter_turn 永远轮不到而空转到超时。
|
||||
# 调用方(workflow.py)已有 session is None 的处理分支。
|
||||
return None
|
||||
self._mark_allocation_cursor(session, route_domain=f"group::{target}")
|
||||
logger.debug(
|
||||
f"TabPool -> {session.id} "
|
||||
f"(task={task_id}, route_group={target}, "
|
||||
f"idx=#{session.persistent_index}, snapshot={self._describe_session(session)})"
|
||||
)
|
||||
return session
|
||||
|
||||
remaining = deadline - time.time()
|
||||
if remaining <= 0:
|
||||
member_state = ", ".join(
|
||||
f"{session.id}(#{session.persistent_index}:{session.status.value})"
|
||||
for session in matching_sessions
|
||||
) or "no live members"
|
||||
logger.warning(
|
||||
f"Timed out waiting for route group '{target}' "
|
||||
f"(task={task_id}, members: {member_state})"
|
||||
)
|
||||
return None
|
||||
|
||||
logger.debug_throttled(
|
||||
f"tab_pool.wait_group.{target}",
|
||||
f"Waiting for route group '{target}' member release...",
|
||||
interval_sec=5.0,
|
||||
)
|
||||
self._condition.wait(
|
||||
timeout=min(remaining, self.ACQUIRE_CANCEL_POLL_SEC)
|
||||
)
|
||||
finally:
|
||||
self._unregister_waiter(
|
||||
waiters,
|
||||
waiter_token,
|
||||
owner_map=self._group_waiters,
|
||||
owner_key=target,
|
||||
)
|
||||
@@ -0,0 +1,338 @@
|
||||
"""TabPoolManager 的一部分(R2-3 拆分):会话封装、移除、请求取消、隔离区(quarantine)与恢复服务。
|
||||
|
||||
方法原样从 manager.py 迁出,行为不变;通过 mixin 组合回 TabPoolManager,
|
||||
类属性与实例状态都在 TabPoolManager(manager.py)里定义。
|
||||
"""
|
||||
|
||||
from typing import Any, Optional
|
||||
|
||||
from app.core.config import logger
|
||||
from app.utils.site_url import extract_remote_site_domain
|
||||
|
||||
from .recovery import TabQuarantineEntry, TabRecoveryService
|
||||
from .session import TabSession
|
||||
|
||||
|
||||
class TabPoolSessionsMixin:
|
||||
"""会话封装、移除、请求取消、隔离区(quarantine)与恢复服务"""
|
||||
|
||||
def _get_domain_abbr(self, url: str) -> str:
|
||||
try:
|
||||
if not url or "://" not in url:
|
||||
return "tab"
|
||||
|
||||
domain = url.split("//")[-1].split("/")[0].lower()
|
||||
clean_domain = domain.replace("www.", "")
|
||||
|
||||
for key, abbr in self.DOMAIN_ABBR_MAP.items():
|
||||
if key in clean_domain:
|
||||
return abbr
|
||||
|
||||
first_part = clean_domain.split(".")[0]
|
||||
return first_part[:10]
|
||||
|
||||
except Exception:
|
||||
return "tab"
|
||||
@staticmethod
|
||||
def _get_tab_ref_id(tab_ref: Any) -> Optional[str]:
|
||||
if tab_ref is None:
|
||||
return None
|
||||
|
||||
if isinstance(tab_ref, tuple) and tab_ref:
|
||||
return str(tab_ref[0] or "").strip() or None
|
||||
if isinstance(tab_ref, dict):
|
||||
target_id = str(tab_ref.get("targetId") or tab_ref.get("tab_id") or "").strip()
|
||||
return target_id or None
|
||||
|
||||
tab_id = getattr(tab_ref, "tab_id", None)
|
||||
if tab_id:
|
||||
return str(tab_id)
|
||||
|
||||
raw = str(tab_ref or "").strip()
|
||||
return raw or None
|
||||
@staticmethod
|
||||
def _get_tab_ref_url(tab_ref: Any) -> str:
|
||||
if isinstance(tab_ref, tuple) and len(tab_ref) >= 2 and isinstance(tab_ref[1], dict):
|
||||
return str(tab_ref[1].get("url") or "").strip()
|
||||
if isinstance(tab_ref, dict):
|
||||
return str(tab_ref.get("url") or "").strip()
|
||||
try:
|
||||
return str(getattr(tab_ref, "url", "") or "").strip()
|
||||
except Exception:
|
||||
return ""
|
||||
@staticmethod
|
||||
def _tab_ref_has_url_field(tab_ref: Any) -> bool:
|
||||
if isinstance(tab_ref, tuple) and len(tab_ref) >= 2 and isinstance(tab_ref[1], dict):
|
||||
return "url" in tab_ref[1]
|
||||
if isinstance(tab_ref, dict):
|
||||
return "url" in tab_ref
|
||||
return False
|
||||
def _resolve_tab_from_ref(self, tab_ref: Any) -> Optional[Any]:
|
||||
if tab_ref is None:
|
||||
return None
|
||||
if isinstance(tab_ref, tuple) and len(tab_ref) >= 3:
|
||||
existing_tab = tab_ref[2]
|
||||
if existing_tab is not None:
|
||||
return existing_tab
|
||||
|
||||
try:
|
||||
existing_tab_id = getattr(tab_ref, "tab_id", None)
|
||||
except Exception:
|
||||
existing_tab_id = None
|
||||
if existing_tab_id:
|
||||
return tab_ref
|
||||
|
||||
raw_tab_id = self._get_tab_ref_id(tab_ref)
|
||||
if not raw_tab_id:
|
||||
return None
|
||||
|
||||
try:
|
||||
browser = self._get_browser_handle()
|
||||
return browser.get_tab(raw_tab_id)
|
||||
except Exception as exc:
|
||||
if self._is_execution_context_error(exc) and self._schedule_context_recovery(raw_tab_id):
|
||||
logger.warning(
|
||||
f"[TabPool] 标签页 {raw_tab_id} 丢失 JS 执行上下文,"
|
||||
"已异步请求 CDP 刷新唤醒"
|
||||
)
|
||||
return None
|
||||
def _wrap_tab(
|
||||
self,
|
||||
tab,
|
||||
raw_tab_id: str = None,
|
||||
*,
|
||||
browser_context_id: Optional[str] = None,
|
||||
is_isolated_context: bool = False,
|
||||
initial_url: str = "",
|
||||
) -> TabSession:
|
||||
self._tab_counter += 1
|
||||
|
||||
url = str(initial_url or "").strip()
|
||||
if not url:
|
||||
try:
|
||||
url = tab.url or ""
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
abbr = self._get_domain_abbr(url)
|
||||
tab_id = f"{abbr}_{self._tab_counter}"
|
||||
|
||||
session = TabSession(
|
||||
id=tab_id,
|
||||
tab=tab,
|
||||
browser_context_id=browser_context_id,
|
||||
is_isolated_context=bool(is_isolated_context),
|
||||
)
|
||||
session._remember_url(url)
|
||||
|
||||
try:
|
||||
session.current_domain = extract_remote_site_domain(url)
|
||||
except:
|
||||
pass
|
||||
|
||||
# 记录底层标签页 ID
|
||||
if raw_tab_id:
|
||||
self._known_tab_ids.add(raw_tab_id)
|
||||
if session.is_isolated_context:
|
||||
self._register_isolated_context(raw_tab_id, browser_context_id)
|
||||
|
||||
# 🆕 分配持久化编号
|
||||
if raw_tab_id not in self._raw_id_to_persistent:
|
||||
persistent_idx = self._next_persistent_index
|
||||
self._next_persistent_index += 1
|
||||
self._raw_id_to_persistent[raw_tab_id] = persistent_idx
|
||||
else:
|
||||
persistent_idx = self._raw_id_to_persistent[raw_tab_id]
|
||||
|
||||
session.persistent_index = persistent_idx
|
||||
self._persistent_to_session_id[persistent_idx] = session.id
|
||||
logger.debug(f"标签页 {session.id} 分配编号 #{persistent_idx}")
|
||||
|
||||
self._apply_preset_overrides(session)
|
||||
try:
|
||||
setattr(tab, "_session", session)
|
||||
except Exception:
|
||||
pass
|
||||
session.notify_navigated(reason="tab_created")
|
||||
return session
|
||||
def _on_session_removed(self, session_id: str):
|
||||
"""Notify command engine after a pool session is removed without blocking pool locks."""
|
||||
session_key = str(session_id or "").strip()
|
||||
if not session_key:
|
||||
return
|
||||
|
||||
def _evict():
|
||||
try:
|
||||
from app.services.command_engine import command_engine
|
||||
|
||||
if hasattr(command_engine, "evict_session"):
|
||||
command_engine.evict_session(session_key)
|
||||
except Exception as e:
|
||||
logger.debug(f"[TabPool] evict session {session_key} failed: {e}")
|
||||
|
||||
executor = self._maintenance_executor
|
||||
if executor is None:
|
||||
return
|
||||
try:
|
||||
executor.submit(_evict)
|
||||
except RuntimeError as e:
|
||||
logger.debug(f"[TabPool] maintenance submit failed (evict:{session_key}): {e}")
|
||||
def _submit_request_cancel(
|
||||
self,
|
||||
task_id: str,
|
||||
reason: str,
|
||||
*,
|
||||
session_id: str = "",
|
||||
detail: str = "",
|
||||
) -> bool:
|
||||
"""Cancel a request from the maintenance worker so pool locks never call outward."""
|
||||
task_key = str(task_id or "").strip()
|
||||
if not task_key:
|
||||
return False
|
||||
reason_key = str(reason or "").strip() or "unknown"
|
||||
session_key = str(session_id or "").strip()
|
||||
detail_text = str(detail or "").strip()
|
||||
|
||||
def _cancel():
|
||||
cancelled = False
|
||||
try:
|
||||
from app.services.request_manager import request_manager
|
||||
|
||||
cancelled = bool(request_manager.cancel_request(task_key, reason_key))
|
||||
except Exception as e:
|
||||
logger.debug(
|
||||
f"[{session_key or 'TabPool'}] async cancel failed "
|
||||
f"(task={task_key}, reason={reason_key}, detail={detail_text or '-'}): {e}"
|
||||
)
|
||||
return
|
||||
|
||||
logger.debug(
|
||||
f"[{session_key or 'TabPool'}] async cancel result "
|
||||
f"(task={task_key}, reason={reason_key}, cancelled={cancelled}, "
|
||||
f"detail={detail_text or '-'})"
|
||||
)
|
||||
|
||||
executor = self._maintenance_executor
|
||||
if executor is None:
|
||||
logger.debug(
|
||||
f"[{session_key or 'TabPool'}] async cancel skipped: maintenance executor unavailable "
|
||||
f"(task={task_key}, reason={reason_key}, detail={detail_text or '-'})"
|
||||
)
|
||||
return False
|
||||
try:
|
||||
executor.submit(_cancel)
|
||||
return True
|
||||
except RuntimeError as e:
|
||||
logger.debug(
|
||||
f"[{session_key or 'TabPool'}] maintenance submit failed "
|
||||
f"(cancel:{task_key}, reason={reason_key}, detail={detail_text or '-'}): {e}"
|
||||
)
|
||||
return False
|
||||
def _close_quarantined_tab(self, tab: Any, raw_tab_id: str) -> bool:
|
||||
"""Close a permanently quarantined browser target outside the pool lock."""
|
||||
close = getattr(tab, "close", None)
|
||||
if callable(close):
|
||||
try:
|
||||
close_result = close()
|
||||
if close_result is not False:
|
||||
return True
|
||||
logger.debug(
|
||||
f"[TabRecovery] 标签页对象 close() 返回失败,尝试 CDP 兜底 "
|
||||
f"(raw={raw_tab_id})"
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.debug(
|
||||
f"[TabRecovery] 关闭隔离标签页对象失败,尝试 CDP 兜底 "
|
||||
f"(raw={raw_tab_id}, error={exc})"
|
||||
)
|
||||
|
||||
try:
|
||||
browser = self._get_browser_handle()
|
||||
run_cdp = getattr(browser, "_run_cdp", None)
|
||||
if callable(run_cdp):
|
||||
result = run_cdp("Target.closeTarget", targetId=str(raw_tab_id))
|
||||
if not isinstance(result, dict) or result.get("success", True):
|
||||
return True
|
||||
except Exception as exc:
|
||||
logger.warning(
|
||||
f"[TabRecovery] 关闭永久隔离标签页失败 "
|
||||
f"(raw={raw_tab_id}, error={exc})"
|
||||
)
|
||||
return False
|
||||
def resolve_quarantine(self, raw_tab_id: str, release: bool) -> bool:
|
||||
"""解除或永久化一个隔离条目(供 TabRecoveryService 回调)。
|
||||
|
||||
release=True:从隔离集合移除,下一次 _scan_new_tabs 自然重新入池;
|
||||
release=False:关闭浏览器标签页并释放强引用;关闭失败时保留不带
|
||||
tab 引用的轻量永久隔离记录,避免扫描期间重新入池。
|
||||
"""
|
||||
raw_id = str(raw_tab_id or "").strip()
|
||||
if not raw_id:
|
||||
return False
|
||||
|
||||
tab_to_close = None
|
||||
entry = None
|
||||
with self._condition:
|
||||
entry = self._quarantined_raw_ids.get(raw_id)
|
||||
if entry is None:
|
||||
return False
|
||||
if release:
|
||||
self._quarantined_raw_ids.pop(raw_id, None)
|
||||
# 恢复任务结束后不再需要保留 DrissionPage tab wrapper。
|
||||
entry.tab = None
|
||||
self._condition.notify_all()
|
||||
return True
|
||||
|
||||
entry.permanent = True
|
||||
tab_to_close = getattr(entry, "tab", None)
|
||||
# 先在锁内摘掉强引用;真实 CDP 调用必须在锁外执行。
|
||||
entry.tab = None
|
||||
|
||||
closed = self._close_quarantined_tab(tab_to_close, raw_id)
|
||||
with self._condition:
|
||||
if self._quarantined_raw_ids.get(raw_id) is entry:
|
||||
if closed:
|
||||
self._quarantined_raw_ids.pop(raw_id, None)
|
||||
self._condition.notify_all()
|
||||
return True
|
||||
def get_quarantine_entry(self, raw_tab_id: str) -> Optional[TabQuarantineEntry]:
|
||||
with self._condition:
|
||||
return self._quarantined_raw_ids.get(str(raw_tab_id or "").strip())
|
||||
def _get_recovery_service(self) -> Optional[TabRecoveryService]:
|
||||
if self._shutdown:
|
||||
return None
|
||||
service = self._recovery_service
|
||||
if service is not None:
|
||||
return service
|
||||
with self._recovery_service_lock:
|
||||
if self._recovery_service is None:
|
||||
self._recovery_service = TabRecoveryService(
|
||||
resolve_quarantine=self.resolve_quarantine,
|
||||
get_quarantine_entry=self.get_quarantine_entry,
|
||||
)
|
||||
return self._recovery_service
|
||||
def _submit_recovery(self, entry: TabQuarantineEntry) -> None:
|
||||
"""把隔离条目投递给恢复服务;持池锁调用也安全(仅入队,不阻塞外呼)。"""
|
||||
|
||||
def _submit() -> None:
|
||||
try:
|
||||
service = self._get_recovery_service()
|
||||
if service is not None:
|
||||
service.submit(entry)
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
f"[TabRecovery] 投递恢复任务失败 ({entry.raw_tab_id}): {e}"
|
||||
)
|
||||
|
||||
executor = self._maintenance_executor
|
||||
if executor is not None:
|
||||
try:
|
||||
executor.submit(_submit)
|
||||
return
|
||||
except RuntimeError as e:
|
||||
logger.debug(
|
||||
f"[TabRecovery] maintenance submit failed "
|
||||
f"(recovery:{entry.raw_tab_id}): {e}"
|
||||
)
|
||||
# 兜底:maintenance executor 不可用时直接入队(submit 本身非阻塞)。
|
||||
_submit()
|
||||
@@ -0,0 +1,405 @@
|
||||
"""TabPoolManager 的一部分(R2-3 拆分):状态查询与管理操作(编号列表、预设/模型名设置、状态与看门狗摘要、按编号终止)。
|
||||
|
||||
方法原样从 manager.py 迁出,行为不变;通过 mixin 组合回 TabPoolManager,
|
||||
类属性与实例状态都在 TabPoolManager(manager.py)里定义。
|
||||
"""
|
||||
|
||||
import time
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from app.core.config import logger
|
||||
from app.utils.tab_route_groups import normalize_route_groups
|
||||
|
||||
from .session import TabSession, TabStatus
|
||||
|
||||
|
||||
class TabPoolStatusMixin:
|
||||
"""状态查询与管理操作(编号列表、预设/模型名设置、状态与看门狗摘要、按编号终止)"""
|
||||
|
||||
def terminate_by_index(
|
||||
self,
|
||||
persistent_index: int,
|
||||
reason: str = "manual_terminate",
|
||||
clear_page: bool = True,
|
||||
scope: str = "task",
|
||||
expected_session_id: str = "",
|
||||
expected_task_id: str = "",
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
按标签页编号终止当前任务,并在旧所有者退出前隔离该成员。
|
||||
|
||||
行为:
|
||||
1) 校验调用方看到的 session/task 所有权快照;
|
||||
2) 设置终止隔离闸门,阻止组/域名/编号路由重新分配;
|
||||
3) 取消该成员关联请求并中断页面活动;
|
||||
4) 等待旧工作线程完成正常释放,超时则继续保持隔离。
|
||||
"""
|
||||
normalized_scope = str(scope or "task").strip().lower()
|
||||
if normalized_scope not in {"task", "loop"}:
|
||||
normalized_scope = "task"
|
||||
expected_session = str(expected_session_id or "").strip()
|
||||
expected_task = str(expected_task_id or "").strip()
|
||||
|
||||
with self._condition:
|
||||
session_id = self._persistent_to_session_id.get(persistent_index)
|
||||
if not session_id:
|
||||
return {"ok": False, "error": "tab_not_found", "tab_index": persistent_index}
|
||||
|
||||
session = self._tabs.get(session_id)
|
||||
if not session:
|
||||
return {"ok": False, "error": "tab_not_found", "tab_index": persistent_index}
|
||||
if expected_session and session.id != expected_session:
|
||||
return {
|
||||
"ok": False,
|
||||
"error": "session_ownership_changed",
|
||||
"tab_index": persistent_index,
|
||||
"expected_session_id": expected_session,
|
||||
"current_session_id": session.id,
|
||||
}
|
||||
|
||||
current_task = str(session.current_task_id or "").strip()
|
||||
if expected_task and current_task != expected_task:
|
||||
return {
|
||||
"ok": False,
|
||||
"error": "task_ownership_changed",
|
||||
"tab_index": persistent_index,
|
||||
"tab_id": session.id,
|
||||
"expected_task_id": expected_task,
|
||||
"current_task_id": current_task,
|
||||
}
|
||||
|
||||
if normalized_scope == "loop":
|
||||
loop_cancelled = session.request_command_loop_cancel(reason)
|
||||
self._condition.notify_all()
|
||||
if not loop_cancelled:
|
||||
return {
|
||||
"ok": False,
|
||||
"error": "no_active_command_loop",
|
||||
"tab_index": persistent_index,
|
||||
"tab_id": session.id,
|
||||
"status": session.status.value,
|
||||
"reason": reason,
|
||||
"scope": normalized_scope,
|
||||
}
|
||||
logger.warning(
|
||||
f"[{session.id}] 手动终止当前循环: idx=#{persistent_index}, "
|
||||
f"status={session.status.value}, reason={reason}, "
|
||||
f"loop={session.get_command_loop_info()}"
|
||||
)
|
||||
return {
|
||||
"ok": True,
|
||||
"tab_index": persistent_index,
|
||||
"tab_id": session.id,
|
||||
"was_busy": session.status == TabStatus.BUSY,
|
||||
"task_id": str(session.current_task_id or "").strip(),
|
||||
"cancelled": True,
|
||||
"status": session.status.value,
|
||||
"reason": reason,
|
||||
"scope": normalized_scope,
|
||||
"loop_cancelled": True,
|
||||
"command_loop": session.get_command_loop_info(),
|
||||
}
|
||||
|
||||
before_snapshot = self._describe_session(session)
|
||||
was_busy = session.status == TabStatus.BUSY
|
||||
termination_state = session.begin_termination(expected_task)
|
||||
if not termination_state.get("ok"):
|
||||
# A second terminate request can race with the first one while
|
||||
# the old owner is still unwinding. That is an idempotent
|
||||
# in-progress result, not a failed termination.
|
||||
if termination_state.get("error") == "termination_in_progress":
|
||||
return {
|
||||
"ok": True,
|
||||
"tab_index": persistent_index,
|
||||
"tab_id": session.id,
|
||||
"was_busy": was_busy,
|
||||
"task_id": str(
|
||||
termination_state.get("current_task_id") or current_task
|
||||
).strip(),
|
||||
"cancelled": False,
|
||||
"status": session.status.value,
|
||||
"pending": True,
|
||||
"released": False,
|
||||
"interrupt_success": False,
|
||||
"already_in_progress": True,
|
||||
"reason": reason,
|
||||
"scope": normalized_scope,
|
||||
}
|
||||
return {
|
||||
"ok": False,
|
||||
**termination_state,
|
||||
"tab_index": persistent_index,
|
||||
"tab_id": session.id,
|
||||
"status": session.status.value,
|
||||
"reason": reason,
|
||||
}
|
||||
task_id = str(termination_state.get("task_id") or "").strip()
|
||||
command_request_id = str(
|
||||
termination_state.get("command_request_id") or ""
|
||||
).strip()
|
||||
command_loop_request_id = str(
|
||||
termination_state.get("command_loop_request_id") or ""
|
||||
).strip()
|
||||
|
||||
cancelled = False
|
||||
cancel_error = ""
|
||||
|
||||
request_ids_to_cancel = []
|
||||
for request_id in (task_id, command_request_id, command_loop_request_id):
|
||||
if request_id and request_id not in request_ids_to_cancel:
|
||||
request_ids_to_cancel.append(request_id)
|
||||
|
||||
if request_ids_to_cancel:
|
||||
try:
|
||||
from app.services.request_manager import request_manager
|
||||
for request_id in request_ids_to_cancel:
|
||||
cancelled = bool(request_manager.cancel_request(request_id, reason)) or cancelled
|
||||
except Exception as e:
|
||||
cancel_error = str(e)
|
||||
logger.debug(f"[{session.id}] 取消任务失败(忽略): {e}")
|
||||
|
||||
self._stop_global_monitor_for_session(session.id, reason=f"terminate:{reason}", wait=True)
|
||||
interrupt_success = session.interrupt_for_termination(clear_page=clear_page)
|
||||
|
||||
with self._condition:
|
||||
wait_deadline = time.monotonic() + self.TERMINATION_RELEASE_WAIT_SEC
|
||||
while (
|
||||
self._tabs.get(session.id) is session
|
||||
and session.is_termination_in_progress()
|
||||
):
|
||||
remaining = wait_deadline - time.monotonic()
|
||||
if remaining <= 0:
|
||||
break
|
||||
self._condition.wait(timeout=min(remaining, 0.1))
|
||||
|
||||
pending = bool(
|
||||
self._tabs.get(session.id) is session
|
||||
and session.is_termination_in_progress()
|
||||
)
|
||||
if not pending and self._tabs.get(session.id) is session and session.status == TabStatus.IDLE:
|
||||
self._start_global_monitor_for_session(session)
|
||||
|
||||
self._condition.notify_all()
|
||||
|
||||
logger.warning(
|
||||
f"[{session.id}] 手动终止: idx=#{persistent_index}, "
|
||||
f"task={task_id or '-'}, cancelled={cancelled}, "
|
||||
f"pending={pending}, interrupt_success={interrupt_success}, "
|
||||
f"status={session.status.value}, reason={reason}, "
|
||||
f"before={before_snapshot}, after={self._describe_session(session)}"
|
||||
)
|
||||
|
||||
result = {
|
||||
"ok": True,
|
||||
"tab_index": persistent_index,
|
||||
"tab_id": session.id,
|
||||
"was_busy": was_busy,
|
||||
"task_id": task_id,
|
||||
"cancelled": cancelled,
|
||||
"status": session.status.value,
|
||||
"pending": pending,
|
||||
"released": not pending and session.status == TabStatus.IDLE,
|
||||
"interrupt_success": interrupt_success,
|
||||
"reason": reason,
|
||||
"scope": normalized_scope,
|
||||
}
|
||||
if cancel_error:
|
||||
result["cancel_error"] = cancel_error
|
||||
return result
|
||||
def get_tabs_with_index(self) -> List[Dict]:
|
||||
"""获取所有标签页及其持久编号(供 API 调用)"""
|
||||
with self._lock:
|
||||
if self._should_scan_for_query():
|
||||
self._scan_new_tabs()
|
||||
|
||||
sessions = list(self._tabs.values())
|
||||
groups_by_session: Dict[str, List[str]] = {}
|
||||
for group in normalize_route_groups(getattr(self, "route_groups", [])):
|
||||
for member_session in self._get_sessions_for_route_group(group["id"], bind=False):
|
||||
groups_by_session.setdefault(member_session.id, []).append(group["id"])
|
||||
|
||||
result = []
|
||||
for session in sessions:
|
||||
info = session.get_info(use_cached_url=True)
|
||||
tab_route_prefix = f"/tab/{session.persistent_index}"
|
||||
route_domain = str(info.get("route_domain") or "").strip()
|
||||
domain_route_prefix = f"/url/{route_domain}" if route_domain else ""
|
||||
preset_route_domain = str(info.get("current_domain") or route_domain).strip()
|
||||
url_route_token = str(info.get("url_route_token") or "").strip()
|
||||
exact_url_route_prefix = f"/tab-url/{url_route_token}" if url_route_token else ""
|
||||
info["tab_route_prefix"] = tab_route_prefix
|
||||
info["domain_route_prefix"] = domain_route_prefix
|
||||
info["preset_route_domain"] = preset_route_domain
|
||||
info["preset_domain_route_prefix"] = f"/url/{preset_route_domain}" if preset_route_domain else ""
|
||||
info["exact_url_route_prefix"] = exact_url_route_prefix
|
||||
info["route_prefix"] = domain_route_prefix or tab_route_prefix
|
||||
info["route_groups"] = groups_by_session.get(session.id, [])
|
||||
self._apply_preset_overrides(session, info)
|
||||
self._apply_exposed_model_name(info)
|
||||
result.append(info)
|
||||
|
||||
# 按编号排序
|
||||
result.sort(key=lambda x: x.get("persistent_index", 0))
|
||||
return result
|
||||
def set_tab_preset(self, persistent_index: int, preset_name: str) -> bool:
|
||||
"""
|
||||
为指定标签页设置预设
|
||||
|
||||
Args:
|
||||
persistent_index: 标签页持久化编号
|
||||
preset_name: 预设名称(None 或空字符串表示恢复为跟随站点默认预设)
|
||||
|
||||
Returns:
|
||||
是否成功
|
||||
"""
|
||||
with self._lock:
|
||||
session_id = self._persistent_to_session_id.get(persistent_index)
|
||||
if not session_id:
|
||||
logger.warning(f"标签页 #{persistent_index} 不存在")
|
||||
return False
|
||||
|
||||
session = self._tabs.get(session_id)
|
||||
if not session:
|
||||
logger.warning(f"标签页 {session_id} 已被移除")
|
||||
return False
|
||||
|
||||
old_preset = session.preset_name
|
||||
session.set_preset(preset_name, source="tab" if preset_name else None)
|
||||
|
||||
logger.debug(
|
||||
f"[{session.id}] 预设切换: "
|
||||
f"'{old_preset or '跟随站点默认预设'}' → '{preset_name or '跟随站点默认预设'}'"
|
||||
)
|
||||
return True
|
||||
def get_tab_preset(self, persistent_index: int) -> Optional[str]:
|
||||
"""获取指定标签页的当前预设名称"""
|
||||
with self._lock:
|
||||
session_id = self._persistent_to_session_id.get(persistent_index)
|
||||
if not session_id:
|
||||
return None
|
||||
|
||||
session = self._tabs.get(session_id)
|
||||
if not session:
|
||||
return None
|
||||
|
||||
return session.preset_name
|
||||
def set_tab_model_name(self, persistent_index: int, model_name: Optional[str]) -> Dict[str, Any]:
|
||||
"""为指定标签页设置临时暴露模型名。空值表示恢复到持久化/默认规则。"""
|
||||
normalized_name = self._normalize_model_name(model_name)
|
||||
with self._lock:
|
||||
session_id = self._persistent_to_session_id.get(persistent_index)
|
||||
if not session_id:
|
||||
return {"ok": False, "error": "tab_not_found", "tab_index": persistent_index}
|
||||
|
||||
session = self._tabs.get(session_id)
|
||||
if not session:
|
||||
return {"ok": False, "error": "tab_not_found", "tab_index": persistent_index}
|
||||
|
||||
session.model_name_override = normalized_name or None
|
||||
info = session.get_info(use_cached_url=True)
|
||||
tab_route_prefix = f"/tab/{session.persistent_index}"
|
||||
route_domain = str(info.get("route_domain") or "").strip()
|
||||
domain_route_prefix = f"/url/{route_domain}" if route_domain else ""
|
||||
preset_route_domain = str(info.get("current_domain") or route_domain).strip()
|
||||
url_route_token = str(info.get("url_route_token") or "").strip()
|
||||
info["tab_route_prefix"] = tab_route_prefix
|
||||
info["domain_route_prefix"] = domain_route_prefix
|
||||
info["preset_route_domain"] = preset_route_domain
|
||||
info["preset_domain_route_prefix"] = f"/url/{preset_route_domain}" if preset_route_domain else ""
|
||||
info["exact_url_route_prefix"] = f"/tab-url/{url_route_token}" if url_route_token else ""
|
||||
info["route_prefix"] = domain_route_prefix or tab_route_prefix
|
||||
self._apply_exposed_model_name(info)
|
||||
|
||||
return {
|
||||
"ok": True,
|
||||
"tab_index": persistent_index,
|
||||
"tab": info,
|
||||
}
|
||||
def get_status(self) -> Dict:
|
||||
with self._lock:
|
||||
sessions = list(self._tabs.values())
|
||||
total = len(sessions)
|
||||
idle = sum(1 for s in sessions if s.status == TabStatus.IDLE)
|
||||
busy = sum(1 for s in sessions if s.status == TabStatus.BUSY)
|
||||
max_tabs = self.max_tabs
|
||||
min_tabs = self.min_tabs
|
||||
idle_timeout = self.idle_timeout
|
||||
acquire_timeout = self.acquire_timeout
|
||||
stuck_timeout = self.stuck_timeout
|
||||
allocation_mode = self.allocation_mode
|
||||
excluded_urls = list(self.excluded_urls)
|
||||
preserve_error_tabs = self.preserve_error_tabs
|
||||
route_groups = normalize_route_groups(getattr(self, "route_groups", []))
|
||||
global_network_enabled = self._global_network_enabled
|
||||
known_raw_tabs = len(self._known_tab_ids)
|
||||
last_scan = round(time.time() - self._last_scan_time, 1)
|
||||
|
||||
tabs_info = [s.get_info(use_cached_url=True) for s in sessions]
|
||||
|
||||
return {
|
||||
"total": total,
|
||||
"idle": idle,
|
||||
"busy": busy,
|
||||
"max_tabs": max_tabs,
|
||||
"min_tabs": min_tabs,
|
||||
"idle_timeout": idle_timeout,
|
||||
"acquire_timeout": acquire_timeout,
|
||||
"stuck_timeout": stuck_timeout,
|
||||
"allocation_mode": allocation_mode,
|
||||
"excluded_urls": excluded_urls,
|
||||
"preserve_error_tabs": preserve_error_tabs,
|
||||
"route_groups": route_groups,
|
||||
"global_network_enabled": global_network_enabled,
|
||||
"known_raw_tabs": known_raw_tabs,
|
||||
"last_scan": last_scan,
|
||||
"tabs": tabs_info
|
||||
}
|
||||
def get_watchdog_summary(self, limit: int = 4) -> Dict[str, Any]:
|
||||
"""Return the lightweight tab pool fields needed by BrowserWatchdog logs."""
|
||||
try:
|
||||
max_items = max(0, int(limit))
|
||||
except Exception:
|
||||
max_items = 4
|
||||
|
||||
with self._lock:
|
||||
sessions = list(self._tabs.values())
|
||||
total = len(sessions)
|
||||
idle = sum(1 for s in sessions if s.status == TabStatus.IDLE)
|
||||
busy = sum(1 for s in sessions if s.status == TabStatus.BUSY)
|
||||
preview = sessions[:max_items]
|
||||
|
||||
tabs = []
|
||||
for session in preview:
|
||||
status = getattr(getattr(session, "status", None), "value", None)
|
||||
tabs.append({
|
||||
"id": getattr(session, "id", None),
|
||||
"status": status or str(getattr(session, "status", "") or "?"),
|
||||
"is_isolated_context": bool(getattr(session, "is_isolated_context", False)),
|
||||
})
|
||||
|
||||
return {
|
||||
"total": total,
|
||||
"idle": idle,
|
||||
"busy": busy,
|
||||
"tabs": tabs,
|
||||
}
|
||||
def get_idle_sessions_snapshot(self) -> List[TabSession]:
|
||||
"""Return a shallow snapshot of currently idle tab sessions."""
|
||||
with self._lock:
|
||||
return [s for s in self._tabs.values() if s.status == TabStatus.IDLE]
|
||||
def get_sessions_snapshot(self) -> List[TabSession]:
|
||||
"""Return a shallow snapshot of all current tab sessions."""
|
||||
with self._lock:
|
||||
return list(self._tabs.values())
|
||||
def get_session_by_tab(self, tab: Any) -> Optional[TabSession]:
|
||||
"""Look up a TabSession for a given tab instance."""
|
||||
if tab is None:
|
||||
return None
|
||||
direct = getattr(tab, "_session", None)
|
||||
if direct is not None and isinstance(direct, TabSession):
|
||||
return direct
|
||||
with self._lock:
|
||||
for session in self._tabs.values():
|
||||
if getattr(session, "tab", None) is tab:
|
||||
return session
|
||||
return None
|
||||
@@ -0,0 +1,646 @@
|
||||
"""TabPoolManager 的一部分(R2-3 拆分):标签页的创建、扫描发现、初始化与刷新。
|
||||
|
||||
方法原样从 manager.py 迁出,行为不变;通过 mixin 组合回 TabPoolManager,
|
||||
类属性与实例状态都在 TabPoolManager(manager.py)里定义。
|
||||
"""
|
||||
|
||||
import time
|
||||
from typing import Any, Dict, Optional
|
||||
|
||||
from app.core.config import logger
|
||||
from app.utils.site_url import extract_remote_site_domain, route_domain_matches
|
||||
|
||||
from ._utils import _should_skip_pool_url
|
||||
from .session import TabStatus
|
||||
|
||||
|
||||
class TabPoolTabsMixin:
|
||||
"""标签页的创建、扫描发现、初始化与刷新"""
|
||||
|
||||
def _build_site_entry_url(self, domain: str) -> str:
|
||||
normalized_domain = str(domain or "").strip()
|
||||
if not normalized_domain:
|
||||
return ""
|
||||
|
||||
for session in self._tabs.values():
|
||||
current_url, actual_domain = session.get_cached_route_snapshot()
|
||||
if route_domain_matches(normalized_domain, actual_domain):
|
||||
if current_url and not _should_skip_pool_url(current_url):
|
||||
return current_url
|
||||
built = session._build_domain_url(current_url, actual_domain)
|
||||
if built:
|
||||
return built
|
||||
|
||||
return f"https://{normalized_domain}/"
|
||||
def _create_isolated_tab(self, url: str, background: bool = False) -> Optional[Dict[str, Any]]:
|
||||
try:
|
||||
browser = self._get_browser_handle()
|
||||
tab = browser.new_tab(background=background, new_context=True, new_window=True)
|
||||
raw_tab_id = getattr(tab, "tab_id", None)
|
||||
browser_context_id = self._get_browser_context_id(raw_tab_id)
|
||||
self._register_isolated_context(raw_tab_id, browser_context_id)
|
||||
if url:
|
||||
tab.get(url)
|
||||
return {
|
||||
"tab": tab,
|
||||
"raw_tab_id": raw_tab_id,
|
||||
"browser_context_id": browser_context_id,
|
||||
"url": str(getattr(tab, "url", "") or url or ""),
|
||||
}
|
||||
except Exception as e:
|
||||
logger.warning(f"[TabPool] create isolated tab failed: {e}")
|
||||
return None
|
||||
def _create_shared_tab(
|
||||
self,
|
||||
url: str,
|
||||
*,
|
||||
background: bool = False,
|
||||
new_window: bool = True,
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
try:
|
||||
browser = self._get_browser_handle()
|
||||
tab = browser.new_tab(url=url or None, background=background, new_window=new_window)
|
||||
raw_tab_id = getattr(tab, "tab_id", None)
|
||||
browser_context_id = self._get_browser_context_id(raw_tab_id)
|
||||
return {
|
||||
"tab": tab,
|
||||
"raw_tab_id": raw_tab_id,
|
||||
"browser_context_id": browser_context_id,
|
||||
"url": str(getattr(tab, "url", "") or url or ""),
|
||||
}
|
||||
except Exception as e:
|
||||
logger.warning(f"[TabPool] create shared tab failed: {e}")
|
||||
return None
|
||||
def _ensure_cookie_isolation_for_new_tab(self, tab: Any, raw_tab_id: str, url: str) -> Dict[str, Any]:
|
||||
final_url = str(url or "").strip()
|
||||
browser_context_id = self._get_browser_context_id(raw_tab_id)
|
||||
if raw_tab_id in self._isolated_context_by_raw_id:
|
||||
return {
|
||||
"tab": tab,
|
||||
"raw_tab_id": raw_tab_id,
|
||||
"browser_context_id": browser_context_id,
|
||||
"is_isolated_context": True,
|
||||
"url": final_url,
|
||||
}
|
||||
|
||||
domain = extract_remote_site_domain(final_url) or ""
|
||||
if not self._is_site_independent_cookie_enabled(domain):
|
||||
return {
|
||||
"tab": tab,
|
||||
"raw_tab_id": raw_tab_id,
|
||||
"browser_context_id": browser_context_id,
|
||||
"is_isolated_context": False,
|
||||
"url": final_url,
|
||||
}
|
||||
|
||||
if not self._is_site_independent_cookie_auto_takeover_enabled(domain):
|
||||
return {
|
||||
"tab": tab,
|
||||
"raw_tab_id": raw_tab_id,
|
||||
"browser_context_id": browser_context_id,
|
||||
"is_isolated_context": False,
|
||||
"url": final_url,
|
||||
}
|
||||
|
||||
logger.warning(
|
||||
f"[TabPool] skipped automatic isolated-cookie takeover because existing tabs are never closed: "
|
||||
f"{domain or final_url}"
|
||||
)
|
||||
return {
|
||||
"tab": tab,
|
||||
"raw_tab_id": raw_tab_id,
|
||||
"browser_context_id": browser_context_id,
|
||||
"is_isolated_context": False,
|
||||
"url": final_url,
|
||||
}
|
||||
def create_isolated_site_tab(self, domain: str) -> Dict[str, Any]:
|
||||
target_domain = str(domain or "").strip()
|
||||
if not target_domain:
|
||||
return {"ok": False, "error": "domain_required"}
|
||||
|
||||
with self._condition:
|
||||
self._scan_new_tabs()
|
||||
if len(self._tabs) >= self.max_tabs:
|
||||
return {"ok": False, "error": "tab_pool_full"}
|
||||
|
||||
target_url = self._build_site_entry_url(target_domain)
|
||||
|
||||
created = self._create_isolated_tab(target_url, background=False)
|
||||
if not created:
|
||||
return {"ok": False, "error": "create_isolated_tab_failed"}
|
||||
|
||||
with self._condition:
|
||||
if len(self._tabs) >= self.max_tabs:
|
||||
logger.warning(
|
||||
"[TabPool] pool reached capacity while creating an isolated tab; "
|
||||
"keeping the newly created tab managed"
|
||||
)
|
||||
|
||||
session = self._wrap_tab(
|
||||
created["tab"],
|
||||
created["raw_tab_id"],
|
||||
browser_context_id=created.get("browser_context_id"),
|
||||
is_isolated_context=True,
|
||||
initial_url=created.get("url") or target_url,
|
||||
)
|
||||
self._tabs[session.id] = session
|
||||
self._start_global_monitor_for_session(session)
|
||||
self._last_scan_time = time.time()
|
||||
self._condition.notify_all()
|
||||
|
||||
info = session.get_info(use_cached_url=True)
|
||||
info["tab_route_prefix"] = f"/tab/{session.persistent_index}"
|
||||
route_domain = str(info.get("route_domain") or "").strip()
|
||||
info["domain_route_prefix"] = f"/url/{route_domain}" if route_domain else ""
|
||||
preset_route_domain = str(info.get("current_domain") or route_domain).strip()
|
||||
info["preset_route_domain"] = preset_route_domain
|
||||
info["preset_domain_route_prefix"] = f"/url/{preset_route_domain}" if preset_route_domain else ""
|
||||
url_route_token = str(info.get("url_route_token") or "").strip()
|
||||
info["exact_url_route_prefix"] = f"/tab-url/{url_route_token}" if url_route_token else ""
|
||||
info["route_prefix"] = info["domain_route_prefix"] or info["tab_route_prefix"]
|
||||
self._apply_exposed_model_name(info)
|
||||
|
||||
return {
|
||||
"ok": True,
|
||||
"domain": target_domain,
|
||||
"message": f"已为 {target_domain} 新建独立 Cookie 标签页",
|
||||
"tab": info,
|
||||
}
|
||||
def create_shared_site_tab(self, domain: str) -> Dict[str, Any]:
|
||||
target_domain = str(domain or "").strip()
|
||||
if not target_domain:
|
||||
return {"ok": False, "error": "domain_required"}
|
||||
|
||||
with self._condition:
|
||||
self._scan_new_tabs()
|
||||
if len(self._tabs) >= self.max_tabs:
|
||||
return {"ok": False, "error": "tab_pool_full"}
|
||||
|
||||
target_url = self._build_site_entry_url(target_domain)
|
||||
|
||||
created = self._create_shared_tab(target_url, background=False, new_window=True)
|
||||
if not created:
|
||||
return {"ok": False, "error": "create_shared_tab_failed"}
|
||||
|
||||
with self._condition:
|
||||
if len(self._tabs) >= self.max_tabs:
|
||||
logger.warning(
|
||||
"[TabPool] pool reached capacity while creating a shared tab; "
|
||||
"keeping the newly created tab managed"
|
||||
)
|
||||
|
||||
session = self._wrap_tab(
|
||||
created["tab"],
|
||||
created["raw_tab_id"],
|
||||
browser_context_id=created.get("browser_context_id"),
|
||||
is_isolated_context=False,
|
||||
initial_url=created.get("url") or target_url,
|
||||
)
|
||||
self._tabs[session.id] = session
|
||||
self._start_global_monitor_for_session(session)
|
||||
self._last_scan_time = time.time()
|
||||
self._condition.notify_all()
|
||||
|
||||
info = session.get_info(use_cached_url=True)
|
||||
info["tab_route_prefix"] = f"/tab/{session.persistent_index}"
|
||||
route_domain = str(info.get("route_domain") or "").strip()
|
||||
info["domain_route_prefix"] = f"/url/{route_domain}" if route_domain else ""
|
||||
preset_route_domain = str(info.get("current_domain") or route_domain).strip()
|
||||
info["preset_route_domain"] = preset_route_domain
|
||||
info["preset_domain_route_prefix"] = f"/url/{preset_route_domain}" if preset_route_domain else ""
|
||||
url_route_token = str(info.get("url_route_token") or "").strip()
|
||||
info["exact_url_route_prefix"] = f"/tab-url/{url_route_token}" if url_route_token else ""
|
||||
info["route_prefix"] = info["domain_route_prefix"] or info["tab_route_prefix"]
|
||||
self._apply_exposed_model_name(info)
|
||||
|
||||
return {
|
||||
"ok": True,
|
||||
"domain": target_domain,
|
||||
"message": f"已为 {target_domain} 打开共享 Cookie 受控窗口",
|
||||
"tab": info,
|
||||
}
|
||||
def _should_scan(self) -> bool:
|
||||
"""检查是否需要扫描新标签页"""
|
||||
return time.time() - self._last_scan_time >= self.SCAN_INTERVAL
|
||||
def _should_scan_for_query(self) -> bool:
|
||||
"""读接口需要较新的浏览器 target 状态,但按 QUERY_SCAN_MIN_INTERVAL_SEC 节流。
|
||||
|
||||
此前恒返回 True,导致每个 /v1/chat/completions、/v1/models 请求都触发
|
||||
一次全量 CDP 扫描(Target.getTargets 等同步网络 I/O),并发时扫描成本
|
||||
线性叠加到首字延迟上;浏览器卡顿时还会拖住调用线程。
|
||||
1 秒内的快照对路由决策而言足够新鲜。
|
||||
"""
|
||||
return time.time() - self._last_scan_time >= self.QUERY_SCAN_MIN_INTERVAL_SEC
|
||||
def _scan_new_tabs(self):
|
||||
"""扫描并添加新标签页(已持有锁)"""
|
||||
try:
|
||||
# 先占住本次扫描窗口,避免唤醒后多个等待线程重复扫描。
|
||||
self._last_scan_time = time.time()
|
||||
# 锁序考量:下面会释放池锁去采集浏览器快照,期间新入池的会话
|
||||
# (create_shared_site_tab / create_isolated_site_tab 等)不会出现在
|
||||
# 这份过期快照里。记录快照起始时间,应用移除逻辑时据此跳过晚入池的会话
|
||||
# (session.created_at 即入池时刻,构造后随即持池锁入池)。
|
||||
snapshot_started_at = time.time()
|
||||
self._condition.release()
|
||||
try:
|
||||
with self._scan_snapshot_lock:
|
||||
(
|
||||
current_tabs,
|
||||
current_tab_ids,
|
||||
current_context_by_raw,
|
||||
current_url_by_raw,
|
||||
current_url_known_raw,
|
||||
) = self._snapshot_current_tab_targets()
|
||||
finally:
|
||||
self._condition.acquire()
|
||||
|
||||
if self._shutdown:
|
||||
return
|
||||
|
||||
current_tab_set = set(current_tab_ids)
|
||||
# 物理关闭成功后,Target.getTargets 可能要到下一轮才反映结果;
|
||||
# 这里清理已永久隔离且不再持有 tab 引用的陈旧元数据,避免隔离
|
||||
# 字典只增不减。
|
||||
for raw_id, entry in list(self._quarantined_raw_ids.items()):
|
||||
if (
|
||||
getattr(entry, "permanent", False)
|
||||
and getattr(entry, "tab", None) is None
|
||||
and raw_id not in current_tab_set
|
||||
):
|
||||
self._quarantined_raw_ids.pop(raw_id, None)
|
||||
self._cleanup_orphaned_isolated_contexts(list(current_context_by_raw.values()))
|
||||
changed = False
|
||||
|
||||
session_raw_by_id: Dict[str, str] = {}
|
||||
for rid, pidx in self._raw_id_to_persistent.items():
|
||||
sid = self._persistent_to_session_id.get(pidx)
|
||||
if sid and sid in self._tabs:
|
||||
session_raw_by_id[sid] = rid
|
||||
|
||||
for session_id, session in self._tabs.items():
|
||||
if session_raw_by_id.get(session_id):
|
||||
continue
|
||||
candidate_raw_id = self._get_tab_ref_id(getattr(session, "tab", None))
|
||||
if not candidate_raw_id or candidate_raw_id not in current_tab_set:
|
||||
continue
|
||||
|
||||
existing_persistent_idx = self._raw_id_to_persistent.get(candidate_raw_id)
|
||||
if existing_persistent_idx is not None:
|
||||
existing_session_id = self._persistent_to_session_id.get(existing_persistent_idx)
|
||||
if (
|
||||
existing_session_id
|
||||
and existing_session_id != session_id
|
||||
and existing_session_id in self._tabs
|
||||
):
|
||||
continue
|
||||
|
||||
persistent_idx = int(
|
||||
existing_persistent_idx
|
||||
or getattr(session, "persistent_index", 0)
|
||||
or 0
|
||||
)
|
||||
if persistent_idx <= 0:
|
||||
persistent_idx = self._next_persistent_index
|
||||
self._next_persistent_index += 1
|
||||
session.persistent_index = persistent_idx
|
||||
|
||||
self._raw_id_to_persistent[candidate_raw_id] = persistent_idx
|
||||
self._persistent_to_session_id[persistent_idx] = session_id
|
||||
self._known_tab_ids.add(candidate_raw_id)
|
||||
if session.is_isolated_context:
|
||||
context_id = (
|
||||
str(session.browser_context_id or "").strip()
|
||||
or current_context_by_raw.get(candidate_raw_id)
|
||||
)
|
||||
if context_id:
|
||||
session.browser_context_id = context_id
|
||||
self._register_isolated_context(candidate_raw_id, context_id)
|
||||
session_raw_by_id[session_id] = candidate_raw_id
|
||||
changed = True
|
||||
logger.warning(
|
||||
f"[{session_id}] repaired missing raw tab mapping "
|
||||
f"(raw={candidate_raw_id}, idx=#{persistent_idx})"
|
||||
)
|
||||
|
||||
for session_id, session in self._tabs.items():
|
||||
raw_id = session_raw_by_id.get(session_id)
|
||||
if not raw_id or raw_id not in current_tab_set:
|
||||
continue
|
||||
if raw_id in current_url_known_raw:
|
||||
current_url = current_url_by_raw.get(raw_id, "")
|
||||
old_url = session.last_known_url
|
||||
session._remember_url(current_url)
|
||||
try:
|
||||
session.current_domain = extract_remote_site_domain(current_url)
|
||||
except Exception:
|
||||
session.current_domain = None
|
||||
if old_url and current_url and old_url != current_url:
|
||||
session.notify_navigated(reason="url_changed")
|
||||
if not session.is_isolated_context:
|
||||
continue
|
||||
if session.status == TabStatus.BUSY:
|
||||
continue
|
||||
bound_raw_id = self._get_tab_ref_id(getattr(session, "tab", None))
|
||||
if bound_raw_id == raw_id:
|
||||
session.clear_transient_disconnect()
|
||||
continue
|
||||
if self._refresh_isolated_session_binding(session, raw_id):
|
||||
logger.info(f"[{session_id}] isolated session target recovered: {raw_id}")
|
||||
continue
|
||||
if session.is_in_transient_disconnect():
|
||||
logger.debug(
|
||||
f"[{session_id}] isolated session still waiting for target recovery: {raw_id}"
|
||||
)
|
||||
|
||||
reserved_raw_ids = {
|
||||
raw_id
|
||||
for raw_id in session_raw_by_id.values()
|
||||
if raw_id in current_tab_set
|
||||
}
|
||||
|
||||
# ===== 第一步:清理已关闭的标签页 =====
|
||||
# 找出池中存在、但浏览器中已消失的标签页
|
||||
sessions_to_remove = []
|
||||
for session_id, session in self._tabs.items():
|
||||
# 查找该 session 对应的 raw_tab_id
|
||||
raw_id = session_raw_by_id.get(session_id)
|
||||
|
||||
if raw_id is None or raw_id not in current_tab_set:
|
||||
# 入池晚于本次快照起始的会话必然不在快照里,不能据此判定"已关闭"
|
||||
# (isolated 会话有 rebind 宽限兜底,shared 会话没有,会被直接移除)。
|
||||
# 跳过本轮,等下一轮新快照再判定。
|
||||
joined_at = float(getattr(session, "created_at", 0.0) or 0.0)
|
||||
if joined_at > snapshot_started_at:
|
||||
logger.debug(
|
||||
f"[{session_id}] 跳过过期快照的移除判定 "
|
||||
f"(入池晚于快照起始 {joined_at - snapshot_started_at:.3f}s, raw={raw_id})"
|
||||
)
|
||||
continue
|
||||
sessions_to_remove.append((session_id, raw_id, session))
|
||||
|
||||
for session_id, raw_id, session in sessions_to_remove:
|
||||
replacement_raw_id = None
|
||||
replacement_tab = None
|
||||
browser_context_id = str(session.browser_context_id or "").strip()
|
||||
if session.is_isolated_context and browser_context_id:
|
||||
for candidate_raw_id in current_tab_ids:
|
||||
if candidate_raw_id in reserved_raw_ids:
|
||||
continue
|
||||
if current_context_by_raw.get(candidate_raw_id) != browser_context_id:
|
||||
continue
|
||||
replacement_tab = self._resolve_tab_from_ref(candidate_raw_id)
|
||||
if not replacement_tab:
|
||||
continue
|
||||
replacement_raw_id = candidate_raw_id
|
||||
break
|
||||
|
||||
if replacement_raw_id and replacement_tab:
|
||||
self._detach_global_monitor_for_session(session_id, reason="target_rebind")
|
||||
p_idx = self._raw_id_to_persistent.pop(raw_id, None) if raw_id else None
|
||||
if p_idx is None:
|
||||
p_idx = int(getattr(session, "persistent_index", 0) or 0) or None
|
||||
if p_idx is not None:
|
||||
self._raw_id_to_persistent[replacement_raw_id] = p_idx
|
||||
self._persistent_to_session_id[p_idx] = session_id
|
||||
if raw_id:
|
||||
self._known_tab_ids.discard(raw_id)
|
||||
self._known_tab_ids.add(replacement_raw_id)
|
||||
if raw_id:
|
||||
self._isolated_context_by_raw_id.pop(raw_id, None)
|
||||
self._register_isolated_context(replacement_raw_id, browser_context_id)
|
||||
session.tab = replacement_tab
|
||||
session.browser_context_id = browser_context_id or current_context_by_raw.get(replacement_raw_id)
|
||||
session.clear_transient_disconnect()
|
||||
try:
|
||||
_cached_url, cached_domain = session.get_cached_route_snapshot()
|
||||
if cached_domain:
|
||||
session.current_domain = cached_domain
|
||||
except Exception:
|
||||
pass
|
||||
reserved_raw_ids.add(replacement_raw_id)
|
||||
if session.status == TabStatus.IDLE:
|
||||
self._restart_global_monitor_for_session_async(session_id, "target_rebind")
|
||||
logger.info(
|
||||
f"[{session_id}] rebound isolated context target: {raw_id} -> {replacement_raw_id}"
|
||||
)
|
||||
changed = True
|
||||
continue
|
||||
|
||||
if session.is_isolated_context and browser_context_id:
|
||||
if self._arm_isolated_rebind_grace(
|
||||
session,
|
||||
reason="target_missing",
|
||||
detail=f"target missing (raw={raw_id})",
|
||||
):
|
||||
continue
|
||||
|
||||
if session.status == TabStatus.BUSY:
|
||||
self._cancel_active_request_for_session(
|
||||
session,
|
||||
"tab_closed",
|
||||
detail=f"raw={raw_id}",
|
||||
)
|
||||
logger.warning(f"[{session_id}] 标签页已关闭但仍在忙碌,标记为关闭")
|
||||
session.mark_closed("tab_closed")
|
||||
self._detach_global_monitor_for_session(session_id, reason="tab_closed")
|
||||
else:
|
||||
logger.info(f"[{session_id}] 标签页已关闭,从池中移除")
|
||||
session.mark_closed("tab_closed")
|
||||
self._detach_global_monitor_for_session(session_id, reason="tab_closed")
|
||||
del self._tabs[session_id]
|
||||
self._on_session_removed(session_id)
|
||||
|
||||
# 清理映射
|
||||
if raw_id:
|
||||
self._known_tab_ids.discard(raw_id)
|
||||
p_idx = self._raw_id_to_persistent.pop(raw_id, None) if raw_id else None
|
||||
if p_idx is None:
|
||||
p_idx = int(getattr(session, "persistent_index", 0) or 0) or None
|
||||
if p_idx is not None:
|
||||
if self._persistent_to_session_id.get(p_idx) == session_id:
|
||||
self._persistent_to_session_id.pop(p_idx, None)
|
||||
browser_context_id = self._isolated_context_by_raw_id.pop(raw_id, None) if raw_id else None
|
||||
if not browser_context_id and session.is_isolated_context:
|
||||
browser_context_id = str(session.browser_context_id or "").strip()
|
||||
if browser_context_id:
|
||||
self._mark_orphaned_isolated_context(browser_context_id)
|
||||
if self._active_session_id == session_id:
|
||||
self._active_session_id = None
|
||||
changed = True
|
||||
|
||||
# 顺手清理已切换到本地页/无效页的空闲标签,避免继续展示和参与调度。
|
||||
self._cleanup_unhealthy_tabs()
|
||||
|
||||
# ===== 第二步:构建"已在池中的 tab 对象"集合 =====
|
||||
tabs_in_pool = set()
|
||||
for rid in self._raw_id_to_persistent:
|
||||
pidx = self._raw_id_to_persistent[rid]
|
||||
sid = self._persistent_to_session_id.get(pidx)
|
||||
if sid and sid in self._tabs:
|
||||
tabs_in_pool.add(rid)
|
||||
|
||||
# ===== 第三步:扫描新标签页 =====
|
||||
new_count = 0
|
||||
for tab_ref in current_tabs:
|
||||
if len(self._tabs) >= self.max_tabs:
|
||||
break
|
||||
|
||||
raw_tab = self._get_tab_ref_id(tab_ref)
|
||||
if not raw_tab:
|
||||
continue
|
||||
|
||||
# 已在池中,跳过
|
||||
if raw_tab in tabs_in_pool:
|
||||
continue
|
||||
|
||||
# 隔离中的错误标签页不重新入池:旧 worker 可能尚未退出,
|
||||
# 由 TabRecoveryService 决定何时解除隔离(或永久排除)。
|
||||
if raw_tab in self._quarantined_raw_ids:
|
||||
logger.debug(f"[TabRecovery] 扫描跳过隔离中的标签页: {raw_tab}")
|
||||
continue
|
||||
|
||||
try:
|
||||
url = current_url_by_raw.get(raw_tab) or self._get_tab_ref_url(tab_ref)
|
||||
|
||||
# 本地页、浏览器内部页、空白页都不纳入标签页池。
|
||||
if _should_skip_pool_url(url):
|
||||
continue
|
||||
|
||||
tab = self._resolve_tab_from_ref(tab_ref)
|
||||
if not tab:
|
||||
continue
|
||||
|
||||
isolation_result = self._ensure_cookie_isolation_for_new_tab(tab, raw_tab, url)
|
||||
tab = isolation_result["tab"]
|
||||
raw_tab = isolation_result["raw_tab_id"]
|
||||
url = isolation_result["url"]
|
||||
|
||||
# 有效页面 - 添加到池
|
||||
session = self._wrap_tab(
|
||||
tab,
|
||||
raw_tab,
|
||||
browser_context_id=isolation_result.get("browser_context_id"),
|
||||
is_isolated_context=isolation_result.get("is_isolated_context", False),
|
||||
initial_url=url,
|
||||
)
|
||||
self._tabs[session.id] = session
|
||||
self._start_global_monitor_for_session(session)
|
||||
new_count += 1
|
||||
changed = True
|
||||
|
||||
display_url = url[:60] + "..." if len(url) > 60 else url
|
||||
logger.debug(f"🆕 发现新标签页: {session.id} -> {display_url}")
|
||||
|
||||
except Exception as e:
|
||||
logger.debug(f"处理标签页出错: {e}")
|
||||
continue
|
||||
|
||||
# Remove unusable pool sessions after discovery without closing targets.
|
||||
self._cleanup_unhealthy_tabs()
|
||||
self._last_scan_time = time.time()
|
||||
if changed:
|
||||
self._condition.notify_all()
|
||||
|
||||
if new_count > 0:
|
||||
logger.info(f"扫描完成: +{new_count} 个,当前共 {len(self._tabs)} 个标签页")
|
||||
|
||||
except Exception as e:
|
||||
logger.warning(f"扫描标签页失败: {e}")
|
||||
def initialize(self):
|
||||
"""初始化标签页池"""
|
||||
with self._lock:
|
||||
if self._initialized:
|
||||
return
|
||||
|
||||
raw_target_count = 0
|
||||
try:
|
||||
browser = self._get_browser_handle()
|
||||
existing_tabs = self._list_current_tab_refs()
|
||||
raw_target_count = len(existing_tabs)
|
||||
logger.debug(f"[TabPool] 检测到 {raw_target_count} 个浏览器 page target")
|
||||
|
||||
for tab_ref in existing_tabs:
|
||||
if len(self._tabs) >= self.max_tabs:
|
||||
break
|
||||
|
||||
try:
|
||||
raw_tab = self._get_tab_ref_id(tab_ref)
|
||||
if not raw_tab:
|
||||
continue
|
||||
|
||||
url = self._get_tab_ref_url(tab_ref)
|
||||
# 初始化时直接跳过本地页和浏览器内部页。
|
||||
if _should_skip_pool_url(url):
|
||||
continue
|
||||
|
||||
tab = self._resolve_tab_from_ref(tab_ref)
|
||||
if not tab:
|
||||
continue
|
||||
|
||||
isolation_result = self._ensure_cookie_isolation_for_new_tab(tab, raw_tab, url)
|
||||
tab = isolation_result["tab"]
|
||||
raw_tab = isolation_result["raw_tab_id"]
|
||||
url = isolation_result["url"]
|
||||
|
||||
# 有效页面 - 添加到池
|
||||
session = self._wrap_tab(
|
||||
tab,
|
||||
raw_tab,
|
||||
browser_context_id=isolation_result.get("browser_context_id"),
|
||||
is_isolated_context=isolation_result.get("is_isolated_context", False),
|
||||
initial_url=url,
|
||||
)
|
||||
self._tabs[session.id] = session
|
||||
|
||||
display_url = url[:60] + "..." if len(url) > 60 else url
|
||||
logger.info(f"TabPool: {session.id} -> {display_url}")
|
||||
except Exception as e:
|
||||
logger.debug(f"处理标签页出错: {e}")
|
||||
continue
|
||||
|
||||
except Exception as e:
|
||||
logger.warning(f"扫描标签页失败: {e}")
|
||||
|
||||
# 重置所有状态为 IDLE
|
||||
for session in self._tabs.values():
|
||||
# 锁序考量:状态迁移须在 session._lock 内进行,与 acquire/release 的
|
||||
# 状态检查互斥。既有锁序为 池锁 → session._lock(release 等路径同序),
|
||||
# session._lock 内不会反向取池锁,无死锁风险。
|
||||
with session._lock:
|
||||
session.status = TabStatus.IDLE
|
||||
session.current_task_id = None
|
||||
self._start_global_monitor_for_session(session)
|
||||
|
||||
self._initialized = True
|
||||
self._last_scan_time = time.time()
|
||||
ignored_count = max(0, raw_target_count - len(self._tabs))
|
||||
logger.info(
|
||||
f"TabPool 就绪: {len(self._tabs)} 个远程网页"
|
||||
+ (f"(已忽略 {ignored_count} 个内部/无效 target)" if ignored_count else "")
|
||||
)
|
||||
def refresh_tabs(self) -> Dict:
|
||||
"""手动刷新标签页列表(供外部调用)"""
|
||||
with self._condition:
|
||||
old_count = len(self._tabs)
|
||||
old_ids = set(self._tabs.keys())
|
||||
|
||||
# 强制扫描(不受时间间隔限制)
|
||||
self._last_scan_time = 0
|
||||
self._scan_new_tabs()
|
||||
|
||||
# 同时清理不健康的标签页
|
||||
self._cleanup_unhealthy_tabs()
|
||||
|
||||
new_ids = set(self._tabs.keys())
|
||||
added = new_ids - old_ids
|
||||
removed = old_ids - new_ids
|
||||
|
||||
if added or removed:
|
||||
self._condition.notify_all()
|
||||
logger.info(f"刷新完成: +{len(added)} -{len(removed)} = {len(self._tabs)} 个标签页")
|
||||
|
||||
return {
|
||||
"added": len(added),
|
||||
"removed": len(removed),
|
||||
"total": len(self._tabs)
|
||||
}
|
||||
@@ -0,0 +1,530 @@
|
||||
"""
|
||||
app/core/tab_pool_parts/recovery.py - 错误标签页隔离与 AI 恢复服务
|
||||
|
||||
职责:
|
||||
- TabQuarantineEntry:错误标签页的隔离记录(由 TabPoolManager 在池锁内维护)
|
||||
- TabRecoveryService:单线程后台服务,等旧 worker 退出后按配置决定
|
||||
截图 → AI 判断 → 刷新 → 解除隔离 / 永久隔离
|
||||
|
||||
锁纪律:
|
||||
- 本模块所有阻塞操作(等待旧 worker、截图、AI HTTP 调用、刷新)都发生在
|
||||
自持的 ThreadPoolExecutor(max_workers=1) 线程里,绝不在事件循环线程执行,
|
||||
也绝不持有 TabPoolManager 的池锁。
|
||||
- 与 TabPoolManager 的交互只通过构造时注入的两个回调
|
||||
(resolve_quarantine / get_quarantine_entry),本模块不 import manager,
|
||||
避免循环导入;request_manager 的 import 放在函数内。
|
||||
"""
|
||||
import base64
|
||||
import json
|
||||
import threading
|
||||
import time
|
||||
import urllib.request
|
||||
from collections import OrderedDict
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Callable, Dict, Optional, Tuple
|
||||
|
||||
from app.core.config import AppConfig, logger
|
||||
|
||||
|
||||
# DrissionPage/CDP 调用可能在底层连接断开时永久阻塞。超时后 daemon
|
||||
# 线程无法被强制终止,因此限制同时存活的超时调用数量,避免每次异常
|
||||
# 都累积一个永不退出的线程。
|
||||
_RECOVERY_CALL_SLOTS = threading.BoundedSemaphore(4)
|
||||
|
||||
|
||||
@dataclass
|
||||
class TabQuarantineEntry:
|
||||
"""错误标签页的隔离记录(在 TabPoolManager 的池锁保护下读写)"""
|
||||
raw_tab_id: str
|
||||
persistent_index: int = 0
|
||||
task_id: str = ""
|
||||
tab: Any = None
|
||||
reason: str = ""
|
||||
since: float = field(default_factory=time.monotonic)
|
||||
permanent: bool = False
|
||||
attempts: int = 0
|
||||
session_id: str = ""
|
||||
url: str = ""
|
||||
|
||||
|
||||
class TabRecoveryService:
|
||||
"""错误标签页 AI 恢复服务(由 TabPoolManager 懒创建并持有,单例风格)"""
|
||||
|
||||
WORKER_POLL_INTERVAL_SEC = 5.0
|
||||
# 拿不到旧 worker 线程引用时的时间宽限:隔离时刻起 180s 视为已退出
|
||||
WORKER_EXIT_FALLBACK_GRACE_SEC = 180.0
|
||||
SCREENSHOT_TIMEOUT_SEC = 30.0
|
||||
REFRESH_TIMEOUT_SEC = 30.0
|
||||
POST_REFRESH_SETTLE_SEC = 3.0
|
||||
MAX_ATTEMPT_CACHE_ENTRIES = 4096
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
resolve_quarantine: Callable[[str, bool], bool],
|
||||
get_quarantine_entry: Callable[[str], Optional[TabQuarantineEntry]],
|
||||
):
|
||||
self._resolve_quarantine = resolve_quarantine
|
||||
self._get_quarantine_entry = get_quarantine_entry
|
||||
self._attempts: OrderedDict[str, int] = OrderedDict()
|
||||
self._attempts_lock = threading.Lock()
|
||||
self._shutdown = False
|
||||
self._executor = ThreadPoolExecutor(
|
||||
max_workers=1,
|
||||
thread_name_prefix="tab-recovery",
|
||||
)
|
||||
|
||||
# ================= 对外接口 =================
|
||||
|
||||
def submit(self, entry: TabQuarantineEntry) -> bool:
|
||||
"""受理一个隔离标签页;仅入队,不阻塞调用线程。"""
|
||||
if entry is None or not str(getattr(entry, "raw_tab_id", "") or "").strip():
|
||||
return False
|
||||
if self._shutdown:
|
||||
return False
|
||||
try:
|
||||
self._executor.submit(self._process_safe, entry)
|
||||
logger.info(
|
||||
f"[TabRecovery] 已受理隔离标签页,排队处理 "
|
||||
f"(raw={entry.raw_tab_id}, tab={entry.session_id or '-'}, "
|
||||
f"idx=#{entry.persistent_index or '-'}, task={entry.task_id or '-'}, "
|
||||
f"reason={entry.reason or '-'})"
|
||||
)
|
||||
return True
|
||||
except RuntimeError as e:
|
||||
logger.debug(f"[TabRecovery] submit failed ({entry.raw_tab_id}): {e}")
|
||||
return False
|
||||
|
||||
def shutdown(self) -> None:
|
||||
self._shutdown = True
|
||||
with self._attempts_lock:
|
||||
self._attempts.clear()
|
||||
try:
|
||||
self._executor.shutdown(wait=False, cancel_futures=True)
|
||||
except Exception as e:
|
||||
logger.debug(f"[TabRecovery] executor shutdown failed: {e}")
|
||||
|
||||
# ================= 主流程 =================
|
||||
|
||||
def _process_safe(self, entry: TabQuarantineEntry) -> None:
|
||||
"""线程池任务入口:吃掉所有异常,绝不向外抛。"""
|
||||
try:
|
||||
self._process(entry)
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
f"[TabRecovery] 恢复流程异常 (raw={entry.raw_tab_id}): {e}"
|
||||
)
|
||||
# 未知内部错误时保守处理:永久隔离,避免把状态不明的标签页重新入池。
|
||||
try:
|
||||
self._resolve(entry, release=False, reason=f"internal_error:{e}")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
def _process(self, entry: TabQuarantineEntry) -> None:
|
||||
raw_id = entry.raw_tab_id
|
||||
logger.info(
|
||||
f"[TabRecovery] 开始处理隔离标签页 "
|
||||
f"(raw={raw_id}, tab={entry.session_id or '-'}, "
|
||||
f"task={entry.task_id or '-'}, reason={entry.reason or '-'})"
|
||||
)
|
||||
|
||||
# 1. 等旧 worker 退出:串扰的根源是旧请求的 worker 线程可能仍阻塞在
|
||||
# 对该 tab 的同步 DrissionPage 调用里。
|
||||
wait_result = self._wait_for_worker_exit(entry)
|
||||
if wait_result == "aborted":
|
||||
logger.debug(f"[TabRecovery] 服务关闭,放弃处理 (raw={raw_id})")
|
||||
return
|
||||
if wait_result == "timeout":
|
||||
self._resolve(entry, release=False, reason="worker_never_exited")
|
||||
return
|
||||
logger.debug(f"[TabRecovery] 旧 worker 已确认退出 (raw={raw_id})")
|
||||
|
||||
# 2. 次数检查:解除隔离后再次错误 → 不再调 API,直接永久隔离。
|
||||
with self._attempts_lock:
|
||||
attempts = int(self._attempts.get(raw_id, 0))
|
||||
if raw_id in self._attempts:
|
||||
# 最近使用的 raw id 保留在缓存尾部,便于后续按上限淘汰。
|
||||
self._attempts.move_to_end(raw_id)
|
||||
entry.attempts = attempts
|
||||
max_attempts = AppConfig.get_tab_recovery_max_attempts()
|
||||
if attempts >= max_attempts:
|
||||
logger.info(
|
||||
f"[TabRecovery] 恢复次数已用尽,不再调用 AI "
|
||||
f"(raw={raw_id}, attempts={attempts}, max={max_attempts})"
|
||||
)
|
||||
self._resolve(entry, release=False, reason="attempts_exhausted")
|
||||
return
|
||||
|
||||
# 3. 功能开关:未启用时不截图不调 API,直接解除隔离——保持既有的
|
||||
# 自动重入池行为,但此时旧 worker 已确认退出,串扰已消除。
|
||||
if not AppConfig.is_tab_recovery_enabled():
|
||||
self._resolve(entry, release=True, reason="recovery_disabled")
|
||||
return
|
||||
|
||||
# 4/5. 截图 + AI 判断;任一失败 → verdict=None(未知)
|
||||
verdict: Optional[bool] = None
|
||||
detail = ""
|
||||
image_b64 = ""
|
||||
try:
|
||||
image_b64 = self._capture_screenshot(entry)
|
||||
except Exception as e:
|
||||
detail = f"screenshot_failed:{e}"
|
||||
logger.warning(f"[TabRecovery] 截图失败 (raw={raw_id}): {e}")
|
||||
|
||||
if image_b64:
|
||||
try:
|
||||
verdict, detail = self._judge_recoverable(entry, image_b64)
|
||||
logger.info(
|
||||
f"[TabRecovery] AI 判断结果 "
|
||||
f"(raw={raw_id}, recoverable={verdict}, detail={detail or '-'})"
|
||||
)
|
||||
except Exception as e:
|
||||
verdict, detail = None, f"api_error:{e}"
|
||||
logger.warning(f"[TabRecovery] AI 判断失败 (raw={raw_id}): {e}")
|
||||
|
||||
# 6. 决策
|
||||
if verdict is False:
|
||||
self._resolve(
|
||||
entry,
|
||||
release=False,
|
||||
reason=f"ai_judged_unrecoverable:{detail or '-'}",
|
||||
)
|
||||
return
|
||||
if verdict is None:
|
||||
if not AppConfig.is_tab_recovery_refresh_on_unknown():
|
||||
self._resolve(
|
||||
entry,
|
||||
release=False,
|
||||
reason=f"judgement_unknown:{detail or '-'}",
|
||||
)
|
||||
return
|
||||
refresh_reason = "unknown_refresh"
|
||||
else:
|
||||
refresh_reason = "ai_judged_recoverable"
|
||||
|
||||
# 7. 刷新与解除
|
||||
self._refresh_and_release(entry, refresh_reason)
|
||||
|
||||
# ================= 步骤实现 =================
|
||||
|
||||
def _wait_for_worker_exit(self, entry: TabQuarantineEntry) -> str:
|
||||
"""轮询旧 worker 是否退出。返回 'exited' / 'timeout' / 'aborted'。"""
|
||||
max_wait = AppConfig.get_tab_recovery_worker_exit_wait_sec()
|
||||
deadline = time.monotonic() + max(0.0, float(max_wait))
|
||||
while True:
|
||||
if self._shutdown:
|
||||
return "aborted"
|
||||
if self._is_worker_exited(entry):
|
||||
return "exited"
|
||||
remaining = deadline - time.monotonic()
|
||||
if remaining <= 0:
|
||||
return "timeout"
|
||||
time.sleep(min(self.WORKER_POLL_INTERVAL_SEC, max(0.1, remaining)))
|
||||
|
||||
def _is_worker_exited(self, entry: TabQuarantineEntry) -> bool:
|
||||
task_id = str(entry.task_id or "").strip()
|
||||
if not task_id:
|
||||
# 隔离时没有绑定任务,无 worker 可等
|
||||
return True
|
||||
|
||||
ctx = None
|
||||
try:
|
||||
from app.services.request_manager import request_manager
|
||||
|
||||
ctx = request_manager.get_request(task_id)
|
||||
except Exception as e:
|
||||
logger.debug(
|
||||
f"[TabRecovery] 查询请求上下文失败 (task={task_id}): {e}"
|
||||
)
|
||||
|
||||
thread = getattr(ctx, "worker_thread", None) if ctx is not None else None
|
||||
if thread is not None and hasattr(thread, "is_alive"):
|
||||
try:
|
||||
return not thread.is_alive()
|
||||
except Exception as e:
|
||||
logger.debug(
|
||||
f"[TabRecovery] 读取 worker 线程状态失败 (task={task_id}): {e}"
|
||||
)
|
||||
|
||||
# 拿不到线程引用(非流式请求 worker 在 request_lifecycle 内创建、
|
||||
# 或请求上下文已被清理):回退时间宽限,隔离时刻起 180s 视为已退出。
|
||||
elapsed = time.monotonic() - float(entry.since or 0.0)
|
||||
return elapsed >= self.WORKER_EXIT_FALLBACK_GRACE_SEC
|
||||
|
||||
def _capture_screenshot(self, entry: TabQuarantineEntry) -> str:
|
||||
tab = entry.tab
|
||||
if tab is None:
|
||||
raise RuntimeError("tab reference unavailable")
|
||||
# DrissionPage 4.x:get_screenshot(as_base64=True) 返回 PNG 的 base64 字符串
|
||||
data = self._call_with_timeout(
|
||||
lambda: tab.get_screenshot(as_base64=True),
|
||||
self.SCREENSHOT_TIMEOUT_SEC,
|
||||
"screenshot",
|
||||
)
|
||||
if isinstance(data, (bytes, bytearray)):
|
||||
data = base64.b64encode(bytes(data)).decode("ascii")
|
||||
text = str(data or "").strip()
|
||||
if not text:
|
||||
raise RuntimeError("empty screenshot payload")
|
||||
return text
|
||||
|
||||
def _judge_recoverable(
|
||||
self,
|
||||
entry: TabQuarantineEntry,
|
||||
image_b64: str,
|
||||
) -> Tuple[Optional[bool], str]:
|
||||
"""调用 OpenAI 兼容接口判断刷新能否恢复。返回 (verdict, detail)。"""
|
||||
api_url = AppConfig.get_tab_recovery_api_url()
|
||||
if api_url:
|
||||
target_url = api_url
|
||||
api_key = AppConfig.get_tab_recovery_api_key()
|
||||
local_mode = False
|
||||
else:
|
||||
# 本地模式:走本服务自身的 /v1/chat/completions,
|
||||
# 由其他空闲标签页完成实际判断
|
||||
target_url = (
|
||||
f"http://127.0.0.1:{AppConfig.get_port()}/v1/chat/completions"
|
||||
)
|
||||
api_key = (
|
||||
AppConfig.get_auth_token() if AppConfig.is_auth_enabled() else ""
|
||||
)
|
||||
local_mode = True
|
||||
|
||||
site_url = str(entry.url or "").strip() or "未知"
|
||||
prompt = (
|
||||
"这是反代服务的一个疑似卡死的网页标签页截图,"
|
||||
f"站点为 {site_url},错误原因 {entry.reason or '未知'}。"
|
||||
"请判断刷新页面是否可能恢复正常使用。只输出 JSON:"
|
||||
"{\"recoverable\": true/false, \"reason\": \"...\"}。"
|
||||
"登录失效、封号提示、人机验证页输出 false;"
|
||||
"加载中、白屏、内容正常、临时错误提示输出 true。"
|
||||
)
|
||||
payload = {
|
||||
"model": AppConfig.get_tab_recovery_model(),
|
||||
"stream": False,
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": prompt},
|
||||
{
|
||||
"type": "image_url",
|
||||
"image_url": {
|
||||
"url": f"data:image/png;base64,{image_b64}",
|
||||
},
|
||||
},
|
||||
],
|
||||
}
|
||||
],
|
||||
}
|
||||
headers = {"Content-Type": "application/json"}
|
||||
if api_key:
|
||||
headers["Authorization"] = f"Bearer {api_key}"
|
||||
|
||||
timeout_sec = AppConfig.get_tab_recovery_timeout_sec()
|
||||
request = urllib.request.Request(
|
||||
target_url,
|
||||
data=json.dumps(payload).encode("utf-8"),
|
||||
headers=headers,
|
||||
method="POST",
|
||||
)
|
||||
logger.debug(
|
||||
f"[TabRecovery] 调用 AI 判断 "
|
||||
f"(raw={entry.raw_tab_id}, url={target_url}, "
|
||||
f"local_mode={local_mode}, timeout={timeout_sec}s)"
|
||||
)
|
||||
if local_mode:
|
||||
# 本地回环地址不应经过环境代理
|
||||
opener = urllib.request.build_opener(urllib.request.ProxyHandler({}))
|
||||
response = opener.open(request, timeout=timeout_sec)
|
||||
else:
|
||||
response = urllib.request.urlopen(request, timeout=timeout_sec)
|
||||
with response:
|
||||
body = response.read().decode("utf-8", "replace")
|
||||
|
||||
data = json.loads(body)
|
||||
choices = data.get("choices") or []
|
||||
if not choices:
|
||||
return None, "empty_choices"
|
||||
message = choices[0].get("message") or {}
|
||||
content = message.get("content")
|
||||
text = self._flatten_message_content(content)
|
||||
parsed = self._extract_first_json_object(text)
|
||||
if not isinstance(parsed, dict):
|
||||
return None, f"parse_failed:{text[:120]!r}"
|
||||
verdict = self._coerce_bool(parsed.get("recoverable"))
|
||||
detail = str(parsed.get("reason") or "").strip()
|
||||
if verdict is None:
|
||||
return None, f"invalid_recoverable_field:{parsed.get('recoverable')!r}"
|
||||
return verdict, detail
|
||||
|
||||
def _refresh_and_release(
|
||||
self,
|
||||
entry: TabQuarantineEntry,
|
||||
refresh_reason: str,
|
||||
) -> None:
|
||||
raw_id = entry.raw_tab_id
|
||||
try:
|
||||
tab = entry.tab
|
||||
if tab is None:
|
||||
raise RuntimeError("tab reference unavailable")
|
||||
self._call_with_timeout(
|
||||
lambda: tab.refresh(),
|
||||
self.REFRESH_TIMEOUT_SEC,
|
||||
"refresh",
|
||||
)
|
||||
time.sleep(max(0.0, float(self.POST_REFRESH_SETTLE_SEC)))
|
||||
current_url = self._call_with_timeout(
|
||||
lambda: str(getattr(tab, "url", "") or ""),
|
||||
self.REFRESH_TIMEOUT_SEC,
|
||||
"health-check",
|
||||
)
|
||||
if not str(current_url or "").strip():
|
||||
raise RuntimeError("empty url after refresh")
|
||||
except Exception as e:
|
||||
logger.warning(f"[TabRecovery] 刷新/健康检查失败 (raw={raw_id}): {e}")
|
||||
self._resolve(entry, release=False, reason=f"refresh_failed:{e}")
|
||||
return
|
||||
|
||||
with self._attempts_lock:
|
||||
self._attempts[raw_id] = int(self._attempts.get(raw_id, 0)) + 1
|
||||
self._attempts.move_to_end(raw_id)
|
||||
entry.attempts = self._attempts[raw_id]
|
||||
while len(self._attempts) > self.MAX_ATTEMPT_CACHE_ENTRIES:
|
||||
self._attempts.popitem(last=False)
|
||||
self._resolve(entry, release=True, reason=refresh_reason)
|
||||
logger.info(
|
||||
f"[TabRecovery] 标签页已恢复并重新入池 "
|
||||
f"(raw={raw_id}, idx=#{entry.persistent_index or '-'}, "
|
||||
f"url={str(current_url)[:80]}, attempts={entry.attempts}, "
|
||||
f"reason={refresh_reason})"
|
||||
)
|
||||
|
||||
# ================= 辅助 =================
|
||||
|
||||
def _resolve(
|
||||
self,
|
||||
entry: TabQuarantineEntry,
|
||||
*,
|
||||
release: bool,
|
||||
reason: str,
|
||||
) -> None:
|
||||
entry.reason = str(reason or "").strip() or entry.reason
|
||||
try:
|
||||
resolved = bool(self._resolve_quarantine(entry.raw_tab_id, release))
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
f"[TabRecovery] 解除/永久隔离操作失败 "
|
||||
f"(raw={entry.raw_tab_id}, release={release}): {e}"
|
||||
)
|
||||
return
|
||||
if release:
|
||||
logger.info(
|
||||
f"[TabRecovery] 解除隔离,标签页将由下次扫描重新入池 "
|
||||
f"(raw={entry.raw_tab_id}, reason={reason}, resolved={resolved})"
|
||||
)
|
||||
else:
|
||||
logger.info(
|
||||
f"[TabRecovery] 永久隔离标签页 "
|
||||
f"(raw={entry.raw_tab_id}, reason={reason}, resolved={resolved})"
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _call_with_timeout(fn: Callable[[], Any], timeout_sec: float, label: str) -> Any:
|
||||
"""在守护线程中执行阻塞调用并限时等待,防止 CDP 调用无限挂起。"""
|
||||
wait_timeout = max(0.1, float(timeout_sec))
|
||||
if not _RECOVERY_CALL_SLOTS.acquire(timeout=wait_timeout):
|
||||
raise TimeoutError(f"{label} call slots exhausted")
|
||||
|
||||
result: Dict[str, Any] = {}
|
||||
done = threading.Event()
|
||||
|
||||
def _runner():
|
||||
try:
|
||||
result["value"] = fn()
|
||||
except BaseException as e: # noqa: BLE001 - 结果原样转抛给调用方
|
||||
result["error"] = e
|
||||
finally:
|
||||
done.set()
|
||||
_RECOVERY_CALL_SLOTS.release()
|
||||
|
||||
thread = threading.Thread(
|
||||
target=_runner,
|
||||
name=f"tab-recovery-{label}",
|
||||
daemon=True,
|
||||
)
|
||||
try:
|
||||
thread.start()
|
||||
except BaseException:
|
||||
_RECOVERY_CALL_SLOTS.release()
|
||||
raise
|
||||
if not done.wait(wait_timeout):
|
||||
raise TimeoutError(f"{label} timed out after {timeout_sec}s")
|
||||
if "error" in result:
|
||||
raise result["error"]
|
||||
return result.get("value")
|
||||
|
||||
@staticmethod
|
||||
def _flatten_message_content(content: Any) -> str:
|
||||
if isinstance(content, str):
|
||||
return content
|
||||
if isinstance(content, list):
|
||||
parts = []
|
||||
for item in content:
|
||||
if isinstance(item, dict):
|
||||
text = item.get("text")
|
||||
if isinstance(text, str):
|
||||
parts.append(text)
|
||||
elif isinstance(item, str):
|
||||
parts.append(item)
|
||||
return "\n".join(parts)
|
||||
return str(content or "")
|
||||
|
||||
@staticmethod
|
||||
def _extract_first_json_object(text: str) -> Optional[Dict[str, Any]]:
|
||||
"""宽松提取文本中第一个平衡的 {...} 并解析为 dict。"""
|
||||
source = str(text or "")
|
||||
start = source.find("{")
|
||||
while start != -1:
|
||||
depth = 0
|
||||
in_string = False
|
||||
escaped = False
|
||||
for index in range(start, len(source)):
|
||||
char = source[index]
|
||||
if in_string:
|
||||
if escaped:
|
||||
escaped = False
|
||||
elif char == "\\":
|
||||
escaped = True
|
||||
elif char == '"':
|
||||
in_string = False
|
||||
continue
|
||||
if char == '"':
|
||||
in_string = True
|
||||
elif char == "{":
|
||||
depth += 1
|
||||
elif char == "}":
|
||||
depth -= 1
|
||||
if depth == 0:
|
||||
candidate = source[start:index + 1]
|
||||
try:
|
||||
parsed = json.loads(candidate)
|
||||
if isinstance(parsed, dict):
|
||||
return parsed
|
||||
except Exception:
|
||||
pass
|
||||
break
|
||||
start = source.find("{", start + 1)
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _coerce_bool(value: Any) -> Optional[bool]:
|
||||
if isinstance(value, bool):
|
||||
return value
|
||||
text = str(value or "").strip().lower()
|
||||
if text in {"true", "1", "yes", "y"}:
|
||||
return True
|
||||
if text in {"false", "0", "no", "n"}:
|
||||
return False
|
||||
return None
|
||||
@@ -0,0 +1,994 @@
|
||||
import threading
|
||||
import time
|
||||
from dataclasses import dataclass, field
|
||||
from enum import Enum
|
||||
from typing import Any, Dict, Optional
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
from app.core.config import logger
|
||||
from app.utils.site_url import (
|
||||
encode_tab_url_route_token,
|
||||
extract_remote_site_domain,
|
||||
get_preferred_route_domain,
|
||||
)
|
||||
|
||||
from ._utils import _TAB_HEALTH_CACHE_TTL_SEC, _should_skip_pool_url
|
||||
from app.core.driver import driver_for_tab
|
||||
|
||||
|
||||
class TabStatus(Enum):
|
||||
"""标签页状态"""
|
||||
IDLE = "idle"
|
||||
BUSY = "busy"
|
||||
ERROR = "error"
|
||||
CLOSED = "closed"
|
||||
|
||||
@dataclass
|
||||
class TabSession:
|
||||
"""标签页会话"""
|
||||
id: str
|
||||
tab: Any
|
||||
status: TabStatus = TabStatus.IDLE
|
||||
current_task_id: Optional[str] = None
|
||||
current_command_name: Optional[str] = None
|
||||
current_domain: Optional[str] = None
|
||||
last_known_url: Optional[str] = None
|
||||
created_at: float = field(default_factory=time.time)
|
||||
last_used_at: float = field(default_factory=time.time)
|
||||
request_count: int = 0
|
||||
error_count: int = 0
|
||||
persistent_index: int = 0 # 🆕 持久化编号(重启前不变)
|
||||
preset_name: Optional[str] = None # 🆕 当前显式指定的预设名称(None = 跟随站点默认预设)
|
||||
preset_override_source: Optional[str] = None # 预设来源(url: 来自 URL 记忆, tab: 标签页临时指定, None: 默认)
|
||||
model_name_override: Optional[str] = None # 当前标签页临时暴露模型名(关闭标签页后失效)
|
||||
browser_context_id: Optional[str] = None
|
||||
is_isolated_context: bool = False
|
||||
transient_disconnect_until: float = 0.0
|
||||
transient_disconnect_reason: Optional[str] = None
|
||||
last_conversation_activity_at: float = 0.0
|
||||
last_conversation_domain: Optional[str] = None
|
||||
last_conversation_preset_name: Optional[str] = None
|
||||
command_loop_started_at: float = 0.0
|
||||
command_loop_iteration: Optional[int] = None
|
||||
command_loop_total: Optional[int] = None
|
||||
command_loop_label: Optional[str] = None
|
||||
command_loop_cancel_requested: bool = False
|
||||
command_loop_cancel_reason: Optional[str] = None
|
||||
command_loop_cancel_requested_at: float = 0.0
|
||||
_health_cache_until: float = field(default=0.0, repr=False)
|
||||
_health_cache_result: bool = field(default=False, repr=False)
|
||||
_health_cache_url: str = field(default="", repr=False)
|
||||
_health_cache_domain: str = field(default="", repr=False)
|
||||
_release_in_progress: bool = field(default=False, repr=False)
|
||||
_termination_in_progress: bool = field(default=False, repr=False)
|
||||
_termination_task_id: Optional[str] = field(default=None, repr=False)
|
||||
_termination_interrupt_done: bool = field(default=False, repr=False)
|
||||
_workflow_parser_id: str = field(default="", repr=False)
|
||||
_workflow_target_side: str = field(default="", repr=False)
|
||||
_workflow_runtime_id: str = field(default="", repr=False)
|
||||
_request_occupied_url: Optional[str] = field(default=None, repr=False)
|
||||
|
||||
_lock: threading.RLock = field(default_factory=threading.RLock, repr=False)
|
||||
|
||||
|
||||
@property
|
||||
def driver(self):
|
||||
"""R2-1:该标签页的统一驱动(按标签页对象复用同一个包装)。迁移中的代码通过它访问浏览器。"""
|
||||
return driver_for_tab(self.tab)
|
||||
|
||||
def is_available(self) -> bool:
|
||||
with self._lock:
|
||||
return self.status == TabStatus.IDLE and not self._termination_in_progress
|
||||
|
||||
def is_in_transient_disconnect(self) -> bool:
|
||||
return self.transient_disconnect_until > time.time()
|
||||
|
||||
def mark_transient_disconnect(self, seconds: float, reason: str = None):
|
||||
with self._lock:
|
||||
self.transient_disconnect_until = time.time() + max(0.1, float(seconds))
|
||||
self.transient_disconnect_reason = str(reason or "").strip() or None
|
||||
|
||||
def clear_transient_disconnect(self):
|
||||
with self._lock:
|
||||
self.transient_disconnect_until = 0.0
|
||||
self.transient_disconnect_reason = None
|
||||
|
||||
def set_preset(self, preset_name: Optional[str], source: Optional[str] = None) -> bool:
|
||||
"""线程安全地设置预设及来源"""
|
||||
with self._lock:
|
||||
old_preset = self.preset_name
|
||||
self.preset_name = str(preset_name).strip() if preset_name else None
|
||||
self.preset_override_source = str(source).strip() if (source and self.preset_name) else None
|
||||
if old_preset != self.preset_name:
|
||||
self._reset_conversation_state_unlocked()
|
||||
return True
|
||||
|
||||
def _clear_health_cache_unlocked(self):
|
||||
self._health_cache_until = 0.0
|
||||
self._health_cache_result = False
|
||||
self._health_cache_url = ""
|
||||
self._health_cache_domain = ""
|
||||
|
||||
def _clear_command_loop_unlocked(self):
|
||||
self.command_loop_started_at = 0.0
|
||||
self.command_loop_iteration = None
|
||||
self.command_loop_total = None
|
||||
self.command_loop_label = None
|
||||
self.command_loop_cancel_requested = False
|
||||
self.command_loop_cancel_reason = None
|
||||
self.command_loop_cancel_requested_at = 0.0
|
||||
|
||||
def begin_command_loop(
|
||||
self,
|
||||
*,
|
||||
iteration: Optional[int] = None,
|
||||
total: Optional[int] = None,
|
||||
label: str = "",
|
||||
) -> Dict[str, Any]:
|
||||
now = time.time()
|
||||
with self._lock:
|
||||
self.command_loop_started_at = now
|
||||
self.command_loop_iteration = iteration if iteration is None else int(iteration)
|
||||
self.command_loop_total = total if total is None else int(total)
|
||||
self.command_loop_label = str(label or "").strip() or None
|
||||
self.command_loop_cancel_requested = False
|
||||
self.command_loop_cancel_reason = None
|
||||
self.command_loop_cancel_requested_at = 0.0
|
||||
return self._command_loop_info_unlocked(now=now)
|
||||
|
||||
def end_command_loop(self, status: str = "completed") -> Dict[str, Any]:
|
||||
now = time.time()
|
||||
with self._lock:
|
||||
info = self._command_loop_info_unlocked(now=now)
|
||||
info["final_status"] = str(status or "").strip() or "completed"
|
||||
self._clear_command_loop_unlocked()
|
||||
return info
|
||||
|
||||
def request_command_loop_cancel(self, reason: str = "manual_loop_cancel") -> bool:
|
||||
now = time.time()
|
||||
with self._lock:
|
||||
if not self.command_loop_started_at:
|
||||
return False
|
||||
self.command_loop_cancel_requested = True
|
||||
self.command_loop_cancel_reason = str(reason or "").strip() or "manual_loop_cancel"
|
||||
self.command_loop_cancel_requested_at = now
|
||||
return True
|
||||
|
||||
def is_command_loop_cancelled(self) -> bool:
|
||||
with self._lock:
|
||||
return bool(self.command_loop_cancel_requested)
|
||||
|
||||
def _command_loop_info_unlocked(self, *, now: Optional[float] = None) -> Dict[str, Any]:
|
||||
current_time = time.time() if now is None else float(now)
|
||||
started_at = float(self.command_loop_started_at or 0.0)
|
||||
active = started_at > 0
|
||||
elapsed = max(0.0, current_time - started_at) if active else 0.0
|
||||
return {
|
||||
"active": active,
|
||||
"iteration": self.command_loop_iteration,
|
||||
"total": self.command_loop_total,
|
||||
"label": self.command_loop_label or "",
|
||||
"started_at": started_at or None,
|
||||
"elapsed_sec": round(elapsed, 1),
|
||||
"cancel_requested": bool(self.command_loop_cancel_requested),
|
||||
"cancel_reason": self.command_loop_cancel_reason or "",
|
||||
}
|
||||
|
||||
def get_command_loop_info(self) -> Dict[str, Any]:
|
||||
with self._lock:
|
||||
return self._command_loop_info_unlocked()
|
||||
|
||||
def touch_activity(self) -> None:
|
||||
with self._lock:
|
||||
self.last_used_at = time.time()
|
||||
|
||||
def get_busy_timeout_info(self, *, now: Optional[float] = None) -> Dict[str, Any]:
|
||||
current_time = time.time() if now is None else float(now)
|
||||
with self._lock:
|
||||
loop_started_at = float(self.command_loop_started_at or 0.0)
|
||||
if loop_started_at > 0:
|
||||
reference_at = loop_started_at
|
||||
basis = "command_loop"
|
||||
else:
|
||||
reference_at = float(self.last_used_at or current_time)
|
||||
basis = "session"
|
||||
return {
|
||||
"basis": basis,
|
||||
"started_at": reference_at,
|
||||
"duration": max(0.0, current_time - reference_at),
|
||||
"command_loop": self._command_loop_info_unlocked(now=current_time),
|
||||
}
|
||||
|
||||
def is_healthy(self, *, allow_live_check: bool = True) -> bool:
|
||||
"""检查标签页是否健康,避免在忙碌时额外触发 live CDP 读取。"""
|
||||
now = time.time()
|
||||
with self._lock:
|
||||
status = self.status
|
||||
cached_url = str(self.last_known_url or "").strip()
|
||||
cached_domain = str(self.current_domain or "").strip()
|
||||
cache_valid = (
|
||||
status == TabStatus.IDLE
|
||||
and self._health_cache_until > now
|
||||
and self._health_cache_url == cached_url
|
||||
and self._health_cache_domain == cached_domain
|
||||
)
|
||||
if cache_valid:
|
||||
return bool(self._health_cache_result)
|
||||
|
||||
if status == TabStatus.CLOSED:
|
||||
return False
|
||||
|
||||
if status == TabStatus.BUSY:
|
||||
if cached_url:
|
||||
return not _should_skip_pool_url(cached_url)
|
||||
return bool(cached_domain)
|
||||
|
||||
if not allow_live_check:
|
||||
if cached_url:
|
||||
return not _should_skip_pool_url(cached_url)
|
||||
return bool(cached_domain)
|
||||
|
||||
url = self._safe_get_url()
|
||||
result = bool(url and not _should_skip_pool_url(url))
|
||||
with self._lock:
|
||||
self._health_cache_result = result
|
||||
self._health_cache_url = str(self.last_known_url or "").strip()
|
||||
self._health_cache_domain = str(self.current_domain or "").strip()
|
||||
self._health_cache_until = time.time() + _TAB_HEALTH_CACHE_TTL_SEC
|
||||
return result
|
||||
|
||||
def get_cached_route_snapshot(self) -> tuple[str, str]:
|
||||
"""Return cached URL/domain without touching the live tab/CDP endpoint."""
|
||||
with self._lock:
|
||||
cached_url = str(self.last_known_url or "").strip()
|
||||
cached_domain = str(self.current_domain or "").strip()
|
||||
if cached_url:
|
||||
cached_domain = self._refresh_current_domain(cached_url)
|
||||
return cached_url, cached_domain
|
||||
|
||||
def _debug_summary_unlocked(self) -> str:
|
||||
status_value = getattr(self.status, "value", str(self.status))
|
||||
return (
|
||||
f"idx=#{self.persistent_index or '-'}, "
|
||||
f"status={status_value}, "
|
||||
f"task={str(self.current_task_id or '').strip() or '-'}, "
|
||||
f"bound_req={str(getattr(self, '_bound_request_id', '') or '').strip() or '-'}, "
|
||||
f"cmd_req={str(getattr(self, '_command_request_id', '') or '').strip() or '-'}, "
|
||||
f"req_count={self.request_count}, "
|
||||
f"domain={str(self.current_domain or '').strip() or '-'}"
|
||||
)
|
||||
|
||||
def debug_summary(self) -> str:
|
||||
with self._lock:
|
||||
return self._debug_summary_unlocked()
|
||||
|
||||
# B5:解冻失败后的冷却期,期间 acquire 直接跳过该会话,避免每次都卡 2 秒 CDP 超时
|
||||
_RESUME_FAILURE_BACKOFF_SEC = 10.0
|
||||
|
||||
def acquire(self, task_id: str) -> bool:
|
||||
if self._resume_recently_failed():
|
||||
return False
|
||||
if not self._acquire_request_unlocked_entry(task_id):
|
||||
return False
|
||||
if not self._resume_after_acquire("acquire"):
|
||||
self._rollback_failed_acquire(task_id, request_mode=True)
|
||||
return False
|
||||
return True
|
||||
|
||||
def acquire_for_command(self, task_id: str) -> bool:
|
||||
"""Acquire tab for command execution without incrementing request counter."""
|
||||
if self._resume_recently_failed():
|
||||
return False
|
||||
if not self._acquire_command_unlocked_entry(task_id):
|
||||
return False
|
||||
if not self._resume_after_acquire("acquire_for_command"):
|
||||
self._rollback_failed_acquire(task_id, request_mode=False)
|
||||
return False
|
||||
return True
|
||||
|
||||
def _resume_recently_failed(self) -> bool:
|
||||
if not getattr(self, "_uwapi_frozen", False):
|
||||
return False
|
||||
failed_at = float(getattr(self, "_uwapi_resume_failed_at", 0.0) or 0.0)
|
||||
return bool(failed_at) and (time.time() - failed_at) < self._RESUME_FAILURE_BACKOFF_SEC
|
||||
|
||||
def _resume_after_acquire(self, reason: str) -> bool:
|
||||
"""P0-6:被空闲冻结的标签页在交给调用方之前恢复为 active。
|
||||
|
||||
B5:返回该会话是否可交付——解冻失败(仍处于冻结态)返回 False。
|
||||
"""
|
||||
try:
|
||||
from .idle_maintenance import resume_if_frozen
|
||||
|
||||
resume_if_frozen(self, reason=reason)
|
||||
except Exception as e:
|
||||
logger.debug(f"[{self.id}] resume frozen tab failed: {e}")
|
||||
return not bool(getattr(self, "_uwapi_frozen", False))
|
||||
|
||||
def _rollback_failed_acquire(self, task_id: str, *, request_mode: bool) -> None:
|
||||
"""B5:解冻失败时撤销本次占用,把会话退回 IDLE(保留冻结标记),不交付给调用方。"""
|
||||
with self._lock:
|
||||
if self.status != TabStatus.BUSY or str(self.current_task_id or "") != str(task_id or ""):
|
||||
return
|
||||
self.status = TabStatus.IDLE
|
||||
self.current_task_id = None
|
||||
if request_mode and self.request_count > 0:
|
||||
self.request_count -= 1
|
||||
self._clear_health_cache_unlocked()
|
||||
logger.warning(
|
||||
f"[{self.id}] 标签页解冻失败,本次不交付该会话(task={task_id}),"
|
||||
f"{self._RESUME_FAILURE_BACKOFF_SEC:.0f}s 内跳过"
|
||||
)
|
||||
|
||||
def _acquire_request_unlocked_entry(self, task_id: str) -> bool:
|
||||
with self._lock:
|
||||
if self.status != TabStatus.IDLE or self._termination_in_progress:
|
||||
return False
|
||||
|
||||
prev_status = self.status.value
|
||||
prev_task = str(self.current_task_id or "").strip()
|
||||
prev_request_count = self.request_count
|
||||
self.status = TabStatus.BUSY
|
||||
self.current_task_id = task_id
|
||||
self._clear_health_cache_unlocked()
|
||||
self._clear_command_loop_unlocked()
|
||||
self._workflow_parser_id = ""
|
||||
self._workflow_target_side = ""
|
||||
self._workflow_runtime_id = ""
|
||||
self._request_occupied_url = None
|
||||
setattr(self, "_last_cancel_request_task_id", None)
|
||||
setattr(self, "_last_cancel_request_reason", None)
|
||||
self.last_used_at = time.time()
|
||||
self.request_count += 1
|
||||
logger.debug(
|
||||
f"[{self.id}] 会话占用: mode=request, idx=#{self.persistent_index or '-'}, "
|
||||
f"prev_status={prev_status}, prev_task={prev_task or '-'}, "
|
||||
f"new_task={str(task_id or '').strip() or '-'}, "
|
||||
f"req_count={prev_request_count}->{self.request_count}"
|
||||
)
|
||||
return True
|
||||
|
||||
def _acquire_command_unlocked_entry(self, task_id: str) -> bool:
|
||||
with self._lock:
|
||||
if self.status != TabStatus.IDLE or self._termination_in_progress:
|
||||
return False
|
||||
prev_status = self.status.value
|
||||
prev_task = str(self.current_task_id or "").strip()
|
||||
self.status = TabStatus.BUSY
|
||||
self.current_task_id = task_id
|
||||
self._clear_health_cache_unlocked()
|
||||
self._clear_command_loop_unlocked()
|
||||
self._workflow_parser_id = ""
|
||||
self._workflow_target_side = ""
|
||||
self._workflow_runtime_id = ""
|
||||
setattr(self, "_last_cancel_request_task_id", None)
|
||||
setattr(self, "_last_cancel_request_reason", None)
|
||||
self.last_used_at = time.time()
|
||||
logger.debug(
|
||||
f"[{self.id}] 会话占用: mode=command, idx=#{self.persistent_index or '-'}, "
|
||||
f"prev_status={prev_status}, prev_task={prev_task or '-'}, "
|
||||
f"new_task={str(task_id or '').strip() or '-'}, "
|
||||
f"req_count={self.request_count}"
|
||||
)
|
||||
return True
|
||||
|
||||
def _begin_release_state(
|
||||
self,
|
||||
*,
|
||||
clear_page: bool,
|
||||
rollback_request_count: bool = False,
|
||||
force: bool = False,
|
||||
expected_task_id: str = "",
|
||||
reject_ownerless: bool = False,
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
with self._lock:
|
||||
if self._release_in_progress:
|
||||
action = "force_release" if force else "release"
|
||||
logger.debug(f"[{self.id}] {action} skipped: another release is already in progress")
|
||||
return None
|
||||
|
||||
expected_task = str(expected_task_id or "").strip()
|
||||
current_task = str(self.current_task_id or "").strip()
|
||||
if current_task and reject_ownerless and not expected_task:
|
||||
logger.warning(
|
||||
f"[{self.id}] release skipped without ownership (current_task={current_task})"
|
||||
)
|
||||
return None
|
||||
if expected_task and current_task != expected_task:
|
||||
logger.warning(
|
||||
f"[{self.id}] release ownership mismatch "
|
||||
f"(expected_task={expected_task}, current_task={current_task or '-'})"
|
||||
)
|
||||
return None
|
||||
|
||||
prev_status_obj = self.status
|
||||
prev_status = prev_status_obj.value
|
||||
prev_task = str(self.current_task_id or "").strip()
|
||||
prev_bound_request_id = str(getattr(self, "_bound_request_id", "") or "").strip()
|
||||
prev_command_request_id = str(getattr(self, "_command_request_id", "") or "").strip()
|
||||
prev_request_count = self.request_count
|
||||
prev_was_error = prev_status_obj == TabStatus.ERROR
|
||||
prev_was_closed = prev_status_obj == TabStatus.CLOSED
|
||||
|
||||
if rollback_request_count and self.request_count > 0 and (
|
||||
prev_task or prev_status_obj == TabStatus.BUSY
|
||||
):
|
||||
self.request_count -= 1
|
||||
|
||||
needs_cleanup_window = True
|
||||
if needs_cleanup_window:
|
||||
self._release_in_progress = True
|
||||
if not prev_was_error and not prev_was_closed:
|
||||
self.status = TabStatus.BUSY
|
||||
elif not prev_was_error and not prev_was_closed:
|
||||
self.status = TabStatus.IDLE
|
||||
|
||||
self.current_task_id = None
|
||||
self.current_command_name = None
|
||||
self._clear_health_cache_unlocked()
|
||||
self._clear_command_loop_unlocked()
|
||||
setattr(self, "_bound_request_id", None)
|
||||
setattr(self, "_command_request_id", None)
|
||||
setattr(self, "_command_loop_request_id", None)
|
||||
setattr(self, "_command_vars", {})
|
||||
self._workflow_parser_id = ""
|
||||
self._workflow_target_side = ""
|
||||
self._workflow_runtime_id = ""
|
||||
self._request_occupied_url = None
|
||||
self.last_used_at = time.time()
|
||||
|
||||
return {
|
||||
"prev_status": prev_status,
|
||||
"prev_task": prev_task,
|
||||
"prev_bound_request_id": prev_bound_request_id,
|
||||
"prev_command_request_id": prev_command_request_id,
|
||||
"prev_request_count": prev_request_count,
|
||||
"new_request_count": self.request_count,
|
||||
"prev_was_error": prev_was_error,
|
||||
"prev_was_closed": prev_was_closed,
|
||||
"needs_cleanup_window": needs_cleanup_window,
|
||||
}
|
||||
|
||||
def _finish_release_state(self, state: Dict[str, Any], success: bool) -> None:
|
||||
if not state.get("needs_cleanup_window"):
|
||||
return
|
||||
|
||||
with self._lock:
|
||||
if success:
|
||||
if (
|
||||
self.status == TabStatus.BUSY
|
||||
and not state.get("prev_was_error")
|
||||
and not state.get("prev_was_closed")
|
||||
):
|
||||
self.status = TabStatus.IDLE
|
||||
elif self.status != TabStatus.CLOSED:
|
||||
self.status = TabStatus.ERROR
|
||||
self.error_count += 1
|
||||
termination_task = str(self._termination_task_id or "").strip()
|
||||
released_task = str(state.get("prev_task") or "").strip()
|
||||
if self._termination_in_progress and self._termination_interrupt_done and (
|
||||
not termination_task or termination_task == released_task
|
||||
):
|
||||
self._termination_in_progress = False
|
||||
self._termination_task_id = None
|
||||
self._termination_interrupt_done = False
|
||||
self._release_in_progress = False
|
||||
|
||||
def begin_termination(self, expected_task_id: str = "") -> Dict[str, Any]:
|
||||
"""Quarantine this session until its current owner finishes releasing it."""
|
||||
with self._lock:
|
||||
expected_task = str(expected_task_id or "").strip()
|
||||
current_task = str(self.current_task_id or "").strip()
|
||||
if expected_task and current_task != expected_task:
|
||||
return {
|
||||
"ok": False,
|
||||
"error": "task_ownership_changed",
|
||||
"expected_task_id": expected_task,
|
||||
"current_task_id": current_task,
|
||||
}
|
||||
if self._release_in_progress:
|
||||
return {"ok": False, "error": "release_in_progress"}
|
||||
if self._termination_in_progress:
|
||||
return {
|
||||
"ok": False,
|
||||
"error": "termination_in_progress",
|
||||
"current_task_id": current_task,
|
||||
}
|
||||
|
||||
self._termination_in_progress = True
|
||||
self._termination_task_id = current_task or expected_task or None
|
||||
self._termination_interrupt_done = False
|
||||
return {
|
||||
"ok": True,
|
||||
"task_id": current_task,
|
||||
"status": self.status.value,
|
||||
"command_request_id": str(
|
||||
getattr(self, "_command_request_id", "") or ""
|
||||
).strip(),
|
||||
"command_loop_request_id": str(
|
||||
getattr(self, "_command_loop_request_id", "") or ""
|
||||
).strip(),
|
||||
}
|
||||
|
||||
def finish_ownerless_termination(self) -> None:
|
||||
"""Release the quarantine for a session that had no active owner."""
|
||||
with self._lock:
|
||||
if self.current_task_id or self._release_in_progress:
|
||||
return
|
||||
self._termination_in_progress = False
|
||||
self._termination_task_id = None
|
||||
self._termination_interrupt_done = False
|
||||
|
||||
def is_termination_in_progress(self) -> bool:
|
||||
with self._lock:
|
||||
return bool(self._termination_in_progress)
|
||||
|
||||
def interrupt_for_termination(self, clear_page: bool = False) -> bool:
|
||||
"""Interrupt page activity without making the session available for reuse."""
|
||||
self.clear_visibility_emulation("terminate")
|
||||
success = True
|
||||
try:
|
||||
if hasattr(self.tab, "stop_loading"):
|
||||
self.tab.stop_loading()
|
||||
driver_for_tab(self.tab).run_js("if (window.stop) { window.stop(); }")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if clear_page:
|
||||
try:
|
||||
self.tab.refresh()
|
||||
self.reset_conversation_state()
|
||||
self.notify_navigated(reason="termination_clear")
|
||||
except Exception as e:
|
||||
logger.warning(f"[{self.id}] terminate refresh failed: {e}")
|
||||
success = False
|
||||
with self._lock:
|
||||
self._termination_interrupt_done = True
|
||||
if not self.current_task_id and not self._release_in_progress:
|
||||
self._termination_in_progress = False
|
||||
self._termination_task_id = None
|
||||
self._termination_interrupt_done = False
|
||||
return success
|
||||
|
||||
def _run_release_from_state(
|
||||
self,
|
||||
state: Dict[str, Any],
|
||||
*,
|
||||
clear_page: bool,
|
||||
check_triggers: bool,
|
||||
rollback_request_count: bool,
|
||||
) -> None:
|
||||
clear_page_success = True
|
||||
if clear_page:
|
||||
try:
|
||||
self.tab.get("about:blank")
|
||||
with self._lock:
|
||||
self.current_domain = None
|
||||
self.last_known_url = None
|
||||
self.preset_name = None
|
||||
self.preset_override_source = None
|
||||
self._clear_health_cache_unlocked()
|
||||
self.last_conversation_activity_at = 0.0
|
||||
self.last_conversation_domain = None
|
||||
self.last_conversation_preset_name = None
|
||||
self.notify_navigated(reason="release_clear_page")
|
||||
except Exception as e:
|
||||
clear_page_success = False
|
||||
logger.debug(f"clear page failed: {e}")
|
||||
|
||||
logger.debug(
|
||||
f"[{self.id}] 会话释放: idx=#{self.persistent_index or '-'}, "
|
||||
f"prev_status={state['prev_status']}, prev_task={state['prev_task'] or '-'}, "
|
||||
f"prev_bound_req={state['prev_bound_request_id'] or '-'}, "
|
||||
f"prev_cmd_req={state['prev_command_request_id'] or '-'}, "
|
||||
f"clear_page={clear_page}, check_triggers={check_triggers}, "
|
||||
f"rollback_request_count={rollback_request_count}, "
|
||||
f"req_count={state['prev_request_count']}->{state['new_request_count']}, "
|
||||
f"clear_page_success={clear_page_success}"
|
||||
)
|
||||
|
||||
self.clear_visibility_emulation("release")
|
||||
self._finish_release_state(state, clear_page_success)
|
||||
|
||||
try:
|
||||
from app.services.command_engine import command_engine
|
||||
if check_triggers:
|
||||
command_engine.check_triggers(self)
|
||||
except Exception as e:
|
||||
logger.debug(f"命令触发检查异常: {e}")
|
||||
|
||||
def release(
|
||||
self,
|
||||
clear_page: bool = False,
|
||||
check_triggers: bool = True,
|
||||
rollback_request_count: bool = False,
|
||||
expected_task_id: str = "",
|
||||
) -> bool:
|
||||
state = self._begin_release_state(
|
||||
clear_page=clear_page,
|
||||
rollback_request_count=rollback_request_count,
|
||||
force=False,
|
||||
expected_task_id=expected_task_id,
|
||||
reject_ownerless=True,
|
||||
)
|
||||
if state is None:
|
||||
return False
|
||||
self._run_release_from_state(
|
||||
state,
|
||||
clear_page=clear_page,
|
||||
check_triggers=check_triggers,
|
||||
rollback_request_count=rollback_request_count,
|
||||
)
|
||||
return self.status == TabStatus.IDLE
|
||||
|
||||
def _run_force_release_from_state(
|
||||
self,
|
||||
state: Dict[str, Any],
|
||||
*,
|
||||
clear_page: bool,
|
||||
check_triggers: bool,
|
||||
) -> None:
|
||||
logger.warning(
|
||||
f"[{self.id}] 强制释放开始: idx=#{self.persistent_index or '-'}, "
|
||||
f"prev_status={state['prev_status']}, prev_task={state['prev_task'] or '-'}, "
|
||||
f"prev_bound_req={state['prev_bound_request_id'] or '-'}, "
|
||||
f"prev_cmd_req={state['prev_command_request_id'] or '-'}, "
|
||||
f"req_count={state['prev_request_count']}, clear_page={clear_page}, "
|
||||
f"check_triggers={check_triggers}"
|
||||
)
|
||||
|
||||
self.clear_visibility_emulation("force_release")
|
||||
|
||||
try:
|
||||
if hasattr(self.tab, "stop_loading"):
|
||||
self.tab.stop_loading()
|
||||
driver_for_tab(self.tab).run_js("if (window.stop) { window.stop(); }")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
reset_success = True
|
||||
if clear_page:
|
||||
try:
|
||||
self.tab.refresh()
|
||||
self.reset_conversation_state()
|
||||
self.notify_navigated(reason="force_release_clear")
|
||||
except Exception as e:
|
||||
logger.warning(f"[{self.id}] force_release refresh failed: {e}")
|
||||
reset_success = False
|
||||
|
||||
self._finish_release_state(state, reset_success)
|
||||
|
||||
with self._lock:
|
||||
final_status = self.status.value
|
||||
final_task = str(self.current_task_id or "").strip()
|
||||
request_count = self.request_count
|
||||
error_count = self.error_count
|
||||
|
||||
if reset_success:
|
||||
logger.info(
|
||||
f"[{self.id}] force_release done "
|
||||
f"(idx=#{self.persistent_index or '-'}, final_status={final_status}, "
|
||||
f"task={final_task or '-'}, req_count={request_count})"
|
||||
)
|
||||
else:
|
||||
logger.warning(
|
||||
f"[{self.id}] force_release failed, set ERROR "
|
||||
f"(idx=#{self.persistent_index or '-'}, req_count={request_count}, "
|
||||
f"error_count={error_count})"
|
||||
)
|
||||
|
||||
if check_triggers:
|
||||
try:
|
||||
from app.services.command_engine import command_engine
|
||||
command_engine.check_triggers(self)
|
||||
except Exception as e:
|
||||
logger.debug(f"command trigger check failed: {e}")
|
||||
|
||||
def force_release(self, clear_page: bool = False, check_triggers: bool = False):
|
||||
"""Force release tab lock and optionally refresh current page."""
|
||||
state = self._begin_release_state(clear_page=clear_page, force=True)
|
||||
if state is None:
|
||||
return
|
||||
self._run_force_release_from_state(
|
||||
state,
|
||||
clear_page=clear_page,
|
||||
check_triggers=check_triggers,
|
||||
)
|
||||
|
||||
def activate(self) -> bool:
|
||||
"""激活标签页(使其成为浏览器焦点)"""
|
||||
try:
|
||||
self.tab.set.activate()
|
||||
logger.debug(f"[{self.id}] 已激活")
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.warning(f"[{self.id}] 激活失败: {e}")
|
||||
return False
|
||||
|
||||
def mark_error(self, reason: str = None):
|
||||
with self._lock:
|
||||
self.status = TabStatus.ERROR
|
||||
self._clear_health_cache_unlocked()
|
||||
self.error_count += 1
|
||||
logger.warning(f"[{self.id}] 标记为错误: {reason}")
|
||||
|
||||
def mark_closed(self, reason: str = None):
|
||||
with self._lock:
|
||||
if self.status == TabStatus.CLOSED:
|
||||
return
|
||||
self.status = TabStatus.CLOSED
|
||||
self.current_task_id = None
|
||||
self._release_in_progress = False
|
||||
self._termination_in_progress = False
|
||||
self._termination_task_id = None
|
||||
self._termination_interrupt_done = False
|
||||
self.transient_disconnect_until = 0.0
|
||||
self.transient_disconnect_reason = None
|
||||
self._clear_health_cache_unlocked()
|
||||
logger.debug(f"[{self.id}] 标记为关闭: {reason or '-'}")
|
||||
|
||||
def _reset_conversation_state_unlocked(self):
|
||||
self.last_conversation_activity_at = 0.0
|
||||
self.last_conversation_domain = None
|
||||
self.last_conversation_preset_name = None
|
||||
|
||||
def reset_conversation_state(self):
|
||||
with self._lock:
|
||||
self._reset_conversation_state_unlocked()
|
||||
|
||||
def mark_conversation_activity(self, domain: str = "", preset_name: str = ""):
|
||||
normalized_domain = str(domain or "").strip().lower() or None
|
||||
normalized_preset = str(preset_name or "").strip() or None
|
||||
now = time.time()
|
||||
with self._lock:
|
||||
self.last_conversation_activity_at = now
|
||||
self.last_conversation_domain = normalized_domain
|
||||
self.last_conversation_preset_name = normalized_preset
|
||||
logger.debug(
|
||||
f"[{self.id}] 记录会话活动: idx=#{self.persistent_index or '-'}, "
|
||||
f"domain={normalized_domain or '-'}, preset={normalized_preset or '-'}, "
|
||||
f"at={round(now, 3)}"
|
||||
)
|
||||
|
||||
def get_conversation_status(
|
||||
self,
|
||||
current_domain: str = "",
|
||||
preset_name: str = "",
|
||||
threshold_seconds: float = 0.0,
|
||||
force_new: bool = False,
|
||||
) -> Dict[str, Any]:
|
||||
normalized_domain = str(current_domain or self.current_domain or "").strip().lower()
|
||||
normalized_preset = str(preset_name or "").strip() or None
|
||||
threshold_value = max(0.0, float(threshold_seconds or 0.0))
|
||||
|
||||
with self._lock:
|
||||
last_activity_at = float(self.last_conversation_activity_at or 0.0)
|
||||
last_domain = str(self.last_conversation_domain or "").strip().lower()
|
||||
last_preset = str(self.last_conversation_preset_name or "").strip() or None
|
||||
|
||||
elapsed_seconds = None
|
||||
if last_activity_at > 0:
|
||||
elapsed_seconds = max(0.0, time.time() - last_activity_at)
|
||||
|
||||
reason = "no_previous_conversation"
|
||||
will_start_new = True
|
||||
|
||||
if force_new:
|
||||
reason = "force_new_enabled"
|
||||
elif threshold_value <= 0:
|
||||
reason = "threshold_disabled_reuse"
|
||||
elif not normalized_domain:
|
||||
reason = "missing_current_domain"
|
||||
elif not last_activity_at:
|
||||
reason = "no_previous_conversation"
|
||||
elif last_domain and normalized_domain != last_domain:
|
||||
reason = "domain_changed"
|
||||
elif normalized_preset and last_preset and normalized_preset != last_preset:
|
||||
reason = "preset_changed"
|
||||
elif elapsed_seconds is not None and elapsed_seconds < threshold_value:
|
||||
will_start_new = False
|
||||
reason = "reuse_existing"
|
||||
else:
|
||||
reason = "timeout_expired"
|
||||
|
||||
return {
|
||||
"id": self.id,
|
||||
"persistent_index": self.persistent_index,
|
||||
"current_domain": normalized_domain,
|
||||
"current_preset_name": normalized_preset,
|
||||
"last_conversation_domain": last_domain or None,
|
||||
"last_conversation_preset_name": last_preset,
|
||||
"last_conversation_at": last_activity_at or None,
|
||||
"elapsed_seconds": round(elapsed_seconds, 1) if elapsed_seconds is not None else None,
|
||||
"threshold_seconds": threshold_value,
|
||||
"force_new_conversation": bool(force_new),
|
||||
"will_start_new_conversation": will_start_new,
|
||||
"reason": reason,
|
||||
}
|
||||
|
||||
def should_start_new_conversation(
|
||||
self,
|
||||
current_domain: str = "",
|
||||
preset_name: str = "",
|
||||
threshold_seconds: float = 0.0,
|
||||
force_new: bool = False,
|
||||
) -> bool:
|
||||
return bool(
|
||||
self.get_conversation_status(
|
||||
current_domain=current_domain,
|
||||
preset_name=preset_name,
|
||||
threshold_seconds=threshold_seconds,
|
||||
force_new=force_new,
|
||||
).get("will_start_new_conversation", True)
|
||||
)
|
||||
|
||||
def get_info(
|
||||
self,
|
||||
*,
|
||||
use_cached_url: bool = False,
|
||||
allow_live_when_busy: bool = False,
|
||||
) -> Dict:
|
||||
with self._lock:
|
||||
status = self.status
|
||||
last_used_at = self.last_used_at
|
||||
current_task_id = self.current_task_id
|
||||
command_request_id = str(getattr(self, "_command_request_id", "") or "").strip()
|
||||
current_command_id = str(getattr(self, "_current_command_id", "") or "").strip()
|
||||
current_command_name = str(self.current_command_name or "").strip()
|
||||
current_command = getattr(self, "_current_command", None)
|
||||
current_domain_snapshot = str(self.current_domain or "").strip()
|
||||
cached_url = str(self.last_known_url or "").strip()
|
||||
request_count = self.request_count
|
||||
preset_name = self.preset_name
|
||||
preset_override_source = self.preset_override_source
|
||||
model_name_override = self.model_name_override
|
||||
is_isolated_context = self.is_isolated_context
|
||||
browser_context_id = self.browser_context_id
|
||||
last_conversation_activity_at = self.last_conversation_activity_at
|
||||
last_conversation_domain = self.last_conversation_domain
|
||||
last_conversation_preset_name = self.last_conversation_preset_name
|
||||
command_loop = self._command_loop_info_unlocked()
|
||||
termination_in_progress = self._termination_in_progress
|
||||
busy_duration = None
|
||||
if status == TabStatus.BUSY:
|
||||
try:
|
||||
busy_duration = round(self.get_busy_timeout_info(now=time.time()).get("duration") or 0.0, 1)
|
||||
except Exception:
|
||||
busy_duration = round(time.time() - last_used_at, 1)
|
||||
|
||||
if use_cached_url:
|
||||
current_url = cached_url
|
||||
if current_url:
|
||||
current_domain = self._refresh_current_domain(current_url)
|
||||
else:
|
||||
current_domain = current_domain_snapshot
|
||||
else:
|
||||
current_url = self._safe_get_url(allow_live_when_busy=allow_live_when_busy)
|
||||
current_domain = self._refresh_current_domain(current_url)
|
||||
|
||||
url_route_token = encode_tab_url_route_token(current_url)
|
||||
|
||||
if not current_command_name and isinstance(current_command, dict):
|
||||
current_command_name = str(current_command.get("name") or "").strip()
|
||||
|
||||
return {
|
||||
"id": self.id,
|
||||
"persistent_index": self.persistent_index,
|
||||
"status": status.value,
|
||||
"current_task": current_task_id,
|
||||
"command_task": command_request_id,
|
||||
"current_command_id": current_command_id,
|
||||
"current_command": current_command_name,
|
||||
"current_domain": current_domain,
|
||||
"route_domain": get_preferred_route_domain(current_domain),
|
||||
"domain_url": self._build_domain_url(current_url, current_domain),
|
||||
"url": current_url,
|
||||
"url_route_token": url_route_token,
|
||||
"request_count": request_count,
|
||||
"busy_duration": busy_duration,
|
||||
"preset_name": preset_name, # 🆕
|
||||
"preset_override_source": preset_override_source,
|
||||
"model_name_override": model_name_override,
|
||||
"is_isolated_context": is_isolated_context,
|
||||
"browser_context_id": browser_context_id,
|
||||
"last_conversation_at": last_conversation_activity_at or None,
|
||||
"last_conversation_domain": last_conversation_domain,
|
||||
"last_conversation_preset_name": last_conversation_preset_name,
|
||||
"command_loop": command_loop,
|
||||
"terminating": termination_in_progress,
|
||||
}
|
||||
|
||||
def _refresh_current_domain(self, url: str = "") -> str:
|
||||
current_url = str(url or "").strip()
|
||||
try:
|
||||
resolved = extract_remote_site_domain(current_url) or ""
|
||||
except Exception:
|
||||
resolved = ""
|
||||
|
||||
if resolved:
|
||||
with self._lock:
|
||||
self.current_domain = resolved
|
||||
return resolved
|
||||
|
||||
with self._lock:
|
||||
fallback = str(self.current_domain or "").strip()
|
||||
if _should_skip_pool_url(current_url) or "://" in current_url:
|
||||
with self._lock:
|
||||
self.current_domain = None
|
||||
return ""
|
||||
return fallback
|
||||
|
||||
@staticmethod
|
||||
def _build_domain_url(url: str, current_domain: str) -> str:
|
||||
source_url = str(url or "").strip()
|
||||
domain = str(current_domain or "").strip()
|
||||
if not source_url or not domain:
|
||||
return ""
|
||||
|
||||
try:
|
||||
parsed = urlsplit(source_url)
|
||||
except Exception:
|
||||
return ""
|
||||
|
||||
scheme = parsed.scheme if parsed.scheme in {"http", "https", "ws", "wss"} else "https"
|
||||
return f"{scheme}://{domain}/"
|
||||
|
||||
def _remember_url(self, url: str) -> str:
|
||||
normalized = str(url or "").strip()
|
||||
with self._lock:
|
||||
if str(self.last_known_url or "").strip() != normalized:
|
||||
self._clear_health_cache_unlocked()
|
||||
if self.preset_override_source == "url":
|
||||
self.preset_name = None
|
||||
self.preset_override_source = None
|
||||
self.last_known_url = normalized or None
|
||||
return normalized
|
||||
|
||||
def get_occupied_url(self) -> Optional[str]:
|
||||
with self._lock:
|
||||
return self._request_occupied_url
|
||||
|
||||
def set_occupied_url(self, url: Optional[str]) -> None:
|
||||
with self._lock:
|
||||
normalized = str(url or "").strip()
|
||||
if normalized and "challenges.cloudflare.com" not in normalized and "challenge-platform" not in normalized and normalized.startswith("http"):
|
||||
self._request_occupied_url = normalized
|
||||
elif not normalized:
|
||||
self._request_occupied_url = None
|
||||
|
||||
def _safe_get_url(self, allow_live_when_busy: bool = False) -> str:
|
||||
with self._lock:
|
||||
status = self.status
|
||||
cached_url = str(self.last_known_url or "").strip()
|
||||
|
||||
if status == TabStatus.CLOSED:
|
||||
return ""
|
||||
if status == TabStatus.BUSY and not allow_live_when_busy:
|
||||
return cached_url
|
||||
|
||||
try:
|
||||
current_url = self._remember_url(self.tab.url or "")
|
||||
if current_url:
|
||||
return current_url
|
||||
except Exception:
|
||||
pass
|
||||
return cached_url
|
||||
|
||||
def clear_visibility_emulation(self, reason: str = "") -> None:
|
||||
try:
|
||||
from app.core.page_lifecycle import restore_visibility_emulation
|
||||
|
||||
restore_visibility_emulation(self.tab, owner=self, reason=reason)
|
||||
except Exception as e:
|
||||
logger.debug(f"[{self.id}] visibility emulation cleanup failed: {e}")
|
||||
|
||||
def notify_navigated(self, reason: str = "page_reload") -> None:
|
||||
"""Notify command engine of page reload/navigation."""
|
||||
try:
|
||||
from app.services.command_engine import command_engine
|
||||
|
||||
if hasattr(command_engine, "notify_tab_navigated"):
|
||||
command_engine.notify_tab_navigated(self, reason=reason)
|
||||
except Exception as e:
|
||||
logger.debug(f"[{self.id}] notify_navigated failed (ignored): {e}")
|
||||
|
||||
Reference in new issue
Block a user