Initial commit

This commit is contained in:
Alex committed 2026-11-02 22:54:14 +08:00
commit 57fc3525eb
7311 files changed
+1291398

No files matched your search

+10
View File
@@ -0,0 +1,10 @@
from .clipboard import get_clipboard_lock
from .manager import TabPoolManager
from .session import TabSession, TabStatus
__all__ = [
"TabStatus",
"TabSession",
"TabPoolManager",
"get_clipboard_lock",
]
+264
View File
@@ -0,0 +1,264 @@
"""
app/core/tab_pool_parts/_arena_snapshot.py
无外部依赖的纯 JavaScript 常量定义模块。
供 network, arena_tab_listener 以及命令引擎沙箱安全引用。
"""
_ARENA_STORE_SNAPSHOT_JS = r"""
return (() => {
function safe(fn, fallback) {
try { return fn(); } catch (error) { return fallback; }
}
function isExplicitDirectUrl(url) {
try {
const u = new URL(url || location.href);
const p = String(u.pathname || '').toLowerCase();
if (
p === '/direct' ||
p.startsWith('/direct/') ||
p === '/text/direct' ||
p.startsWith('/text/direct/') ||
p === '/image/direct' ||
p.startsWith('/image/direct/') ||
p === '/code/direct' ||
p.startsWith('/code/direct/') ||
p === '/search/direct' ||
p.startsWith('/search/direct/') ||
p.endsWith('/direct') ||
u.searchParams.get('mode') === 'direct'
) return true;
return false;
} catch (e) {
const s = String(url || location.href || '').toLowerCase();
return s.includes('/direct') || s.includes('mode=direct');
}
}
function textOf(value) {
if (typeof value === 'string') return value;
if (Array.isArray(value)) {
return value.map(item => {
if (typeof item === 'string') return item;
if (item && typeof item === 'object') return item.text || item.content || '';
return '';
}).filter(Boolean).join('\n');
}
return '';
}
function preferredModelName(value) {
if (!value || typeof value !== 'object') return '';
return String(value.displayName || value.publicName || value.name || value.modelName || value.slug || '').trim();
}
function modelIdOf(message) {
if (!message || typeof message !== 'object') return '';
const model = message.model;
if (model && typeof model === 'object') {
return String(model.id || model.modelId || '').trim();
}
return String(message.modelId || message.model || message.modelName || message.modelSlug || '').trim();
}
function getPageModelMap() {
const html = safe(() => document.documentElement.outerHTML || '', '');
const cache = window.__arenaDetectorModelMap;
if (cache && cache.htmlLength === html.length && cache.map) return cache.map;
const map = {};
const re = /\{\\"id\\":\\"[a-f0-9-]+\\"/g;
let match;
while ((match = re.exec(html)) && Object.keys(map).length < 2000) {
const start = match.index;
let openBraces = 0;
let end = -1;
const limit = Math.min(html.length, start + 20000);
for (let i = start; i < limit; i += 1) {
if (html[i] === '{') openBraces += 1;
else if (html[i] === '}') {
openBraces -= 1;
if (openBraces === 0) {
end = i + 1;
break;
}
}
}
if (end < 0) continue;
const raw = html.slice(start, end).replace(/\\"/g, '"').replace(/\\\\/g, '\\');
const item = safe(() => JSON.parse(raw), null);
if (!item || typeof item !== 'object' || !item.id) continue;
const name = preferredModelName(item);
if (name) map[String(item.id).trim()] = name;
}
window.__arenaDetectorModelMap = { htmlLength: html.length, map };
return map;
}
function modelNameOf(message) {
if (!message || typeof message !== 'object') return '';
const direct = preferredModelName(message.model) || preferredModelName(message);
if (direct) return direct;
const modelId = modelIdOf(message);
if (!modelId) return '';
return getPageModelMap()[modelId] || modelId;
}
function findReactFiber(el) {
if (!el) return null;
const key = Object.keys(el).find(k => k.startsWith('__reactFiber$') || k.startsWith('__reactInternalInstance$'));
return key ? el[key] : null;
}
function looksLikeArenaStore(value) {
if (!value || typeof value !== 'object') return false;
if (typeof value.getState !== 'function') return false;
const state = safe(() => value.getState(), null);
return !!(state && typeof state === 'object' && Array.isArray(state.messages) && typeof state.id === 'string');
}
function findArenaStoreIn(value, depth, seen) {
if (!value || typeof value !== 'object' || depth < 0 || seen.has(value)) return null;
seen.add(value);
if (looksLikeArenaStore(value)) return value;
const keys = safe(() => Object.keys(value), []);
for (const key of keys.slice(0, 100)) {
if (['_owner', 'return', 'child', 'sibling', 'alternate'].includes(key)) continue;
const found = findArenaStoreIn(value[key], depth - 1, seen);
if (found) return found;
}
return null;
}
function findStoreFromFiber() {
const roots = [
document.querySelector('main'),
document.querySelector('form'),
document.body,
].filter(Boolean);
for (const root of roots) {
const fiber = findReactFiber(root);
for (let cur = fiber, depth = 0; cur && depth < 100; depth += 1, cur = cur.return) {
const found = findArenaStoreIn(cur.memoizedProps, 5, new WeakSet())
|| findArenaStoreIn(cur.memoizedState, 5, new WeakSet());
if (found) return found;
}
}
return null;
}
const store = findStoreFromFiber();
const state = store && safe(() => store.getState(), null);
const stateMode = String((state && (state.evaluationMode || state.mode || state.sessionMode || state.type)) || '').toLowerCase();
const isDirect = stateMode.includes('direct') || stateMode.includes('single') || isExplicitDirectUrl(location.href);
if (isDirect) {
return {
is_direct: true,
mode: 'direct',
url: location.href,
conversation_id: String(state && state.id || ''),
message_id_a: '',
message_id_b: '',
status_a: '',
status_b: '',
model_a: '',
model_b: '',
model_id_a: '',
model_id_b: '',
response_a: '',
response_b: '',
};
}
const messages = state && Array.isArray(state.messages) ? state.messages : [];
const byId = new Map(messages.map(message => [String(message && message.id || ''), message]));
let assistantIds = Array.isArray(state && state.lastMessageIds)
? state.lastMessageIds.map(id => String(id || '')).filter(id => byId.get(id) && byId.get(id).role === 'assistant')
: [];
// 严格校验双侧对战:必须有成对的 2 个 assistant 消息
if (assistantIds.length < 2) {
const lastAssistants = messages.filter(m => m && m.role === 'assistant');
if (lastAssistants.length >= 2) {
const cand1 = lastAssistants[lastAssistants.length - 1];
const cand2 = lastAssistants[lastAssistants.length - 2];
const p1 = String((Array.isArray(cand1.parentMessageIds) ? cand1.parentMessageIds[0] : cand1.parentId) || '');
const p2 = String((Array.isArray(cand2.parentMessageIds) ? cand2.parentMessageIds[0] : cand2.parentId) || '');
const s1 = String(cand1.side || cand1.modelSide || '').toLowerCase();
const s2 = String(cand2.side || cand2.modelSide || '').toLowerCase();
const m1 = modelIdOf(cand1);
const m2 = modelIdOf(cand2);
const isBattlePair = (p1 && p2 && p1 === p2) && (
(s1 && s2 && s1 !== s2) ||
(m1 && m2 && m1 !== m2)
);
if (isBattlePair) {
assistantIds = [String(cand2.id || ''), String(cand1.id || '')];
}
}
}
if (assistantIds.length < 2) {
return {
is_direct: false,
mode: stateMode || 'arena',
url: location.href,
conversation_id: String(state && state.id || ''),
message_id_a: '',
message_id_b: '',
status_a: '',
status_b: '',
model_a: '',
model_b: '',
model_id_a: '',
model_id_b: '',
response_a: '',
response_b: '',
};
}
const a = byId.get(assistantIds[0]) || null;
const b = byId.get(assistantIds[1]) || null;
const parentIds = []
.concat(Array.isArray(a && a.parentMessageIds) ? a.parentMessageIds : [])
.concat(Array.isArray(b && b.parentMessageIds) ? b.parentMessageIds : []);
let userMessage = null;
for (const parentId of parentIds) {
const candidate = byId.get(String(parentId || ''));
if (candidate && candidate.role === 'user') {
userMessage = candidate;
break;
}
}
if (!userMessage) {
for (let i = messages.length - 1; i >= 0; i -= 1) {
if (messages[i] && messages[i].role === 'user') {
userMessage = messages[i];
break;
}
}
}
return {
is_direct: false,
mode: stateMode || 'arena',
url: location.href,
conversation_id: String(state && state.id || ''),
prompt: textOf(userMessage && userMessage.content),
message_id_a: String(a && a.id || ''),
message_id_b: String(b && b.id || ''),
status_a: String(a && a.status || ''),
status_b: String(b && b.status || ''),
model_a: modelNameOf(a),
model_b: modelNameOf(b),
model_id_a: modelIdOf(a),
model_id_b: modelIdOf(b),
response_a: textOf(a && a.content),
response_b: textOf(b && b.content),
};
})()
""".strip()
# P0-2:轮询用的 JSON 版本。页面内 JSON.stringify 后只回传一个字符串,
# 不会在渲染进程里留下 RemoteObject,也省掉 DrissionPage 额外的一次序列化往返。
# 原常量保留(命令沙箱脚本可能依赖它返回 dict)。调用方用 json.loads 解码。
_ARENA_STORE_SNAPSHOT_JSON_JS = (
"function(){"
"const __uwapiResult = (function(){\n"
+ _ARENA_STORE_SNAPSHOT_JS
+ "\n}).apply(this, arguments);"
"return JSON.stringify(__uwapiResult === undefined ? null : __uwapiResult);"
"}"
)
__all__ = ["_ARENA_STORE_SNAPSHOT_JS", "_ARENA_STORE_SNAPSHOT_JSON_JS"]
+52
View File
@@ -0,0 +1,52 @@
from typing import Any
from app.utils.site_url import is_remote_site_url
_POOL_SKIP_URL_PREFIXES = (
"about:",
"chrome://",
"chrome-devtools://",
"devtools://",
"edge://",
"brave://",
"javascript:",
"data:",
"blob:",
)
_POOL_SKIP_URL_CONTAINS = (
"chrome-error://",
"about:neterror",
)
_TAB_HEALTH_CACHE_TTL_SEC = 5.0
def _looks_like_transient_local_debug_error(error: Any) -> bool:
text = str(error or "").strip().lower()
if not text:
return False
if "winerror 10048" in text:
return True
if "max retries exceeded" in text and "127.0.0.1" in text and "/json" in text:
return True
if "failed to establish a new connection" in text and "127.0.0.1" in text:
return True
return False
def _should_skip_pool_url(url: str) -> bool:
raw = str(url or "").strip()
if not raw:
return True
lowered = raw.lower()
if lowered.startswith(_POOL_SKIP_URL_PREFIXES):
return True
if any(marker in lowered for marker in _POOL_SKIP_URL_CONTAINS):
return True
return not is_remote_site_url(lowered)
+8
View File
@@ -0,0 +1,8 @@
import threading
_clipboard_lock = threading.Lock()
def get_clipboard_lock() -> threading.Lock:
return _clipboard_lock
+904
View File
@@ -0,0 +1,904 @@
"""
app/core/tab_pool_parts/idle_maintenance.py - 空闲标签页维护(P0-3 / P0-6)
P0-3 CDP 会话回收(recycle)
DrissionPage 执行 run_js / 元素查找时产生的 RemoteObject、DOM agent 的
nodeId 映射都挂在 CDP 会话上,只有会话断开才会整体释放;它们会把早已从
页面移除的旧对话 DOM 钉在渲染进程里。空闲标签页满足触发条件时,断开并
重新建立该标签页的 CDP 连接(不刷新页面、不影响登录态),释放这些引用。
顺序:maint 占用会话 → 停止全局网络监听 → tab.disconnect() →
tab._driver_init(target_id) → tab._get_document() →
重置 listener 驱动状态 / 会话级注入脚本标记 → 释放会话(释放流程会恢复
可见性模拟状态,全局监听随后自动重启)。
触发(任一满足,且自上次回收后标签页确有使用):
- 自上次回收后已处理 BROWSER_CDP_RECYCLE_AFTER_REQUESTS 个请求(默认 20)
- 距上次回收超过 BROWSER_CDP_RECYCLE_INTERVAL_SEC(默认 1800s)
- Memory.getDOMCounters 节点数 ≥ BROWSER_CDP_RECYCLE_DOM_NODES(默认 150000)
P0-6 空闲冻结(freeze)
空闲 ≥ BROWSER_IDLE_FREEZE_AFTER_SEC(默认 60s)、且真实 visibilityState 为
hidden 的标签页执行 Page.setWebLifecycleState=frozen(计时器 / rAF / 页面任务
全部暂停)。被占用(acquire / acquire_for_command)或 page_check 唤醒时
先恢复为 active。可见(用户正在看)的标签页永远不会被冻结。
环境变量:
BROWSER_IDLE_FREEZE_ENABLED 默认 true
BROWSER_IDLE_FREEZE_AFTER_SEC 默认 60(最小 15)
BROWSER_IDLE_FREEZE_REQUIRE_HIDDEN 默认 true(仅冻结后台/被遮挡的标签页)
BROWSER_CDP_RECYCLE_ENABLED 默认 true
BROWSER_CDP_RECYCLE_IDLE_SEC 默认 30(最小 10)
BROWSER_CDP_RECYCLE_AFTER_REQUESTS 默认 20(0 = 不按请求数触发)
BROWSER_CDP_RECYCLE_INTERVAL_SEC 默认 1800(0 = 不按时间触发)
BROWSER_CDP_RECYCLE_DOM_NODES 默认 150000(0 = 不按节点数触发)
BROWSER_CDP_RECYCLE_DOM_CHECK_SEC 默认 300
"""
from __future__ import annotations
import json
import math
import os
import threading
import time
from dataclasses import dataclass
from typing import Any, Optional
from app.core.config import logger
from app.core.driver import browser_driver, driver_for_tab
# 真实可见性:可见性模拟在 document 实例上定义了 own property,
# 这里直接调用 Document.prototype 上的原生 getter,得到浏览器真实状态。
REAL_VISIBILITY_JS = (
"return (function(){try{"
"var d=Object.getOwnPropertyDescriptor(Document.prototype,'visibilityState');"
"return String(d&&d.get?d.get.call(document):document.visibilityState);"
"}catch(e){return 'unknown';}})();"
)
_LOCK_FACTORY_GUARD = threading.Lock()
def _env_bool(name: str, default: bool) -> bool:
raw = os.getenv(name)
if raw is None or not str(raw).strip():
return default
return str(raw).strip().lower() in {"1", "true", "yes", "y", "on"}
def _env_float(name: str, default: float, minimum: float = 0.0) -> float:
raw = os.getenv(name)
try:
value = float(str(raw).strip()) if raw not in (None, "") else float(default)
except (TypeError, ValueError):
value = float(default)
if not math.isfinite(value):
# B6:inf / nan 不是有效配置(int(inf) 会 OverflowError,nan 让所有比较为假)。
# 想要「禁用」请写 0。
logger.warning(f"[IdleMaintenance] {name}={raw!r} 不是有限数值,使用默认值 {default}(禁用请设为 0)")
value = float(default)
if value <= 0:
return 0.0
return max(minimum, value)
def _env_int(name: str, default: int, minimum: int = 0) -> int:
return int(_env_float(name, float(default), float(minimum)))
@dataclass
class IdleMaintenanceConfig:
freeze_enabled: bool = True
freeze_after_sec: float = 60.0
freeze_require_hidden: bool = True
recycle_enabled: bool = True
recycle_idle_sec: float = 30.0
recycle_after_requests: int = 20
recycle_interval_sec: float = 1800.0
recycle_dom_nodes: int = 150000
dom_check_interval_sec: float = 300.0
@classmethod
def from_env(cls) -> "IdleMaintenanceConfig":
return cls(
freeze_enabled=_env_bool("BROWSER_IDLE_FREEZE_ENABLED", True),
freeze_after_sec=_env_float("BROWSER_IDLE_FREEZE_AFTER_SEC", 60.0, 15.0) or 60.0,
freeze_require_hidden=_env_bool("BROWSER_IDLE_FREEZE_REQUIRE_HIDDEN", True),
recycle_enabled=_env_bool("BROWSER_CDP_RECYCLE_ENABLED", True),
recycle_idle_sec=_env_float("BROWSER_CDP_RECYCLE_IDLE_SEC", 30.0, 10.0) or 30.0,
recycle_after_requests=_env_int("BROWSER_CDP_RECYCLE_AFTER_REQUESTS", 20, 1),
recycle_interval_sec=_env_float("BROWSER_CDP_RECYCLE_INTERVAL_SEC", 1800.0, 60.0),
recycle_dom_nodes=_env_int("BROWSER_CDP_RECYCLE_DOM_NODES", 150000, 1000),
dom_check_interval_sec=_env_float("BROWSER_CDP_RECYCLE_DOM_CHECK_SEC", 300.0, 30.0) or 300.0,
)
def is_idle_freeze_enabled() -> bool:
return _env_bool("BROWSER_IDLE_FREEZE_ENABLED", True)
# --------------------------------------------------------------------------- #
# helpers
# --------------------------------------------------------------------------- #
def _run_cdp(tab: Any, method: str, *, timeout: float = 2.0, **params):
runner = getattr(tab, "run_cdp", None)
if not callable(runner):
runner = getattr(tab, "_run_cdp", None)
if not callable(runner):
raise AttributeError(f"CDP runner unavailable for {method}")
try:
return runner(method, _timeout=max(0.1, float(timeout)), **params)
except TypeError as exc:
if "timeout" not in str(exc).lower():
raise
return runner(method, **params)
def lifecycle_lock(session: Any) -> threading.Lock:
lock = getattr(session, "_uwapi_lifecycle_lock", None)
if lock is None:
with _LOCK_FACTORY_GUARD:
lock = getattr(session, "_uwapi_lifecycle_lock", None)
if lock is None:
lock = threading.Lock()
setattr(session, "_uwapi_lifecycle_lock", lock)
return lock
def _status_name(session: Any) -> str:
status = getattr(session, "status", None)
return str(getattr(status, "value", status) or "").lower()
def _is_idle(session: Any) -> bool:
return _status_name(session) == "idle" and not bool(getattr(session, "_termination_in_progress", False))
def last_activity_at(session: Any) -> float:
"""Most recent sign of life: request/command use, page_check wake, network event."""
values = [
getattr(session, "last_used_at", 0.0),
getattr(session, "_pc_last_wake_at", 0.0),
getattr(session, "_uwapi_last_network_event_at", 0.0),
getattr(session, "_uwapi_last_resumed_at", 0.0),
]
best = 0.0
for value in values:
try:
best = max(best, float(value or 0.0))
except (TypeError, ValueError):
continue
return best
def note_network_activity(session: Any) -> None:
try:
setattr(session, "_uwapi_last_network_event_at", time.time())
except Exception:
pass
# --------------------------------------------------------------------------- #
# freeze / resume (P0-6)
# --------------------------------------------------------------------------- #
#
# 注意:DrissionPage 在 _driver_init 中对每个标签页调用
# Emulation.setFocusEmulationEnabled(true)。Chromium 用“增加 capturer 计数”
# 实现该仿真,结果是被控制的标签页永远被视为 visible —— 后台标签页既不会被
# 浏览器降频,Page.setWebLifecycleState=frozen 也会被静默忽略(实测:冻结后
# 计时器照常运行)。因此冻结前必须先在本会话上关闭焦点仿真,读取真实可见性;
# 恢复时重新打开。
def _set_focus_emulation(tab: Any, enabled: bool) -> bool:
try:
_run_cdp(tab, "Emulation.setFocusEmulationEnabled", enabled=bool(enabled), timeout=1.5)
return True
except Exception:
return False
def _read_real_visibility(tab: Any) -> str:
try:
return str(driver_for_tab(tab).run_js(REAL_VISIBILITY_JS, timeout=1.0) or "").strip().lower()
except Exception:
return "unknown"
def _wait_real_hidden(tab: Any, timeout_sec: float = 0.6, should_abort=None) -> str:
"""关闭焦点仿真后,可见性变化是异步传到渲染进程的,短暂轮询。"""
deadline = time.time() + max(0.0, timeout_sec)
state = _read_real_visibility(tab)
while state != "hidden" and time.time() < deadline:
if should_abort is not None and should_abort():
return "aborted"
time.sleep(0.05)
state = _read_real_visibility(tab)
return state
# 冻结核验:CDP 冻结可能被 Chrome 静默忽略或在我们背后解除(发起冻结的 DevTools 会话断开、
# 用户切回标签页、与其它会话附加并发等,均已实测)。冻结前在页面挂一次性的标准生命周期事件
# freeze / resume 监听(不可枚举属性,不影响页面逻辑),用同步 run_js 读取——冻结的页面也能执行。
_FREEZE_MARK_INSTALL_JS = (
"return (function(t){try{var st={t:t,f:0,r:0};"
"Object.defineProperty(window,'__uwapiFz',{value:st,configurable:true,enumerable:false,writable:true});"
"document.addEventListener('freeze',function(){st.f=Date.now();},{once:true});"
"document.addEventListener('resume',function(){st.r=Date.now();},{once:true});"
"return 1;}catch(e){return 0;}})(arguments[0]);"
)
_FREEZE_MARK_READ_JS = (
"return (function(){try{var s=window.__uwapiFz;"
"return s?JSON.stringify({t:s.t,f:s.f,r:s.r}):'';}catch(e){return '';}})();"
)
def _read_freeze_mark(tab: Any) -> Optional[dict]:
"""``None`` = could not read; ``{}`` = marker missing (document replaced)."""
try:
raw = driver_for_tab(tab).run_js(_FREEZE_MARK_READ_JS, timeout=1.0)
except Exception:
return None
if not raw:
return {}
try:
data = json.loads(raw)
return data if isinstance(data, dict) else {}
except Exception:
return {}
def _freeze_took_effect(tab: Any, token: str, timeout_sec: float = 0.4, should_abort=None) -> bool:
deadline = time.time() + max(0.0, timeout_sec)
while True:
mark = _read_freeze_mark(tab)
if mark is None:
return True # 读不到(非真实页面对象等)时不否定冻结,交给后续巡检
if mark.get("t") == token and mark.get("f") and not mark.get("r"):
return True
if should_abort is not None and should_abort():
return True # 已被占用:不再核验,按已冻结登记,由等锁的占用方立刻解冻
if time.time() >= deadline:
return False
time.sleep(0.03)
def freeze_was_lifted(session: Any) -> bool:
"""True when the page is known to be running again although we think it is frozen."""
tab = getattr(session, "tab", None)
token = str(getattr(session, "_uwapi_freeze_token", "") or "")
if tab is None or not token:
return False
mark = _read_freeze_mark(tab)
if mark is None:
return False
return mark.get("t") != token or bool(mark.get("r"))
def _front_tab_hint(tab: Any) -> str:
"""Guess whether the user can see ``tab`` WITHOUT touching focus emulation.
Toggling focus emulation off/on (the authoritative probe) fires ``blur`` /
``focus`` on a tab the user is looking at, and sites with refetch-on-focus
react to that. So first use browser-side facts that have no page-visible
side effects:
* ``"minimized"`` – the tab's window is minimized (hidden for sure)
* ``"front"`` – the tab is the most recently activated page of its
(non-minimized) window, i.e. the tab currently shown there. The DevTools
``/json`` list is MRU-ordered (DrissionPage's ``latest_tab`` relies on it).
* ``"background"`` – another page is in front of it in the same window
* ``"unknown"`` – cannot tell (ws-only mode, errors, …) → caller probes
"""
try:
info = _run_cdp(tab, "Browser.getWindowForTarget", timeout=1.0) or {}
window_id = info.get("windowId")
window_state = str((info.get("bounds") or {}).get("windowState") or "").lower()
except Exception:
return "unknown"
if window_state == "minimized":
return "minimized"
browser = getattr(tab, "browser", None)
my_id = str(getattr(tab, "tab_id", "") or "")
if browser is None or not my_id or window_id is None or getattr(browser, "_ws_only", False):
return "unknown"
try:
mru_ids = list(browser.tab_ids)
except Exception:
return "unknown"
for target_id in mru_ids[:32]:
if str(target_id) == my_id:
return "front"
try:
other = browser_driver(browser).run_cdp("Browser.getWindowForTarget", targetId=target_id) or {}
except Exception:
continue
if other.get("windowId") == window_id:
return "background"
return "unknown"
_VISIBLE_BACKOFF_MAX_SEC = 900.0
def _schedule_visible_retry(session: Any, config: "IdleMaintenanceConfig", *, probed: bool) -> None:
"""Back off after finding the tab visible.
A side-effect-free ``front`` hint retries at the normal cadence; an actual
probe (which toggles focus emulation) backs off exponentially up to 15 min
so a tab the user keeps looking at sees at most a handful of blur/focus
pairs. Any real activity resets the streak (see schedule_idle_maintenance).
"""
base = max(30.0, float(config.freeze_after_sec))
if not probed:
setattr(session, "_uwapi_freeze_retry_at", time.time() + base)
return
activity = last_activity_at(session)
streak = int(getattr(session, "_uwapi_visible_probe_streak", 0) or 0)
if activity > float(getattr(session, "_uwapi_visible_probe_mark", 0.0) or 0.0):
streak = 0
streak += 1
setattr(session, "_uwapi_visible_probe_streak", streak)
setattr(session, "_uwapi_visible_probe_mark", activity)
delay = min(_VISIBLE_BACKOFF_MAX_SEC, base * (2 ** (streak - 1)))
setattr(session, "_uwapi_freeze_retry_at", time.time() + delay)
def _restore_focus_emulation(session: Any, tab: Any) -> None:
if getattr(session, "_uwapi_focus_emulation_off", False):
_set_focus_emulation(tab, True)
setattr(session, "_uwapi_focus_emulation_off", False)
def resume_if_frozen(session: Any, reason: str = "") -> bool:
"""Set a frozen tab back to ``active``. Cheap no-op when not frozen.
Always goes through the lifecycle lock so a concurrent freeze cannot land
after the session has been acquired.
"""
if session is None:
return False
if (
not getattr(session, "_uwapi_frozen", False)
and not getattr(session, "_uwapi_freeze_in_progress", False)
and not getattr(session, "_uwapi_focus_emulation_off", False)
):
return False
with lifecycle_lock(session):
tab = getattr(session, "tab", None)
if not getattr(session, "_uwapi_frozen", False):
_restore_focus_emulation(session, tab)
return False
ok = False
try:
_run_cdp(tab, "Page.setWebLifecycleState", state="active", timeout=2.0)
ok = True
except Exception as exc:
logger.warning(
f"[{getattr(session, 'id', '?')}] 解除标签页冻结失败(reason={reason or '-'}): {exc}"
)
finally:
_restore_focus_emulation(session, tab)
if ok:
setattr(session, "_uwapi_frozen", False)
setattr(session, "_uwapi_freeze_token", "")
setattr(session, "_uwapi_last_resumed_at", time.time())
setattr(session, "_uwapi_resume_failed_at", 0.0)
else:
# B5:解冻失败时保留「待确认冻结」状态,调用方据此拒绝交付该会话;
# 旧实现无条件清除冻结标志,acquire() 会把仍冻结的页面当作可用标签页交出去。
setattr(session, "_uwapi_resume_failed_at", time.time())
frozen_at = float(getattr(session, "_uwapi_frozen_at", 0.0) or 0.0)
if ok:
logger.debug(
f"[{getattr(session, 'id', '?')}] 标签页已解除冻结 "
f"(reason={reason or '-'}, frozen_for={max(0.0, time.time() - frozen_at):.0f}s)"
)
return ok
def freeze_session(manager: Any, session: Any, config: IdleMaintenanceConfig) -> bool:
"""Freeze one idle, hidden tab. Returns True when the tab was frozen."""
if session is None or getattr(manager, "_shutdown", False):
return False
lock = lifecycle_lock(session)
if not lock.acquire(timeout=0.5):
return False
try:
setattr(session, "_uwapi_freeze_in_progress", True)
session_lock = getattr(session, "_lock", None)
if session_lock is not None:
with session_lock:
eligible = _is_idle(session)
else:
eligible = _is_idle(session)
if not eligible or getattr(session, "_uwapi_frozen", False):
return False
if getattr(session, "_cdp_recycle_in_progress", False):
return False
if (time.time() - last_activity_at(session)) < config.freeze_after_sec:
return False
tab = getattr(session, "tab", None)
if tab is None:
return False
def claimed() -> bool:
# 冻结给占用让路:acquire 先把状态置为 BUSY 再在 lifecycle 锁上等待解冻,
# 这里每一步都检查,发现被占用立刻收手,不让请求等冻结流程走完
return (not _is_idle(session)) or bool(getattr(manager, "_shutdown", False))
if config.freeze_require_hidden and _front_tab_hint(tab) == "front":
# 用户很可能正看着它:不切换焦点仿真(避免给页面发 blur/focus),直接跳过
_schedule_visible_retry(session, config, probed=False)
return False
if claimed():
return False
# 先关闭焦点仿真,否则 Chromium 视其为 visible,冻结无效(见上方说明)
setattr(session, "_uwapi_focus_emulation_off", True)
_set_focus_emulation(tab, False)
if config.freeze_require_hidden:
state = _wait_real_hidden(tab, should_abort=claimed)
if state == "aborted" or claimed():
_restore_focus_emulation(session, tab)
return False
if state != "hidden":
# 用户正在看这个标签页(前台 / 独立窗口未最小化)或无法判断:不冻结
_restore_focus_emulation(session, tab)
_schedule_visible_retry(session, config, probed=True)
return False
token = f"{getattr(session, 'id', '?')}:{time.time_ns()}"
mark_installed = False
try:
mark_installed = bool(driver_for_tab(tab).run_js(_FREEZE_MARK_INSTALL_JS, token, timeout=1.0))
except Exception:
mark_installed = False
if claimed():
_restore_focus_emulation(session, tab)
return False
# 此后若被占用,占用方会在 lifecycle 锁上等待并解冻
try:
_run_cdp(tab, "Page.setWebLifecycleState", state="frozen", timeout=1.5)
except Exception:
_restore_focus_emulation(session, tab)
raise
if mark_installed and not _freeze_took_effect(tab, token, should_abort=claimed):
# Chrome 没有真正冻结(页面仍在跑):撤销,稍后重试
try:
_run_cdp(tab, "Page.setWebLifecycleState", state="active", timeout=1.5)
except Exception:
pass
_restore_focus_emulation(session, tab)
setattr(session, "_uwapi_freeze_retry_at", time.time() + 30.0)
setattr(session, "_uwapi_freeze_ineffective_count",
int(getattr(session, "_uwapi_freeze_ineffective_count", 0) or 0) + 1)
logger.debug(f"[{getattr(session, 'id', '?')}] 冻结未生效(页面仍在运行),30s 后重试")
return False
setattr(session, "_uwapi_freeze_token", token if mark_installed else "")
setattr(session, "_uwapi_frozen", True)
setattr(session, "_uwapi_frozen_at", time.time())
setattr(session, "_uwapi_frozen_check_at", time.time())
setattr(session, "_uwapi_visible_probe_streak", 0)
logger.debug(f"[{getattr(session, 'id', '?')}] 空闲标签页已冻结 (Page.setWebLifecycleState=frozen)")
return True
except Exception as exc:
logger.debug_throttled(
f"tab_pool.freeze_error.{getattr(session, 'id', '?')}",
f"[{getattr(session, 'id', '?')}] 空闲标签页冻结失败(忽略): {exc}",
interval_sec=300.0,
)
setattr(session, "_uwapi_freeze_retry_at", time.time() + 300.0)
return False
finally:
setattr(session, "_uwapi_freeze_in_progress", False)
lock.release()
# --------------------------------------------------------------------------- #
# CDP session recycle (P0-3)
# --------------------------------------------------------------------------- #
def _supports_recycle(tab: Any) -> bool:
return all(callable(getattr(tab, name, None)) for name in ("disconnect", "_driver_init", "_get_document"))
def recycle_tab_connection(tab: Any) -> None:
"""Drop and re-create the tab's CDP websocket without touching the page.
Chrome detaches the old DevTools session, which releases every RemoteObject
and DOM-agent node mapping held for it.
"""
target_id = getattr(tab, "_target_id", None) or getattr(tab, "tab_id", None)
if not target_id:
raise RuntimeError("tab target id unavailable")
listener = getattr(tab, "_listener", None)
# 每次尝试都从 disconnect 开始:_driver_init 第一步就取新驱动,中途失败会留下一个
# 半初始化的驱动(_is_loading 卡住、load 回调未注册、文档根未获取)。实测半初始化 / 取驱动失败 /
# _get_document 失败三种情况下,标签页都会在后续请求里报错或空等 10s+,必须完整重建。
# 另外 DrissionPage 的 _get_document() 失败时不抛异常:读不到返回 False;
# 另一线程(页面加载事件)正在读时直接返回 None;读的过程中抛异常则会让 _is_reading 标志一直卡住,
# 之后每次都“直接返回 None”。只看异常会把这些情况当成功、留下失效的文档根(run_js 报 ElementLostError)。
last_exc: Optional[BaseException] = None
for attempt in range(RECYCLE_REBUILD_ATTEMPTS):
try:
tab.disconnect()
except Exception:
pass
try:
if getattr(tab, "_is_reading", False):
tab._is_reading = False # 旧会话上中断的读取留下的标志,新驱动上重新读取
tab._driver_init(target_id)
doc = tab._get_document()
if doc is None:
# 新驱动的页面加载事件正在读取文档:等它读完
deadline = time.monotonic() + 5.0
while getattr(tab, "_is_reading", False) and time.monotonic() < deadline:
time.sleep(0.05)
elif doc is False:
raise RuntimeError("document root unavailable after reconnect")
if not _tab_connection_healthy(tab):
raise RuntimeError("tab does not respond after reconnect")
last_exc = None
break
except Exception as exc:
last_exc = exc
if attempt + 1 < RECYCLE_REBUILD_ATTEMPTS:
time.sleep(0.5 * (attempt + 1))
if last_exc is not None:
raise last_exc
# 监听器(补丁版 _reuse_driver 模式)引用的是旧驱动
if listener is not None:
for attr, value in (("_driver", None), ("_network_enabled", False), ("listening", False)):
try:
if hasattr(listener, attr):
setattr(listener, attr, value)
except Exception:
pass
# 会话级注入脚本随旧会话一起失效
try:
init_jss = getattr(tab, "_init_jss", None)
if isinstance(init_jss, list):
init_jss.clear()
except Exception:
pass
for attr in ("_kimi_fetch_init_js_id",):
try:
if hasattr(tab, attr):
delattr(tab, attr)
except Exception:
pass
def _reinstall_command_init_scripts(session: Any, tab: Any) -> int:
"""Re-register command ``run_js_file`` new-document scripts on the fresh CDP session.
``Page.addScriptToEvaluateOnNewDocument`` registrations belong to the CDP session and die
with it, while the command engine's per-session registry (and the bootstrap mtime cache)
still say "installed" -- so page-resilience / monitoring scripts would silently stop being
injected after the next reload. Re-register the same source and update the identifier.
The page itself was not reloaded, so the script is NOT executed again in the current document.
"""
registry = getattr(session, "_command_init_js_registry", None)
if not isinstance(registry, dict) or not registry:
return 0
restored = 0
lost = False
for key, entry in list(registry.items()):
source = str(entry.get("source", "") or "") if isinstance(entry, dict) else ""
if not source:
continue
try:
result = _run_cdp(tab, "Page.addScriptToEvaluateOnNewDocument", timeout=5.0, source=source)
identifier = ""
if isinstance(result, dict):
identifier = str(result.get("identifier") or result.get("scriptId") or "").strip()
entry["identifier"] = identifier
restored += 1
except Exception as exc:
# 让命令引擎下一轮重新同步(启动脚本会重新注册)
registry.pop(key, None)
lost = True
logger.warning(
f"[{getattr(session, 'id', '?')}] CDP 回收后重新注册命令预注入脚本失败: "
f"{entry.get('path', key) if isinstance(entry, dict) else key}: {exc}"
)
if lost:
state = getattr(session, "_command_bootstrap_js_files", None)
if isinstance(state, dict):
state.clear()
return restored
RECYCLE_REBUILD_ATTEMPTS = 3
def _tab_connection_healthy(tab: Any) -> bool:
try:
return driver_for_tab(tab).run_js("return 1", timeout=2.0) == 1
except Exception:
return False
def _quarantine_broken_tab(session: Any, tab: Any, reason: str) -> None:
"""Recycle could not rebuild the connection: never hand this tab out again as-is.
Stops whatever half-initialised driver is left, drops the DrissionPage singleton so a
later pool rescan builds a fresh tab object (instead of getting the same broken one back),
and marks the session ERROR so the pool's existing cleanup / quarantine takes over.
"""
try:
tab.disconnect()
except Exception:
pass
from app.core.driver.drission_internals import forget_tab_object
forget_tab_object(tab)
try:
session.mark_error(reason)
except Exception:
pass
def _dom_node_count(tab: Any) -> int:
try:
result = _run_cdp(tab, "Memory.getDOMCounters", timeout=1.0)
except Exception:
return 0
if isinstance(result, dict):
try:
return int(result.get("nodes") or 0)
except (TypeError, ValueError):
return 0
return 0
def _recycle_reason(session: Any, config: IdleMaintenanceConfig, now: float) -> str:
if not config.recycle_enabled:
return ""
tab = getattr(session, "tab", None)
if tab is None or not _supports_recycle(tab):
return ""
if now < float(getattr(session, "_uwapi_recycle_retry_at", 0.0) or 0.0):
return ""
last_recycle = getattr(session, "_uwapi_last_recycle_at", None)
if last_recycle is None:
# 从未回收:以会话创建为起点,创建以来处理过的请求都计数
setattr(session, "_uwapi_last_recycle_at", float(getattr(session, "created_at", now) or now))
setattr(session, "_uwapi_recycle_request_mark", 0)
last_recycle = getattr(session, "_uwapi_last_recycle_at")
last_recycle = float(last_recycle or 0.0)
# 回收只对“上次回收后有过使用”的标签页有意义
used_since = float(getattr(session, "last_used_at", 0.0) or 0.0) > last_recycle + 1.0 or float(
getattr(session, "_pc_last_wake_at", 0.0) or 0.0
) > last_recycle + 1.0
if not used_since:
return ""
if (now - float(getattr(session, "last_used_at", 0.0) or 0.0)) < config.recycle_idle_sec:
return ""
if config.recycle_after_requests > 0:
mark = int(getattr(session, "_uwapi_recycle_request_mark", 0) or 0)
served = int(getattr(session, "request_count", 0) or 0) - mark
if served >= config.recycle_after_requests:
return f"requests={served}"
if config.recycle_interval_sec > 0 and (now - last_recycle) >= config.recycle_interval_sec:
return f"interval={now - last_recycle:.0f}s"
if config.recycle_dom_nodes > 0:
last_check = float(getattr(session, "_uwapi_dom_check_at", 0.0) or 0.0)
if (now - last_check) >= config.dom_check_interval_sec:
setattr(session, "_uwapi_dom_check_at", now)
nodes = _dom_node_count(tab)
if nodes >= config.recycle_dom_nodes:
return f"dom_nodes={nodes}"
return ""
def recycle_session(manager: Any, session: Any, reason: str) -> bool:
"""Recycle one idle session's CDP connection under a maintenance lease."""
if session is None or getattr(manager, "_shutdown", False):
return False
tabs = getattr(manager, "_tabs", {}) or {}
if tabs.get(getattr(session, "id", None)) is not session:
return False
tab = getattr(session, "tab", None)
if tab is None or not _supports_recycle(tab):
return False
task_id = f"maint:cdp_recycle:{int(time.time() * 1000)}"
saved_last_used = float(getattr(session, "last_used_at", 0.0) or 0.0)
# 本次若因条件不满足而跳过,60s 内不再尝试回收(否则每个 tick 都选中回收,冻结永远轮不到)
setattr(session, "_uwapi_recycle_retry_at", time.time() + 60.0)
if not session.acquire_for_command(task_id):
return False
started = time.perf_counter()
ok = False
setattr(session, "_cdp_recycle_in_progress", True)
try:
stop = getattr(manager, "_stop_global_monitor_for_session", None)
if callable(stop) and not stop(session.id, reason="cdp_recycle", wait=True):
logger.debug(f"[{session.id}] 全局监听未能及时停止,跳过本次 CDP 回收")
return False
listener = getattr(tab, "_listener", None)
if listener is not None and getattr(listener, "listening", False):
logger.debug(f"[{session.id}] 标签页仍在监听网络,跳过本次 CDP 回收")
return False
recycle_tab_connection(tab)
_reinstall_command_init_scripts(session, tab)
# 旧会话注册的可见性模拟脚本已随会话失效;标签页释放时本来就会恢复页面状态
try:
from app.core.page_lifecycle import _clear_visibility_emulation_attrs
_clear_visibility_emulation_attrs(tab)
_clear_visibility_emulation_attrs(session)
except Exception:
pass
try:
setattr(session, "_audio_capture_init_script_source", None)
except Exception:
pass
setattr(session, "_uwapi_last_recycle_at", time.time())
setattr(session, "_uwapi_recycle_request_mark", int(getattr(session, "request_count", 0) or 0))
setattr(session, "_uwapi_recycle_count", int(getattr(session, "_uwapi_recycle_count", 0) or 0) + 1)
setattr(session, "_uwapi_recycle_retry_at", 0.0)
ok = True
logger.debug(
f"[{session.id}] 空闲标签页 CDP 会话已回收({reason},"
f"耗时 {time.perf_counter() - started:.2f}s)"
)
return True
except Exception as exc:
# 避免失败后每个 tick 重试
setattr(session, "_uwapi_last_recycle_at", time.time())
setattr(session, "_uwapi_recycle_failures", int(getattr(session, "_uwapi_recycle_failures", 0) or 0) + 1)
if _tab_connection_healthy(tab):
logger.warning(f"[{session.id}] 空闲标签页 CDP 会话回收失败,但连接仍可用,继续使用: {exc}")
else:
logger.warning(f"[{session.id}] 空闲标签页 CDP 会话回收失败且连接不可用,标记为错误交给池清理: {exc}")
_quarantine_broken_tab(session, tab, "cdp_recycle_failed")
return False
finally:
setattr(session, "_cdp_recycle_in_progress", False)
try:
manager.release(session.id, check_triggers=False, expected_task_id=task_id)
except Exception as exc:
logger.warning(f"[{session.id}] CDP 回收后释放标签页失败: {exc}")
# 维护占用不算“使用”:恢复空闲计时,冻结计时不被推迟
try:
session_lock = getattr(session, "_lock", None)
if session_lock is not None:
with session_lock:
if _is_idle(session):
session.last_used_at = saved_last_used
except Exception:
pass
# --------------------------------------------------------------------------- #
# scheduling (called from TabPoolManager.run_watchdog_tick)
# --------------------------------------------------------------------------- #
FROZEN_VISIBILITY_CHECK_SEC = 10.0
def sync_frozen_visibility(session: Any) -> bool:
"""Chrome lifts a CDP freeze by itself when the user switches to the tab or
restores the window (measured). Bring our bookkeeping in line: mark the tab
resumed and turn focus emulation back on, so it can be re-frozen later once
it is hidden and idle again. Returns True when the tab was found visible.
"""
if session is None or not getattr(session, "_uwapi_frozen", False):
return False
tab = getattr(session, "tab", None)
if tab is None:
return False
# 冻结状态下同步 run_js 仍可执行;此时焦点仿真已关闭,读到的是真实可见性
if _read_real_visibility(tab) == "visible":
resume_if_frozen(session, reason="user_visible")
return True
# 页面已在我们背后恢复运行(会话断开 / 用户切回又切走 / 导航):记账改回,之后可重新冻结
if freeze_was_lifted(session):
resume_if_frozen(session, reason="lifted_externally")
setattr(session, "_uwapi_freeze_retry_at", 0.0)
return False
def _run_maintenance(manager: Any, session: Any, config: IdleMaintenanceConfig, action: str, reason: str) -> None:
try:
if action == "recycle":
recycle_session(manager, session, reason)
elif action == "freeze":
freeze_session(manager, session, config)
elif action == "sync":
sync_frozen_visibility(session)
finally:
setattr(session, "_uwapi_maint_pending", False)
def schedule_idle_maintenance(manager: Any, config: Optional[IdleMaintenanceConfig] = None) -> int:
"""Pick idle sessions that need recycle/freeze and submit them to the maintenance pool."""
config = config or getattr(manager, "_idle_maintenance_config", None) or IdleMaintenanceConfig.from_env()
if not (config.freeze_enabled or config.recycle_enabled):
return 0
if getattr(manager, "_shutdown", False):
return 0
executor = getattr(manager, "_maintenance_executor", None)
if executor is None:
return 0
lock = getattr(manager, "_lock", None)
if lock is not None:
with lock:
sessions = list((getattr(manager, "_tabs", {}) or {}).values())
else:
sessions = list((getattr(manager, "_tabs", {}) or {}).values())
now = time.time()
scheduled = 0
for session in sessions:
try:
if getattr(session, "_uwapi_maint_pending", False):
continue
if not _is_idle(session) or getattr(session, "_cdp_recycle_in_progress", False):
continue
action = ""
reason = ""
frozen = bool(getattr(session, "_uwapi_frozen", False))
if frozen:
last_check = float(getattr(session, "_uwapi_frozen_check_at", 0.0) or 0.0)
if now - last_check >= FROZEN_VISIBILITY_CHECK_SEC:
setattr(session, "_uwapi_frozen_check_at", now)
action = "sync"
else:
reason = _recycle_reason(session, config, now)
if reason:
action = "recycle"
if not action and config.freeze_enabled and not frozen:
retry_at = float(getattr(session, "_uwapi_freeze_retry_at", 0.0) or 0.0)
activity = last_activity_at(session)
# 被“可见”退避推迟时,一旦有新活动就不再等退避期满
if int(getattr(session, "_uwapi_visible_probe_streak", 0) or 0) > 0 and activity > float(
getattr(session, "_uwapi_visible_probe_mark", 0.0) or 0.0
):
retry_at = min(retry_at, activity + config.freeze_after_sec)
if now >= retry_at and (now - activity) >= config.freeze_after_sec:
action = "freeze"
if not action:
continue
setattr(session, "_uwapi_maint_pending", True)
try:
executor.submit(_run_maintenance, manager, session, config, action, reason)
scheduled += 1
except RuntimeError:
setattr(session, "_uwapi_maint_pending", False)
break
except Exception as exc:
logger.debug(f"[TabPool] 空闲维护调度异常(忽略): {exc}")
return scheduled
__all__ = [
"IdleMaintenanceConfig",
"REAL_VISIBILITY_JS",
"freeze_session",
"is_idle_freeze_enabled",
"last_activity_at",
"lifecycle_lock",
"note_network_activity",
"recycle_session",
"recycle_tab_connection",
"resume_if_frozen",
"schedule_idle_maintenance",
"sync_frozen_visibility",
]
+273
View File
@@ -0,0 +1,273 @@
"""标签页池管理器。
R2-3:原 4500 行的单个类按职责拆成 mixin(pool_*.py),这里只保留类属性、__init__ 与 shutdown;
对外接口(TabPoolManager 的全部方法与属性)保持不变。
"""
import os
import threading
from collections import OrderedDict, deque
from concurrent.futures import ThreadPoolExecutor
from typing import Any, Dict, List, Optional
from app.core.config import BrowserConstants, logger
from app.utils.tab_route_groups import normalize_route_groups
from .idle_maintenance import IdleMaintenanceConfig
from .network import _GlobalNetworkInterceptionManager
from .recovery import TabQuarantineEntry, TabRecoveryService
from .session import TabSession
from .pool_config import TabPoolConfigMixin
from .pool_contexts import TabPoolContextMixin
from .pool_tabs import TabPoolTabsMixin
from .pool_sessions import TabPoolSessionsMixin
from .pool_allocation import TabPoolAllocationMixin
from .pool_maintenance import TabPoolMaintenanceMixin
from .pool_acquire import TabPoolAcquireMixin
from .pool_routing import TabPoolRoutingMixin
from .pool_status import TabPoolStatusMixin
class TabPoolManager(
TabPoolConfigMixin,
TabPoolContextMixin,
TabPoolTabsMixin,
TabPoolSessionsMixin,
TabPoolAllocationMixin,
TabPoolMaintenanceMixin,
TabPoolAcquireMixin,
TabPoolRoutingMixin,
TabPoolStatusMixin,
):
"""标签页池管理器"""
DOMAIN_ABBR_MAP = {
"chatgpt": "gpt",
"openai": "gpt",
"gemini": "gemini",
"aistudio": "aistudio",
"claude": "claude",
"anthropic": "claude",
"poe": "poe",
"bing": "bing",
"copilot": "copilot",
"perplexity": "pplx",
"lmarena": "lmarena",
"chat": "chat",
}
# 卡死超时时间(秒)
STUCK_TIMEOUT = 180
# 新标签页扫描间隔(秒)
SCAN_INTERVAL = 10
QUERY_SCAN_MIN_INTERVAL_SEC = 1.0
ISOLATED_CONTEXT_ORPHAN_GRACE_SEC = 3.0
ISOLATED_CONTEXT_REBIND_GRACE_SEC = 20.0
GET_TABS_FAILURE_COOLDOWN_SEC = 5.0
GET_TABS_WARNING_INTERVAL_SEC = 10.0
ROUTE_CURSOR_LIMIT = 1000
MAINTENANCE_WORKER_LIMIT = 4
TERMINATION_RELEASE_WAIT_SEC = 5.0
IDLE_MEMORY_PURGE_MIN_SEC = 60.0
CONTEXT_RECOVERY_CDP_TIMEOUT_SEC = 2.0
CONTEXT_RECOVERY_COOLDOWN_SEC = 30.0
CONTEXT_RECOVERY_CACHE_LIMIT = 2048
# A cancelled HTTP request must not leave its worker blocked in a 60s
# acquire. Polling this frequently keeps cancellation cleanup bounded
# without adding a second notification channel from RequestManager.
ACQUIRE_CANCEL_POLL_SEC = 0.25
def __init__(
self,
browser_page,
max_tabs: int = 5,
min_tabs: int = 1,
idle_timeout: float = 300,
acquire_timeout: float = 60,
stuck_timeout: float = STUCK_TIMEOUT,
allocation_mode: str = "first_idle",
excluded_urls: Optional[List[str]] = None,
preserve_error_tabs: bool = False,
auto_remember_url_presets: bool = False,
model_name_overrides: Optional[Dict[str, Any]] = None,
preset_overrides: Optional[Dict[str, Any]] = None,
route_groups: Optional[List[Dict[str, Any]]] = None,
):
self.page = browser_page
self.max_tabs = max(1, int(max_tabs))
self.min_tabs = min(self.max_tabs, max(1, int(min_tabs)))
self.idle_timeout = max(1.0, float(idle_timeout))
self.acquire_timeout = max(1.0, float(acquire_timeout))
self.stuck_timeout = max(1.0, float(stuck_timeout))
self.allocation_mode = self._normalize_allocation_mode(allocation_mode)
self.excluded_urls = self._normalize_excluded_urls(excluded_urls)
self.preserve_error_tabs = self._to_bool(preserve_error_tabs, False)
self.auto_remember_url_presets = self._to_bool(auto_remember_url_presets, False)
self.model_name_overrides = self._normalize_model_name_overrides(model_name_overrides)
self.preset_overrides = self._normalize_preset_overrides(preset_overrides)
self.route_groups = normalize_route_groups(route_groups)
self._tabs: Dict[str, TabSession] = {}
self._lock = threading.RLock()
self._condition = threading.Condition(self._lock)
self._scan_snapshot_lock = threading.Lock()
self._global_monitor_transition_lock = threading.RLock()
self._initialized = False
self._shutdown = False
self._tab_counter = 0
self._last_scan_time: float = 0
self._get_tabs_retry_after: float = 0.0
self._last_get_tabs_warning_at: float = 0.0
self._context_recovery_lock = threading.Lock()
self._context_recovery_pending: set[str] = set()
self._context_recovery_requested_at: Dict[str, float] = {}
# 记录已知的标签页底层 ID(用于检测新标签页)
self._known_tab_ids: set = set()
# 🆕 记录当前活动的标签页 ID(避免重复激活)
self._active_session_id: Optional[str] = None
self._auto_activate_on_acquire = self._to_bool(
os.getenv("TAB_AUTO_ACTIVATE_ON_ACQUIRE"), False
)
self._acquire_waiters: deque[str] = deque()
self._index_waiters: Dict[int, deque[str]] = {}
self._route_waiters: Dict[str, deque[str]] = {}
self._group_waiters: Dict[str, deque[str]] = {}
self._route_group_bindings: Dict[str, Dict[str, str]] = {}
self._waiter_counter = 0
# H9:所有 acquire 等待者(通用 / 按编号 / 按域名 / 按路由组)的总量上限,0 = 不限
try:
self._max_acquire_waiters = max(0, int(os.getenv("TAB_ACQUIRE_MAX_WAITERS", "128") or 128))
except ValueError:
self._max_acquire_waiters = 128
self._queue_full_rejections: "OrderedDict[str, float]" = OrderedDict()
# 🆕 持久化编号系统
self._next_persistent_index: int = 1 # 下一个可分配的编号
self._raw_id_to_persistent: Dict[str, int] = {} # raw_tab_id → persistent_index
self._persistent_to_session_id: Dict[int, str] = {} # persistent_index → session.id
self._isolated_context_by_raw_id: Dict[str, str] = {}
self._orphaned_isolated_contexts: Dict[str, float] = {}
self._round_robin_cursor: int = 0
self._route_round_robin_cursor: OrderedDict[str, int] = OrderedDict()
self._preserved_error_session_ids = set()
self._maintenance_executor: Optional[ThreadPoolExecutor] = ThreadPoolExecutor(
max_workers=self.MAINTENANCE_WORKER_LIMIT,
thread_name_prefix="tab-maint",
)
# acquire 专用线程池(懒创建):阻塞式 acquire 可占线程最长 acquire_timeout,
# 若复用 asyncio 默认 executor,池满时一批被取消/慢的 acquire 会占满默认
# executor,拖垮事件循环上所有 to_thread 调用。创建用独立小锁保护,
# 避免事件循环线程为建池去竞争可能被长持有的池锁。
self._acquire_executor: Optional[ThreadPoolExecutor] = None
self._acquire_executor_lock = threading.Lock()
# 错误标签页隔离集合(池锁 self._condition 保护下读写)。
# ERROR 会话被移出池但浏览器标签页保留时,其 raw id 先进隔离集合,
# 阻止 _scan_new_tabs 立即把同一 tab 当"新标签页"重新入池——
# 旧请求的 worker 线程可能仍阻塞在对该 tab 的同步 DrissionPage 调用里,
# 直接重入池会造成新旧工作流并发驱动同一页面。
self._quarantined_raw_ids: Dict[str, TabQuarantineEntry] = {}
# 恢复服务懒创建:创建仅涉及线程池构造(非阻塞),
# 用独立小锁保护,不与池锁竞争。
self._recovery_service: Optional[TabRecoveryService] = None
self._recovery_service_lock = threading.Lock()
# 全局常驻网络监听(可配置)
self._global_network_enabled = self._to_bool(
BrowserConstants.get("GLOBAL_NETWORK_INTERCEPTION_ENABLED"), False
)
self._global_network_listen_pattern = str(
BrowserConstants.get("GLOBAL_NETWORK_INTERCEPTION_LISTEN_PATTERN") or "http"
).strip() or "http"
self._global_network_wait_timeout = max(
0.1,
self._to_float(BrowserConstants.get("GLOBAL_NETWORK_INTERCEPTION_WAIT_TIMEOUT"), 0.5),
)
self._global_network_retry_delay = max(
0.2,
self._to_float(BrowserConstants.get("GLOBAL_NETWORK_INTERCEPTION_RETRY_DELAY"), 1.0),
)
# A JavaScript heap purge does not freeze or discard a page. Run it
# only for sessions that have been idle for a while, so DOM clicks and
# active workflows keep their existing timing behavior.
self._idle_memory_purge_enabled = self._to_bool(
os.getenv("BROWSER_IDLE_MEMORY_PURGE_ENABLED"),
True,
)
self._idle_memory_purge_after_sec = max(
self.IDLE_MEMORY_PURGE_MIN_SEC,
self._to_float(os.getenv("BROWSER_IDLE_MEMORY_PURGE_AFTER_SEC"), 300.0),
)
self._idle_memory_purge_interval_sec = max(
self.IDLE_MEMORY_PURGE_MIN_SEC,
self._to_float(os.getenv("BROWSER_IDLE_MEMORY_PURGE_INTERVAL_SEC"), 180.0),
)
self._idle_maintenance_config = IdleMaintenanceConfig.from_env()
self._global_network_monitor: Optional[_GlobalNetworkInterceptionManager] = None
if self._global_network_enabled:
self._global_network_monitor = _GlobalNetworkInterceptionManager(
get_session_fn=self._get_session_for_monitor_snapshot,
is_shutdown_fn=lambda: self._shutdown,
listen_pattern=self._global_network_listen_pattern,
wait_timeout=self._global_network_wait_timeout,
retry_delay=self._global_network_retry_delay,
res_types=BrowserConstants.get("GLOBAL_NETWORK_INTERCEPTION_RES_TYPES"),
)
try:
from app.services.arena_tab_listener import register_arena_tab_listener
register_arena_tab_listener(self._global_network_monitor)
except Exception as e:
logger.debug(f"[TabPool] 注册 Arena 扩展监听器失败(忽略): {e}")
logger.debug(
f"TabPoolManager 初始化 (max={max_tabs}, stuck_timeout={self.stuck_timeout}s, "
f"allocation_mode={self.allocation_mode}, preserve_error_tabs={self.preserve_error_tabs})"
)
def shutdown(self):
monitor = None
maintenance_executor = None
with self._lock:
self._shutdown = True
self._condition.notify_all()
monitor = self._global_network_monitor
self._global_network_monitor = None
maintenance_executor = self._maintenance_executor
self._maintenance_executor = None
# acquire 专用线程池随池一起关闭(不等待:阻塞中的 acquire 线程会因
# _shutdown + notify_all 自行退出)。
with self._acquire_executor_lock:
acquire_executor = self._acquire_executor
self._acquire_executor = None
with self._recovery_service_lock:
recovery_service = self._recovery_service
self._recovery_service = None
if monitor:
monitor.shutdown()
if maintenance_executor:
maintenance_executor.shutdown(wait=False, cancel_futures=True)
if acquire_executor:
acquire_executor.shutdown(wait=False, cancel_futures=True)
if recovery_service:
# 内部同样以 wait=False, cancel_futures=True 关闭其线程池
recovery_service.shutdown()
with self._lock:
self._tabs.clear()
self._known_tab_ids.clear()
self._quarantined_raw_ids.clear()
self._active_session_id = None # 🆕 重置活动标签页记录
recovery_lock = getattr(self, "_context_recovery_lock", None)
if recovery_lock is not None:
with recovery_lock:
getattr(self, "_context_recovery_pending", set()).clear()
getattr(self, "_context_recovery_requested_at", {}).clear()
# 🆕 清理编号映射
self._raw_id_to_persistent.clear()
self._persistent_to_session_id.clear()
self._isolated_context_by_raw_id.clear()
self._orphaned_isolated_contexts.clear()
self._round_robin_cursor = 0
self._route_round_robin_cursor.clear()
self._next_persistent_index = 1
logger.info("TabPoolManager 已关闭")
+619
View File
@@ -0,0 +1,619 @@
import threading
import time
from dataclasses import dataclass
from typing import Any, Callable, Dict, List, Optional, Tuple, Union
from app.core.config import logger
from ._arena_snapshot import _ARENA_STORE_SNAPSHOT_JS
from .idle_maintenance import note_network_activity
from .session import TabSession
from app.core.driver import driver_for_tab
ResponseListenerCallback = Callable[[TabSession, Any, Dict[str, Any], threading.Event], None]
ResponseFilter = Union[str, Callable[[Dict[str, Any]], bool], None]
@dataclass
class _GlobalNetworkWorker:
"""单个标签页的全局网络监听工作线程。"""
session_id: str
thread: threading.Thread
stop_event: threading.Event
class _GlobalNetworkInterceptionManager:
"""
全局常驻网络事件监听。
设计要点:
- 仅在标签页空闲时运行;
- 标签页被任务占用时暂停,让位给工作流内监听器;
- 命令触发逻辑仍由 CommandEngine 决定;
- 通过 register_response_listener 机制支持插件化挂载各种响应监听器(例如 Arena 翻牌探测等)。
"""
LISTENER_STOP_TIMEOUT_SEC = 2.0
LISTENER_CLEAR_INTERVAL_SEC = 60.0
LISTENER_CLEAR_EVENT_INTERVAL = 200
def __init__(
self,
get_session_fn,
is_shutdown_fn,
listen_pattern: str = "http",
wait_timeout: float = 0.5,
retry_delay: float = 1.0,
res_types: Any = None,
):
self._get_session = get_session_fn
self._is_shutdown = is_shutdown_fn
self._listen_pattern = str(listen_pattern or "http").strip() or "http"
# P0-4:只捕获 API 类资源。补丁版 DrissionPage 会对每个命中的包附加
# streamResourceContent 并缓存响应体,放开到脚本/图片/样式会显著增加内存与 CPU。
self._res_types = self.parse_res_types(res_types)
self._wait_timeout = max(0.1, float(wait_timeout or 0.5))
self._retry_delay = max(0.2, float(retry_delay or 1.0))
self._workers: Dict[str, _GlobalNetworkWorker] = {}
self._lock = threading.RLock()
self._stop_join_timeout = max(2.0, self._wait_timeout + self._retry_delay + 0.2)
self._response_listeners: List[Tuple[ResponseFilter, ResponseListenerCallback]] = []
self._listeners_lock = threading.RLock()
def register_response_listener(
self,
pattern_or_filter: ResponseFilter,
callback: ResponseListenerCallback,
) -> Callable[[], None]:
"""
注册全局网络响应观察者。
Args:
pattern_or_filter: 匹配规则。可为 URL 匹配子串、Callable[[event_dict], bool] 或 None(匹配全部)。
callback: 响应处理回调,签名 (session, response, event, stop_event) -> None。
Returns:
注销函数。
"""
with self._listeners_lock:
entry = (pattern_or_filter, callback)
self._response_listeners.append(entry)
def _unregister():
self.unregister_response_listener(callback)
return _unregister
def unregister_response_listener(self, callback: ResponseListenerCallback) -> bool:
"""注销全局网络响应观察者。"""
with self._listeners_lock:
original_len = len(self._response_listeners)
self._response_listeners = [
item for item in self._response_listeners if item[1] != callback
]
return len(self._response_listeners) < original_len
@staticmethod
def _extract_event(response: Any) -> Dict[str, Any]:
req = getattr(response, "request", None)
resp = getattr(response, "response", None)
url = (
getattr(req, "url", None)
or getattr(resp, "url", None)
or getattr(response, "url", None)
or ""
)
method = (
getattr(req, "method", None)
or getattr(response, "method", None)
or ""
)
status = (
getattr(resp, "status", None)
or getattr(resp, "status_code", None)
or getattr(response, "status", None)
or 0
)
try:
status = int(status)
except (TypeError, ValueError, OverflowError):
status = 0
return {
"url": str(url or ""),
"method": str(method or "").upper(),
"status": status,
"timestamp": time.time(),
}
@staticmethod
def _is_expected_stop_error(error: Any) -> bool:
text = str(error or "").strip().lower()
if not text:
return False
expected_markers = (
"监听未启动或已停止",
"target closed",
"invalid session",
"no such window",
"not connected",
"connection refused",
"disconnected",
)
if "nonetype" in text and "is_running" in text:
return True
return any(marker in text for marker in expected_markers)
@staticmethod
def _force_reset_listen_state(tab: Any) -> bool:
listener = getattr(tab, "listen", None)
if listener is None:
return True
ok = True
for attr, value in (
("listening", False),
("_network_enabled", False),
("_driver", None),
):
try:
if hasattr(listener, attr):
setattr(listener, attr, value)
except Exception:
ok = False
try:
clear = getattr(listener, "clear", None)
if callable(clear):
clear()
except Exception:
ok = False
return ok
@staticmethod
def _force_stop_listener_driver(listener: Any) -> bool:
driver = getattr(listener, "_driver", None)
if driver is None:
return False
stopped = False
for method_name in ("stop", "close", "disconnect"):
method = getattr(driver, method_name, None)
if not callable(method):
continue
try:
method()
stopped = True
break
except Exception:
pass
return stopped
@staticmethod
def _listener_is_marked_active(listener: Any) -> bool:
try:
return bool(getattr(listener, "listening", False))
except Exception:
return False
@classmethod
def _safe_stop_listen(cls, tab: Any) -> bool:
listener = getattr(tab, "listen", None)
if listener is None:
return True
try:
if cls._listener_is_marked_active(listener):
stop_result: Dict[str, Any] = {"error": None}
def _stop_listener():
try:
listener.stop()
except Exception as e:
stop_result["error"] = e
stop_thread = threading.Thread(
target=_stop_listener,
daemon=True,
name="global-net-listen-stop",
)
stop_thread.start()
stop_thread.join(timeout=cls.LISTENER_STOP_TIMEOUT_SEC)
if stop_thread.is_alive():
logger.warning(
f"[GlobalNet] listen.stop timed out after "
f"{cls.LISTENER_STOP_TIMEOUT_SEC:.1f}s; "
"listener remains unavailable"
)
# The stop thread can still mutate the listener when it eventually
# returns. Never report success or reset its private fields here,
# otherwise a new owner could start listening before that late write.
return False
if stop_result["error"] is not None:
raise stop_result["error"]
except Exception as e:
reset_ok = cls._force_reset_listen_state(tab)
log = logger.debug if cls._is_expected_stop_error(e) else logger.warning
log(f"[GlobalNet] listen.stop failed; forced_reset={reset_ok}: {e}")
return reset_ok and not cls._listener_is_marked_active(listener)
try:
if cls._listener_is_marked_active(listener):
reset_ok = cls._force_reset_listen_state(tab)
if cls._listener_is_marked_active(listener):
logger.warning(
f"[GlobalNet] listen.stop returned but listener is still active "
f"(forced_reset={reset_ok})"
)
return False
logger.debug("[GlobalNet] listener state reset after stop")
except Exception as e:
reset_ok = cls._force_reset_listen_state(tab)
log = logger.debug if cls._is_expected_stop_error(e) else logger.warning
log(f"[GlobalNet] listen state check failed; forced_reset={reset_ok}: {e}")
return reset_ok
try:
clear = getattr(listener, "clear", None)
if callable(clear):
clear()
except Exception:
pass
cls._reset_listener_res_type(listener)
return True
@staticmethod
def _reset_listener_res_type(listener: Any) -> None:
"""tab.listen 是共享对象:让出时把资源类型过滤恢复为“全部”,
避免工作流/命令脚本里不带 res_type 的 listen.start() 继承全局监听的过滤。"""
try:
if getattr(listener, "_res_type", True) is not True:
listener._res_type = True
except Exception:
pass
@staticmethod
def _safe_clear_listener(tab: Any, session_id: str, reason: str) -> bool:
listener = getattr(tab, "listen", None)
if listener is None:
return True
try:
clear = getattr(listener, "clear", None)
if callable(clear):
clear()
return True
except Exception as e:
logger.debug(f"[GlobalNet] 清理监听残留失败: {session_id}, reason={reason}, err={e}")
return False
return False
def _should_cleanup_worker_listener(
self,
session_id: str,
stop_event: threading.Event,
tab: Any = None,
) -> bool:
with self._lock:
current = self._workers.get(session_id)
if current is None or current.stop_event is stop_event:
return True
if tab is None:
return False
try:
session = self._get_session(session_id)
return session is None or getattr(session, "tab", None) is not tab
except Exception:
return False
def _dispatch_event(self, session: TabSession, event: Dict[str, Any]):
try:
from app.services.command_engine import command_engine
command_engine.handle_network_event(session, event)
except Exception as e:
logger.debug(f"[GlobalNet] 事件上报失败(忽略): {e}")
@classmethod
def extract_request_post_data(cls, response: Any) -> Any:
req = getattr(response, "request", None)
if req is None:
return None
for attr in ("postData", "post_data", "body"):
try:
value = getattr(req, attr, None)
except Exception:
value = None
if value:
return value
return None
_extract_request_post_data = extract_request_post_data
@classmethod
def read_response_body(cls, response: Any, stop_event: Optional[threading.Event] = None) -> tuple[Any, str]:
if getattr(response, "_stream_enabled", False):
deadline = time.time() + 120.0
while time.time() < deadline and (stop_event is None or not stop_event.is_set()):
stream = getattr(response, "_stream", None)
if isinstance(stream, dict) and stream.get("complete"):
break
time.sleep(0.1)
stream = getattr(response, "_stream", None)
if isinstance(stream, dict):
return stream.get("fullText") or "", "drission_stream"
return "", "drission_stream"
resp = getattr(response, "response", None)
if resp is None:
return "", "response_missing"
try:
body = getattr(resp, "body", "")
return body or "", "response_body"
except Exception as e:
logger.debug(f"[GlobalNet] 读取候选响应体失败: {e}")
return "", "response_body_error"
_read_response_body = read_response_body
def _dispatch_response_listeners(
self,
session: TabSession,
response: Any,
event: Dict[str, Any],
stop_event: threading.Event,
) -> None:
with self._listeners_lock:
listeners = list(self._response_listeners)
for pattern_or_filter, callback in listeners:
if stop_event.is_set() or self._is_shutdown():
break
try:
matched = False
if pattern_or_filter is None or pattern_or_filter == "*":
matched = True
elif isinstance(pattern_or_filter, str):
matched = pattern_or_filter.lower() in str(event.get("url", "")).lower()
elif callable(pattern_or_filter):
matched = bool(pattern_or_filter(event))
if matched and callable(callback):
callback(session, response, event, stop_event)
except Exception as e:
logger.debug(f"[GlobalNet] 响应监听器回调执行异常(忽略): {e}")
def _forget_worker_if_current(self, worker: _GlobalNetworkWorker) -> None:
with self._lock:
current = self._workers.get(worker.session_id)
if current is worker:
self._workers.pop(worker.session_id, None)
def _join_stopping_worker(self, worker: _GlobalNetworkWorker, reason: str) -> bool:
if not worker.thread.is_alive():
self._forget_worker_if_current(worker)
return True
if worker.thread is threading.current_thread():
return False
worker.thread.join(timeout=self._stop_join_timeout)
if worker.thread.is_alive():
logger.warning(
f"[GlobalNet] previous worker still stopping: {worker.session_id} "
f"(reason={reason or '-'})"
)
return False
self._forget_worker_if_current(worker)
return True
DEFAULT_RES_TYPES = ("XHR", "Fetch", "EventSource", "Document")
@classmethod
def parse_res_types(cls, value: Any) -> Any:
"""``None`` → default API types; ``*``/``all``/``True`` → no filter."""
if value is None:
return tuple(cls.DEFAULT_RES_TYPES)
if value is True:
return True
if isinstance(value, str):
text = value.strip()
if not text:
return tuple(cls.DEFAULT_RES_TYPES)
if text.lower() in {"*", "all", "true", "any"}:
return True
items = [part.strip() for part in text.replace(";", ",").split(",")]
elif isinstance(value, (list, tuple, set)):
items = [str(part).strip() for part in value]
else:
return tuple(cls.DEFAULT_RES_TYPES)
items = [item for item in items if item]
return tuple(items) if items else tuple(cls.DEFAULT_RES_TYPES)
def _start_listen(self, tab: Any) -> None:
try:
driver_for_tab(tab).listener.start(self._listen_pattern, res_type=self._res_types)
except TypeError:
# 简化的测试替身 / 旧版 Listener 不支持 res_type
driver_for_tab(tab).listener.start(self._listen_pattern)
def start_for_session(self, session: TabSession) -> bool:
if not session:
return False
while True:
with self._lock:
existing = self._workers.get(session.id)
if existing is None:
break
if not existing.thread.is_alive():
self._workers.pop(session.id, None)
break
if not existing.stop_event.is_set():
return True
if not self._join_stopping_worker(existing, "start"):
return False
with self._lock:
existing = self._workers.get(session.id)
if existing and existing.thread.is_alive():
return True
stop_event = threading.Event()
thread = threading.Thread(
target=self._worker_loop,
args=(session.id, stop_event),
daemon=True,
name=f"global-net-{session.id}",
)
self._workers[session.id] = _GlobalNetworkWorker(
session_id=session.id,
thread=thread,
stop_event=stop_event,
)
thread.start()
# logger.debug(f"[GlobalNet] 启动监听: {session.id} pattern={self._listen_pattern!r}")
return True
def stop_for_session(self, session_id: str, reason: str = "", join: bool = False) -> bool:
if not session_id:
return True
with self._lock:
worker = self._workers.get(session_id)
if not worker:
return True
worker.stop_event.set()
session = self._get_session(session_id)
stop_ok = True
if session is not None:
stop_ok = self._safe_stop_listen(session.tab)
should_join = bool(join or not stop_ok)
if should_join and worker.thread.is_alive() and worker.thread is not threading.current_thread():
worker.thread.join(timeout=self._stop_join_timeout)
if worker.thread.is_alive():
logger.warning(
f"[GlobalNet] worker did not stop promptly: {session_id} "
f"(reason={reason or '-'}, stop_listen_ok={stop_ok}, "
f"requested_join={join})"
)
return False
if not worker.thread.is_alive():
self._forget_worker_if_current(worker)
return True
def request_stop_for_session(
self,
session_id: str,
reason: str = "",
*,
detach: bool = False,
) -> bool:
if not session_id:
return True
with self._lock:
worker = self._workers.get(session_id)
if not worker:
return True
worker.stop_event.set()
return True
def shutdown(self):
with self._lock:
session_ids = list(self._workers.keys())
for session_id in session_ids:
self.stop_for_session(session_id, reason="shutdown", join=True)
logger.info("[GlobalNet] 全局网络监听已关闭")
def _worker_loop(self, session_id: str, stop_event: threading.Event):
tab = None
listening = False
last_listener_clear_at = time.monotonic()
events_since_listener_clear = 0
try:
while not stop_event.is_set():
if self._is_shutdown():
break
session = self._get_session(session_id)
if session is None:
break
tab = session.tab
if not listening:
try:
# 复用连接,降低对 CDP session 的额外占用
driver_for_tab(tab).listener.reuse_driver = True
self._start_listen(tab)
listening = True
last_listener_clear_at = time.monotonic()
events_since_listener_clear = 0
except Exception as e:
logger.debug(f"[GlobalNet] 启动监听失败: {session_id}, err={e}")
stop_event.wait(self._retry_delay)
continue
now = time.monotonic()
if listening and now - last_listener_clear_at >= self.LISTENER_CLEAR_INTERVAL_SEC:
if self._safe_clear_listener(tab, session_id, "interval"):
last_listener_clear_at = now
events_since_listener_clear = 0
try:
response = driver_for_tab(tab).listener.wait(timeout=self._wait_timeout)
except Exception as e:
if stop_event.is_set() or self._is_shutdown():
break
err_text = str(e)
if "NoneType" in err_text and "is_running" in err_text:
logger.debug(f"[GlobalNet] 监听状态失效,准备重启: {session_id}")
else:
logger.debug(f"[GlobalNet] wait 异常: {session_id}, err={e}")
listening = False
if not self._safe_stop_listen(tab):
logger.warning(
f"[GlobalNet] listener could not stop safely; "
f"ending worker: {session_id}"
)
break
stop_event.wait(self._retry_delay)
continue
if response is None or response is False:
continue
if stop_event.is_set() or self._is_shutdown():
break
note_network_activity(session)
event = self._extract_event(response)
self._dispatch_event(session, event)
self._dispatch_response_listeners(session, response, event, stop_event)
events_since_listener_clear += 1
if events_since_listener_clear >= self.LISTENER_CLEAR_EVENT_INTERVAL:
if self._safe_clear_listener(tab, session_id, "event_budget"):
last_listener_clear_at = time.monotonic()
events_since_listener_clear = 0
finally:
if tab is not None and self._should_cleanup_worker_listener(session_id, stop_event, tab):
self._safe_stop_listen(tab)
with self._lock:
current = self._workers.get(session_id)
if current is not None and current.stop_event is stop_event:
self._workers.pop(session_id, None)
+730
View File
@@ -0,0 +1,730 @@
"""TabPoolManager 的一部分(R2-3 拆分):借用与归还:等待队列、通用/按编号/按原始 ID 借用、释放。
方法原样从 manager.py 迁出,行为不变;通过 mixin 组合回 TabPoolManager,
类属性与实例状态都在 TabPoolManager(manager.py)里定义。
"""
import asyncio
import threading
import time
from collections import OrderedDict, deque
from concurrent.futures import ThreadPoolExecutor
from contextlib import asynccontextmanager
from typing import Any, Dict, List, Optional
from app.core.config import logger
from .session import TabSession, TabStatus
class TabPoolAcquireMixin:
"""借用与归还:等待队列、通用/按编号/按原始 ID 借用、释放"""
def _normalize_acquire_timeout(self, timeout: Optional[float]) -> float:
"""统一各 acquire 路径的 timeout 语义。
原先大多数路径写 `timeout = timeout or self.acquire_timeout`,于是
timeout=0(调用方想表达"不等待,立刻失败")会被当成"未传"而退化成默认
60 秒阻塞,只有 acquire_by_route_group 一条路径按显式值处理。
统一成:None -> 池默认;其余按显式值处理,负数收敛到 0。
"""
if timeout is None:
return float(self.acquire_timeout)
try:
return max(0.0, float(timeout))
except (TypeError, ValueError):
return float(self.acquire_timeout)
@staticmethod
def _request_should_stop(task_id: str) -> bool:
"""Return whether the request owning a blocking acquire was cancelled."""
request_id = str(task_id or "").strip()
if not request_id:
return False
try:
from app.services.request_manager import request_manager
context = request_manager.get_request(request_id)
return bool(context is not None and context.should_stop())
except Exception:
# Pool acquisition is also used by command/internal callers that do
# not have a RequestContext; cancellation lookup must stay best effort.
return False
def _total_acquire_waiters(self) -> int:
total = len(self._acquire_waiters)
for group in (self._index_waiters, self._route_waiters, self._group_waiters):
for waiters in group.values():
total += len(waiters)
return total
def _acquire_queue_full(self, task_id: str) -> bool:
"""H9:所有 acquire 等待队列的总量上限(调用方持有 self._condition)。
满了立即返回 True:调用方直接返回 None,并记下该 task 是「队列满」而不是
「没有标签页」,工作流层据此给出明确的 503(可重试)而不是再排 60 秒。
"""
limit = int(getattr(self, "_max_acquire_waiters", 0) or 0)
if limit <= 0:
return False
total = self._total_acquire_waiters()
if total < limit:
return False
rejections = getattr(self, "_queue_full_rejections", None)
if rejections is None:
rejections = OrderedDict()
self._queue_full_rejections = rejections
rejections[str(task_id or "")] = time.time()
while len(rejections) > 256:
rejections.popitem(last=False)
logger.warning(
f"[TabPool] acquire 等待队列已满({total}/{limit}),立即拒绝 task={task_id}"
)
return True
def consume_queue_full_rejection(self, task_id: str) -> bool:
"""工作流层查询:该 task 的 acquire 是否因等待队列已满被拒(查询即清除)。"""
rejections = getattr(self, "_queue_full_rejections", None)
if not rejections:
return False
with self._condition:
return rejections.pop(str(task_id or ""), None) is not None
def _next_waiter_token(self, task_id: str) -> str:
self._waiter_counter += 1
base = str(task_id or "task").strip() or "task"
return f"{base}#{self._waiter_counter}"
@staticmethod
def _is_waiter_turn(waiters: deque[str], waiter_token: str) -> bool:
return not waiters or waiters[0] == waiter_token
@staticmethod
def _count_waiters_ahead(waiters: deque[str], waiter_token: str) -> int:
try:
return waiters.index(waiter_token)
except ValueError:
return 0
def _unregister_waiter(
self,
waiters: deque[str],
waiter_token: str,
*,
owner_map: Optional[Dict[Any, deque[str]]] = None,
owner_key: Optional[Any] = None,
) -> bool:
removed = False
try:
waiters.remove(waiter_token)
removed = True
except ValueError:
removed = False
# 只有当 owner_map 里登记的仍是同一个队列对象时才摘除。
# 成功路径会先显式 _unregister_waiter 一次(此时队列可能已空并被摘除),
# 随后 finally 再调一次;期间池锁可能被 _complete_acquired_session_for_return
# 短暂释放,其他线程会用同一个 key 建出**新的**队列。若这里无条件 pop,
# 就会把新队列连同里面排队的等待者一起丢掉,破坏 FIFO 公平性。
if (
owner_map is not None
and owner_key is not None
and not waiters
and owner_map.get(owner_key) is waiters
):
owner_map.pop(owner_key, None)
if removed:
self._condition.notify_all()
return removed
@staticmethod
def _describe_session(session: Optional[TabSession]) -> str:
if session is None:
return "session=-"
try:
return session.debug_summary()
except Exception as e:
return f"session={getattr(session, 'id', '-')}, snapshot_error={e}"
def _describe_index_wait_state(
self,
persistent_index: int,
waiters: deque[str],
waiter_token: str,
) -> str:
session_id = self._persistent_to_session_id.get(persistent_index)
session = self._tabs.get(session_id) if session_id else None
ahead = self._count_waiters_ahead(waiters, waiter_token)
busy_for = "-"
if session is not None:
try:
if getattr(session, "status", None) == TabStatus.BUSY:
busy_for = (
f"{max(0.0, time.time() - float(getattr(session, 'last_used_at', 0.0) or 0.0)):.1f}s"
)
except Exception:
busy_for = "?"
return (
f"idx=#{persistent_index}, ahead={ahead}, waiters={len(waiters)}, "
f"busy_for={busy_for}, holder={self._describe_session(session)}"
)
async def acquire_async(self, task_id: str, timeout: float = None) -> Optional[TabSession]:
return await self._run_cancellable_acquire(
lambda: self.acquire(task_id, timeout),
task_id,
)
def _ensure_acquire_executor(self) -> ThreadPoolExecutor:
# 双重检查懒创建:读属性无锁,创建走专用小锁(不碰池锁,事件循环线程不会
# 被池维护路径阻塞)。max_workers=32 覆盖"池满 + 一批客户端断连"的堆积场景。
executor = self._acquire_executor
if executor is not None:
return executor
with self._acquire_executor_lock:
if self._acquire_executor is None:
self._acquire_executor = ThreadPoolExecutor(
max_workers=32,
thread_name_prefix="tab-acquire",
)
return self._acquire_executor
async def _run_cancellable_acquire(self, acquire_fn, task_id: str) -> Optional[TabSession]:
"""Release a session acquired after the awaiting task was cancelled."""
# 改用专用 executor:默认 executor 线程数有限(约 min(32, cpu+4)),被取消的
# acquire 仍会占线程直到超时,占满后所有 to_thread 排不上队。
# run_in_executor 返回的 Future 同样兼容 shield 与 add_done_callback,
# shield 语义不变:外层取消不打断底层 acquire 线程,结果由回调兜底释放。
loop = asyncio.get_running_loop()
acquire_task = loop.run_in_executor(self._ensure_acquire_executor(), acquire_fn)
try:
return await asyncio.shield(acquire_task)
except asyncio.CancelledError:
def _release_late_session(completed_task) -> None:
try:
session = completed_task.result()
except BaseException:
return
if session is None:
return
def _do_release() -> None:
try:
self.release(
session.id,
check_triggers=False,
rollback_request_count=True,
expected_task_id=task_id,
)
except Exception as exc:
logger.warning(
f"Cancelled acquire cleanup failed (task={task_id}, "
f"tab={getattr(session, 'id', '-')}, error={exc})"
)
# 锁序考量:done callback 在事件循环线程执行,release 含池锁竞争与
# CDP 调用(visibility 恢复各 0.5s 超时),不能同步跑在事件循环里。
# 投递到 maintenance executor;submit 失败(已关闭)回退 daemon 线程。
executor = self._maintenance_executor
submitted = False
if executor is not None:
try:
executor.submit(_do_release)
submitted = True
except RuntimeError as exc:
logger.debug(
f"[TabPool] maintenance submit failed "
f"(late-release:{getattr(session, 'id', '-')}): {exc}"
)
if not submitted:
threading.Thread(
target=_do_release,
name="tab-late-release",
daemon=True,
).start()
acquire_task.add_done_callback(_release_late_session)
raise
def release(
self,
tab_id: str,
clear_page: bool = False,
check_triggers: bool = True,
rollback_request_count: bool = False,
expected_task_id: str = "",
) -> bool:
"""释放标签页"""
with self._condition:
session = self._tabs.get(tab_id)
if not session:
return False
before_snapshot = self._describe_session(session)
expected_task = str(expected_task_id or "").strip()
current_task = str(getattr(session, "current_task_id", "") or "").strip()
session_status = getattr(getattr(session, "status", None), "value", "")
if current_task and not expected_task:
logger.warning(
f"[{tab_id}] 跳过无所有权释放 "
f"(current_task={current_task}, snapshot={before_snapshot})"
)
return False
if expected_task:
if current_task and current_task != expected_task:
logger.warning(
f"[{tab_id}] 跳过释放:标签页已被其他任务接管 "
f"(expected_task={expected_task}, current_task={current_task}, "
f"snapshot={before_snapshot})"
)
return False
if not current_task:
logger.warning(
f"[{tab_id}] 跳过释放:标签页 task_id 已丢失 "
f"(expected_task={expected_task}, status={session_status or 'unknown'}, "
f"snapshot={before_snapshot})"
)
return False
release_state = session._begin_release_state(
clear_page=clear_page,
rollback_request_count=rollback_request_count,
force=False,
expected_task_id=expected_task,
reject_ownerless=True,
)
if release_state is None:
self._condition.notify_all()
return False
session._run_release_from_state(
release_state,
clear_page=clear_page,
check_triggers=check_triggers,
rollback_request_count=rollback_request_count,
)
with self._condition:
if self._tabs.get(tab_id) is session:
# 锁序考量:不在持池锁时同步启动 monitor——start_for_session 可能
# join 停止中的 worker 最长 2s(本文件他处已注明持池锁禁止 join worker)。
# 复用 _restart_global_monitor_for_session_async 投递到 maintenance
# executor:它会在池锁外复核会话仍存在且 IDLE 再启动,start_for_session
# 本身幂等(已有存活 worker 时直接返回);若会话已被再次 acquire,
# 复核发现非 IDLE 即放弃,acquire 侧的 handoff stop 亦会兜底摘除。
self._restart_global_monitor_for_session_async(tab_id, "release")
self._condition.notify_all()
logger.debug(
f"[{tab_id}] 已释放 "
f"(expected_task={expected_task or '-'}, clear_page={clear_page}, "
f"check_triggers={check_triggers}, rollback_request_count={rollback_request_count}, "
f"before={before_snapshot}, after={self._describe_session(session)})"
)
return session.status == TabStatus.IDLE
return False
def force_release_all(self):
"""强制释放所有标签页(调试用)"""
with self._condition:
pending: List[tuple[TabSession, Dict[str, Any]]] = []
for session in self._tabs.values():
if session.status == TabStatus.BUSY:
# 先请求取消旧工作流再强制释放:否则 BUSY 会话被洗成 IDLE 后,
# 旧工作流尚未退出,同一 tab 可能立刻被再分配,新旧任务并发操作
# 同一页面。锁序安全:_cancel_active_request_for_session 只向
# maintenance executor 投递,持池锁时不外呼。
busy_task = str(session.current_task_id or "").strip()
cancel_submitted = self._cancel_active_request_for_session(
session,
"force_release_all",
detail=f"task={busy_task or '-'}",
)
logger.warning(
f"[{session.id}] force_release_all 强制释放忙碌会话 "
f"(task={busy_task or '-'}, cancel_submitted={cancel_submitted})"
)
release_state = session._begin_release_state(
clear_page=False,
force=True,
)
if release_state is not None:
pending.append((session, release_state))
for session, release_state in pending:
session._run_force_release_from_state(
release_state,
clear_page=False,
check_triggers=False,
)
with self._condition:
for session, _release_state in pending:
if self._tabs.get(session.id) is session and session.status == TabStatus.IDLE:
self._start_global_monitor_for_session(session)
self._condition.notify_all()
count = len(pending)
logger.info(f"强制释放 {count} 个标签页")
return count
@asynccontextmanager
async def get_tab(self, task_id: str, timeout: float = None):
session = await self.acquire_async(task_id, timeout)
if session is None:
raise TimeoutError(f"获取标签页超时 (task: {task_id})")
try:
yield session
except Exception as e:
session.mark_error(str(e))
raise
finally:
# 锁序考量:release 含池锁竞争与 CDP 调用(visibility 恢复各 0.5s 超时),
# 移出事件循环线程执行。shield 保证 __aexit__ 期间再次被取消时,
# release 仍在后台任务/线程中完成,不会泄漏标签页占用。
await asyncio.shield(
asyncio.to_thread(self.release, session.id, expected_task_id=task_id)
)
def acquire_by_raw_tab_id(
self,
raw_tab_id: str,
task_id: str,
timeout: float = None,
count_request: bool = True,
activate: Optional[bool] = None,
) -> Optional[TabSession]:
"""
根据底层浏览器标签页 ID 获取指定标签页会话。
这用于外部已经明确知道目标标签页时,复用 TabPool 的运行时上下文,
保持与正常工作流执行一致的监听、占用与释放行为。
"""
raw_tab_id = str(raw_tab_id or "").strip()
if not raw_tab_id:
return None
timeout = self._normalize_acquire_timeout(timeout)
deadline = time.time() + timeout
with self._condition:
while True:
if self._shutdown:
return None
if self._request_should_stop(task_id):
logger.debug(f"Raw tab acquire cancelled (task={task_id})")
return None
if self._should_scan():
self._scan_new_tabs()
self._check_stuck_tabs()
self._cleanup_unhealthy_tabs()
persistent_index = self._raw_id_to_persistent.get(raw_tab_id)
if not persistent_index:
logger.warning(f"底层标签页 ID 不存在: {raw_tab_id}")
return None
session_id = self._persistent_to_session_id.get(persistent_index)
if not session_id:
logger.warning(f"底层标签页 {raw_tab_id} 未绑定会话")
return None
session = self._tabs.get(session_id)
if not session:
logger.warning(f"底层标签页 {raw_tab_id} 对应会话已移除")
return None
if not session.is_healthy(allow_live_check=False):
logger.warning(f"[{session.id}] 标签页不健康")
return None
if self._should_defer_to_command(session, task_id):
logger.debug(f"[{session.id}] defer raw-tab acquire to high-priority command")
remaining = deadline - time.time()
if remaining <= 0:
return None
self._condition.wait(timeout=min(remaining, 0.5))
continue
if session.status == TabStatus.IDLE:
acquired = (
session.acquire(task_id)
if count_request
else session.acquire_for_command(task_id)
)
if acquired:
if not self._complete_acquired_session_for_return(
session,
"acquire_by_raw_tab_id",
task_id,
rollback_request_count=count_request,
activate=(
self._auto_activate_on_acquire and session.id != self._active_session_id
if activate is None
else bool(activate)
),
):
continue
logger.debug(
f"TabPool → {session.id} "
f"(task={task_id}, raw_tab_id={raw_tab_id}, idx=#{persistent_index}, "
f"count_request={count_request}, snapshot={self._describe_session(session)})"
)
return session
remaining = deadline - time.time()
if remaining <= 0:
logger.warning(
f"获取底层标签页 {raw_tab_id} 超时 "
f"(task={task_id}, 当前状态: {session.status.value}, "
f"snapshot={self._describe_session(session)})"
)
return None
logger.debug_throttled(
f"tab_pool.wait_raw.{raw_tab_id}",
f"等待底层标签页 {raw_tab_id} 释放...",
interval_sec=5.0,
)
self._condition.wait(
timeout=min(remaining, self.ACQUIRE_CANCEL_POLL_SEC)
)
async def acquire_by_index_async(self, persistent_index: int, task_id: str, timeout: float = None) -> Optional[TabSession]:
"""异步版本的按编号获取"""
return await self._run_cancellable_acquire(
lambda: self.acquire_by_index(persistent_index, task_id, timeout),
task_id,
)
def acquire(self, task_id: str, timeout: float = None) -> Optional[TabSession]:
"""ASCII-safe fair acquire for generic request routing."""
timeout = self._normalize_acquire_timeout(timeout)
deadline = time.time() + timeout
logged_waiting = False
first_iteration = True
with self._condition:
if self._acquire_queue_full(task_id):
return None
waiter_token = self._next_waiter_token(task_id)
self._acquire_waiters.append(waiter_token)
try:
while True:
if self._shutdown:
return None
if self._request_should_stop(task_id):
logger.debug(f"Acquire cancelled (task={task_id})")
return None
if first_iteration or self._should_scan():
self._scan_new_tabs()
first_iteration = False
self._check_stuck_tabs()
self._cleanup_unhealthy_tabs()
if not self._is_waiter_turn(self._acquire_waiters, waiter_token):
remaining = deadline - time.time()
if remaining <= 0:
busy_info = [
f"{s.id}({s.current_task_id})"
for s in self._tabs.values()
if s.status == TabStatus.BUSY
]
unhealthy_count = sum(
1 for s in self._tabs.values()
if s.status == TabStatus.IDLE and not s.is_healthy(allow_live_check=False)
)
logger.warning(
f"Acquire tab timed out (task={task_id}, busy: {', '.join(busy_info) or 'none'}, "
f"unhealthy: {unhealthy_count})"
)
return None
if not logged_waiting:
busy_tabs = [s.id for s in self._tabs.values() if s.status == TabStatus.BUSY]
ahead = self._count_waiters_ahead(self._acquire_waiters, waiter_token)
if busy_tabs:
logger.debug(f"Waiting for tab (busy: {', '.join(busy_tabs)})")
elif ahead > 0:
logger.debug(f"Waiting in queue (ahead: {ahead})")
logged_waiting = True
self._condition.wait(
timeout=min(remaining, self.ACQUIRE_CANCEL_POLL_SEC)
)
continue
dynamic_sessions = self._get_sessions_for_dynamic_routing()
if not dynamic_sessions and self._tabs:
logger.warning(
f"Acquire tab skipped: all open tabs are excluded from dynamic routing "
f"(task={task_id})"
)
return None
session = self._try_acquire_session_for_request(
dynamic_sessions,
task_id,
)
if session is not None:
self._unregister_waiter(self._acquire_waiters, waiter_token)
if not self._complete_acquired_session_for_return(
session,
"acquire",
task_id,
activate=self._auto_activate_on_acquire and session.id != self._active_session_id,
):
return None
self._mark_allocation_cursor(session)
if logged_waiting:
logger.debug(
f"Acquire finished -> {session.id} "
f"(task={task_id}, snapshot={self._describe_session(session)})"
)
else:
logger.debug(
f"TabPool -> {session.id} "
f"(task={task_id}, snapshot={self._describe_session(session)})"
)
return session
remaining = deadline - time.time()
if remaining <= 0:
busy_info = [
f"{s.id}({s.current_task_id})"
for s in self._tabs.values()
if s.status == TabStatus.BUSY
]
unhealthy_count = sum(
1 for s in self._tabs.values()
if s.status == TabStatus.IDLE and not s.is_healthy(allow_live_check=False)
)
logger.warning(
f"Acquire tab timed out (task={task_id}, busy: {', '.join(busy_info) or 'none'}, "
f"unhealthy: {unhealthy_count})"
)
return None
if not logged_waiting:
busy_tabs = [s.id for s in self._tabs.values() if s.status == TabStatus.BUSY]
if busy_tabs:
logger.debug(f"Waiting for tab (busy: {', '.join(busy_tabs)})")
logged_waiting = True
self._condition.wait(
timeout=min(remaining, self.ACQUIRE_CANCEL_POLL_SEC)
)
finally:
self._unregister_waiter(self._acquire_waiters, waiter_token)
def acquire_by_index(self, persistent_index: int, task_id: str, timeout: float = None) -> Optional[TabSession]:
"""ASCII-safe fair acquire for a fixed persistent tab index."""
timeout = self._normalize_acquire_timeout(timeout)
deadline = time.time() + timeout
with self._condition:
if self._acquire_queue_full(task_id):
return None
waiters = self._index_waiters.setdefault(persistent_index, deque())
waiter_token = self._next_waiter_token(task_id)
waiters.append(waiter_token)
try:
while True:
if self._shutdown:
return None
if self._request_should_stop(task_id):
logger.debug(f"Index acquire cancelled (task={task_id})")
return None
if self._should_scan():
self._scan_new_tabs()
# Keep fixed-index acquisition aligned with other wait paths so a wedged
# holder can be cancelled and released instead of blocking the queue.
released_stuck = self._check_stuck_tabs()
self._cleanup_unhealthy_tabs()
if released_stuck:
logger.debug(
f"Rechecking tab #{persistent_index} after stuck-tab maintenance "
f"(task={task_id}, wait_state={self._describe_index_wait_state(persistent_index, waiters, waiter_token)})"
)
if not self._is_waiter_turn(waiters, waiter_token):
remaining = deadline - time.time()
if remaining <= 0:
logger.warning(
f"Timed out waiting for tab #{persistent_index} "
f"(task={task_id}, wait_state={self._describe_index_wait_state(persistent_index, waiters, waiter_token)})"
)
return None
logger.debug_throttled(
f"tab_pool.wait_index.{persistent_index}",
f"Waiting for tab #{persistent_index} release... "
f"({self._describe_index_wait_state(persistent_index, waiters, waiter_token)})",
interval_sec=5.0,
)
self._condition.wait(
timeout=min(remaining, self.ACQUIRE_CANCEL_POLL_SEC)
)
continue
session_id = self._persistent_to_session_id.get(persistent_index)
if not session_id:
logger.warning(f"Persistent tab #{persistent_index} does not exist")
return None
session = self._tabs.get(session_id)
if not session:
logger.warning(f"Tab {session_id} (#{persistent_index}) was removed")
return None
if not session.is_healthy(allow_live_check=False):
logger.warning(
f"[{session.id}] tab is unhealthy "
f"(task={task_id}, snapshot={self._describe_session(session)})"
)
return None
if self._should_defer_to_command(session, task_id):
logger.debug(
f"[{session.id}] defer by index acquire to high-priority command "
f"(task={task_id}, snapshot={self._describe_session(session)})"
)
remaining = deadline - time.time()
if remaining <= 0:
return None
self._condition.wait(timeout=min(remaining, 0.5))
continue
if session.status == TabStatus.IDLE and session.acquire(task_id):
self._unregister_waiter(
waiters,
waiter_token,
owner_map=self._index_waiters,
owner_key=persistent_index,
)
if not self._complete_acquired_session_for_return(
session,
"acquire_by_index",
task_id,
activate=self._auto_activate_on_acquire and session.id != self._active_session_id,
):
return None
logger.debug(
f"TabPool -> {session.id} "
f"(task={task_id}, idx=#{persistent_index}, "
f"snapshot={self._describe_session(session)})"
)
return session
remaining = deadline - time.time()
if remaining <= 0:
logger.warning(
f"Timed out waiting for tab #{persistent_index} "
f"(task={task_id}, status: {session.status.value}, "
f"wait_state={self._describe_index_wait_state(persistent_index, waiters, waiter_token)}, "
f"snapshot={self._describe_session(session)})"
)
return None
logger.debug_throttled(
f"tab_pool.wait_index.{persistent_index}",
f"Waiting for tab #{persistent_index} release... "
f"({self._describe_index_wait_state(persistent_index, waiters, waiter_token)})",
interval_sec=5.0,
)
self._condition.wait(
timeout=min(remaining, self.ACQUIRE_CANCEL_POLL_SEC)
)
finally:
self._unregister_waiter(
waiters,
waiter_token,
owner_map=self._index_waiters,
owner_key=persistent_index,
)
+366
View File
@@ -0,0 +1,366 @@
"""TabPoolManager 的一部分(R2-3 拆分):分配顺序与游标、全局网络监听的交接(借出/归还时的启停)。
方法原样从 manager.py 迁出,行为不变;通过 mixin 组合回 TabPoolManager,
类属性与实例状态都在 TabPoolManager(manager.py)里定义。
"""
import random
import threading
import time
from typing import Any, List, Optional
from app.core.config import logger
from .session import TabSession, TabStatus
class TabPoolAllocationMixin:
"""分配顺序与游标、全局网络监听的交接(借出/归还时的启停)"""
def _order_sessions_for_allocation(
self,
sessions: List[TabSession],
*,
route_domain: Optional[str] = None,
allocation_mode: Optional[str] = None,
) -> List[TabSession]:
ordered = sorted(list(sessions or []), key=self._session_allocation_key)
mode = self._normalize_allocation_mode(allocation_mode or self.allocation_mode)
if mode == "random" and len(ordered) > 1:
randomized = ordered[:]
random.shuffle(randomized)
return randomized
if mode != "round_robin" or len(ordered) <= 1:
return ordered
cursor = (
self._route_round_robin_cursor.get(route_domain, 0)
if route_domain
else self._round_robin_cursor
)
cursor_index = self._persistent_index_value(cursor)
next_items = [
item for item in ordered
if self._persistent_index_value(item.persistent_index) > cursor_index
]
return next_items + [
item for item in ordered
if self._persistent_index_value(item.persistent_index) <= cursor_index
]
@staticmethod
def _persistent_index_value(value: Any) -> int:
try:
return int(value or 0)
except (TypeError, ValueError):
return 0
@classmethod
def _session_allocation_key(cls, session: TabSession):
return (
cls._persistent_index_value(getattr(session, "persistent_index", 0)),
str(getattr(session, "id", "") or ""),
)
def _select_first_idle_acquire_candidate(
self,
sessions,
task_id: str,
attempted_ids: set,
*,
defer_context: str = "acquire",
) -> Optional[TabSession]:
selected = None
selected_key = None
for session in sessions or []:
session_id = str(getattr(session, "id", "") or "")
if session_id in attempted_ids:
continue
if session.status != TabStatus.IDLE:
continue
if not session.is_healthy(allow_live_check=False):
logger.warning(f"[{session.id}] skip unhealthy tab")
continue
if self._should_defer_to_command(session, task_id):
logger.debug(f"[{session.id}] defer {defer_context} to high-priority command")
continue
candidate_key = self._session_allocation_key(session)
if selected is None or candidate_key < selected_key:
selected = session
selected_key = candidate_key
return selected
def _try_acquire_session_for_request(
self,
sessions,
task_id: str,
*,
route_domain: Optional[str] = None,
allocation_mode: Optional[str] = None,
defer_context: str = "acquire",
) -> Optional[TabSession]:
mode = self._normalize_allocation_mode(allocation_mode or self.allocation_mode)
if mode == "first_idle":
attempted_ids = set()
while True:
session = self._select_first_idle_acquire_candidate(
sessions,
task_id,
attempted_ids,
defer_context=defer_context,
)
if session is None:
return None
attempted_ids.add(str(session.id or ""))
if session.acquire(task_id):
return session
for session in self._order_sessions_for_allocation(
sessions,
route_domain=route_domain,
allocation_mode=mode,
):
if session.status != TabStatus.IDLE:
continue
if not session.is_healthy(allow_live_check=False):
logger.warning(f"[{session.id}] skip unhealthy tab")
continue
if self._should_defer_to_command(session, task_id):
logger.debug(f"[{session.id}] defer {defer_context} to high-priority command")
continue
if session.acquire(task_id):
return session
return None
def _mark_allocation_cursor(self, session: TabSession, route_domain: Optional[str] = None) -> None:
current_index = int(session.persistent_index or 0)
if route_domain:
self._route_round_robin_cursor[route_domain] = current_index
self._route_round_robin_cursor.move_to_end(route_domain)
while len(self._route_round_robin_cursor) > self.ROUTE_CURSOR_LIMIT:
self._route_round_robin_cursor.popitem(last=False)
else:
self._round_robin_cursor = current_index
def _get_session_for_monitor(self, session_id: str) -> Optional[TabSession]:
with self._lock:
return self._tabs.get(session_id)
def _get_session_for_monitor_snapshot(self, session_id: str) -> Optional[TabSession]:
# Global monitor workers may be joined while the pool lock is held elsewhere.
# Use a lock-free snapshot lookup here so the worker can observe removal and exit
# instead of blocking behind the pool lock during shutdown/rebind paths.
return self._tabs.get(session_id)
def _start_global_monitor_for_session(self, session: Optional[TabSession]) -> bool:
if not session or not self._global_network_monitor:
return False
transition_lock = getattr(self, "_global_monitor_transition_lock", None)
if transition_lock is None:
transition_lock = self._global_monitor_transition_lock = threading.RLock()
# Start and stop use one transition lock. If acquisition changes the
# session to BUSY during a delayed start, its handoff stop waits here and
# removes the newly registered worker before returning the session.
with transition_lock:
if self._shutdown:
return False
if self._tabs.get(session.id) is not session:
return False
# 仅在空闲标签页常驻监听,任务执行时让位
if session.status != TabStatus.IDLE:
return False
return bool(self._global_network_monitor.start_for_session(session))
def _restart_global_monitor_for_session_async(self, session_id: str, reason: str) -> None:
"""Restart an idle session monitor outside the pool lock after a target rebind."""
session_key = str(session_id or "").strip()
executor = self._maintenance_executor
if not session_key or executor is None or not self._global_network_monitor:
return
def _restart() -> None:
deadline = time.monotonic() + 5.0
while True:
with self._lock:
if self._shutdown:
return
session = self._tabs.get(session_key)
if session is None or session.status != TabStatus.IDLE:
return
if self._start_global_monitor_for_session(session):
# logger.debug(f"[GlobalNet] rebound listener restarted: {session_key} ({reason})")
return
if time.monotonic() >= deadline:
logger.warning(f"[GlobalNet] rebound listener restart failed: {session_key} ({reason})")
return
time.sleep(0.1)
try:
executor.submit(_restart)
except RuntimeError as e:
logger.debug(f"[GlobalNet] restart submit failed ({session_key}): {e}")
def _stop_global_monitor_for_session(self, session_id: str, reason: str = "", wait: bool = False) -> bool:
if not self._global_network_monitor:
return True
transition_lock = getattr(self, "_global_monitor_transition_lock", None)
if transition_lock is None:
transition_lock = self._global_monitor_transition_lock = threading.RLock()
# Do not hold the pool lock while joining a worker: command callbacks can
# legitimately need that lock while the worker is winding down.
with transition_lock:
monitor = self._global_network_monitor
if not monitor:
return True
return bool(monitor.stop_for_session(session_id, reason=reason, join=wait))
def _request_global_monitor_stop_for_session(
self,
session_id: str,
reason: str = "",
*,
detach: bool = False,
) -> bool:
if not self._global_network_monitor:
return True
request_stop = getattr(self._global_network_monitor, "request_stop_for_session", None)
if callable(request_stop):
return bool(request_stop(session_id, reason=reason, detach=detach))
return self._stop_global_monitor_for_session(session_id, reason=reason, wait=False)
def _detach_global_monitor_for_session(
self,
session_id: str,
reason: str = "",
) -> bool:
session_id = str(session_id or "").strip()
if not session_id:
return True
if not self._global_network_monitor:
return True
return self._request_global_monitor_stop_for_session(
session_id,
reason=reason,
detach=True,
)
def _prepare_acquired_session_for_handoff(
self,
session: TabSession,
reason: str,
*,
rollback_request_count: bool = True,
) -> bool:
self._request_global_monitor_stop_for_session(session.id, reason=reason)
return True
def _finish_acquired_session_handoff(
self,
session: TabSession,
reason: str,
task_id: str,
*,
rollback_request_count: bool = True,
) -> bool:
if self._stop_global_monitor_for_session(session.id, reason=reason, wait=True):
return True
logger.warning(
f"[{session.id}] global monitor did not stop before handoff "
f"(reason={reason}); marking session unhealthy"
)
expected_task = str(task_id or "").strip()
release_state = None
with self._condition:
current = self._tabs.get(session.id)
current_task = str(getattr(session, "current_task_id", "") or "").strip()
if (
current is session
and session.status == TabStatus.BUSY
and (not expected_task or current_task == expected_task)
):
release_state = session._begin_release_state(
clear_page=False,
rollback_request_count=rollback_request_count,
force=False,
)
if release_state is not None:
session._run_release_from_state(
release_state,
clear_page=False,
check_triggers=False,
rollback_request_count=rollback_request_count,
)
with self._condition:
current = self._tabs.get(session.id)
current_task = str(getattr(session, "current_task_id", "") or "").strip()
if current is session and (not expected_task or not current_task or current_task == expected_task):
session.mark_error("global_monitor_stop_timeout")
self._condition.notify_all()
return False
def _complete_acquired_session_for_return(
self,
session: TabSession,
reason: str,
task_id: str,
*,
rollback_request_count: bool = True,
activate: bool = False,
) -> bool:
self._prepare_acquired_session_for_handoff(
session,
reason,
rollback_request_count=rollback_request_count,
)
self._condition.release()
try:
handoff_ok = self._finish_acquired_session_handoff(
session,
reason,
task_id,
rollback_request_count=rollback_request_count,
)
finally:
self._condition.acquire()
if not handoff_ok:
return False
current_task = str(getattr(session, "current_task_id", "") or "").strip()
expected_task = str(task_id or "").strip()
if (
self._tabs.get(session.id) is not session
or session.status != TabStatus.BUSY
or (expected_task and current_task != expected_task)
):
logger.warning(
f"[{session.id}] acquire handoff lost ownership "
f"(reason={reason}, expected_task={expected_task or '-'}, "
f"current_task={current_task or '-'}, status={session.status.value})"
)
return False
if activate:
self._condition.release()
try:
session.activate()
finally:
self._condition.acquire()
current_task = str(getattr(session, "current_task_id", "") or "").strip()
if (
self._tabs.get(session.id) is not session
or session.status != TabStatus.BUSY
or (expected_task and current_task != expected_task)
):
logger.warning(
f"[{session.id}] acquire activation lost ownership "
f"(reason={reason}, expected_task={expected_task or '-'}, "
f"current_task={current_task or '-'}, status={session.status.value})"
)
return False
self._active_session_id = session.id
return True
def suspend_global_network_monitor(self, tab_id: str, reason: str = "manual"):
self._stop_global_monitor_for_session(tab_id, reason=reason)
def resume_global_network_monitor(self, tab_id: str, reason: str = "manual"):
with self._lock:
session = self._tabs.get(tab_id)
if not session:
return
if session.status != TabStatus.IDLE or not session.is_healthy():
return
self._start_global_monitor_for_session(session)
logger.debug(f"[GlobalNet] 恢复监听: {tab_id} ({reason})")
+289
View File
@@ -0,0 +1,289 @@
"""TabPoolManager 的一部分(R2-3 拆分):配置解析与运行时配置(排除 URL、模型名/预设覆盖、动态路由候选)。
方法原样从 manager.py 迁出,行为不变;通过 mixin 组合回 TabPoolManager,
类属性与实例状态都在 TabPoolManager(manager.py)里定义。
"""
from typing import Any, Dict, List, Optional
from app.core.config import logger
from app.utils.site_url import normalize_exact_tab_url, normalize_route_domain, tab_url_matches
from app.utils.tab_route_groups import normalize_route_groups, route_groups_by_id
from .session import TabSession
class TabPoolConfigMixin:
"""配置解析与运行时配置(排除 URL、模型名/预设覆盖、动态路由候选)"""
@staticmethod
def _to_bool(value: Any, default: bool = False) -> bool:
if isinstance(value, bool):
return value
if value is None:
return default
return str(value).strip().lower() in {"1", "true", "yes", "y", "on"}
@staticmethod
def _to_float(value: Any, default: float) -> float:
try:
return float(value)
except (TypeError, ValueError, OverflowError):
return default
@staticmethod
def _normalize_allocation_mode(value: Any) -> str:
normalized = str(value or "").strip().lower()
if normalized in {"round_robin", "random"}:
return normalized
return "first_idle"
@staticmethod
def _normalize_excluded_urls(value: Any) -> List[str]:
if not isinstance(value, list):
return []
normalized: List[str] = []
seen = set()
for item in value:
text = str(item or "").strip()
normalized_text = normalize_exact_tab_url(text) or text
if not normalized_text or normalized_text in seen:
continue
seen.add(normalized_text)
normalized.append(normalized_text)
return normalized
@staticmethod
def _normalize_model_name(value: Any) -> str:
return str(value or "").strip()
@classmethod
def _normalize_model_name_overrides(cls, value: Any) -> Dict[str, Dict[str, str]]:
payload = value if isinstance(value, dict) else {}
normalized = {"sites": {}, "urls": {}}
sites = payload.get("sites") if isinstance(payload, dict) else {}
if isinstance(sites, dict):
for key, model_name in sites.items():
route_key = normalize_route_domain(key)
display_name = cls._normalize_model_name(model_name)
if route_key and display_name:
normalized["sites"][route_key] = display_name
urls = payload.get("urls") if isinstance(payload, dict) else {}
if isinstance(urls, dict):
for key, model_name in urls.items():
url_key = normalize_exact_tab_url(str(key or "").strip())
display_name = cls._normalize_model_name(model_name)
if url_key and display_name:
normalized["urls"][url_key] = display_name
return normalized
@classmethod
def _normalize_preset_overrides(cls, value: Any) -> Dict[str, Dict[str, str]]:
payload = value if isinstance(value, dict) else {}
normalized = {"urls": {}}
urls = payload.get("urls") if isinstance(payload, dict) else {}
if isinstance(urls, dict):
for key, preset_name in urls.items():
url_key = normalize_exact_tab_url(str(key or "").strip())
display_preset = str(preset_name or "").strip()
if url_key and display_preset:
normalized["urls"][url_key] = display_preset
return normalized
def _apply_preset_overrides(self, session: TabSession, info: Optional[Dict[str, Any]] = None) -> None:
"""如果开启了自动按 URL 记忆预设,自动为标签页恢复该 URL 上次保存的预设"""
if not getattr(self, "auto_remember_url_presets", False):
return
current_url = str((info.get("url") if info else None) or session.last_known_url or "").strip()
if not current_url:
try:
current_url = str(session.tab.url or "").strip()
except Exception:
pass
normalized_url = normalize_exact_tab_url(current_url)
if not normalized_url:
return
overrides = self._normalize_preset_overrides(
getattr(self, "preset_overrides", None)
)
saved_preset = overrides["urls"].get(normalized_url)
if saved_preset:
applied = False
with session._lock:
if session.preset_name is None or session.preset_name == saved_preset:
session.set_preset(saved_preset, source="url")
applied = True
if applied and info is not None:
info["preset_name"] = saved_preset
info["preset_override_source"] = "url"
@staticmethod
def _default_model_name_for_info(info: Dict[str, Any]) -> str:
route_domain = str(info.get("route_domain") or "").strip()
current_domain = str(info.get("current_domain") or "").strip()
tab_id = str(info.get("id") or "").strip()
return route_domain or current_domain or tab_id or "web-browser"
def _apply_exposed_model_name(self, info: Dict[str, Any]) -> None:
default_name = self._default_model_name_for_info(info)
override_source = ""
override_name = self._normalize_model_name(info.get("model_name_override"))
if override_name:
override_source = "tab"
else:
overrides = self._normalize_model_name_overrides(
getattr(self, "model_name_overrides", None)
)
current_url = str(info.get("url") or "").strip()
normalized_url = normalize_exact_tab_url(current_url)
if normalized_url:
override_name = overrides["urls"].get(normalized_url, "")
if override_name:
override_source = "url"
if not override_name:
route_domain = normalize_route_domain(
info.get("route_domain") or info.get("current_domain") or ""
)
if route_domain:
override_name = overrides["sites"].get(route_domain, "")
if override_name:
override_source = "site"
info["default_model_name"] = default_name
info["exposed_model_name"] = override_name or default_name
info["model_name_override_source"] = override_source
def _get_session_override_model_name(self, session: TabSession) -> str:
"""获取 session 当前被用户显式覆盖的重命名模型名(若未重命名则返回空串)。"""
override_name = self._normalize_model_name(getattr(session, "model_name_override", None))
if override_name:
return override_name
try:
current_url, route_domain = session.get_cached_route_snapshot()
except Exception:
current_url, route_domain = "", ""
overrides = self._normalize_model_name_overrides(
getattr(self, "model_name_overrides", None)
)
normalized_url = normalize_exact_tab_url(current_url)
if normalized_url and normalized_url in overrides.get("urls", {}):
return str(overrides["urls"][normalized_url] or "").strip()
norm_domain = normalize_route_domain(route_domain)
if norm_domain and norm_domain in overrides.get("sites", {}):
return str(overrides["sites"][norm_domain] or "").strip()
return ""
def is_url_excluded(self, url: str) -> bool:
current_url = str(url or "").strip()
if not current_url:
return False
with self._lock:
excluded_urls = list(self.excluded_urls)
return any(tab_url_matches(excluded_url, current_url) for excluded_url in excluded_urls)
def _is_session_excluded_from_dynamic_routing(
self,
session: TabSession,
current_url: Optional[str] = None,
) -> bool:
if current_url is None:
try:
current_url, _actual_domain = session.get_cached_route_snapshot()
except Exception:
current_url = ""
return self.is_url_excluded(current_url)
def _get_sessions_for_dynamic_routing(self) -> List[TabSession]:
return [
session
for session in self._tabs.values()
if not self._is_session_excluded_from_dynamic_routing(session)
]
def apply_runtime_config(
self,
*,
max_tabs: Optional[int] = None,
min_tabs: Optional[int] = None,
idle_timeout: Optional[float] = None,
acquire_timeout: Optional[float] = None,
stuck_timeout: Optional[float] = None,
allocation_mode: Optional[str] = None,
excluded_urls: Optional[List[str]] = None,
preserve_error_tabs: Optional[bool] = None,
auto_remember_url_presets: Optional[bool] = None,
model_name_overrides: Optional[Dict[str, Any]] = None,
preset_overrides: Optional[Dict[str, Any]] = None,
route_groups: Optional[List[Dict[str, Any]]] = None,
) -> Dict[str, Any]:
"""同步更新运行中的标签页池参数。"""
with self._lock:
new_max_tabs = self.max_tabs if max_tabs is None else max(1, int(max_tabs))
new_min_tabs = self.min_tabs if min_tabs is None else max(1, int(min_tabs))
if new_min_tabs > new_max_tabs:
new_min_tabs = new_max_tabs
self.max_tabs = new_max_tabs
self.min_tabs = new_min_tabs
if idle_timeout is not None:
self.idle_timeout = max(1.0, float(idle_timeout))
if acquire_timeout is not None:
self.acquire_timeout = max(1.0, float(acquire_timeout))
if stuck_timeout is not None:
self.stuck_timeout = max(1.0, float(stuck_timeout))
if allocation_mode is not None:
self.allocation_mode = self._normalize_allocation_mode(allocation_mode)
if excluded_urls is not None:
self.excluded_urls = self._normalize_excluded_urls(excluded_urls)
if preserve_error_tabs is not None:
self.preserve_error_tabs = self._to_bool(preserve_error_tabs, False)
if auto_remember_url_presets is not None:
self.auto_remember_url_presets = self._to_bool(auto_remember_url_presets, False)
if model_name_overrides is not None:
self.model_name_overrides = self._normalize_model_name_overrides(model_name_overrides)
if preset_overrides is not None:
self.preset_overrides = self._normalize_preset_overrides(preset_overrides)
if route_groups is not None:
self.route_groups = normalize_route_groups(route_groups)
valid_groups = route_groups_by_id(self.route_groups)
self._route_group_bindings = {
group_id: bindings
for group_id, bindings in self._route_group_bindings.items()
if group_id in valid_groups
}
self._condition.notify_all()
updated = {
"max_tabs": self.max_tabs,
"min_tabs": self.min_tabs,
"idle_timeout": self.idle_timeout,
"acquire_timeout": self.acquire_timeout,
"stuck_timeout": self.stuck_timeout,
"allocation_mode": self.allocation_mode,
"excluded_urls": list(self.excluded_urls),
"preserve_error_tabs": self.preserve_error_tabs,
"auto_remember_url_presets": self.auto_remember_url_presets,
"model_name_overrides": {
"sites": dict(getattr(self, "model_name_overrides", {}).get("sites", {})),
"urls": dict(getattr(self, "model_name_overrides", {}).get("urls", {})),
},
"preset_overrides": {
"urls": dict(getattr(self, "preset_overrides", {}).get("urls", {})),
},
"route_groups": normalize_route_groups(self.route_groups),
}
logger.info(
"[TabPool] 运行时配置已更新: "
f"max_tabs={self.max_tabs}, min_tabs={self.min_tabs}, "
f"idle_timeout={self.idle_timeout}, acquire_timeout={self.acquire_timeout}, "
f"stuck_timeout={self.stuck_timeout}, allocation_mode={self.allocation_mode}, "
f"excluded_urls={len(self.excluded_urls)}, "
f"preserve_error_tabs={self.preserve_error_tabs}, "
f"auto_remember_url_presets={self.auto_remember_url_presets}, "
f"model_name_overrides="
f"{len(updated['model_name_overrides']['sites'])}/"
f"{len(updated['model_name_overrides']['urls'])}, "
f"preset_overrides={len(updated['preset_overrides']['urls'])}"
f", route_groups={len(updated['route_groups'])}"
)
return updated
+574
View File
@@ -0,0 +1,574 @@
"""TabPoolManager 的一部分(R2-3 拆分):浏览器上下文与 CDP:独立 Cookie 上下文、目标快照、强制重载、上下文恢复。
方法原样从 manager.py 迁出,行为不变;通过 mixin 组合回 TabPoolManager,
类属性与实例状态都在 TabPoolManager(manager.py)里定义。
"""
import json
import threading
import time
from typing import Any, Dict, List, Optional
from app.core.config import logger
from ._utils import _looks_like_transient_local_debug_error
from .session import TabSession, TabStatus
from app.core.driver import browser_driver
class TabPoolContextMixin:
"""浏览器上下文与 CDP:独立 Cookie 上下文、目标快照、强制重载、上下文恢复"""
def _get_target_info(self, raw_tab_id: str) -> Dict[str, Any]:
if not raw_tab_id:
return {}
try:
browser = self._get_browser_handle()
if browser is None or not hasattr(browser, "_run_cdp"):
return {}
result = browser_driver(browser).run_cdp("Target.getTargetInfo", targetId=raw_tab_id) or {}
info = result.get("targetInfo") or {}
return info if isinstance(info, dict) else {}
except Exception as e:
logger.debug(f"[TabPool] get target info failed ({raw_tab_id}): {e}")
return {}
def _get_browser_context_id(self, raw_tab_id: str) -> Optional[str]:
if raw_tab_id in self._isolated_context_by_raw_id:
return self._isolated_context_by_raw_id.get(raw_tab_id)
info = self._get_target_info(raw_tab_id)
browser_context_id = str(info.get("browserContextId") or "").strip()
return browser_context_id or None
def _snapshot_current_tab_targets(
self,
) -> tuple[List[Any], List[str], Dict[str, str], Dict[str, str], set[str]]:
current_tabs = self._list_current_tab_refs()
current_tab_ids = [
raw_id for raw_id in
(self._get_tab_ref_id(tab_ref) for tab_ref in current_tabs)
if raw_id
]
current_context_by_raw: Dict[str, str] = {}
current_url_by_raw: Dict[str, str] = {}
current_url_known_raw: set[str] = set()
for tab_ref in current_tabs:
raw_id = self._get_tab_ref_id(tab_ref)
if not raw_id:
continue
if self._tab_ref_has_url_field(tab_ref):
current_url_known_raw.add(raw_id)
url = self._get_tab_ref_url(tab_ref)
if url:
current_url_by_raw[raw_id] = url
target_info_by_raw: Dict[str, Dict[str, Any]] = {}
for tab_ref in current_tabs:
raw_id = self._get_tab_ref_id(tab_ref)
if not raw_id:
continue
if isinstance(tab_ref, tuple) and len(tab_ref) >= 2 and isinstance(tab_ref[1], dict):
target_info_by_raw[raw_id] = tab_ref[1]
elif isinstance(tab_ref, dict):
target_info_by_raw[raw_id] = tab_ref
with self._lock:
cached_contexts = dict(self._isolated_context_by_raw_id)
for raw_id in current_tab_ids:
cached_context = str(cached_contexts.get(raw_id) or "").strip()
if cached_context:
current_context_by_raw[raw_id] = cached_context
continue
target_info = target_info_by_raw.get(raw_id) or {}
context_id = str(target_info.get("browserContextId") or "").strip()
if context_id:
current_context_by_raw[raw_id] = context_id
if raw_id in current_context_by_raw:
continue
try:
info = self._get_target_info(raw_id)
context_id = str(info.get("browserContextId") or "").strip()
if context_id:
current_context_by_raw[raw_id] = context_id
except Exception as e:
logger.debug(f"[TabPool] skip context lookup for {raw_id}: {e}")
return current_tabs, current_tab_ids, current_context_by_raw, current_url_by_raw, current_url_known_raw
def _is_site_independent_cookie_enabled(self, domain: str) -> bool:
normalized_domain = str(domain or "").strip()
if not normalized_domain:
return False
try:
from app.services.config_engine import config_engine
advanced = config_engine.get_site_advanced_config(normalized_domain)
return bool(advanced.get("independent_cookies", False))
except Exception as e:
logger.debug(f"[TabPool] load site advanced config failed ({normalized_domain}): {e}")
return False
def _is_site_independent_cookie_auto_takeover_enabled(self, domain: str) -> bool:
normalized_domain = str(domain or "").strip()
if not normalized_domain:
return False
try:
from app.services.config_engine import config_engine
advanced = config_engine.get_site_advanced_config(normalized_domain)
return bool(advanced.get("independent_cookies_auto_takeover", False))
except Exception as e:
logger.debug(f"[TabPool] load site advanced config failed ({normalized_domain}): {e}")
return False
def _register_isolated_context(self, raw_tab_id: str, browser_context_id: Optional[str]) -> None:
context_id = str(browser_context_id or "").strip()
if raw_tab_id and context_id:
self._isolated_context_by_raw_id[raw_tab_id] = context_id
self._orphaned_isolated_contexts.pop(context_id, None)
def _mark_orphaned_isolated_context(self, browser_context_id: Optional[str]) -> None:
context_id = str(browser_context_id or "").strip()
if context_id:
self._orphaned_isolated_contexts[context_id] = (
time.time() + self.ISOLATED_CONTEXT_ORPHAN_GRACE_SEC
)
def _cleanup_orphaned_isolated_contexts(
self,
current_context_ids: Optional[List[str]] = None,
) -> None:
if not self._orphaned_isolated_contexts:
return
now = time.time()
active_contexts = {
str(context_id).strip()
for context_id in (current_context_ids or [])
if str(context_id).strip()
}
active_contexts.update(
str(context_id).strip()
for context_id in self._isolated_context_by_raw_id.values()
if str(context_id).strip()
)
for context_id, expire_at in list(self._orphaned_isolated_contexts.items()):
if context_id in active_contexts:
self._orphaned_isolated_contexts.pop(context_id, None)
continue
if now < expire_at:
continue
self._orphaned_isolated_contexts.pop(context_id, None)
self._dispose_browser_context_async(context_id)
logger.info(
f"[TabPool] disposed orphaned isolated context after grace: {context_id}"
)
def _dispose_browser_context(self, browser_context_id: Optional[str]) -> None:
context_id = str(browser_context_id or "").strip()
if not context_id:
return
try:
browser = self._get_browser_handle()
if browser is None or not hasattr(browser, "_run_cdp"):
return
browser_driver(browser).run_cdp("Target.disposeBrowserContext", browserContextId=context_id)
except Exception as e:
logger.debug(f"[TabPool] dispose browser context failed ({context_id}): {e}")
def _dispose_browser_context_async(self, browser_context_id: Optional[str]) -> None:
context_id = str(browser_context_id or "").strip()
if not context_id:
return
executor = self._maintenance_executor
if executor is None:
return
try:
executor.submit(self._dispose_browser_context, context_id)
except RuntimeError as e:
logger.debug(f"[TabPool] maintenance submit failed (dispose:{context_id}): {e}")
def _arm_isolated_rebind_grace(self, session: TabSession, reason: str, detail: str) -> bool:
now = time.time()
same_reason = session.transient_disconnect_reason == reason
if same_reason and session.transient_disconnect_until > now:
return True
if same_reason and session.transient_disconnect_until > 0:
logger.warning(
f"[{session.id}] isolated session grace expired ({detail}), removing from pool"
)
session.clear_transient_disconnect()
return False
session.mark_transient_disconnect(self.ISOLATED_CONTEXT_REBIND_GRACE_SEC, reason=reason)
logger.warning(
f"[{session.id}] isolated session entered {self.ISOLATED_CONTEXT_REBIND_GRACE_SEC:.0f}s "
f"rebind grace ({detail})"
)
return True
def _refresh_isolated_session_binding(self, session: TabSession, raw_tab_id: str) -> bool:
if not session.is_isolated_context or not raw_tab_id:
return False
replacement_tab = self._resolve_tab_from_ref(raw_tab_id)
if not replacement_tab:
return False
self._detach_global_monitor_for_session(session.id, reason="target_rebind")
session.tab = replacement_tab
session.clear_transient_disconnect()
try:
_cached_url, cached_domain = session.get_cached_route_snapshot()
if cached_domain:
session.current_domain = cached_domain
except Exception:
pass
if session.status == TabStatus.IDLE:
self._restart_global_monitor_for_session_async(session.id, "target_rebind")
return True
def _get_browser_handle(self):
browser = getattr(self.page, "browser", None)
if browser is not None:
return browser
return self.page
@staticmethod
def _run_cdp_compat(target: Any, method: str, *, timeout: float = 2.0, **params):
"""Call DrissionPage CDP APIs across minor 4.x signature differences."""
runner = getattr(target, "run_cdp", None)
if not callable(runner):
runner = getattr(target, "_run_cdp", None)
if not callable(runner):
raise AttributeError(f"CDP runner unavailable for {method}")
timeout_value = max(0.1, float(timeout))
try:
return runner(method, _timeout=timeout_value, **params)
except TypeError as exc:
# A few older wrappers do not expose DrissionPage's private timeout
# keyword. Retry only when the exception clearly concerns that kwarg.
error_text = str(exc).lower()
if "timeout" not in error_text:
raise
return runner(method, **params)
@staticmethod
def _is_execution_context_error(error: Any) -> bool:
text = str(error or "").strip().lower()
if not text:
return False
return any(
marker in text
for marker in (
"cannot find default execution context",
"execution context was destroyed",
"execution context is not available",
"context was destroyed",
)
)
@staticmethod
def _is_unsupported_cdp_method(error: Any) -> bool:
text = str(error or "").strip().lower()
return any(
marker in text
for marker in (
"unknown method",
"method not found",
"not supported",
"unsupported command",
)
)
def _find_idle_tab_by_raw_id(self, raw_tab_id: str) -> Optional[Any]:
"""Find an existing idle wrapper before opening another CDP session."""
raw_id = str(raw_tab_id or "").strip()
if not raw_id:
return None
with self._lock:
sessions = list(getattr(self, "_tabs", {}).values())
for session in sessions:
try:
with session._lock:
if session.status != TabStatus.IDLE or session._termination_in_progress:
continue
tab = getattr(session, "tab", None)
if self._get_tab_ref_id(tab) == raw_id:
return tab
except Exception:
continue
return None
def _force_reload_flat_via_cdp(self, browser: Any, target_id: str) -> bool:
"""Fallback for protocol versions where nested Target messaging is unavailable."""
driver = getattr(browser, "_driver", None)
send = getattr(driver, "_send", None)
if not callable(send):
return False
session_id = None
try:
result = self._run_cdp_compat(
browser,
"Target.attachToTarget",
timeout=self.CONTEXT_RECOVERY_CDP_TIMEOUT_SEC,
targetId=target_id,
flatten=True,
) or {}
session_id = result.get("sessionId") if isinstance(result, dict) else None
if not session_id:
return False
message = {
"method": "Page.reload",
"params": {},
"sessionId": session_id,
}
try:
response = send(message, timeout=self.CONTEXT_RECOVERY_CDP_TIMEOUT_SEC)
except TypeError as exc:
if "timeout" not in str(exc).lower():
raise
response = send(message)
return isinstance(response, dict) and "error" not in response
except Exception as exc:
logger.debug(f"[TabPool] flat CDP 刷新唤醒失败 (target={target_id}): {exc}")
return False
finally:
if session_id:
try:
self._run_cdp_compat(
browser,
"Target.detachFromTarget",
timeout=self.CONTEXT_RECOVERY_CDP_TIMEOUT_SEC,
sessionId=session_id,
)
except Exception as exc:
logger.debug(
f"[TabPool] flat CDP session detach 失败 "
f"(target={target_id}, session={session_id}): {exc}"
)
def _force_reload_tab_via_cdp(self, target_id: str) -> bool:
"""Best-effort reload that works with old nested and newer flat CDP sessions."""
raw_id = str(target_id or "").strip()
if not raw_id or self._shutdown:
return False
existing_tab = self._find_idle_tab_by_raw_id(raw_id)
if existing_tab is not None:
try:
reload_result = self._run_cdp_compat(
existing_tab,
"Page.reload",
timeout=self.CONTEXT_RECOVERY_CDP_TIMEOUT_SEC,
)
if isinstance(reload_result, dict) and reload_result.get("error"):
raise RuntimeError(str(reload_result.get("error")))
return True
except Exception as exc:
logger.debug(
f"[TabPool] 复用现有 tab wrapper 刷新失败,改用 browser CDP "
f"(target={raw_id}): {exc}"
)
try:
browser = self._get_browser_handle()
if browser is None:
return False
run_cdp = getattr(browser, "_run_cdp", None)
if not callable(run_cdp):
# Some older/public wrappers expose only run_cdp().
run_cdp = getattr(browser, "run_cdp", None)
if not callable(run_cdp):
return False
except Exception:
return False
session_id = None
use_flat_fallback = False
try:
# Omitting flatten keeps the legacy nested-session path compatible
# with older Chromium versions. Current Chrome accepts this too.
result = self._run_cdp_compat(
browser,
"Target.attachToTarget",
timeout=self.CONTEXT_RECOVERY_CDP_TIMEOUT_SEC,
targetId=raw_id,
) or {}
session_id = result.get("sessionId") if isinstance(result, dict) else None
if not session_id:
return False
message = json.dumps(
{"id": 1001, "method": "Page.reload", "params": {}},
separators=(",", ":"),
)
try:
nested_result = self._run_cdp_compat(
browser,
"Target.sendMessageToTarget",
timeout=self.CONTEXT_RECOVERY_CDP_TIMEOUT_SEC,
message=message,
sessionId=session_id,
)
if isinstance(nested_result, dict) and nested_result.get("error"):
raise RuntimeError(str(nested_result.get("error")))
return True
except Exception as exc:
use_flat_fallback = self._is_unsupported_cdp_method(exc)
if not use_flat_fallback:
logger.debug(
f"[TabPool] 嵌套 CDP 刷新唤醒失败 "
f"(target={raw_id}): {exc}"
)
return False
except Exception as exc:
logger.debug(f"[TabPool] CDP attach 刷新唤醒失败 (target={raw_id}): {exc}")
return False
finally:
if session_id:
try:
self._run_cdp_compat(
browser,
"Target.detachFromTarget",
timeout=self.CONTEXT_RECOVERY_CDP_TIMEOUT_SEC,
sessionId=session_id,
)
except Exception as exc:
logger.debug(
f"[TabPool] CDP session detach 失败 "
f"(target={raw_id}, session={session_id}): {exc}"
)
return use_flat_fallback and self._force_reload_flat_via_cdp(browser, raw_id)
def _run_context_recovery(self, raw_tab_id: str) -> None:
try:
self._force_reload_tab_via_cdp(raw_tab_id)
except Exception as exc:
logger.debug(f"[TabPool] 上下文恢复任务失败 (target={raw_tab_id}): {exc}")
finally:
recovery_lock = getattr(self, "_context_recovery_lock", None)
if recovery_lock is not None:
with recovery_lock:
getattr(self, "_context_recovery_pending", set()).discard(raw_tab_id)
def _schedule_context_recovery(self, raw_tab_id: str) -> bool:
"""Schedule one reload per target cooldown without holding the pool lock."""
raw_id = str(raw_tab_id or "").strip()
if not raw_id or self._shutdown:
return False
recovery_lock = getattr(self, "_context_recovery_lock", None)
if recovery_lock is None:
recovery_lock = threading.Lock()
self._context_recovery_lock = recovery_lock
pending = getattr(self, "_context_recovery_pending", None)
requested_at = getattr(self, "_context_recovery_requested_at", None)
if pending is None:
pending = set()
self._context_recovery_pending = pending
if requested_at is None:
requested_at = {}
self._context_recovery_requested_at = requested_at
now = time.monotonic()
with recovery_lock:
last_requested = float(requested_at.get(raw_id, 0.0) or 0.0)
if raw_id in pending or (
last_requested
and now - last_requested < self.CONTEXT_RECOVERY_COOLDOWN_SEC
):
return False
requested_at[raw_id] = now
pending.add(raw_id)
if len(requested_at) > self.CONTEXT_RECOVERY_CACHE_LIMIT:
oldest_id = min(requested_at, key=requested_at.get)
requested_at.pop(oldest_id, None)
executor = getattr(self, "_maintenance_executor", None)
if executor is None:
with recovery_lock:
pending.discard(raw_id)
return False
try:
executor.submit(self._run_context_recovery, raw_id)
return True
except RuntimeError as exc:
with recovery_lock:
pending.discard(raw_id)
logger.debug(f"[TabPool] 上下文恢复任务提交失败 (target={raw_id}): {exc}")
return False
def _list_current_tab_ids_via_cdp(self) -> List[str]:
return [
item[0]
for item in (self._list_current_tab_targets_via_cdp() or [])
if item and item[0]
]
def _list_current_tab_targets_via_cdp(self) -> Optional[List[tuple[str, Dict[str, Any]]]]:
browser = self._get_browser_handle()
if browser is None or not hasattr(browser, "_run_cdp"):
return None
try:
result = browser_driver(browser).run_cdp("Target.getTargets")
if not isinstance(result, dict) or not isinstance(result.get("targetInfos"), list):
logger.debug("[TabPool] Target.getTargets returned an incomplete snapshot")
return None
target_infos = result["targetInfos"]
except Exception as e:
logger.debug(f"[TabPool] Target.getTargets 失败: {e}")
return None
targets: List[tuple[str, Dict[str, Any]]] = []
for info in target_infos:
if not isinstance(info, dict):
continue
if str(info.get("type") or "").strip().lower() != "page":
continue
target_id = str(info.get("targetId") or "").strip()
if target_id:
targets.append((target_id, info))
return targets
def _log_get_tabs_warning(self, message: str) -> None:
now = time.time()
if now - self._last_get_tabs_warning_at < self.GET_TABS_WARNING_INTERVAL_SEC:
return
self._last_get_tabs_warning_at = now
logger.warning(message)
def _list_current_tab_refs(self) -> List[Any]:
browser = self._get_browser_handle()
if browser is None:
raise RuntimeError("browser_handle_unavailable")
cdp_targets = self._list_current_tab_targets_via_cdp()
if cdp_targets is not None:
self._get_tabs_retry_after = 0.0
return cdp_targets
now = time.time()
if now < self._get_tabs_retry_after:
raise RuntimeError("tab_snapshot_temporarily_unavailable")
try:
tabs = browser.get_tabs()
if tabs is None:
raise RuntimeError("browser.get_tabs returned no snapshot")
self._get_tabs_retry_after = 0.0
return list(tabs)
except Exception as e:
self._get_tabs_retry_after = max(
self._get_tabs_retry_after,
time.time() + self.GET_TABS_FAILURE_COOLDOWN_SEC,
)
cdp_targets = self._list_current_tab_targets_via_cdp()
if cdp_targets is not None:
message = f"[TabPool] browser.get_tabs() 失败,回退到 Target.getTargets 扫描: {e}"
if _looks_like_transient_local_debug_error(e):
logger.debug(message)
else:
self._log_get_tabs_warning(message)
return cdp_targets
fallback_ids = list(getattr(browser, "tab_ids", []) or [])
if fallback_ids:
message = f"[TabPool] browser.get_tabs() 失败,回退到 tab_ids 扫描: {e}"
if _looks_like_transient_local_debug_error(e):
logger.debug(message)
else:
self._log_get_tabs_warning(message)
return fallback_ids
raise
+381
View File
@@ -0,0 +1,381 @@
"""TabPoolManager 的一部分(R2-3 拆分):看门狗、卡死检测、空闲维护与内存清理、不健康标签页清理。
方法原样从 manager.py 迁出,行为不变;通过 mixin 组合回 TabPoolManager,
类属性与实例状态都在 TabPoolManager(manager.py)里定义。
"""
import time
from typing import Dict, List, Optional
from app.core.config import logger
from .idle_maintenance import schedule_idle_maintenance
from .recovery import TabQuarantineEntry
from .session import TabSession, TabStatus
class TabPoolMaintenanceMixin:
"""看门狗、卡死检测、空闲维护与内存清理、不健康标签页清理"""
def _check_stuck_tabs(self):
"""检查并释放卡死的标签页"""
now = time.time()
released_any = False
for session in self._tabs.values():
if session.status == TabStatus.BUSY:
try:
timeout_info = session.get_busy_timeout_info(now=now)
except Exception:
timeout_info = {
"basis": "session",
"duration": now - session.last_used_at,
"command_loop": {},
}
busy_duration = float(timeout_info.get("duration") or 0.0)
timeout_basis = str(timeout_info.get("basis") or "session")
command_loop = timeout_info.get("command_loop") if isinstance(timeout_info, dict) else {}
if busy_duration > self.stuck_timeout:
task_id = session.current_task_id or ""
detail_parts = [
f"busy_duration={busy_duration:.0f}s",
f"basis={timeout_basis}",
]
if isinstance(command_loop, dict) and command_loop.get("active"):
loop_iteration = command_loop.get("iteration")
loop_total = command_loop.get("total")
loop_label = str(command_loop.get("label") or "").strip()
loop_text = f"loop={loop_iteration or '-'}"
if loop_total:
loop_text = f"{loop_text}/{loop_total}"
if loop_label:
loop_text = f"{loop_label}:{loop_text}"
detail_parts.append(loop_text)
cancel_submitted = self._cancel_active_request_for_session(
session,
"stuck_timeout",
detail=", ".join(detail_parts),
)
snapshot = self._describe_session(session)
action_label = "record stuck session" if self.preserve_error_tabs else "retire session"
logger.warning(
f"[{session.id}] stuck for {busy_duration:.0f}s "
f"(basis={timeout_basis}), {action_label} "
f"(task={task_id or '-'}, cancel_submitted={cancel_submitted}, "
f"snapshot={snapshot})"
)
session.mark_error("stuck_timeout")
released_any = True
if released_any:
self._condition.notify_all()
return released_any
def run_watchdog_tick(self) -> bool:
"""Run periodic stuck-tab maintenance from an external watchdog thread."""
with self._condition:
if self._shutdown:
return False
changed = bool(self._check_stuck_tabs())
self._cleanup_unhealthy_tabs()
self._schedule_idle_memory_purge()
self._schedule_idle_maintenance()
return changed
def _schedule_idle_maintenance(self) -> int:
"""P0-3 / P0-6:空闲标签页 CDP 会话回收与冻结(在 maintenance 线程池执行)。"""
try:
return schedule_idle_maintenance(self, getattr(self, "_idle_maintenance_config", None))
except Exception as e:
logger.debug(f"[TabPool] idle maintenance scheduling failed: {e}")
return 0
def _schedule_idle_memory_purge(self) -> bool:
"""Request best-effort JS GC for long-idle tabs without holding pool locks."""
if not self._idle_memory_purge_enabled:
return False
now_wall = time.time()
now_mono = time.monotonic()
with self._lock:
if self._shutdown:
return False
sessions = list(self._tabs.values())
candidates: List[TabSession] = []
for session in sessions:
try:
with session._lock:
if session.status != TabStatus.IDLE or session._termination_in_progress:
continue
idle_for = now_wall - float(session.last_used_at or now_wall)
last_requested = float(
getattr(session, "_last_memory_purge_requested_at", 0.0) or 0.0
)
if idle_for < self._idle_memory_purge_after_sec:
continue
if last_requested and (now_mono - last_requested) < self._idle_memory_purge_interval_sec:
continue
setattr(session, "_last_memory_purge_requested_at", now_mono)
candidates.append(session)
except Exception:
continue
if not candidates:
return False
executor = self._maintenance_executor
if executor is None:
return False
scheduled = False
for session in candidates:
try:
executor.submit(self._purge_idle_session_memory, session)
scheduled = True
except RuntimeError:
break
return scheduled
def _purge_idle_session_memory(self, session: TabSession) -> None:
"""Force renderer GC for one idle session; never changes page lifecycle."""
if self._shutdown:
return
heap_profiler_enabled = False
try:
with session._lock:
if session.status != TabStatus.IDLE or session._termination_in_progress:
return
try:
self._run_cdp_compat(
session.tab,
"HeapProfiler.enable",
timeout=1.0,
)
heap_profiler_enabled = True
except Exception:
# collectGarbage is available without enabling the domain on
# some older Chromium/DrissionPage combinations.
pass
self._run_cdp_compat(
session.tab,
"HeapProfiler.collectGarbage",
timeout=1.0,
)
logger.debug_throttled(
f"tab_pool.memory_purge.{session.id}",
f"[{session.id}] 空闲标签页已执行 JavaScript 内存回收",
interval_sec=self._idle_memory_purge_interval_sec,
)
except Exception as e:
logger.debug_throttled(
f"tab_pool.memory_purge_error.{session.id}",
f"[{session.id}] 空闲标签页内存回收失败(忽略): {e}",
interval_sec=self._idle_memory_purge_interval_sec,
)
finally:
if heap_profiler_enabled:
try:
self._run_cdp_compat(
session.tab,
"HeapProfiler.disable",
timeout=1.0,
)
except Exception as e:
logger.debug_throttled(
f"tab_pool.memory_purge_disable_error.{session.id}",
f"[{session.id}] HeapProfiler 关闭失败(忽略): {e}",
interval_sec=self._idle_memory_purge_interval_sec,
)
def _cancel_active_request_for_session(
self,
session: Optional[TabSession],
reason: str,
*,
detail: str = "",
) -> bool:
"""Request the bound workflow to stop when a session becomes unusable."""
if session is None:
return False
request_ids = []
for attr_name in ("current_task_id", "_command_request_id", "_bound_request_id", "_command_loop_request_id"):
try:
request_id = str(getattr(session, attr_name, "") or "").strip()
except Exception:
request_id = ""
if request_id and request_id not in request_ids:
request_ids.append(request_id)
if not request_ids:
return False
task_id = request_ids[0]
cancel_key = "|".join(request_ids)
if getattr(session, "_last_cancel_request_task_id", None) == cancel_key:
logger.debug(
f"[{session.id}] duplicate cancel skipped "
f"(task={cancel_key}, reason={reason}, "
f"previous_reason={getattr(session, '_last_cancel_request_reason', '-')}, "
f"detail={detail or '-'})"
)
return False
try:
setattr(session, "_workflow_stop_reason", reason)
setattr(session, "_last_cancel_request_task_id", cancel_key)
setattr(session, "_last_cancel_request_reason", reason)
except Exception:
pass
cancel_submitted = False
for request_id in request_ids:
cancel_submitted = bool(self._submit_request_cancel(
request_id,
reason,
session_id=session.id,
detail=detail,
)) or cancel_submitted
logger.warning(
f"[{session.id}] 会话失效,已请求取消任务 "
f"(task={task_id}, requests={','.join(request_ids)}, reason={reason}, "
f"cancel_submitted={cancel_submitted}, detail={detail or '-'})"
)
return cancel_submitted
def _cleanup_unhealthy_tabs(self):
"""清理不健康的空闲标签页、关闭状态和错误状态的标签页"""
to_remove: Dict[str, str] = {}
for tab_id, session in list(self._tabs.items()):
try:
if session.is_isolated_context and session.is_healthy(allow_live_check=False):
session.clear_transient_disconnect()
if session.status != TabStatus.ERROR:
self._preserved_error_session_ids.discard(tab_id)
if session.status == TabStatus.CLOSED:
to_remove[tab_id] = "closed"
# 清理 ERROR 状态的标签页(包括强制释放失败的)
elif session.status == TabStatus.ERROR:
if self.preserve_error_tabs:
if tab_id not in self._preserved_error_session_ids:
logger.warning(
f"[{tab_id}] 错误状态已记录,按配置保留标签页 "
"(preserve_error_tabs=True)"
)
self._preserved_error_session_ids.add(tab_id)
continue
to_remove[tab_id] = "error"
# 清理空闲但不健康的标签页
elif session.status == TabStatus.IDLE and not session.is_healthy(allow_live_check=False):
if session.is_isolated_context:
if session.is_in_transient_disconnect():
continue
if self._arm_isolated_rebind_grace(
session,
reason="health_probe_failed",
detail="health probe failed",
):
continue
to_remove[tab_id] = "unhealthy"
except Exception as e:
logger.warning(f"[TabPool] cleanup check failed for tab {tab_id}: {e}")
to_remove[tab_id] = "health_check_failed"
newly_quarantined: List[TabQuarantineEntry] = []
for tab_id, removal_reason in list(to_remove.items()):
try:
session = self._tabs.get(tab_id)
if not session:
continue
# mark_closed 会清空 current_task_id,先捕获用于隔离条目
quarantine_task_id = str(
getattr(session, "current_task_id", "") or ""
).strip()
self._cancel_active_request_for_session(
session,
removal_reason,
detail=(
f"status={getattr(getattr(session, 'status', None), 'value', 'unknown')}, "
f"reason={removal_reason}"
),
)
logger.warning(
f"[{tab_id}] 不健康或错误状态,从池中移除但保留浏览器标签页 "
f"({removal_reason})"
)
session.mark_closed(removal_reason)
self._detach_global_monitor_for_session(tab_id, reason=removal_reason)
# 清理映射表,允许相同 raw_tab_id 被重新扫描
raw_ids_to_remove = [
raw_id for raw_id, p_idx in list(self._raw_id_to_persistent.items())
if self._persistent_to_session_id.get(p_idx) == tab_id
]
for raw_id in raw_ids_to_remove:
self._known_tab_ids.discard(raw_id)
self._raw_id_to_persistent.pop(raw_id, None)
browser_context_id = self._isolated_context_by_raw_id.pop(raw_id, None)
if browser_context_id:
self._mark_orphaned_isolated_context(browser_context_id)
if removal_reason == "error":
# ERROR 会话移出池但浏览器标签页保留:旧 worker 可能仍
# 阻塞在该 tab 的同步调用里。先隔离 raw id,阻止下次扫描
# 立即重新入池(双工作流串扰的根源);由 TabRecoveryService
# 在锁外确认旧 worker 退出后决定解除或永久隔离。
entry = TabQuarantineEntry(
raw_tab_id=raw_id,
persistent_index=int(
getattr(session, "persistent_index", 0) or 0
),
task_id=quarantine_task_id,
tab=getattr(session, "tab", None),
session_id=tab_id,
reason=str(
getattr(session, "_workflow_stop_reason", "") or ""
).strip() or removal_reason,
url=str(getattr(session, "last_known_url", "") or ""),
)
self._quarantined_raw_ids[raw_id] = entry
newly_quarantined.append(entry)
logger.info(
f"[TabRecovery] 已隔离错误标签页 "
f"(raw={raw_id}, tab={tab_id}, "
f"task={quarantine_task_id or '-'}, "
f"reason={entry.reason})"
)
# 清理持久编号映射
p_idx = session.persistent_index
if p_idx and self._persistent_to_session_id.get(p_idx) == tab_id:
self._persistent_to_session_id.pop(p_idx, None)
# 清理活动标签页记录
if self._active_session_id == tab_id:
self._active_session_id = None
self._tabs.pop(tab_id, None)
self._preserved_error_session_ids.discard(tab_id)
self._on_session_removed(tab_id)
except Exception as e:
logger.warning(f"[TabPool] cleanup remove failed for tab {tab_id}: {e}")
# 投递到 maintenance executor 由恢复服务处理(submit 仅入队,
# 持池锁调用不会阻塞,也不触发任何 CDP/网络操作)。
for entry in newly_quarantined:
self._submit_recovery(entry)
if to_remove:
self._condition.notify_all()
def _should_defer_to_command(self, session: TabSession, task_id: str) -> bool:
"""Whether request acquisition should defer to high-priority pending/running commands."""
task = str(task_id or "").strip().lower()
if task.startswith("cmd_") or task.startswith("group_"):
return False
try:
from app.services.command_engine import command_engine
if hasattr(command_engine, "should_block_request_for_session"):
return bool(command_engine.should_block_request_for_session(session, task_id=task_id))
except Exception:
return False
return False
+603
View File
@@ -0,0 +1,603 @@
"""TabPoolManager 的一部分(R2-3 拆分):按域名、路由组、精确 URL 借用标签页。
方法原样从 manager.py 迁出,行为不变;通过 mixin 组合回 TabPoolManager,
类属性与实例状态都在 TabPoolManager(manager.py)里定义。
"""
import time
from collections import deque
from typing import Any, Dict, List, Optional
from app.core.config import logger
from app.utils.site_url import (
encode_tab_url_route_token,
normalize_exact_tab_url,
normalize_route_domain,
route_domain_matches,
tab_url_matches,
)
from app.utils.tab_route_groups import (
normalize_route_group_id,
normalize_route_groups,
route_group_member_key,
route_groups_by_id,
)
from .session import TabSession, TabStatus
class TabPoolRoutingMixin:
"""按域名、路由组、精确 URL 借用标签页"""
def _get_sessions_for_route_domain(self, route_domain: str) -> List[TabSession]:
target = normalize_route_domain(route_domain)
if not target:
return []
matches: List[TabSession] = []
for session in self._tabs.values():
current_url, actual_domain = session.get_cached_route_snapshot()
if route_domain_matches(target, actual_domain):
if self._is_session_excluded_from_dynamic_routing(session, current_url):
continue
matches.append(session)
return matches
def _get_route_group_config(self, group_id: str) -> Optional[Dict[str, Any]]:
normalized_id = normalize_route_group_id(group_id)
if not normalized_id:
return None
return route_groups_by_id(getattr(self, "route_groups", [])).get(normalized_id)
@staticmethod
def _session_matches_route_group_member(
session: TabSession,
member: Dict[str, Any],
) -> bool:
current_url, _actual_domain = session.get_cached_route_snapshot()
member_url = str(member.get("url") or "").strip()
member_token = str(member.get("url_token") or "").strip().lower()
if member_url and tab_url_matches(member_url, current_url):
return True
if member_token and encode_tab_url_route_token(current_url) == member_token:
return True
return False
def _get_sessions_for_route_group(
self,
group_id: str,
*,
bind: bool = True,
) -> List[TabSession]:
"""解析路由组当前的成员标签页。
bind=False 供只读查询(/api/tabs 面板轮询、组快照)使用:只做匹配,
不写 _route_group_bindings。否则一次与请求无关的面板轮询就会替真实请求
提前把成员钉死在某个标签页上(绑定是粘滞的),后续请求再也轮不到同 URL
的其他标签页。
"""
group = self._get_route_group_config(group_id)
if not group:
return []
normalized_id = group["id"]
members = group.get("members") or []
valid_member_keys = {route_group_member_key(member) for member in members}
if bind:
bindings = self._route_group_bindings.setdefault(normalized_id, {})
for member_key in list(bindings):
session_id = bindings.get(member_key)
if member_key not in valid_member_keys or session_id not in self._tabs:
bindings.pop(member_key, None)
else:
bindings = {
member_key: session_id
for member_key, session_id in self._route_group_bindings.get(normalized_id, {}).items()
if member_key in valid_member_keys and session_id in self._tabs
}
sessions: List[TabSession] = []
used_session_ids = set()
ordered_sessions = sorted(self._tabs.values(), key=self._session_allocation_key)
for member in members:
member_key = route_group_member_key(member)
bound_session = self._tabs.get(bindings.get(member_key, ""))
if bound_session is not None and bound_session.id not in used_session_ids:
sessions.append(bound_session)
used_session_ids.add(bound_session.id)
continue
matching = [
session
for session in ordered_sessions
if session.id not in used_session_ids
and self._session_matches_route_group_member(session, member)
]
try:
preferred_index = int(member.get("tab_index") or 0)
except (TypeError, ValueError):
preferred_index = 0
if preferred_index > 0:
matching.sort(
key=lambda session: (
int(session.persistent_index or 0) != preferred_index,
self._session_allocation_key(session),
)
)
if not matching:
bindings.pop(member_key, None)
continue
selected = matching[0]
bindings[member_key] = selected.id
sessions.append(selected)
used_session_ids.add(selected.id)
return sessions
def get_route_groups_snapshot(self) -> List[Dict[str, Any]]:
with self._lock:
groups = normalize_route_groups(getattr(self, "route_groups", []))
result = []
for group in groups:
sessions = self._get_sessions_for_route_group(group["id"], bind=False)
item = dict(group)
item["live_member_count"] = len(sessions)
item["idle_member_count"] = sum(
1 for session in sessions if session.status == TabStatus.IDLE
)
item["busy_member_count"] = sum(
1 for session in sessions if session.status == TabStatus.BUSY
)
item["live_tab_indices"] = [
int(session.persistent_index or 0) for session in sessions
]
result.append(item)
return result
def _refresh_route_snapshots_for_sessions(self, sessions: List[TabSession]) -> None:
"""Refresh live tab URL/domain snapshots; callers must not hold self._condition."""
for session in sessions:
try:
# BUSY 标签页跳过实时读取:
# 1) 该 tab 正被别的工作流线程用同步 DrissionPage 调用驱动,
# 这里再发一次 CDP 是并发访问同一 target(_scan_new_tabs 同样避开 BUSY);
# 2) 工作流中途导航时 tab.url 会瞬时变成 about:blank,
# _refresh_current_domain 会据此把 current_domain 清空,
# 使并发的 /url/<domain> 请求匹配不到这个标签页而误报 404。
# BUSY 标签页的 URL 已由 _scan_new_tabs 从 Target.getTargets 快照刷新。
if session.status == TabStatus.BUSY:
continue
current_url = session._safe_get_url(allow_live_when_busy=True)
if current_url:
session._refresh_current_domain(current_url)
except Exception as e:
logger.debug(f"[{session.id}] failed to refresh route snapshot: {e}")
def _refresh_route_snapshots_unlocked_once(self) -> None:
"""Refresh route snapshots once without holding the pool condition."""
sessions_to_refresh = list(self._tabs.values())
self._condition.release()
try:
self._refresh_route_snapshots_for_sessions(sessions_to_refresh)
finally:
self._condition.acquire()
async def acquire_by_route_domain_async(
self,
route_domain: str,
task_id: str,
timeout: float = None,
allocation_mode: Optional[str] = None,
) -> Optional[TabSession]:
"""异步版本的按域名路由获取。"""
return await self._run_cancellable_acquire(
lambda: self.acquire_by_route_domain(
route_domain,
task_id,
timeout,
allocation_mode,
),
task_id,
)
async def acquire_by_route_group_async(
self,
group_id: str,
task_id: str,
timeout: float = None,
allocation_mode: Optional[str] = None,
requested_model: Optional[str] = None,
) -> Optional[TabSession]:
return await self._run_cancellable_acquire(
lambda: self.acquire_by_route_group(
group_id,
task_id,
timeout,
allocation_mode,
requested_model=requested_model,
),
task_id,
)
def _get_sessions_for_exact_url(self, exact_url: str) -> List[TabSession]:
target = str(exact_url or "").strip()
if not target:
return []
matches: List[TabSession] = []
for session in self._tabs.values():
current_url, _actual_domain = session.get_cached_route_snapshot()
if tab_url_matches(target, current_url):
matches.append(session)
return matches
def acquire_by_exact_url(self, exact_url: str, task_id: str, timeout: float = None) -> Optional[TabSession]:
"""Acquire a tab by strict full-URL match, round-robin within identical URLs."""
raw_target = str(exact_url or "").strip()
if not raw_target:
logger.warning("Exact tab URL is empty; cannot acquire a tab")
return None
# 队列 key / 轮询游标 key 必须用归一化后的 URL:匹配侧 tab_url_matches 会剥掉
# 默认端口等差异,若这里仍用原始串,"https://x/a" 与 "https://x:443/a"
# 会落进两条互不相干的等待队列,各自都是队首,FIFO 互斥形同虚设。
target = normalize_exact_tab_url(raw_target) or raw_target
waiter_key = f"url::{target}"
timeout = self._normalize_acquire_timeout(timeout)
deadline = time.time() + timeout
with self._condition:
if self._acquire_queue_full(task_id):
return None
waiters = self._route_waiters.setdefault(waiter_key, deque())
waiter_token = self._next_waiter_token(task_id)
waiters.append(waiter_token)
try:
while True:
if self._shutdown:
return None
if self._request_should_stop(task_id):
logger.debug(f"Exact URL acquire cancelled (task={task_id})")
return None
if self._should_scan():
self._scan_new_tabs()
self._check_stuck_tabs()
self._cleanup_unhealthy_tabs()
if not self._is_waiter_turn(waiters, waiter_token):
remaining = deadline - time.time()
if remaining <= 0:
logger.warning(
f"Timed out waiting for exact URL '{target}' (task={task_id})"
)
return None
logger.debug_throttled(
f"tab_pool.wait_exact_url.{target}",
f"Waiting for exact URL '{target}' release...",
interval_sec=5.0,
)
self._condition.wait(
timeout=min(remaining, self.ACQUIRE_CANCEL_POLL_SEC)
)
continue
self._refresh_route_snapshots_unlocked_once()
matching_sessions = self._get_sessions_for_exact_url(target)
if not matching_sessions:
logger.warning(f"No tab matches exact URL '{target}'")
return None
session = self._try_acquire_session_for_request(
matching_sessions,
task_id,
route_domain=waiter_key,
defer_context="exact-url acquire",
)
if session is not None:
self._unregister_waiter(
waiters,
waiter_token,
owner_map=self._route_waiters,
owner_key=waiter_key,
)
if not self._complete_acquired_session_for_return(
session,
"acquire_by_exact_url",
task_id,
activate=self._auto_activate_on_acquire and session.id != self._active_session_id,
):
return None
self._mark_allocation_cursor(session, route_domain=waiter_key)
logger.debug(
f"TabPool -> {session.id} "
f"(task={task_id}, exact_url={target}, "
f"idx=#{session.persistent_index}, snapshot={self._describe_session(session)})"
)
return session
remaining = deadline - time.time()
if remaining <= 0:
logger.warning(
f"Timed out waiting for exact URL '{target}' "
f"(task={task_id}, matching tabs: "
f"{', '.join(f'{s.id}(#{s.persistent_index}:{s.status.value})' for s in matching_sessions) or 'none'})"
)
return None
logger.debug_throttled(
f"tab_pool.wait_exact_url.{target}",
f"Waiting for exact URL '{target}' release...",
interval_sec=5.0,
)
self._condition.wait(
timeout=min(remaining, self.ACQUIRE_CANCEL_POLL_SEC)
)
finally:
self._unregister_waiter(
waiters,
waiter_token,
owner_map=self._route_waiters,
owner_key=waiter_key,
)
def acquire_by_route_domain(
self,
route_domain: str,
task_id: str,
timeout: float = None,
allocation_mode: Optional[str] = None,
) -> Optional[TabSession]:
"""ASCII-safe fair acquire for tabs matching the same route domain."""
target = normalize_route_domain(route_domain)
if not target:
logger.warning("Route domain is empty; cannot acquire a tab")
return None
timeout = self._normalize_acquire_timeout(timeout)
deadline = time.time() + timeout
with self._condition:
if self._acquire_queue_full(task_id):
return None
waiters = self._route_waiters.setdefault(target, deque())
waiter_token = self._next_waiter_token(task_id)
waiters.append(waiter_token)
try:
while True:
if self._shutdown:
return None
if self._request_should_stop(task_id):
logger.debug(f"Route domain acquire cancelled (task={task_id})")
return None
if self._should_scan():
self._scan_new_tabs()
self._check_stuck_tabs()
self._cleanup_unhealthy_tabs()
if not self._is_waiter_turn(waiters, waiter_token):
remaining = deadline - time.time()
if remaining <= 0:
logger.warning(
f"Timed out waiting for route domain '{target}' (task={task_id})"
)
return None
logger.debug_throttled(
f"tab_pool.wait_route.{target}",
f"Waiting for route domain '{target}' release...",
interval_sec=5.0,
)
self._condition.wait(
timeout=min(remaining, self.ACQUIRE_CANCEL_POLL_SEC)
)
continue
self._refresh_route_snapshots_unlocked_once()
matching_sessions = self._get_sessions_for_route_domain(target)
if not matching_sessions:
logger.warning(f"No tab matches route domain '{target}'")
return None
session = self._try_acquire_session_for_request(
matching_sessions,
task_id,
route_domain=target,
allocation_mode=allocation_mode,
defer_context="route-domain acquire",
)
if session is not None:
self._unregister_waiter(
waiters,
waiter_token,
owner_map=self._route_waiters,
owner_key=target,
)
if not self._complete_acquired_session_for_return(
session,
"acquire_by_route_domain",
task_id,
activate=self._auto_activate_on_acquire and session.id != self._active_session_id,
):
return None
self._mark_allocation_cursor(session, route_domain=target)
logger.debug(
f"TabPool -> {session.id} "
f"(task={task_id}, route_domain={target}, "
f"idx=#{session.persistent_index}, snapshot={self._describe_session(session)})"
)
return session
remaining = deadline - time.time()
if remaining <= 0:
busy_info = [
f"{session.id}(#{session.persistent_index}:{session.status.value})"
for session in matching_sessions
]
logger.warning(
f"Timed out waiting for route domain '{target}' "
f"(task={task_id}, matching tabs: {', '.join(busy_info) or 'none'})"
)
return None
logger.debug_throttled(
f"tab_pool.wait_route.{target}",
f"Waiting for route domain '{target}' release...",
interval_sec=5.0,
)
self._condition.wait(
timeout=min(remaining, self.ACQUIRE_CANCEL_POLL_SEC)
)
finally:
self._unregister_waiter(
waiters,
waiter_token,
owner_map=self._route_waiters,
owner_key=target,
)
def acquire_by_route_group(
self,
group_id: str,
task_id: str,
timeout: float = None,
allocation_mode: Optional[str] = None,
requested_model: Optional[str] = None,
) -> Optional[TabSession]:
target = normalize_route_group_id(group_id)
if not target:
logger.warning("Route group ID is invalid")
return None
timeout = self._normalize_acquire_timeout(timeout)
deadline = time.time() + timeout
first_iteration = True
with self._condition:
group = self._get_route_group_config(target)
if not group:
logger.warning(f"Route group '{target}' does not exist")
return None
if self._acquire_queue_full(task_id):
return None
waiters = self._group_waiters.setdefault(target, deque())
waiter_token = self._next_waiter_token(task_id)
waiters.append(waiter_token)
try:
while True:
if self._shutdown:
return None
if self._request_should_stop(task_id):
logger.debug(f"Route group acquire cancelled (task={task_id})")
return None
# 组配置每轮重新读取:apply_runtime_config 可能在等待期间
# 删除/改名该组或改掉分配模式。用循环外的旧快照会让等待者既不
# 报"组不存在"也匹配不到成员,只能空转到超时。
group = self._get_route_group_config(target)
if not group:
logger.warning(
f"Route group '{target}' no longer exists (task={task_id})"
)
return None
if first_iteration or self._should_scan():
self._scan_new_tabs()
first_iteration = False
self._check_stuck_tabs()
self._cleanup_unhealthy_tabs()
if not self._is_waiter_turn(waiters, waiter_token):
remaining = deadline - time.time()
if remaining <= 0:
logger.warning(
f"Timed out waiting in route group '{target}' (task={task_id})"
)
return None
logger.debug_throttled(
f"tab_pool.wait_group.{target}",
f"Waiting in route group '{target}' queue...",
interval_sec=5.0,
)
self._condition.wait(
timeout=min(remaining, self.ACQUIRE_CANCEL_POLL_SEC)
)
continue
matching_sessions = self._get_sessions_for_route_group(target)
# 空组没有任何可能在后续释放的成员。立即返回,让上游的
# 多 URL 轮询跳到下一个 URL,而不是按默认 60 秒超时等待。
if not matching_sessions:
logger.info(
f"Route group '{target}' has no live members; skip immediately "
f"(task={task_id})"
)
return None
if requested_model:
clean_req_model = str(requested_model).strip().lower()
model_specific = [
s for s in matching_sessions
if self._get_session_override_model_name(s).strip().lower() == clean_req_model
]
if model_specific:
matching_sessions = model_specific
mode = allocation_mode or group.get("allocation_mode") or self.allocation_mode
session = self._try_acquire_session_for_request(
matching_sessions,
task_id,
route_domain=f"group::{target}",
allocation_mode=mode,
defer_context=f"route-group {target} acquire",
)
if session is not None:
self._unregister_waiter(
waiters,
waiter_token,
owner_map=self._group_waiters,
owner_key=target,
)
if not self._complete_acquired_session_for_return(
session,
"acquire_by_route_group",
task_id,
activate=self._auto_activate_on_acquire and session.id != self._active_session_id,
):
# 与 acquire/acquire_by_route_domain 等路径保持一致:
# handoff 失败直接返回 None。此处 waiter 已出队,若 continue
# 会因 _is_waiter_turn 永远轮不到而空转到超时。
# 调用方(workflow.py)已有 session is None 的处理分支。
return None
self._mark_allocation_cursor(session, route_domain=f"group::{target}")
logger.debug(
f"TabPool -> {session.id} "
f"(task={task_id}, route_group={target}, "
f"idx=#{session.persistent_index}, snapshot={self._describe_session(session)})"
)
return session
remaining = deadline - time.time()
if remaining <= 0:
member_state = ", ".join(
f"{session.id}(#{session.persistent_index}:{session.status.value})"
for session in matching_sessions
) or "no live members"
logger.warning(
f"Timed out waiting for route group '{target}' "
f"(task={task_id}, members: {member_state})"
)
return None
logger.debug_throttled(
f"tab_pool.wait_group.{target}",
f"Waiting for route group '{target}' member release...",
interval_sec=5.0,
)
self._condition.wait(
timeout=min(remaining, self.ACQUIRE_CANCEL_POLL_SEC)
)
finally:
self._unregister_waiter(
waiters,
waiter_token,
owner_map=self._group_waiters,
owner_key=target,
)
+338
View File
@@ -0,0 +1,338 @@
"""TabPoolManager 的一部分(R2-3 拆分):会话封装、移除、请求取消、隔离区(quarantine)与恢复服务。
方法原样从 manager.py 迁出,行为不变;通过 mixin 组合回 TabPoolManager,
类属性与实例状态都在 TabPoolManager(manager.py)里定义。
"""
from typing import Any, Optional
from app.core.config import logger
from app.utils.site_url import extract_remote_site_domain
from .recovery import TabQuarantineEntry, TabRecoveryService
from .session import TabSession
class TabPoolSessionsMixin:
"""会话封装、移除、请求取消、隔离区(quarantine)与恢复服务"""
def _get_domain_abbr(self, url: str) -> str:
try:
if not url or "://" not in url:
return "tab"
domain = url.split("//")[-1].split("/")[0].lower()
clean_domain = domain.replace("www.", "")
for key, abbr in self.DOMAIN_ABBR_MAP.items():
if key in clean_domain:
return abbr
first_part = clean_domain.split(".")[0]
return first_part[:10]
except Exception:
return "tab"
@staticmethod
def _get_tab_ref_id(tab_ref: Any) -> Optional[str]:
if tab_ref is None:
return None
if isinstance(tab_ref, tuple) and tab_ref:
return str(tab_ref[0] or "").strip() or None
if isinstance(tab_ref, dict):
target_id = str(tab_ref.get("targetId") or tab_ref.get("tab_id") or "").strip()
return target_id or None
tab_id = getattr(tab_ref, "tab_id", None)
if tab_id:
return str(tab_id)
raw = str(tab_ref or "").strip()
return raw or None
@staticmethod
def _get_tab_ref_url(tab_ref: Any) -> str:
if isinstance(tab_ref, tuple) and len(tab_ref) >= 2 and isinstance(tab_ref[1], dict):
return str(tab_ref[1].get("url") or "").strip()
if isinstance(tab_ref, dict):
return str(tab_ref.get("url") or "").strip()
try:
return str(getattr(tab_ref, "url", "") or "").strip()
except Exception:
return ""
@staticmethod
def _tab_ref_has_url_field(tab_ref: Any) -> bool:
if isinstance(tab_ref, tuple) and len(tab_ref) >= 2 and isinstance(tab_ref[1], dict):
return "url" in tab_ref[1]
if isinstance(tab_ref, dict):
return "url" in tab_ref
return False
def _resolve_tab_from_ref(self, tab_ref: Any) -> Optional[Any]:
if tab_ref is None:
return None
if isinstance(tab_ref, tuple) and len(tab_ref) >= 3:
existing_tab = tab_ref[2]
if existing_tab is not None:
return existing_tab
try:
existing_tab_id = getattr(tab_ref, "tab_id", None)
except Exception:
existing_tab_id = None
if existing_tab_id:
return tab_ref
raw_tab_id = self._get_tab_ref_id(tab_ref)
if not raw_tab_id:
return None
try:
browser = self._get_browser_handle()
return browser.get_tab(raw_tab_id)
except Exception as exc:
if self._is_execution_context_error(exc) and self._schedule_context_recovery(raw_tab_id):
logger.warning(
f"[TabPool] 标签页 {raw_tab_id} 丢失 JS 执行上下文,"
"已异步请求 CDP 刷新唤醒"
)
return None
def _wrap_tab(
self,
tab,
raw_tab_id: str = None,
*,
browser_context_id: Optional[str] = None,
is_isolated_context: bool = False,
initial_url: str = "",
) -> TabSession:
self._tab_counter += 1
url = str(initial_url or "").strip()
if not url:
try:
url = tab.url or ""
except Exception:
pass
abbr = self._get_domain_abbr(url)
tab_id = f"{abbr}_{self._tab_counter}"
session = TabSession(
id=tab_id,
tab=tab,
browser_context_id=browser_context_id,
is_isolated_context=bool(is_isolated_context),
)
session._remember_url(url)
try:
session.current_domain = extract_remote_site_domain(url)
except:
pass
# 记录底层标签页 ID
if raw_tab_id:
self._known_tab_ids.add(raw_tab_id)
if session.is_isolated_context:
self._register_isolated_context(raw_tab_id, browser_context_id)
# 🆕 分配持久化编号
if raw_tab_id not in self._raw_id_to_persistent:
persistent_idx = self._next_persistent_index
self._next_persistent_index += 1
self._raw_id_to_persistent[raw_tab_id] = persistent_idx
else:
persistent_idx = self._raw_id_to_persistent[raw_tab_id]
session.persistent_index = persistent_idx
self._persistent_to_session_id[persistent_idx] = session.id
logger.debug(f"标签页 {session.id} 分配编号 #{persistent_idx}")
self._apply_preset_overrides(session)
try:
setattr(tab, "_session", session)
except Exception:
pass
session.notify_navigated(reason="tab_created")
return session
def _on_session_removed(self, session_id: str):
"""Notify command engine after a pool session is removed without blocking pool locks."""
session_key = str(session_id or "").strip()
if not session_key:
return
def _evict():
try:
from app.services.command_engine import command_engine
if hasattr(command_engine, "evict_session"):
command_engine.evict_session(session_key)
except Exception as e:
logger.debug(f"[TabPool] evict session {session_key} failed: {e}")
executor = self._maintenance_executor
if executor is None:
return
try:
executor.submit(_evict)
except RuntimeError as e:
logger.debug(f"[TabPool] maintenance submit failed (evict:{session_key}): {e}")
def _submit_request_cancel(
self,
task_id: str,
reason: str,
*,
session_id: str = "",
detail: str = "",
) -> bool:
"""Cancel a request from the maintenance worker so pool locks never call outward."""
task_key = str(task_id or "").strip()
if not task_key:
return False
reason_key = str(reason or "").strip() or "unknown"
session_key = str(session_id or "").strip()
detail_text = str(detail or "").strip()
def _cancel():
cancelled = False
try:
from app.services.request_manager import request_manager
cancelled = bool(request_manager.cancel_request(task_key, reason_key))
except Exception as e:
logger.debug(
f"[{session_key or 'TabPool'}] async cancel failed "
f"(task={task_key}, reason={reason_key}, detail={detail_text or '-'}): {e}"
)
return
logger.debug(
f"[{session_key or 'TabPool'}] async cancel result "
f"(task={task_key}, reason={reason_key}, cancelled={cancelled}, "
f"detail={detail_text or '-'})"
)
executor = self._maintenance_executor
if executor is None:
logger.debug(
f"[{session_key or 'TabPool'}] async cancel skipped: maintenance executor unavailable "
f"(task={task_key}, reason={reason_key}, detail={detail_text or '-'})"
)
return False
try:
executor.submit(_cancel)
return True
except RuntimeError as e:
logger.debug(
f"[{session_key or 'TabPool'}] maintenance submit failed "
f"(cancel:{task_key}, reason={reason_key}, detail={detail_text or '-'}): {e}"
)
return False
def _close_quarantined_tab(self, tab: Any, raw_tab_id: str) -> bool:
"""Close a permanently quarantined browser target outside the pool lock."""
close = getattr(tab, "close", None)
if callable(close):
try:
close_result = close()
if close_result is not False:
return True
logger.debug(
f"[TabRecovery] 标签页对象 close() 返回失败,尝试 CDP 兜底 "
f"(raw={raw_tab_id})"
)
except Exception as exc:
logger.debug(
f"[TabRecovery] 关闭隔离标签页对象失败,尝试 CDP 兜底 "
f"(raw={raw_tab_id}, error={exc})"
)
try:
browser = self._get_browser_handle()
run_cdp = getattr(browser, "_run_cdp", None)
if callable(run_cdp):
result = run_cdp("Target.closeTarget", targetId=str(raw_tab_id))
if not isinstance(result, dict) or result.get("success", True):
return True
except Exception as exc:
logger.warning(
f"[TabRecovery] 关闭永久隔离标签页失败 "
f"(raw={raw_tab_id}, error={exc})"
)
return False
def resolve_quarantine(self, raw_tab_id: str, release: bool) -> bool:
"""解除或永久化一个隔离条目(供 TabRecoveryService 回调)。
release=True:从隔离集合移除,下一次 _scan_new_tabs 自然重新入池;
release=False:关闭浏览器标签页并释放强引用;关闭失败时保留不带
tab 引用的轻量永久隔离记录,避免扫描期间重新入池。
"""
raw_id = str(raw_tab_id or "").strip()
if not raw_id:
return False
tab_to_close = None
entry = None
with self._condition:
entry = self._quarantined_raw_ids.get(raw_id)
if entry is None:
return False
if release:
self._quarantined_raw_ids.pop(raw_id, None)
# 恢复任务结束后不再需要保留 DrissionPage tab wrapper。
entry.tab = None
self._condition.notify_all()
return True
entry.permanent = True
tab_to_close = getattr(entry, "tab", None)
# 先在锁内摘掉强引用;真实 CDP 调用必须在锁外执行。
entry.tab = None
closed = self._close_quarantined_tab(tab_to_close, raw_id)
with self._condition:
if self._quarantined_raw_ids.get(raw_id) is entry:
if closed:
self._quarantined_raw_ids.pop(raw_id, None)
self._condition.notify_all()
return True
def get_quarantine_entry(self, raw_tab_id: str) -> Optional[TabQuarantineEntry]:
with self._condition:
return self._quarantined_raw_ids.get(str(raw_tab_id or "").strip())
def _get_recovery_service(self) -> Optional[TabRecoveryService]:
if self._shutdown:
return None
service = self._recovery_service
if service is not None:
return service
with self._recovery_service_lock:
if self._recovery_service is None:
self._recovery_service = TabRecoveryService(
resolve_quarantine=self.resolve_quarantine,
get_quarantine_entry=self.get_quarantine_entry,
)
return self._recovery_service
def _submit_recovery(self, entry: TabQuarantineEntry) -> None:
"""把隔离条目投递给恢复服务;持池锁调用也安全(仅入队,不阻塞外呼)。"""
def _submit() -> None:
try:
service = self._get_recovery_service()
if service is not None:
service.submit(entry)
except Exception as e:
logger.warning(
f"[TabRecovery] 投递恢复任务失败 ({entry.raw_tab_id}): {e}"
)
executor = self._maintenance_executor
if executor is not None:
try:
executor.submit(_submit)
return
except RuntimeError as e:
logger.debug(
f"[TabRecovery] maintenance submit failed "
f"(recovery:{entry.raw_tab_id}): {e}"
)
# 兜底:maintenance executor 不可用时直接入队(submit 本身非阻塞)。
_submit()
+405
View File
@@ -0,0 +1,405 @@
"""TabPoolManager 的一部分(R2-3 拆分):状态查询与管理操作(编号列表、预设/模型名设置、状态与看门狗摘要、按编号终止)。
方法原样从 manager.py 迁出,行为不变;通过 mixin 组合回 TabPoolManager,
类属性与实例状态都在 TabPoolManager(manager.py)里定义。
"""
import time
from typing import Any, Dict, List, Optional
from app.core.config import logger
from app.utils.tab_route_groups import normalize_route_groups
from .session import TabSession, TabStatus
class TabPoolStatusMixin:
"""状态查询与管理操作(编号列表、预设/模型名设置、状态与看门狗摘要、按编号终止)"""
def terminate_by_index(
self,
persistent_index: int,
reason: str = "manual_terminate",
clear_page: bool = True,
scope: str = "task",
expected_session_id: str = "",
expected_task_id: str = "",
) -> Dict[str, Any]:
"""
按标签页编号终止当前任务,并在旧所有者退出前隔离该成员。
行为:
1) 校验调用方看到的 session/task 所有权快照;
2) 设置终止隔离闸门,阻止组/域名/编号路由重新分配;
3) 取消该成员关联请求并中断页面活动;
4) 等待旧工作线程完成正常释放,超时则继续保持隔离。
"""
normalized_scope = str(scope or "task").strip().lower()
if normalized_scope not in {"task", "loop"}:
normalized_scope = "task"
expected_session = str(expected_session_id or "").strip()
expected_task = str(expected_task_id or "").strip()
with self._condition:
session_id = self._persistent_to_session_id.get(persistent_index)
if not session_id:
return {"ok": False, "error": "tab_not_found", "tab_index": persistent_index}
session = self._tabs.get(session_id)
if not session:
return {"ok": False, "error": "tab_not_found", "tab_index": persistent_index}
if expected_session and session.id != expected_session:
return {
"ok": False,
"error": "session_ownership_changed",
"tab_index": persistent_index,
"expected_session_id": expected_session,
"current_session_id": session.id,
}
current_task = str(session.current_task_id or "").strip()
if expected_task and current_task != expected_task:
return {
"ok": False,
"error": "task_ownership_changed",
"tab_index": persistent_index,
"tab_id": session.id,
"expected_task_id": expected_task,
"current_task_id": current_task,
}
if normalized_scope == "loop":
loop_cancelled = session.request_command_loop_cancel(reason)
self._condition.notify_all()
if not loop_cancelled:
return {
"ok": False,
"error": "no_active_command_loop",
"tab_index": persistent_index,
"tab_id": session.id,
"status": session.status.value,
"reason": reason,
"scope": normalized_scope,
}
logger.warning(
f"[{session.id}] 手动终止当前循环: idx=#{persistent_index}, "
f"status={session.status.value}, reason={reason}, "
f"loop={session.get_command_loop_info()}"
)
return {
"ok": True,
"tab_index": persistent_index,
"tab_id": session.id,
"was_busy": session.status == TabStatus.BUSY,
"task_id": str(session.current_task_id or "").strip(),
"cancelled": True,
"status": session.status.value,
"reason": reason,
"scope": normalized_scope,
"loop_cancelled": True,
"command_loop": session.get_command_loop_info(),
}
before_snapshot = self._describe_session(session)
was_busy = session.status == TabStatus.BUSY
termination_state = session.begin_termination(expected_task)
if not termination_state.get("ok"):
# A second terminate request can race with the first one while
# the old owner is still unwinding. That is an idempotent
# in-progress result, not a failed termination.
if termination_state.get("error") == "termination_in_progress":
return {
"ok": True,
"tab_index": persistent_index,
"tab_id": session.id,
"was_busy": was_busy,
"task_id": str(
termination_state.get("current_task_id") or current_task
).strip(),
"cancelled": False,
"status": session.status.value,
"pending": True,
"released": False,
"interrupt_success": False,
"already_in_progress": True,
"reason": reason,
"scope": normalized_scope,
}
return {
"ok": False,
**termination_state,
"tab_index": persistent_index,
"tab_id": session.id,
"status": session.status.value,
"reason": reason,
}
task_id = str(termination_state.get("task_id") or "").strip()
command_request_id = str(
termination_state.get("command_request_id") or ""
).strip()
command_loop_request_id = str(
termination_state.get("command_loop_request_id") or ""
).strip()
cancelled = False
cancel_error = ""
request_ids_to_cancel = []
for request_id in (task_id, command_request_id, command_loop_request_id):
if request_id and request_id not in request_ids_to_cancel:
request_ids_to_cancel.append(request_id)
if request_ids_to_cancel:
try:
from app.services.request_manager import request_manager
for request_id in request_ids_to_cancel:
cancelled = bool(request_manager.cancel_request(request_id, reason)) or cancelled
except Exception as e:
cancel_error = str(e)
logger.debug(f"[{session.id}] 取消任务失败(忽略): {e}")
self._stop_global_monitor_for_session(session.id, reason=f"terminate:{reason}", wait=True)
interrupt_success = session.interrupt_for_termination(clear_page=clear_page)
with self._condition:
wait_deadline = time.monotonic() + self.TERMINATION_RELEASE_WAIT_SEC
while (
self._tabs.get(session.id) is session
and session.is_termination_in_progress()
):
remaining = wait_deadline - time.monotonic()
if remaining <= 0:
break
self._condition.wait(timeout=min(remaining, 0.1))
pending = bool(
self._tabs.get(session.id) is session
and session.is_termination_in_progress()
)
if not pending and self._tabs.get(session.id) is session and session.status == TabStatus.IDLE:
self._start_global_monitor_for_session(session)
self._condition.notify_all()
logger.warning(
f"[{session.id}] 手动终止: idx=#{persistent_index}, "
f"task={task_id or '-'}, cancelled={cancelled}, "
f"pending={pending}, interrupt_success={interrupt_success}, "
f"status={session.status.value}, reason={reason}, "
f"before={before_snapshot}, after={self._describe_session(session)}"
)
result = {
"ok": True,
"tab_index": persistent_index,
"tab_id": session.id,
"was_busy": was_busy,
"task_id": task_id,
"cancelled": cancelled,
"status": session.status.value,
"pending": pending,
"released": not pending and session.status == TabStatus.IDLE,
"interrupt_success": interrupt_success,
"reason": reason,
"scope": normalized_scope,
}
if cancel_error:
result["cancel_error"] = cancel_error
return result
def get_tabs_with_index(self) -> List[Dict]:
"""获取所有标签页及其持久编号(供 API 调用)"""
with self._lock:
if self._should_scan_for_query():
self._scan_new_tabs()
sessions = list(self._tabs.values())
groups_by_session: Dict[str, List[str]] = {}
for group in normalize_route_groups(getattr(self, "route_groups", [])):
for member_session in self._get_sessions_for_route_group(group["id"], bind=False):
groups_by_session.setdefault(member_session.id, []).append(group["id"])
result = []
for session in sessions:
info = session.get_info(use_cached_url=True)
tab_route_prefix = f"/tab/{session.persistent_index}"
route_domain = str(info.get("route_domain") or "").strip()
domain_route_prefix = f"/url/{route_domain}" if route_domain else ""
preset_route_domain = str(info.get("current_domain") or route_domain).strip()
url_route_token = str(info.get("url_route_token") or "").strip()
exact_url_route_prefix = f"/tab-url/{url_route_token}" if url_route_token else ""
info["tab_route_prefix"] = tab_route_prefix
info["domain_route_prefix"] = domain_route_prefix
info["preset_route_domain"] = preset_route_domain
info["preset_domain_route_prefix"] = f"/url/{preset_route_domain}" if preset_route_domain else ""
info["exact_url_route_prefix"] = exact_url_route_prefix
info["route_prefix"] = domain_route_prefix or tab_route_prefix
info["route_groups"] = groups_by_session.get(session.id, [])
self._apply_preset_overrides(session, info)
self._apply_exposed_model_name(info)
result.append(info)
# 按编号排序
result.sort(key=lambda x: x.get("persistent_index", 0))
return result
def set_tab_preset(self, persistent_index: int, preset_name: str) -> bool:
"""
为指定标签页设置预设
Args:
persistent_index: 标签页持久化编号
preset_name: 预设名称(None 或空字符串表示恢复为跟随站点默认预设)
Returns:
是否成功
"""
with self._lock:
session_id = self._persistent_to_session_id.get(persistent_index)
if not session_id:
logger.warning(f"标签页 #{persistent_index} 不存在")
return False
session = self._tabs.get(session_id)
if not session:
logger.warning(f"标签页 {session_id} 已被移除")
return False
old_preset = session.preset_name
session.set_preset(preset_name, source="tab" if preset_name else None)
logger.debug(
f"[{session.id}] 预设切换: "
f"'{old_preset or '跟随站点默认预设'}' → '{preset_name or '跟随站点默认预设'}'"
)
return True
def get_tab_preset(self, persistent_index: int) -> Optional[str]:
"""获取指定标签页的当前预设名称"""
with self._lock:
session_id = self._persistent_to_session_id.get(persistent_index)
if not session_id:
return None
session = self._tabs.get(session_id)
if not session:
return None
return session.preset_name
def set_tab_model_name(self, persistent_index: int, model_name: Optional[str]) -> Dict[str, Any]:
"""为指定标签页设置临时暴露模型名。空值表示恢复到持久化/默认规则。"""
normalized_name = self._normalize_model_name(model_name)
with self._lock:
session_id = self._persistent_to_session_id.get(persistent_index)
if not session_id:
return {"ok": False, "error": "tab_not_found", "tab_index": persistent_index}
session = self._tabs.get(session_id)
if not session:
return {"ok": False, "error": "tab_not_found", "tab_index": persistent_index}
session.model_name_override = normalized_name or None
info = session.get_info(use_cached_url=True)
tab_route_prefix = f"/tab/{session.persistent_index}"
route_domain = str(info.get("route_domain") or "").strip()
domain_route_prefix = f"/url/{route_domain}" if route_domain else ""
preset_route_domain = str(info.get("current_domain") or route_domain).strip()
url_route_token = str(info.get("url_route_token") or "").strip()
info["tab_route_prefix"] = tab_route_prefix
info["domain_route_prefix"] = domain_route_prefix
info["preset_route_domain"] = preset_route_domain
info["preset_domain_route_prefix"] = f"/url/{preset_route_domain}" if preset_route_domain else ""
info["exact_url_route_prefix"] = f"/tab-url/{url_route_token}" if url_route_token else ""
info["route_prefix"] = domain_route_prefix or tab_route_prefix
self._apply_exposed_model_name(info)
return {
"ok": True,
"tab_index": persistent_index,
"tab": info,
}
def get_status(self) -> Dict:
with self._lock:
sessions = list(self._tabs.values())
total = len(sessions)
idle = sum(1 for s in sessions if s.status == TabStatus.IDLE)
busy = sum(1 for s in sessions if s.status == TabStatus.BUSY)
max_tabs = self.max_tabs
min_tabs = self.min_tabs
idle_timeout = self.idle_timeout
acquire_timeout = self.acquire_timeout
stuck_timeout = self.stuck_timeout
allocation_mode = self.allocation_mode
excluded_urls = list(self.excluded_urls)
preserve_error_tabs = self.preserve_error_tabs
route_groups = normalize_route_groups(getattr(self, "route_groups", []))
global_network_enabled = self._global_network_enabled
known_raw_tabs = len(self._known_tab_ids)
last_scan = round(time.time() - self._last_scan_time, 1)
tabs_info = [s.get_info(use_cached_url=True) for s in sessions]
return {
"total": total,
"idle": idle,
"busy": busy,
"max_tabs": max_tabs,
"min_tabs": min_tabs,
"idle_timeout": idle_timeout,
"acquire_timeout": acquire_timeout,
"stuck_timeout": stuck_timeout,
"allocation_mode": allocation_mode,
"excluded_urls": excluded_urls,
"preserve_error_tabs": preserve_error_tabs,
"route_groups": route_groups,
"global_network_enabled": global_network_enabled,
"known_raw_tabs": known_raw_tabs,
"last_scan": last_scan,
"tabs": tabs_info
}
def get_watchdog_summary(self, limit: int = 4) -> Dict[str, Any]:
"""Return the lightweight tab pool fields needed by BrowserWatchdog logs."""
try:
max_items = max(0, int(limit))
except Exception:
max_items = 4
with self._lock:
sessions = list(self._tabs.values())
total = len(sessions)
idle = sum(1 for s in sessions if s.status == TabStatus.IDLE)
busy = sum(1 for s in sessions if s.status == TabStatus.BUSY)
preview = sessions[:max_items]
tabs = []
for session in preview:
status = getattr(getattr(session, "status", None), "value", None)
tabs.append({
"id": getattr(session, "id", None),
"status": status or str(getattr(session, "status", "") or "?"),
"is_isolated_context": bool(getattr(session, "is_isolated_context", False)),
})
return {
"total": total,
"idle": idle,
"busy": busy,
"tabs": tabs,
}
def get_idle_sessions_snapshot(self) -> List[TabSession]:
"""Return a shallow snapshot of currently idle tab sessions."""
with self._lock:
return [s for s in self._tabs.values() if s.status == TabStatus.IDLE]
def get_sessions_snapshot(self) -> List[TabSession]:
"""Return a shallow snapshot of all current tab sessions."""
with self._lock:
return list(self._tabs.values())
def get_session_by_tab(self, tab: Any) -> Optional[TabSession]:
"""Look up a TabSession for a given tab instance."""
if tab is None:
return None
direct = getattr(tab, "_session", None)
if direct is not None and isinstance(direct, TabSession):
return direct
with self._lock:
for session in self._tabs.values():
if getattr(session, "tab", None) is tab:
return session
return None
+646
View File
@@ -0,0 +1,646 @@
"""TabPoolManager 的一部分(R2-3 拆分):标签页的创建、扫描发现、初始化与刷新。
方法原样从 manager.py 迁出,行为不变;通过 mixin 组合回 TabPoolManager,
类属性与实例状态都在 TabPoolManager(manager.py)里定义。
"""
import time
from typing import Any, Dict, Optional
from app.core.config import logger
from app.utils.site_url import extract_remote_site_domain, route_domain_matches
from ._utils import _should_skip_pool_url
from .session import TabStatus
class TabPoolTabsMixin:
"""标签页的创建、扫描发现、初始化与刷新"""
def _build_site_entry_url(self, domain: str) -> str:
normalized_domain = str(domain or "").strip()
if not normalized_domain:
return ""
for session in self._tabs.values():
current_url, actual_domain = session.get_cached_route_snapshot()
if route_domain_matches(normalized_domain, actual_domain):
if current_url and not _should_skip_pool_url(current_url):
return current_url
built = session._build_domain_url(current_url, actual_domain)
if built:
return built
return f"https://{normalized_domain}/"
def _create_isolated_tab(self, url: str, background: bool = False) -> Optional[Dict[str, Any]]:
try:
browser = self._get_browser_handle()
tab = browser.new_tab(background=background, new_context=True, new_window=True)
raw_tab_id = getattr(tab, "tab_id", None)
browser_context_id = self._get_browser_context_id(raw_tab_id)
self._register_isolated_context(raw_tab_id, browser_context_id)
if url:
tab.get(url)
return {
"tab": tab,
"raw_tab_id": raw_tab_id,
"browser_context_id": browser_context_id,
"url": str(getattr(tab, "url", "") or url or ""),
}
except Exception as e:
logger.warning(f"[TabPool] create isolated tab failed: {e}")
return None
def _create_shared_tab(
self,
url: str,
*,
background: bool = False,
new_window: bool = True,
) -> Optional[Dict[str, Any]]:
try:
browser = self._get_browser_handle()
tab = browser.new_tab(url=url or None, background=background, new_window=new_window)
raw_tab_id = getattr(tab, "tab_id", None)
browser_context_id = self._get_browser_context_id(raw_tab_id)
return {
"tab": tab,
"raw_tab_id": raw_tab_id,
"browser_context_id": browser_context_id,
"url": str(getattr(tab, "url", "") or url or ""),
}
except Exception as e:
logger.warning(f"[TabPool] create shared tab failed: {e}")
return None
def _ensure_cookie_isolation_for_new_tab(self, tab: Any, raw_tab_id: str, url: str) -> Dict[str, Any]:
final_url = str(url or "").strip()
browser_context_id = self._get_browser_context_id(raw_tab_id)
if raw_tab_id in self._isolated_context_by_raw_id:
return {
"tab": tab,
"raw_tab_id": raw_tab_id,
"browser_context_id": browser_context_id,
"is_isolated_context": True,
"url": final_url,
}
domain = extract_remote_site_domain(final_url) or ""
if not self._is_site_independent_cookie_enabled(domain):
return {
"tab": tab,
"raw_tab_id": raw_tab_id,
"browser_context_id": browser_context_id,
"is_isolated_context": False,
"url": final_url,
}
if not self._is_site_independent_cookie_auto_takeover_enabled(domain):
return {
"tab": tab,
"raw_tab_id": raw_tab_id,
"browser_context_id": browser_context_id,
"is_isolated_context": False,
"url": final_url,
}
logger.warning(
f"[TabPool] skipped automatic isolated-cookie takeover because existing tabs are never closed: "
f"{domain or final_url}"
)
return {
"tab": tab,
"raw_tab_id": raw_tab_id,
"browser_context_id": browser_context_id,
"is_isolated_context": False,
"url": final_url,
}
def create_isolated_site_tab(self, domain: str) -> Dict[str, Any]:
target_domain = str(domain or "").strip()
if not target_domain:
return {"ok": False, "error": "domain_required"}
with self._condition:
self._scan_new_tabs()
if len(self._tabs) >= self.max_tabs:
return {"ok": False, "error": "tab_pool_full"}
target_url = self._build_site_entry_url(target_domain)
created = self._create_isolated_tab(target_url, background=False)
if not created:
return {"ok": False, "error": "create_isolated_tab_failed"}
with self._condition:
if len(self._tabs) >= self.max_tabs:
logger.warning(
"[TabPool] pool reached capacity while creating an isolated tab; "
"keeping the newly created tab managed"
)
session = self._wrap_tab(
created["tab"],
created["raw_tab_id"],
browser_context_id=created.get("browser_context_id"),
is_isolated_context=True,
initial_url=created.get("url") or target_url,
)
self._tabs[session.id] = session
self._start_global_monitor_for_session(session)
self._last_scan_time = time.time()
self._condition.notify_all()
info = session.get_info(use_cached_url=True)
info["tab_route_prefix"] = f"/tab/{session.persistent_index}"
route_domain = str(info.get("route_domain") or "").strip()
info["domain_route_prefix"] = f"/url/{route_domain}" if route_domain else ""
preset_route_domain = str(info.get("current_domain") or route_domain).strip()
info["preset_route_domain"] = preset_route_domain
info["preset_domain_route_prefix"] = f"/url/{preset_route_domain}" if preset_route_domain else ""
url_route_token = str(info.get("url_route_token") or "").strip()
info["exact_url_route_prefix"] = f"/tab-url/{url_route_token}" if url_route_token else ""
info["route_prefix"] = info["domain_route_prefix"] or info["tab_route_prefix"]
self._apply_exposed_model_name(info)
return {
"ok": True,
"domain": target_domain,
"message": f"已为 {target_domain} 新建独立 Cookie 标签页",
"tab": info,
}
def create_shared_site_tab(self, domain: str) -> Dict[str, Any]:
target_domain = str(domain or "").strip()
if not target_domain:
return {"ok": False, "error": "domain_required"}
with self._condition:
self._scan_new_tabs()
if len(self._tabs) >= self.max_tabs:
return {"ok": False, "error": "tab_pool_full"}
target_url = self._build_site_entry_url(target_domain)
created = self._create_shared_tab(target_url, background=False, new_window=True)
if not created:
return {"ok": False, "error": "create_shared_tab_failed"}
with self._condition:
if len(self._tabs) >= self.max_tabs:
logger.warning(
"[TabPool] pool reached capacity while creating a shared tab; "
"keeping the newly created tab managed"
)
session = self._wrap_tab(
created["tab"],
created["raw_tab_id"],
browser_context_id=created.get("browser_context_id"),
is_isolated_context=False,
initial_url=created.get("url") or target_url,
)
self._tabs[session.id] = session
self._start_global_monitor_for_session(session)
self._last_scan_time = time.time()
self._condition.notify_all()
info = session.get_info(use_cached_url=True)
info["tab_route_prefix"] = f"/tab/{session.persistent_index}"
route_domain = str(info.get("route_domain") or "").strip()
info["domain_route_prefix"] = f"/url/{route_domain}" if route_domain else ""
preset_route_domain = str(info.get("current_domain") or route_domain).strip()
info["preset_route_domain"] = preset_route_domain
info["preset_domain_route_prefix"] = f"/url/{preset_route_domain}" if preset_route_domain else ""
url_route_token = str(info.get("url_route_token") or "").strip()
info["exact_url_route_prefix"] = f"/tab-url/{url_route_token}" if url_route_token else ""
info["route_prefix"] = info["domain_route_prefix"] or info["tab_route_prefix"]
self._apply_exposed_model_name(info)
return {
"ok": True,
"domain": target_domain,
"message": f"已为 {target_domain} 打开共享 Cookie 受控窗口",
"tab": info,
}
def _should_scan(self) -> bool:
"""检查是否需要扫描新标签页"""
return time.time() - self._last_scan_time >= self.SCAN_INTERVAL
def _should_scan_for_query(self) -> bool:
"""读接口需要较新的浏览器 target 状态,但按 QUERY_SCAN_MIN_INTERVAL_SEC 节流。
此前恒返回 True,导致每个 /v1/chat/completions、/v1/models 请求都触发
一次全量 CDP 扫描(Target.getTargets 等同步网络 I/O),并发时扫描成本
线性叠加到首字延迟上;浏览器卡顿时还会拖住调用线程。
1 秒内的快照对路由决策而言足够新鲜。
"""
return time.time() - self._last_scan_time >= self.QUERY_SCAN_MIN_INTERVAL_SEC
def _scan_new_tabs(self):
"""扫描并添加新标签页(已持有锁)"""
try:
# 先占住本次扫描窗口,避免唤醒后多个等待线程重复扫描。
self._last_scan_time = time.time()
# 锁序考量:下面会释放池锁去采集浏览器快照,期间新入池的会话
# (create_shared_site_tab / create_isolated_site_tab 等)不会出现在
# 这份过期快照里。记录快照起始时间,应用移除逻辑时据此跳过晚入池的会话
# (session.created_at 即入池时刻,构造后随即持池锁入池)。
snapshot_started_at = time.time()
self._condition.release()
try:
with self._scan_snapshot_lock:
(
current_tabs,
current_tab_ids,
current_context_by_raw,
current_url_by_raw,
current_url_known_raw,
) = self._snapshot_current_tab_targets()
finally:
self._condition.acquire()
if self._shutdown:
return
current_tab_set = set(current_tab_ids)
# 物理关闭成功后,Target.getTargets 可能要到下一轮才反映结果;
# 这里清理已永久隔离且不再持有 tab 引用的陈旧元数据,避免隔离
# 字典只增不减。
for raw_id, entry in list(self._quarantined_raw_ids.items()):
if (
getattr(entry, "permanent", False)
and getattr(entry, "tab", None) is None
and raw_id not in current_tab_set
):
self._quarantined_raw_ids.pop(raw_id, None)
self._cleanup_orphaned_isolated_contexts(list(current_context_by_raw.values()))
changed = False
session_raw_by_id: Dict[str, str] = {}
for rid, pidx in self._raw_id_to_persistent.items():
sid = self._persistent_to_session_id.get(pidx)
if sid and sid in self._tabs:
session_raw_by_id[sid] = rid
for session_id, session in self._tabs.items():
if session_raw_by_id.get(session_id):
continue
candidate_raw_id = self._get_tab_ref_id(getattr(session, "tab", None))
if not candidate_raw_id or candidate_raw_id not in current_tab_set:
continue
existing_persistent_idx = self._raw_id_to_persistent.get(candidate_raw_id)
if existing_persistent_idx is not None:
existing_session_id = self._persistent_to_session_id.get(existing_persistent_idx)
if (
existing_session_id
and existing_session_id != session_id
and existing_session_id in self._tabs
):
continue
persistent_idx = int(
existing_persistent_idx
or getattr(session, "persistent_index", 0)
or 0
)
if persistent_idx <= 0:
persistent_idx = self._next_persistent_index
self._next_persistent_index += 1
session.persistent_index = persistent_idx
self._raw_id_to_persistent[candidate_raw_id] = persistent_idx
self._persistent_to_session_id[persistent_idx] = session_id
self._known_tab_ids.add(candidate_raw_id)
if session.is_isolated_context:
context_id = (
str(session.browser_context_id or "").strip()
or current_context_by_raw.get(candidate_raw_id)
)
if context_id:
session.browser_context_id = context_id
self._register_isolated_context(candidate_raw_id, context_id)
session_raw_by_id[session_id] = candidate_raw_id
changed = True
logger.warning(
f"[{session_id}] repaired missing raw tab mapping "
f"(raw={candidate_raw_id}, idx=#{persistent_idx})"
)
for session_id, session in self._tabs.items():
raw_id = session_raw_by_id.get(session_id)
if not raw_id or raw_id not in current_tab_set:
continue
if raw_id in current_url_known_raw:
current_url = current_url_by_raw.get(raw_id, "")
old_url = session.last_known_url
session._remember_url(current_url)
try:
session.current_domain = extract_remote_site_domain(current_url)
except Exception:
session.current_domain = None
if old_url and current_url and old_url != current_url:
session.notify_navigated(reason="url_changed")
if not session.is_isolated_context:
continue
if session.status == TabStatus.BUSY:
continue
bound_raw_id = self._get_tab_ref_id(getattr(session, "tab", None))
if bound_raw_id == raw_id:
session.clear_transient_disconnect()
continue
if self._refresh_isolated_session_binding(session, raw_id):
logger.info(f"[{session_id}] isolated session target recovered: {raw_id}")
continue
if session.is_in_transient_disconnect():
logger.debug(
f"[{session_id}] isolated session still waiting for target recovery: {raw_id}"
)
reserved_raw_ids = {
raw_id
for raw_id in session_raw_by_id.values()
if raw_id in current_tab_set
}
# ===== 第一步:清理已关闭的标签页 =====
# 找出池中存在、但浏览器中已消失的标签页
sessions_to_remove = []
for session_id, session in self._tabs.items():
# 查找该 session 对应的 raw_tab_id
raw_id = session_raw_by_id.get(session_id)
if raw_id is None or raw_id not in current_tab_set:
# 入池晚于本次快照起始的会话必然不在快照里,不能据此判定"已关闭"
# (isolated 会话有 rebind 宽限兜底,shared 会话没有,会被直接移除)。
# 跳过本轮,等下一轮新快照再判定。
joined_at = float(getattr(session, "created_at", 0.0) or 0.0)
if joined_at > snapshot_started_at:
logger.debug(
f"[{session_id}] 跳过过期快照的移除判定 "
f"(入池晚于快照起始 {joined_at - snapshot_started_at:.3f}s, raw={raw_id})"
)
continue
sessions_to_remove.append((session_id, raw_id, session))
for session_id, raw_id, session in sessions_to_remove:
replacement_raw_id = None
replacement_tab = None
browser_context_id = str(session.browser_context_id or "").strip()
if session.is_isolated_context and browser_context_id:
for candidate_raw_id in current_tab_ids:
if candidate_raw_id in reserved_raw_ids:
continue
if current_context_by_raw.get(candidate_raw_id) != browser_context_id:
continue
replacement_tab = self._resolve_tab_from_ref(candidate_raw_id)
if not replacement_tab:
continue
replacement_raw_id = candidate_raw_id
break
if replacement_raw_id and replacement_tab:
self._detach_global_monitor_for_session(session_id, reason="target_rebind")
p_idx = self._raw_id_to_persistent.pop(raw_id, None) if raw_id else None
if p_idx is None:
p_idx = int(getattr(session, "persistent_index", 0) or 0) or None
if p_idx is not None:
self._raw_id_to_persistent[replacement_raw_id] = p_idx
self._persistent_to_session_id[p_idx] = session_id
if raw_id:
self._known_tab_ids.discard(raw_id)
self._known_tab_ids.add(replacement_raw_id)
if raw_id:
self._isolated_context_by_raw_id.pop(raw_id, None)
self._register_isolated_context(replacement_raw_id, browser_context_id)
session.tab = replacement_tab
session.browser_context_id = browser_context_id or current_context_by_raw.get(replacement_raw_id)
session.clear_transient_disconnect()
try:
_cached_url, cached_domain = session.get_cached_route_snapshot()
if cached_domain:
session.current_domain = cached_domain
except Exception:
pass
reserved_raw_ids.add(replacement_raw_id)
if session.status == TabStatus.IDLE:
self._restart_global_monitor_for_session_async(session_id, "target_rebind")
logger.info(
f"[{session_id}] rebound isolated context target: {raw_id} -> {replacement_raw_id}"
)
changed = True
continue
if session.is_isolated_context and browser_context_id:
if self._arm_isolated_rebind_grace(
session,
reason="target_missing",
detail=f"target missing (raw={raw_id})",
):
continue
if session.status == TabStatus.BUSY:
self._cancel_active_request_for_session(
session,
"tab_closed",
detail=f"raw={raw_id}",
)
logger.warning(f"[{session_id}] 标签页已关闭但仍在忙碌,标记为关闭")
session.mark_closed("tab_closed")
self._detach_global_monitor_for_session(session_id, reason="tab_closed")
else:
logger.info(f"[{session_id}] 标签页已关闭,从池中移除")
session.mark_closed("tab_closed")
self._detach_global_monitor_for_session(session_id, reason="tab_closed")
del self._tabs[session_id]
self._on_session_removed(session_id)
# 清理映射
if raw_id:
self._known_tab_ids.discard(raw_id)
p_idx = self._raw_id_to_persistent.pop(raw_id, None) if raw_id else None
if p_idx is None:
p_idx = int(getattr(session, "persistent_index", 0) or 0) or None
if p_idx is not None:
if self._persistent_to_session_id.get(p_idx) == session_id:
self._persistent_to_session_id.pop(p_idx, None)
browser_context_id = self._isolated_context_by_raw_id.pop(raw_id, None) if raw_id else None
if not browser_context_id and session.is_isolated_context:
browser_context_id = str(session.browser_context_id or "").strip()
if browser_context_id:
self._mark_orphaned_isolated_context(browser_context_id)
if self._active_session_id == session_id:
self._active_session_id = None
changed = True
# 顺手清理已切换到本地页/无效页的空闲标签,避免继续展示和参与调度。
self._cleanup_unhealthy_tabs()
# ===== 第二步:构建"已在池中的 tab 对象"集合 =====
tabs_in_pool = set()
for rid in self._raw_id_to_persistent:
pidx = self._raw_id_to_persistent[rid]
sid = self._persistent_to_session_id.get(pidx)
if sid and sid in self._tabs:
tabs_in_pool.add(rid)
# ===== 第三步:扫描新标签页 =====
new_count = 0
for tab_ref in current_tabs:
if len(self._tabs) >= self.max_tabs:
break
raw_tab = self._get_tab_ref_id(tab_ref)
if not raw_tab:
continue
# 已在池中,跳过
if raw_tab in tabs_in_pool:
continue
# 隔离中的错误标签页不重新入池:旧 worker 可能尚未退出,
# 由 TabRecoveryService 决定何时解除隔离(或永久排除)。
if raw_tab in self._quarantined_raw_ids:
logger.debug(f"[TabRecovery] 扫描跳过隔离中的标签页: {raw_tab}")
continue
try:
url = current_url_by_raw.get(raw_tab) or self._get_tab_ref_url(tab_ref)
# 本地页、浏览器内部页、空白页都不纳入标签页池。
if _should_skip_pool_url(url):
continue
tab = self._resolve_tab_from_ref(tab_ref)
if not tab:
continue
isolation_result = self._ensure_cookie_isolation_for_new_tab(tab, raw_tab, url)
tab = isolation_result["tab"]
raw_tab = isolation_result["raw_tab_id"]
url = isolation_result["url"]
# 有效页面 - 添加到池
session = self._wrap_tab(
tab,
raw_tab,
browser_context_id=isolation_result.get("browser_context_id"),
is_isolated_context=isolation_result.get("is_isolated_context", False),
initial_url=url,
)
self._tabs[session.id] = session
self._start_global_monitor_for_session(session)
new_count += 1
changed = True
display_url = url[:60] + "..." if len(url) > 60 else url
logger.debug(f"🆕 发现新标签页: {session.id} -> {display_url}")
except Exception as e:
logger.debug(f"处理标签页出错: {e}")
continue
# Remove unusable pool sessions after discovery without closing targets.
self._cleanup_unhealthy_tabs()
self._last_scan_time = time.time()
if changed:
self._condition.notify_all()
if new_count > 0:
logger.info(f"扫描完成: +{new_count} 个,当前共 {len(self._tabs)} 个标签页")
except Exception as e:
logger.warning(f"扫描标签页失败: {e}")
def initialize(self):
"""初始化标签页池"""
with self._lock:
if self._initialized:
return
raw_target_count = 0
try:
browser = self._get_browser_handle()
existing_tabs = self._list_current_tab_refs()
raw_target_count = len(existing_tabs)
logger.debug(f"[TabPool] 检测到 {raw_target_count} 个浏览器 page target")
for tab_ref in existing_tabs:
if len(self._tabs) >= self.max_tabs:
break
try:
raw_tab = self._get_tab_ref_id(tab_ref)
if not raw_tab:
continue
url = self._get_tab_ref_url(tab_ref)
# 初始化时直接跳过本地页和浏览器内部页。
if _should_skip_pool_url(url):
continue
tab = self._resolve_tab_from_ref(tab_ref)
if not tab:
continue
isolation_result = self._ensure_cookie_isolation_for_new_tab(tab, raw_tab, url)
tab = isolation_result["tab"]
raw_tab = isolation_result["raw_tab_id"]
url = isolation_result["url"]
# 有效页面 - 添加到池
session = self._wrap_tab(
tab,
raw_tab,
browser_context_id=isolation_result.get("browser_context_id"),
is_isolated_context=isolation_result.get("is_isolated_context", False),
initial_url=url,
)
self._tabs[session.id] = session
display_url = url[:60] + "..." if len(url) > 60 else url
logger.info(f"TabPool: {session.id} -> {display_url}")
except Exception as e:
logger.debug(f"处理标签页出错: {e}")
continue
except Exception as e:
logger.warning(f"扫描标签页失败: {e}")
# 重置所有状态为 IDLE
for session in self._tabs.values():
# 锁序考量:状态迁移须在 session._lock 内进行,与 acquire/release 的
# 状态检查互斥。既有锁序为 池锁 → session._lock(release 等路径同序),
# session._lock 内不会反向取池锁,无死锁风险。
with session._lock:
session.status = TabStatus.IDLE
session.current_task_id = None
self._start_global_monitor_for_session(session)
self._initialized = True
self._last_scan_time = time.time()
ignored_count = max(0, raw_target_count - len(self._tabs))
logger.info(
f"TabPool 就绪: {len(self._tabs)} 个远程网页"
+ (f"(已忽略 {ignored_count} 个内部/无效 target)" if ignored_count else "")
)
def refresh_tabs(self) -> Dict:
"""手动刷新标签页列表(供外部调用)"""
with self._condition:
old_count = len(self._tabs)
old_ids = set(self._tabs.keys())
# 强制扫描(不受时间间隔限制)
self._last_scan_time = 0
self._scan_new_tabs()
# 同时清理不健康的标签页
self._cleanup_unhealthy_tabs()
new_ids = set(self._tabs.keys())
added = new_ids - old_ids
removed = old_ids - new_ids
if added or removed:
self._condition.notify_all()
logger.info(f"刷新完成: +{len(added)} -{len(removed)} = {len(self._tabs)} 个标签页")
return {
"added": len(added),
"removed": len(removed),
"total": len(self._tabs)
}
+530
View File
@@ -0,0 +1,530 @@
"""
app/core/tab_pool_parts/recovery.py - 错误标签页隔离与 AI 恢复服务
职责:
- TabQuarantineEntry:错误标签页的隔离记录(由 TabPoolManager 在池锁内维护)
- TabRecoveryService:单线程后台服务,等旧 worker 退出后按配置决定
截图 → AI 判断 → 刷新 → 解除隔离 / 永久隔离
锁纪律:
- 本模块所有阻塞操作(等待旧 worker、截图、AI HTTP 调用、刷新)都发生在
自持的 ThreadPoolExecutor(max_workers=1) 线程里,绝不在事件循环线程执行,
也绝不持有 TabPoolManager 的池锁。
- 与 TabPoolManager 的交互只通过构造时注入的两个回调
(resolve_quarantine / get_quarantine_entry),本模块不 import manager,
避免循环导入;request_manager 的 import 放在函数内。
"""
import base64
import json
import threading
import time
import urllib.request
from collections import OrderedDict
from concurrent.futures import ThreadPoolExecutor
from dataclasses import dataclass, field
from typing import Any, Callable, Dict, Optional, Tuple
from app.core.config import AppConfig, logger
# DrissionPage/CDP 调用可能在底层连接断开时永久阻塞。超时后 daemon
# 线程无法被强制终止,因此限制同时存活的超时调用数量,避免每次异常
# 都累积一个永不退出的线程。
_RECOVERY_CALL_SLOTS = threading.BoundedSemaphore(4)
@dataclass
class TabQuarantineEntry:
"""错误标签页的隔离记录(在 TabPoolManager 的池锁保护下读写)"""
raw_tab_id: str
persistent_index: int = 0
task_id: str = ""
tab: Any = None
reason: str = ""
since: float = field(default_factory=time.monotonic)
permanent: bool = False
attempts: int = 0
session_id: str = ""
url: str = ""
class TabRecoveryService:
"""错误标签页 AI 恢复服务(由 TabPoolManager 懒创建并持有,单例风格)"""
WORKER_POLL_INTERVAL_SEC = 5.0
# 拿不到旧 worker 线程引用时的时间宽限:隔离时刻起 180s 视为已退出
WORKER_EXIT_FALLBACK_GRACE_SEC = 180.0
SCREENSHOT_TIMEOUT_SEC = 30.0
REFRESH_TIMEOUT_SEC = 30.0
POST_REFRESH_SETTLE_SEC = 3.0
MAX_ATTEMPT_CACHE_ENTRIES = 4096
def __init__(
self,
*,
resolve_quarantine: Callable[[str, bool], bool],
get_quarantine_entry: Callable[[str], Optional[TabQuarantineEntry]],
):
self._resolve_quarantine = resolve_quarantine
self._get_quarantine_entry = get_quarantine_entry
self._attempts: OrderedDict[str, int] = OrderedDict()
self._attempts_lock = threading.Lock()
self._shutdown = False
self._executor = ThreadPoolExecutor(
max_workers=1,
thread_name_prefix="tab-recovery",
)
# ================= 对外接口 =================
def submit(self, entry: TabQuarantineEntry) -> bool:
"""受理一个隔离标签页;仅入队,不阻塞调用线程。"""
if entry is None or not str(getattr(entry, "raw_tab_id", "") or "").strip():
return False
if self._shutdown:
return False
try:
self._executor.submit(self._process_safe, entry)
logger.info(
f"[TabRecovery] 已受理隔离标签页,排队处理 "
f"(raw={entry.raw_tab_id}, tab={entry.session_id or '-'}, "
f"idx=#{entry.persistent_index or '-'}, task={entry.task_id or '-'}, "
f"reason={entry.reason or '-'})"
)
return True
except RuntimeError as e:
logger.debug(f"[TabRecovery] submit failed ({entry.raw_tab_id}): {e}")
return False
def shutdown(self) -> None:
self._shutdown = True
with self._attempts_lock:
self._attempts.clear()
try:
self._executor.shutdown(wait=False, cancel_futures=True)
except Exception as e:
logger.debug(f"[TabRecovery] executor shutdown failed: {e}")
# ================= 主流程 =================
def _process_safe(self, entry: TabQuarantineEntry) -> None:
"""线程池任务入口:吃掉所有异常,绝不向外抛。"""
try:
self._process(entry)
except Exception as e:
logger.warning(
f"[TabRecovery] 恢复流程异常 (raw={entry.raw_tab_id}): {e}"
)
# 未知内部错误时保守处理:永久隔离,避免把状态不明的标签页重新入池。
try:
self._resolve(entry, release=False, reason=f"internal_error:{e}")
except Exception:
pass
def _process(self, entry: TabQuarantineEntry) -> None:
raw_id = entry.raw_tab_id
logger.info(
f"[TabRecovery] 开始处理隔离标签页 "
f"(raw={raw_id}, tab={entry.session_id or '-'}, "
f"task={entry.task_id or '-'}, reason={entry.reason or '-'})"
)
# 1. 等旧 worker 退出:串扰的根源是旧请求的 worker 线程可能仍阻塞在
# 对该 tab 的同步 DrissionPage 调用里。
wait_result = self._wait_for_worker_exit(entry)
if wait_result == "aborted":
logger.debug(f"[TabRecovery] 服务关闭,放弃处理 (raw={raw_id})")
return
if wait_result == "timeout":
self._resolve(entry, release=False, reason="worker_never_exited")
return
logger.debug(f"[TabRecovery] 旧 worker 已确认退出 (raw={raw_id})")
# 2. 次数检查:解除隔离后再次错误 → 不再调 API,直接永久隔离。
with self._attempts_lock:
attempts = int(self._attempts.get(raw_id, 0))
if raw_id in self._attempts:
# 最近使用的 raw id 保留在缓存尾部,便于后续按上限淘汰。
self._attempts.move_to_end(raw_id)
entry.attempts = attempts
max_attempts = AppConfig.get_tab_recovery_max_attempts()
if attempts >= max_attempts:
logger.info(
f"[TabRecovery] 恢复次数已用尽,不再调用 AI "
f"(raw={raw_id}, attempts={attempts}, max={max_attempts})"
)
self._resolve(entry, release=False, reason="attempts_exhausted")
return
# 3. 功能开关:未启用时不截图不调 API,直接解除隔离——保持既有的
# 自动重入池行为,但此时旧 worker 已确认退出,串扰已消除。
if not AppConfig.is_tab_recovery_enabled():
self._resolve(entry, release=True, reason="recovery_disabled")
return
# 4/5. 截图 + AI 判断;任一失败 → verdict=None(未知)
verdict: Optional[bool] = None
detail = ""
image_b64 = ""
try:
image_b64 = self._capture_screenshot(entry)
except Exception as e:
detail = f"screenshot_failed:{e}"
logger.warning(f"[TabRecovery] 截图失败 (raw={raw_id}): {e}")
if image_b64:
try:
verdict, detail = self._judge_recoverable(entry, image_b64)
logger.info(
f"[TabRecovery] AI 判断结果 "
f"(raw={raw_id}, recoverable={verdict}, detail={detail or '-'})"
)
except Exception as e:
verdict, detail = None, f"api_error:{e}"
logger.warning(f"[TabRecovery] AI 判断失败 (raw={raw_id}): {e}")
# 6. 决策
if verdict is False:
self._resolve(
entry,
release=False,
reason=f"ai_judged_unrecoverable:{detail or '-'}",
)
return
if verdict is None:
if not AppConfig.is_tab_recovery_refresh_on_unknown():
self._resolve(
entry,
release=False,
reason=f"judgement_unknown:{detail or '-'}",
)
return
refresh_reason = "unknown_refresh"
else:
refresh_reason = "ai_judged_recoverable"
# 7. 刷新与解除
self._refresh_and_release(entry, refresh_reason)
# ================= 步骤实现 =================
def _wait_for_worker_exit(self, entry: TabQuarantineEntry) -> str:
"""轮询旧 worker 是否退出。返回 'exited' / 'timeout' / 'aborted'。"""
max_wait = AppConfig.get_tab_recovery_worker_exit_wait_sec()
deadline = time.monotonic() + max(0.0, float(max_wait))
while True:
if self._shutdown:
return "aborted"
if self._is_worker_exited(entry):
return "exited"
remaining = deadline - time.monotonic()
if remaining <= 0:
return "timeout"
time.sleep(min(self.WORKER_POLL_INTERVAL_SEC, max(0.1, remaining)))
def _is_worker_exited(self, entry: TabQuarantineEntry) -> bool:
task_id = str(entry.task_id or "").strip()
if not task_id:
# 隔离时没有绑定任务,无 worker 可等
return True
ctx = None
try:
from app.services.request_manager import request_manager
ctx = request_manager.get_request(task_id)
except Exception as e:
logger.debug(
f"[TabRecovery] 查询请求上下文失败 (task={task_id}): {e}"
)
thread = getattr(ctx, "worker_thread", None) if ctx is not None else None
if thread is not None and hasattr(thread, "is_alive"):
try:
return not thread.is_alive()
except Exception as e:
logger.debug(
f"[TabRecovery] 读取 worker 线程状态失败 (task={task_id}): {e}"
)
# 拿不到线程引用(非流式请求 worker 在 request_lifecycle 内创建、
# 或请求上下文已被清理):回退时间宽限,隔离时刻起 180s 视为已退出。
elapsed = time.monotonic() - float(entry.since or 0.0)
return elapsed >= self.WORKER_EXIT_FALLBACK_GRACE_SEC
def _capture_screenshot(self, entry: TabQuarantineEntry) -> str:
tab = entry.tab
if tab is None:
raise RuntimeError("tab reference unavailable")
# DrissionPage 4.x:get_screenshot(as_base64=True) 返回 PNG 的 base64 字符串
data = self._call_with_timeout(
lambda: tab.get_screenshot(as_base64=True),
self.SCREENSHOT_TIMEOUT_SEC,
"screenshot",
)
if isinstance(data, (bytes, bytearray)):
data = base64.b64encode(bytes(data)).decode("ascii")
text = str(data or "").strip()
if not text:
raise RuntimeError("empty screenshot payload")
return text
def _judge_recoverable(
self,
entry: TabQuarantineEntry,
image_b64: str,
) -> Tuple[Optional[bool], str]:
"""调用 OpenAI 兼容接口判断刷新能否恢复。返回 (verdict, detail)。"""
api_url = AppConfig.get_tab_recovery_api_url()
if api_url:
target_url = api_url
api_key = AppConfig.get_tab_recovery_api_key()
local_mode = False
else:
# 本地模式:走本服务自身的 /v1/chat/completions,
# 由其他空闲标签页完成实际判断
target_url = (
f"http://127.0.0.1:{AppConfig.get_port()}/v1/chat/completions"
)
api_key = (
AppConfig.get_auth_token() if AppConfig.is_auth_enabled() else ""
)
local_mode = True
site_url = str(entry.url or "").strip() or "未知"
prompt = (
"这是反代服务的一个疑似卡死的网页标签页截图,"
f"站点为 {site_url},错误原因 {entry.reason or '未知'}。"
"请判断刷新页面是否可能恢复正常使用。只输出 JSON:"
"{\"recoverable\": true/false, \"reason\": \"...\"}。"
"登录失效、封号提示、人机验证页输出 false;"
"加载中、白屏、内容正常、临时错误提示输出 true。"
)
payload = {
"model": AppConfig.get_tab_recovery_model(),
"stream": False,
"messages": [
{
"role": "user",
"content": [
{"type": "text", "text": prompt},
{
"type": "image_url",
"image_url": {
"url": f"data:image/png;base64,{image_b64}",
},
},
],
}
],
}
headers = {"Content-Type": "application/json"}
if api_key:
headers["Authorization"] = f"Bearer {api_key}"
timeout_sec = AppConfig.get_tab_recovery_timeout_sec()
request = urllib.request.Request(
target_url,
data=json.dumps(payload).encode("utf-8"),
headers=headers,
method="POST",
)
logger.debug(
f"[TabRecovery] 调用 AI 判断 "
f"(raw={entry.raw_tab_id}, url={target_url}, "
f"local_mode={local_mode}, timeout={timeout_sec}s)"
)
if local_mode:
# 本地回环地址不应经过环境代理
opener = urllib.request.build_opener(urllib.request.ProxyHandler({}))
response = opener.open(request, timeout=timeout_sec)
else:
response = urllib.request.urlopen(request, timeout=timeout_sec)
with response:
body = response.read().decode("utf-8", "replace")
data = json.loads(body)
choices = data.get("choices") or []
if not choices:
return None, "empty_choices"
message = choices[0].get("message") or {}
content = message.get("content")
text = self._flatten_message_content(content)
parsed = self._extract_first_json_object(text)
if not isinstance(parsed, dict):
return None, f"parse_failed:{text[:120]!r}"
verdict = self._coerce_bool(parsed.get("recoverable"))
detail = str(parsed.get("reason") or "").strip()
if verdict is None:
return None, f"invalid_recoverable_field:{parsed.get('recoverable')!r}"
return verdict, detail
def _refresh_and_release(
self,
entry: TabQuarantineEntry,
refresh_reason: str,
) -> None:
raw_id = entry.raw_tab_id
try:
tab = entry.tab
if tab is None:
raise RuntimeError("tab reference unavailable")
self._call_with_timeout(
lambda: tab.refresh(),
self.REFRESH_TIMEOUT_SEC,
"refresh",
)
time.sleep(max(0.0, float(self.POST_REFRESH_SETTLE_SEC)))
current_url = self._call_with_timeout(
lambda: str(getattr(tab, "url", "") or ""),
self.REFRESH_TIMEOUT_SEC,
"health-check",
)
if not str(current_url or "").strip():
raise RuntimeError("empty url after refresh")
except Exception as e:
logger.warning(f"[TabRecovery] 刷新/健康检查失败 (raw={raw_id}): {e}")
self._resolve(entry, release=False, reason=f"refresh_failed:{e}")
return
with self._attempts_lock:
self._attempts[raw_id] = int(self._attempts.get(raw_id, 0)) + 1
self._attempts.move_to_end(raw_id)
entry.attempts = self._attempts[raw_id]
while len(self._attempts) > self.MAX_ATTEMPT_CACHE_ENTRIES:
self._attempts.popitem(last=False)
self._resolve(entry, release=True, reason=refresh_reason)
logger.info(
f"[TabRecovery] 标签页已恢复并重新入池 "
f"(raw={raw_id}, idx=#{entry.persistent_index or '-'}, "
f"url={str(current_url)[:80]}, attempts={entry.attempts}, "
f"reason={refresh_reason})"
)
# ================= 辅助 =================
def _resolve(
self,
entry: TabQuarantineEntry,
*,
release: bool,
reason: str,
) -> None:
entry.reason = str(reason or "").strip() or entry.reason
try:
resolved = bool(self._resolve_quarantine(entry.raw_tab_id, release))
except Exception as e:
logger.warning(
f"[TabRecovery] 解除/永久隔离操作失败 "
f"(raw={entry.raw_tab_id}, release={release}): {e}"
)
return
if release:
logger.info(
f"[TabRecovery] 解除隔离,标签页将由下次扫描重新入池 "
f"(raw={entry.raw_tab_id}, reason={reason}, resolved={resolved})"
)
else:
logger.info(
f"[TabRecovery] 永久隔离标签页 "
f"(raw={entry.raw_tab_id}, reason={reason}, resolved={resolved})"
)
@staticmethod
def _call_with_timeout(fn: Callable[[], Any], timeout_sec: float, label: str) -> Any:
"""在守护线程中执行阻塞调用并限时等待,防止 CDP 调用无限挂起。"""
wait_timeout = max(0.1, float(timeout_sec))
if not _RECOVERY_CALL_SLOTS.acquire(timeout=wait_timeout):
raise TimeoutError(f"{label} call slots exhausted")
result: Dict[str, Any] = {}
done = threading.Event()
def _runner():
try:
result["value"] = fn()
except BaseException as e: # noqa: BLE001 - 结果原样转抛给调用方
result["error"] = e
finally:
done.set()
_RECOVERY_CALL_SLOTS.release()
thread = threading.Thread(
target=_runner,
name=f"tab-recovery-{label}",
daemon=True,
)
try:
thread.start()
except BaseException:
_RECOVERY_CALL_SLOTS.release()
raise
if not done.wait(wait_timeout):
raise TimeoutError(f"{label} timed out after {timeout_sec}s")
if "error" in result:
raise result["error"]
return result.get("value")
@staticmethod
def _flatten_message_content(content: Any) -> str:
if isinstance(content, str):
return content
if isinstance(content, list):
parts = []
for item in content:
if isinstance(item, dict):
text = item.get("text")
if isinstance(text, str):
parts.append(text)
elif isinstance(item, str):
parts.append(item)
return "\n".join(parts)
return str(content or "")
@staticmethod
def _extract_first_json_object(text: str) -> Optional[Dict[str, Any]]:
"""宽松提取文本中第一个平衡的 {...} 并解析为 dict。"""
source = str(text or "")
start = source.find("{")
while start != -1:
depth = 0
in_string = False
escaped = False
for index in range(start, len(source)):
char = source[index]
if in_string:
if escaped:
escaped = False
elif char == "\\":
escaped = True
elif char == '"':
in_string = False
continue
if char == '"':
in_string = True
elif char == "{":
depth += 1
elif char == "}":
depth -= 1
if depth == 0:
candidate = source[start:index + 1]
try:
parsed = json.loads(candidate)
if isinstance(parsed, dict):
return parsed
except Exception:
pass
break
start = source.find("{", start + 1)
return None
@staticmethod
def _coerce_bool(value: Any) -> Optional[bool]:
if isinstance(value, bool):
return value
text = str(value or "").strip().lower()
if text in {"true", "1", "yes", "y"}:
return True
if text in {"false", "0", "no", "n"}:
return False
return None
+994
View File
@@ -0,0 +1,994 @@
import threading
import time
from dataclasses import dataclass, field
from enum import Enum
from typing import Any, Dict, Optional
from urllib.parse import urlsplit
from app.core.config import logger
from app.utils.site_url import (
encode_tab_url_route_token,
extract_remote_site_domain,
get_preferred_route_domain,
)
from ._utils import _TAB_HEALTH_CACHE_TTL_SEC, _should_skip_pool_url
from app.core.driver import driver_for_tab
class TabStatus(Enum):
"""标签页状态"""
IDLE = "idle"
BUSY = "busy"
ERROR = "error"
CLOSED = "closed"
@dataclass
class TabSession:
"""标签页会话"""
id: str
tab: Any
status: TabStatus = TabStatus.IDLE
current_task_id: Optional[str] = None
current_command_name: Optional[str] = None
current_domain: Optional[str] = None
last_known_url: Optional[str] = None
created_at: float = field(default_factory=time.time)
last_used_at: float = field(default_factory=time.time)
request_count: int = 0
error_count: int = 0
persistent_index: int = 0 # 🆕 持久化编号(重启前不变)
preset_name: Optional[str] = None # 🆕 当前显式指定的预设名称(None = 跟随站点默认预设)
preset_override_source: Optional[str] = None # 预设来源(url: 来自 URL 记忆, tab: 标签页临时指定, None: 默认)
model_name_override: Optional[str] = None # 当前标签页临时暴露模型名(关闭标签页后失效)
browser_context_id: Optional[str] = None
is_isolated_context: bool = False
transient_disconnect_until: float = 0.0
transient_disconnect_reason: Optional[str] = None
last_conversation_activity_at: float = 0.0
last_conversation_domain: Optional[str] = None
last_conversation_preset_name: Optional[str] = None
command_loop_started_at: float = 0.0
command_loop_iteration: Optional[int] = None
command_loop_total: Optional[int] = None
command_loop_label: Optional[str] = None
command_loop_cancel_requested: bool = False
command_loop_cancel_reason: Optional[str] = None
command_loop_cancel_requested_at: float = 0.0
_health_cache_until: float = field(default=0.0, repr=False)
_health_cache_result: bool = field(default=False, repr=False)
_health_cache_url: str = field(default="", repr=False)
_health_cache_domain: str = field(default="", repr=False)
_release_in_progress: bool = field(default=False, repr=False)
_termination_in_progress: bool = field(default=False, repr=False)
_termination_task_id: Optional[str] = field(default=None, repr=False)
_termination_interrupt_done: bool = field(default=False, repr=False)
_workflow_parser_id: str = field(default="", repr=False)
_workflow_target_side: str = field(default="", repr=False)
_workflow_runtime_id: str = field(default="", repr=False)
_request_occupied_url: Optional[str] = field(default=None, repr=False)
_lock: threading.RLock = field(default_factory=threading.RLock, repr=False)
@property
def driver(self):
"""R2-1:该标签页的统一驱动(按标签页对象复用同一个包装)。迁移中的代码通过它访问浏览器。"""
return driver_for_tab(self.tab)
def is_available(self) -> bool:
with self._lock:
return self.status == TabStatus.IDLE and not self._termination_in_progress
def is_in_transient_disconnect(self) -> bool:
return self.transient_disconnect_until > time.time()
def mark_transient_disconnect(self, seconds: float, reason: str = None):
with self._lock:
self.transient_disconnect_until = time.time() + max(0.1, float(seconds))
self.transient_disconnect_reason = str(reason or "").strip() or None
def clear_transient_disconnect(self):
with self._lock:
self.transient_disconnect_until = 0.0
self.transient_disconnect_reason = None
def set_preset(self, preset_name: Optional[str], source: Optional[str] = None) -> bool:
"""线程安全地设置预设及来源"""
with self._lock:
old_preset = self.preset_name
self.preset_name = str(preset_name).strip() if preset_name else None
self.preset_override_source = str(source).strip() if (source and self.preset_name) else None
if old_preset != self.preset_name:
self._reset_conversation_state_unlocked()
return True
def _clear_health_cache_unlocked(self):
self._health_cache_until = 0.0
self._health_cache_result = False
self._health_cache_url = ""
self._health_cache_domain = ""
def _clear_command_loop_unlocked(self):
self.command_loop_started_at = 0.0
self.command_loop_iteration = None
self.command_loop_total = None
self.command_loop_label = None
self.command_loop_cancel_requested = False
self.command_loop_cancel_reason = None
self.command_loop_cancel_requested_at = 0.0
def begin_command_loop(
self,
*,
iteration: Optional[int] = None,
total: Optional[int] = None,
label: str = "",
) -> Dict[str, Any]:
now = time.time()
with self._lock:
self.command_loop_started_at = now
self.command_loop_iteration = iteration if iteration is None else int(iteration)
self.command_loop_total = total if total is None else int(total)
self.command_loop_label = str(label or "").strip() or None
self.command_loop_cancel_requested = False
self.command_loop_cancel_reason = None
self.command_loop_cancel_requested_at = 0.0
return self._command_loop_info_unlocked(now=now)
def end_command_loop(self, status: str = "completed") -> Dict[str, Any]:
now = time.time()
with self._lock:
info = self._command_loop_info_unlocked(now=now)
info["final_status"] = str(status or "").strip() or "completed"
self._clear_command_loop_unlocked()
return info
def request_command_loop_cancel(self, reason: str = "manual_loop_cancel") -> bool:
now = time.time()
with self._lock:
if not self.command_loop_started_at:
return False
self.command_loop_cancel_requested = True
self.command_loop_cancel_reason = str(reason or "").strip() or "manual_loop_cancel"
self.command_loop_cancel_requested_at = now
return True
def is_command_loop_cancelled(self) -> bool:
with self._lock:
return bool(self.command_loop_cancel_requested)
def _command_loop_info_unlocked(self, *, now: Optional[float] = None) -> Dict[str, Any]:
current_time = time.time() if now is None else float(now)
started_at = float(self.command_loop_started_at or 0.0)
active = started_at > 0
elapsed = max(0.0, current_time - started_at) if active else 0.0
return {
"active": active,
"iteration": self.command_loop_iteration,
"total": self.command_loop_total,
"label": self.command_loop_label or "",
"started_at": started_at or None,
"elapsed_sec": round(elapsed, 1),
"cancel_requested": bool(self.command_loop_cancel_requested),
"cancel_reason": self.command_loop_cancel_reason or "",
}
def get_command_loop_info(self) -> Dict[str, Any]:
with self._lock:
return self._command_loop_info_unlocked()
def touch_activity(self) -> None:
with self._lock:
self.last_used_at = time.time()
def get_busy_timeout_info(self, *, now: Optional[float] = None) -> Dict[str, Any]:
current_time = time.time() if now is None else float(now)
with self._lock:
loop_started_at = float(self.command_loop_started_at or 0.0)
if loop_started_at > 0:
reference_at = loop_started_at
basis = "command_loop"
else:
reference_at = float(self.last_used_at or current_time)
basis = "session"
return {
"basis": basis,
"started_at": reference_at,
"duration": max(0.0, current_time - reference_at),
"command_loop": self._command_loop_info_unlocked(now=current_time),
}
def is_healthy(self, *, allow_live_check: bool = True) -> bool:
"""检查标签页是否健康,避免在忙碌时额外触发 live CDP 读取。"""
now = time.time()
with self._lock:
status = self.status
cached_url = str(self.last_known_url or "").strip()
cached_domain = str(self.current_domain or "").strip()
cache_valid = (
status == TabStatus.IDLE
and self._health_cache_until > now
and self._health_cache_url == cached_url
and self._health_cache_domain == cached_domain
)
if cache_valid:
return bool(self._health_cache_result)
if status == TabStatus.CLOSED:
return False
if status == TabStatus.BUSY:
if cached_url:
return not _should_skip_pool_url(cached_url)
return bool(cached_domain)
if not allow_live_check:
if cached_url:
return not _should_skip_pool_url(cached_url)
return bool(cached_domain)
url = self._safe_get_url()
result = bool(url and not _should_skip_pool_url(url))
with self._lock:
self._health_cache_result = result
self._health_cache_url = str(self.last_known_url or "").strip()
self._health_cache_domain = str(self.current_domain or "").strip()
self._health_cache_until = time.time() + _TAB_HEALTH_CACHE_TTL_SEC
return result
def get_cached_route_snapshot(self) -> tuple[str, str]:
"""Return cached URL/domain without touching the live tab/CDP endpoint."""
with self._lock:
cached_url = str(self.last_known_url or "").strip()
cached_domain = str(self.current_domain or "").strip()
if cached_url:
cached_domain = self._refresh_current_domain(cached_url)
return cached_url, cached_domain
def _debug_summary_unlocked(self) -> str:
status_value = getattr(self.status, "value", str(self.status))
return (
f"idx=#{self.persistent_index or '-'}, "
f"status={status_value}, "
f"task={str(self.current_task_id or '').strip() or '-'}, "
f"bound_req={str(getattr(self, '_bound_request_id', '') or '').strip() or '-'}, "
f"cmd_req={str(getattr(self, '_command_request_id', '') or '').strip() or '-'}, "
f"req_count={self.request_count}, "
f"domain={str(self.current_domain or '').strip() or '-'}"
)
def debug_summary(self) -> str:
with self._lock:
return self._debug_summary_unlocked()
# B5:解冻失败后的冷却期,期间 acquire 直接跳过该会话,避免每次都卡 2 秒 CDP 超时
_RESUME_FAILURE_BACKOFF_SEC = 10.0
def acquire(self, task_id: str) -> bool:
if self._resume_recently_failed():
return False
if not self._acquire_request_unlocked_entry(task_id):
return False
if not self._resume_after_acquire("acquire"):
self._rollback_failed_acquire(task_id, request_mode=True)
return False
return True
def acquire_for_command(self, task_id: str) -> bool:
"""Acquire tab for command execution without incrementing request counter."""
if self._resume_recently_failed():
return False
if not self._acquire_command_unlocked_entry(task_id):
return False
if not self._resume_after_acquire("acquire_for_command"):
self._rollback_failed_acquire(task_id, request_mode=False)
return False
return True
def _resume_recently_failed(self) -> bool:
if not getattr(self, "_uwapi_frozen", False):
return False
failed_at = float(getattr(self, "_uwapi_resume_failed_at", 0.0) or 0.0)
return bool(failed_at) and (time.time() - failed_at) < self._RESUME_FAILURE_BACKOFF_SEC
def _resume_after_acquire(self, reason: str) -> bool:
"""P0-6:被空闲冻结的标签页在交给调用方之前恢复为 active。
B5:返回该会话是否可交付——解冻失败(仍处于冻结态)返回 False。
"""
try:
from .idle_maintenance import resume_if_frozen
resume_if_frozen(self, reason=reason)
except Exception as e:
logger.debug(f"[{self.id}] resume frozen tab failed: {e}")
return not bool(getattr(self, "_uwapi_frozen", False))
def _rollback_failed_acquire(self, task_id: str, *, request_mode: bool) -> None:
"""B5:解冻失败时撤销本次占用,把会话退回 IDLE(保留冻结标记),不交付给调用方。"""
with self._lock:
if self.status != TabStatus.BUSY or str(self.current_task_id or "") != str(task_id or ""):
return
self.status = TabStatus.IDLE
self.current_task_id = None
if request_mode and self.request_count > 0:
self.request_count -= 1
self._clear_health_cache_unlocked()
logger.warning(
f"[{self.id}] 标签页解冻失败,本次不交付该会话(task={task_id}),"
f"{self._RESUME_FAILURE_BACKOFF_SEC:.0f}s 内跳过"
)
def _acquire_request_unlocked_entry(self, task_id: str) -> bool:
with self._lock:
if self.status != TabStatus.IDLE or self._termination_in_progress:
return False
prev_status = self.status.value
prev_task = str(self.current_task_id or "").strip()
prev_request_count = self.request_count
self.status = TabStatus.BUSY
self.current_task_id = task_id
self._clear_health_cache_unlocked()
self._clear_command_loop_unlocked()
self._workflow_parser_id = ""
self._workflow_target_side = ""
self._workflow_runtime_id = ""
self._request_occupied_url = None
setattr(self, "_last_cancel_request_task_id", None)
setattr(self, "_last_cancel_request_reason", None)
self.last_used_at = time.time()
self.request_count += 1
logger.debug(
f"[{self.id}] 会话占用: mode=request, idx=#{self.persistent_index or '-'}, "
f"prev_status={prev_status}, prev_task={prev_task or '-'}, "
f"new_task={str(task_id or '').strip() or '-'}, "
f"req_count={prev_request_count}->{self.request_count}"
)
return True
def _acquire_command_unlocked_entry(self, task_id: str) -> bool:
with self._lock:
if self.status != TabStatus.IDLE or self._termination_in_progress:
return False
prev_status = self.status.value
prev_task = str(self.current_task_id or "").strip()
self.status = TabStatus.BUSY
self.current_task_id = task_id
self._clear_health_cache_unlocked()
self._clear_command_loop_unlocked()
self._workflow_parser_id = ""
self._workflow_target_side = ""
self._workflow_runtime_id = ""
setattr(self, "_last_cancel_request_task_id", None)
setattr(self, "_last_cancel_request_reason", None)
self.last_used_at = time.time()
logger.debug(
f"[{self.id}] 会话占用: mode=command, idx=#{self.persistent_index or '-'}, "
f"prev_status={prev_status}, prev_task={prev_task or '-'}, "
f"new_task={str(task_id or '').strip() or '-'}, "
f"req_count={self.request_count}"
)
return True
def _begin_release_state(
self,
*,
clear_page: bool,
rollback_request_count: bool = False,
force: bool = False,
expected_task_id: str = "",
reject_ownerless: bool = False,
) -> Optional[Dict[str, Any]]:
with self._lock:
if self._release_in_progress:
action = "force_release" if force else "release"
logger.debug(f"[{self.id}] {action} skipped: another release is already in progress")
return None
expected_task = str(expected_task_id or "").strip()
current_task = str(self.current_task_id or "").strip()
if current_task and reject_ownerless and not expected_task:
logger.warning(
f"[{self.id}] release skipped without ownership (current_task={current_task})"
)
return None
if expected_task and current_task != expected_task:
logger.warning(
f"[{self.id}] release ownership mismatch "
f"(expected_task={expected_task}, current_task={current_task or '-'})"
)
return None
prev_status_obj = self.status
prev_status = prev_status_obj.value
prev_task = str(self.current_task_id or "").strip()
prev_bound_request_id = str(getattr(self, "_bound_request_id", "") or "").strip()
prev_command_request_id = str(getattr(self, "_command_request_id", "") or "").strip()
prev_request_count = self.request_count
prev_was_error = prev_status_obj == TabStatus.ERROR
prev_was_closed = prev_status_obj == TabStatus.CLOSED
if rollback_request_count and self.request_count > 0 and (
prev_task or prev_status_obj == TabStatus.BUSY
):
self.request_count -= 1
needs_cleanup_window = True
if needs_cleanup_window:
self._release_in_progress = True
if not prev_was_error and not prev_was_closed:
self.status = TabStatus.BUSY
elif not prev_was_error and not prev_was_closed:
self.status = TabStatus.IDLE
self.current_task_id = None
self.current_command_name = None
self._clear_health_cache_unlocked()
self._clear_command_loop_unlocked()
setattr(self, "_bound_request_id", None)
setattr(self, "_command_request_id", None)
setattr(self, "_command_loop_request_id", None)
setattr(self, "_command_vars", {})
self._workflow_parser_id = ""
self._workflow_target_side = ""
self._workflow_runtime_id = ""
self._request_occupied_url = None
self.last_used_at = time.time()
return {
"prev_status": prev_status,
"prev_task": prev_task,
"prev_bound_request_id": prev_bound_request_id,
"prev_command_request_id": prev_command_request_id,
"prev_request_count": prev_request_count,
"new_request_count": self.request_count,
"prev_was_error": prev_was_error,
"prev_was_closed": prev_was_closed,
"needs_cleanup_window": needs_cleanup_window,
}
def _finish_release_state(self, state: Dict[str, Any], success: bool) -> None:
if not state.get("needs_cleanup_window"):
return
with self._lock:
if success:
if (
self.status == TabStatus.BUSY
and not state.get("prev_was_error")
and not state.get("prev_was_closed")
):
self.status = TabStatus.IDLE
elif self.status != TabStatus.CLOSED:
self.status = TabStatus.ERROR
self.error_count += 1
termination_task = str(self._termination_task_id or "").strip()
released_task = str(state.get("prev_task") or "").strip()
if self._termination_in_progress and self._termination_interrupt_done and (
not termination_task or termination_task == released_task
):
self._termination_in_progress = False
self._termination_task_id = None
self._termination_interrupt_done = False
self._release_in_progress = False
def begin_termination(self, expected_task_id: str = "") -> Dict[str, Any]:
"""Quarantine this session until its current owner finishes releasing it."""
with self._lock:
expected_task = str(expected_task_id or "").strip()
current_task = str(self.current_task_id or "").strip()
if expected_task and current_task != expected_task:
return {
"ok": False,
"error": "task_ownership_changed",
"expected_task_id": expected_task,
"current_task_id": current_task,
}
if self._release_in_progress:
return {"ok": False, "error": "release_in_progress"}
if self._termination_in_progress:
return {
"ok": False,
"error": "termination_in_progress",
"current_task_id": current_task,
}
self._termination_in_progress = True
self._termination_task_id = current_task or expected_task or None
self._termination_interrupt_done = False
return {
"ok": True,
"task_id": current_task,
"status": self.status.value,
"command_request_id": str(
getattr(self, "_command_request_id", "") or ""
).strip(),
"command_loop_request_id": str(
getattr(self, "_command_loop_request_id", "") or ""
).strip(),
}
def finish_ownerless_termination(self) -> None:
"""Release the quarantine for a session that had no active owner."""
with self._lock:
if self.current_task_id or self._release_in_progress:
return
self._termination_in_progress = False
self._termination_task_id = None
self._termination_interrupt_done = False
def is_termination_in_progress(self) -> bool:
with self._lock:
return bool(self._termination_in_progress)
def interrupt_for_termination(self, clear_page: bool = False) -> bool:
"""Interrupt page activity without making the session available for reuse."""
self.clear_visibility_emulation("terminate")
success = True
try:
if hasattr(self.tab, "stop_loading"):
self.tab.stop_loading()
driver_for_tab(self.tab).run_js("if (window.stop) { window.stop(); }")
except Exception:
pass
if clear_page:
try:
self.tab.refresh()
self.reset_conversation_state()
self.notify_navigated(reason="termination_clear")
except Exception as e:
logger.warning(f"[{self.id}] terminate refresh failed: {e}")
success = False
with self._lock:
self._termination_interrupt_done = True
if not self.current_task_id and not self._release_in_progress:
self._termination_in_progress = False
self._termination_task_id = None
self._termination_interrupt_done = False
return success
def _run_release_from_state(
self,
state: Dict[str, Any],
*,
clear_page: bool,
check_triggers: bool,
rollback_request_count: bool,
) -> None:
clear_page_success = True
if clear_page:
try:
self.tab.get("about:blank")
with self._lock:
self.current_domain = None
self.last_known_url = None
self.preset_name = None
self.preset_override_source = None
self._clear_health_cache_unlocked()
self.last_conversation_activity_at = 0.0
self.last_conversation_domain = None
self.last_conversation_preset_name = None
self.notify_navigated(reason="release_clear_page")
except Exception as e:
clear_page_success = False
logger.debug(f"clear page failed: {e}")
logger.debug(
f"[{self.id}] 会话释放: idx=#{self.persistent_index or '-'}, "
f"prev_status={state['prev_status']}, prev_task={state['prev_task'] or '-'}, "
f"prev_bound_req={state['prev_bound_request_id'] or '-'}, "
f"prev_cmd_req={state['prev_command_request_id'] or '-'}, "
f"clear_page={clear_page}, check_triggers={check_triggers}, "
f"rollback_request_count={rollback_request_count}, "
f"req_count={state['prev_request_count']}->{state['new_request_count']}, "
f"clear_page_success={clear_page_success}"
)
self.clear_visibility_emulation("release")
self._finish_release_state(state, clear_page_success)
try:
from app.services.command_engine import command_engine
if check_triggers:
command_engine.check_triggers(self)
except Exception as e:
logger.debug(f"命令触发检查异常: {e}")
def release(
self,
clear_page: bool = False,
check_triggers: bool = True,
rollback_request_count: bool = False,
expected_task_id: str = "",
) -> bool:
state = self._begin_release_state(
clear_page=clear_page,
rollback_request_count=rollback_request_count,
force=False,
expected_task_id=expected_task_id,
reject_ownerless=True,
)
if state is None:
return False
self._run_release_from_state(
state,
clear_page=clear_page,
check_triggers=check_triggers,
rollback_request_count=rollback_request_count,
)
return self.status == TabStatus.IDLE
def _run_force_release_from_state(
self,
state: Dict[str, Any],
*,
clear_page: bool,
check_triggers: bool,
) -> None:
logger.warning(
f"[{self.id}] 强制释放开始: idx=#{self.persistent_index or '-'}, "
f"prev_status={state['prev_status']}, prev_task={state['prev_task'] or '-'}, "
f"prev_bound_req={state['prev_bound_request_id'] or '-'}, "
f"prev_cmd_req={state['prev_command_request_id'] or '-'}, "
f"req_count={state['prev_request_count']}, clear_page={clear_page}, "
f"check_triggers={check_triggers}"
)
self.clear_visibility_emulation("force_release")
try:
if hasattr(self.tab, "stop_loading"):
self.tab.stop_loading()
driver_for_tab(self.tab).run_js("if (window.stop) { window.stop(); }")
except Exception:
pass
reset_success = True
if clear_page:
try:
self.tab.refresh()
self.reset_conversation_state()
self.notify_navigated(reason="force_release_clear")
except Exception as e:
logger.warning(f"[{self.id}] force_release refresh failed: {e}")
reset_success = False
self._finish_release_state(state, reset_success)
with self._lock:
final_status = self.status.value
final_task = str(self.current_task_id or "").strip()
request_count = self.request_count
error_count = self.error_count
if reset_success:
logger.info(
f"[{self.id}] force_release done "
f"(idx=#{self.persistent_index or '-'}, final_status={final_status}, "
f"task={final_task or '-'}, req_count={request_count})"
)
else:
logger.warning(
f"[{self.id}] force_release failed, set ERROR "
f"(idx=#{self.persistent_index or '-'}, req_count={request_count}, "
f"error_count={error_count})"
)
if check_triggers:
try:
from app.services.command_engine import command_engine
command_engine.check_triggers(self)
except Exception as e:
logger.debug(f"command trigger check failed: {e}")
def force_release(self, clear_page: bool = False, check_triggers: bool = False):
"""Force release tab lock and optionally refresh current page."""
state = self._begin_release_state(clear_page=clear_page, force=True)
if state is None:
return
self._run_force_release_from_state(
state,
clear_page=clear_page,
check_triggers=check_triggers,
)
def activate(self) -> bool:
"""激活标签页(使其成为浏览器焦点)"""
try:
self.tab.set.activate()
logger.debug(f"[{self.id}] 已激活")
return True
except Exception as e:
logger.warning(f"[{self.id}] 激活失败: {e}")
return False
def mark_error(self, reason: str = None):
with self._lock:
self.status = TabStatus.ERROR
self._clear_health_cache_unlocked()
self.error_count += 1
logger.warning(f"[{self.id}] 标记为错误: {reason}")
def mark_closed(self, reason: str = None):
with self._lock:
if self.status == TabStatus.CLOSED:
return
self.status = TabStatus.CLOSED
self.current_task_id = None
self._release_in_progress = False
self._termination_in_progress = False
self._termination_task_id = None
self._termination_interrupt_done = False
self.transient_disconnect_until = 0.0
self.transient_disconnect_reason = None
self._clear_health_cache_unlocked()
logger.debug(f"[{self.id}] 标记为关闭: {reason or '-'}")
def _reset_conversation_state_unlocked(self):
self.last_conversation_activity_at = 0.0
self.last_conversation_domain = None
self.last_conversation_preset_name = None
def reset_conversation_state(self):
with self._lock:
self._reset_conversation_state_unlocked()
def mark_conversation_activity(self, domain: str = "", preset_name: str = ""):
normalized_domain = str(domain or "").strip().lower() or None
normalized_preset = str(preset_name or "").strip() or None
now = time.time()
with self._lock:
self.last_conversation_activity_at = now
self.last_conversation_domain = normalized_domain
self.last_conversation_preset_name = normalized_preset
logger.debug(
f"[{self.id}] 记录会话活动: idx=#{self.persistent_index or '-'}, "
f"domain={normalized_domain or '-'}, preset={normalized_preset or '-'}, "
f"at={round(now, 3)}"
)
def get_conversation_status(
self,
current_domain: str = "",
preset_name: str = "",
threshold_seconds: float = 0.0,
force_new: bool = False,
) -> Dict[str, Any]:
normalized_domain = str(current_domain or self.current_domain or "").strip().lower()
normalized_preset = str(preset_name or "").strip() or None
threshold_value = max(0.0, float(threshold_seconds or 0.0))
with self._lock:
last_activity_at = float(self.last_conversation_activity_at or 0.0)
last_domain = str(self.last_conversation_domain or "").strip().lower()
last_preset = str(self.last_conversation_preset_name or "").strip() or None
elapsed_seconds = None
if last_activity_at > 0:
elapsed_seconds = max(0.0, time.time() - last_activity_at)
reason = "no_previous_conversation"
will_start_new = True
if force_new:
reason = "force_new_enabled"
elif threshold_value <= 0:
reason = "threshold_disabled_reuse"
elif not normalized_domain:
reason = "missing_current_domain"
elif not last_activity_at:
reason = "no_previous_conversation"
elif last_domain and normalized_domain != last_domain:
reason = "domain_changed"
elif normalized_preset and last_preset and normalized_preset != last_preset:
reason = "preset_changed"
elif elapsed_seconds is not None and elapsed_seconds < threshold_value:
will_start_new = False
reason = "reuse_existing"
else:
reason = "timeout_expired"
return {
"id": self.id,
"persistent_index": self.persistent_index,
"current_domain": normalized_domain,
"current_preset_name": normalized_preset,
"last_conversation_domain": last_domain or None,
"last_conversation_preset_name": last_preset,
"last_conversation_at": last_activity_at or None,
"elapsed_seconds": round(elapsed_seconds, 1) if elapsed_seconds is not None else None,
"threshold_seconds": threshold_value,
"force_new_conversation": bool(force_new),
"will_start_new_conversation": will_start_new,
"reason": reason,
}
def should_start_new_conversation(
self,
current_domain: str = "",
preset_name: str = "",
threshold_seconds: float = 0.0,
force_new: bool = False,
) -> bool:
return bool(
self.get_conversation_status(
current_domain=current_domain,
preset_name=preset_name,
threshold_seconds=threshold_seconds,
force_new=force_new,
).get("will_start_new_conversation", True)
)
def get_info(
self,
*,
use_cached_url: bool = False,
allow_live_when_busy: bool = False,
) -> Dict:
with self._lock:
status = self.status
last_used_at = self.last_used_at
current_task_id = self.current_task_id
command_request_id = str(getattr(self, "_command_request_id", "") or "").strip()
current_command_id = str(getattr(self, "_current_command_id", "") or "").strip()
current_command_name = str(self.current_command_name or "").strip()
current_command = getattr(self, "_current_command", None)
current_domain_snapshot = str(self.current_domain or "").strip()
cached_url = str(self.last_known_url or "").strip()
request_count = self.request_count
preset_name = self.preset_name
preset_override_source = self.preset_override_source
model_name_override = self.model_name_override
is_isolated_context = self.is_isolated_context
browser_context_id = self.browser_context_id
last_conversation_activity_at = self.last_conversation_activity_at
last_conversation_domain = self.last_conversation_domain
last_conversation_preset_name = self.last_conversation_preset_name
command_loop = self._command_loop_info_unlocked()
termination_in_progress = self._termination_in_progress
busy_duration = None
if status == TabStatus.BUSY:
try:
busy_duration = round(self.get_busy_timeout_info(now=time.time()).get("duration") or 0.0, 1)
except Exception:
busy_duration = round(time.time() - last_used_at, 1)
if use_cached_url:
current_url = cached_url
if current_url:
current_domain = self._refresh_current_domain(current_url)
else:
current_domain = current_domain_snapshot
else:
current_url = self._safe_get_url(allow_live_when_busy=allow_live_when_busy)
current_domain = self._refresh_current_domain(current_url)
url_route_token = encode_tab_url_route_token(current_url)
if not current_command_name and isinstance(current_command, dict):
current_command_name = str(current_command.get("name") or "").strip()
return {
"id": self.id,
"persistent_index": self.persistent_index,
"status": status.value,
"current_task": current_task_id,
"command_task": command_request_id,
"current_command_id": current_command_id,
"current_command": current_command_name,
"current_domain": current_domain,
"route_domain": get_preferred_route_domain(current_domain),
"domain_url": self._build_domain_url(current_url, current_domain),
"url": current_url,
"url_route_token": url_route_token,
"request_count": request_count,
"busy_duration": busy_duration,
"preset_name": preset_name, # 🆕
"preset_override_source": preset_override_source,
"model_name_override": model_name_override,
"is_isolated_context": is_isolated_context,
"browser_context_id": browser_context_id,
"last_conversation_at": last_conversation_activity_at or None,
"last_conversation_domain": last_conversation_domain,
"last_conversation_preset_name": last_conversation_preset_name,
"command_loop": command_loop,
"terminating": termination_in_progress,
}
def _refresh_current_domain(self, url: str = "") -> str:
current_url = str(url or "").strip()
try:
resolved = extract_remote_site_domain(current_url) or ""
except Exception:
resolved = ""
if resolved:
with self._lock:
self.current_domain = resolved
return resolved
with self._lock:
fallback = str(self.current_domain or "").strip()
if _should_skip_pool_url(current_url) or "://" in current_url:
with self._lock:
self.current_domain = None
return ""
return fallback
@staticmethod
def _build_domain_url(url: str, current_domain: str) -> str:
source_url = str(url or "").strip()
domain = str(current_domain or "").strip()
if not source_url or not domain:
return ""
try:
parsed = urlsplit(source_url)
except Exception:
return ""
scheme = parsed.scheme if parsed.scheme in {"http", "https", "ws", "wss"} else "https"
return f"{scheme}://{domain}/"
def _remember_url(self, url: str) -> str:
normalized = str(url or "").strip()
with self._lock:
if str(self.last_known_url or "").strip() != normalized:
self._clear_health_cache_unlocked()
if self.preset_override_source == "url":
self.preset_name = None
self.preset_override_source = None
self.last_known_url = normalized or None
return normalized
def get_occupied_url(self) -> Optional[str]:
with self._lock:
return self._request_occupied_url
def set_occupied_url(self, url: Optional[str]) -> None:
with self._lock:
normalized = str(url or "").strip()
if normalized and "challenges.cloudflare.com" not in normalized and "challenge-platform" not in normalized and normalized.startswith("http"):
self._request_occupied_url = normalized
elif not normalized:
self._request_occupied_url = None
def _safe_get_url(self, allow_live_when_busy: bool = False) -> str:
with self._lock:
status = self.status
cached_url = str(self.last_known_url or "").strip()
if status == TabStatus.CLOSED:
return ""
if status == TabStatus.BUSY and not allow_live_when_busy:
return cached_url
try:
current_url = self._remember_url(self.tab.url or "")
if current_url:
return current_url
except Exception:
pass
return cached_url
def clear_visibility_emulation(self, reason: str = "") -> None:
try:
from app.core.page_lifecycle import restore_visibility_emulation
restore_visibility_emulation(self.tab, owner=self, reason=reason)
except Exception as e:
logger.debug(f"[{self.id}] visibility emulation cleanup failed: {e}")
def notify_navigated(self, reason: str = "page_reload") -> None:
"""Notify command engine of page reload/navigation."""
try:
from app.services.command_engine import command_engine
if hasattr(command_engine, "notify_tab_navigated"):
command_engine.notify_tab_navigated(self, reason=reason)
except Exception as e:
logger.debug(f"[{self.id}] notify_navigated failed (ignored): {e}")