Files
2026-11-02 22:54:14 +08:00

531 lines
20 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""
app/core/tab_pool_parts/recovery.py - 错误标签页隔离与 AI 恢复服务
职责:
- TabQuarantineEntry:错误标签页的隔离记录(由 TabPoolManager 在池锁内维护)
- TabRecoveryService:单线程后台服务,等旧 worker 退出后按配置决定
截图 → AI 判断 → 刷新 → 解除隔离 / 永久隔离
锁纪律:
- 本模块所有阻塞操作(等待旧 worker、截图、AI HTTP 调用、刷新)都发生在
自持的 ThreadPoolExecutor(max_workers=1) 线程里,绝不在事件循环线程执行,
也绝不持有 TabPoolManager 的池锁。
- 与 TabPoolManager 的交互只通过构造时注入的两个回调
(resolve_quarantine / get_quarantine_entry),本模块不 import manager,
避免循环导入;request_manager 的 import 放在函数内。
"""
import base64
import json
import threading
import time
import urllib.request
from collections import OrderedDict
from concurrent.futures import ThreadPoolExecutor
from dataclasses import dataclass, field
from typing import Any, Callable, Dict, Optional, Tuple
from app.core.config import AppConfig, logger
# DrissionPage/CDP 调用可能在底层连接断开时永久阻塞。超时后 daemon
# 线程无法被强制终止,因此限制同时存活的超时调用数量,避免每次异常
# 都累积一个永不退出的线程。
_RECOVERY_CALL_SLOTS = threading.BoundedSemaphore(4)
@dataclass
class TabQuarantineEntry:
"""错误标签页的隔离记录(在 TabPoolManager 的池锁保护下读写)"""
raw_tab_id: str
persistent_index: int = 0
task_id: str = ""
tab: Any = None
reason: str = ""
since: float = field(default_factory=time.monotonic)
permanent: bool = False
attempts: int = 0
session_id: str = ""
url: str = ""
class TabRecoveryService:
"""错误标签页 AI 恢复服务(由 TabPoolManager 懒创建并持有,单例风格)"""
WORKER_POLL_INTERVAL_SEC = 5.0
# 拿不到旧 worker 线程引用时的时间宽限:隔离时刻起 180s 视为已退出
WORKER_EXIT_FALLBACK_GRACE_SEC = 180.0
SCREENSHOT_TIMEOUT_SEC = 30.0
REFRESH_TIMEOUT_SEC = 30.0
POST_REFRESH_SETTLE_SEC = 3.0
MAX_ATTEMPT_CACHE_ENTRIES = 4096
def __init__(
self,
*,
resolve_quarantine: Callable[[str, bool], bool],
get_quarantine_entry: Callable[[str], Optional[TabQuarantineEntry]],
):
self._resolve_quarantine = resolve_quarantine
self._get_quarantine_entry = get_quarantine_entry
self._attempts: OrderedDict[str, int] = OrderedDict()
self._attempts_lock = threading.Lock()
self._shutdown = False
self._executor = ThreadPoolExecutor(
max_workers=1,
thread_name_prefix="tab-recovery",
)
# ================= 对外接口 =================
def submit(self, entry: TabQuarantineEntry) -> bool:
"""受理一个隔离标签页;仅入队,不阻塞调用线程。"""
if entry is None or not str(getattr(entry, "raw_tab_id", "") or "").strip():
return False
if self._shutdown:
return False
try:
self._executor.submit(self._process_safe, entry)
logger.info(
f"[TabRecovery] 已受理隔离标签页,排队处理 "
f"(raw={entry.raw_tab_id}, tab={entry.session_id or '-'}, "
f"idx=#{entry.persistent_index or '-'}, task={entry.task_id or '-'}, "
f"reason={entry.reason or '-'})"
)
return True
except RuntimeError as e:
logger.debug(f"[TabRecovery] submit failed ({entry.raw_tab_id}): {e}")
return False
def shutdown(self) -> None:
self._shutdown = True
with self._attempts_lock:
self._attempts.clear()
try:
self._executor.shutdown(wait=False, cancel_futures=True)
except Exception as e:
logger.debug(f"[TabRecovery] executor shutdown failed: {e}")
# ================= 主流程 =================
def _process_safe(self, entry: TabQuarantineEntry) -> None:
"""线程池任务入口:吃掉所有异常,绝不向外抛。"""
try:
self._process(entry)
except Exception as e:
logger.warning(
f"[TabRecovery] 恢复流程异常 (raw={entry.raw_tab_id}): {e}"
)
# 未知内部错误时保守处理:永久隔离,避免把状态不明的标签页重新入池。
try:
self._resolve(entry, release=False, reason=f"internal_error:{e}")
except Exception:
pass
def _process(self, entry: TabQuarantineEntry) -> None:
raw_id = entry.raw_tab_id
logger.info(
f"[TabRecovery] 开始处理隔离标签页 "
f"(raw={raw_id}, tab={entry.session_id or '-'}, "
f"task={entry.task_id or '-'}, reason={entry.reason or '-'})"
)
# 1. 等旧 worker 退出:串扰的根源是旧请求的 worker 线程可能仍阻塞在
# 对该 tab 的同步 DrissionPage 调用里。
wait_result = self._wait_for_worker_exit(entry)
if wait_result == "aborted":
logger.debug(f"[TabRecovery] 服务关闭,放弃处理 (raw={raw_id})")
return
if wait_result == "timeout":
self._resolve(entry, release=False, reason="worker_never_exited")
return
logger.debug(f"[TabRecovery] 旧 worker 已确认退出 (raw={raw_id})")
# 2. 次数检查:解除隔离后再次错误 → 不再调 API,直接永久隔离。
with self._attempts_lock:
attempts = int(self._attempts.get(raw_id, 0))
if raw_id in self._attempts:
# 最近使用的 raw id 保留在缓存尾部,便于后续按上限淘汰。
self._attempts.move_to_end(raw_id)
entry.attempts = attempts
max_attempts = AppConfig.get_tab_recovery_max_attempts()
if attempts >= max_attempts:
logger.info(
f"[TabRecovery] 恢复次数已用尽,不再调用 AI "
f"(raw={raw_id}, attempts={attempts}, max={max_attempts})"
)
self._resolve(entry, release=False, reason="attempts_exhausted")
return
# 3. 功能开关:未启用时不截图不调 API,直接解除隔离——保持既有的
# 自动重入池行为,但此时旧 worker 已确认退出,串扰已消除。
if not AppConfig.is_tab_recovery_enabled():
self._resolve(entry, release=True, reason="recovery_disabled")
return
# 4/5. 截图 + AI 判断;任一失败 → verdict=None(未知)
verdict: Optional[bool] = None
detail = ""
image_b64 = ""
try:
image_b64 = self._capture_screenshot(entry)
except Exception as e:
detail = f"screenshot_failed:{e}"
logger.warning(f"[TabRecovery] 截图失败 (raw={raw_id}): {e}")
if image_b64:
try:
verdict, detail = self._judge_recoverable(entry, image_b64)
logger.info(
f"[TabRecovery] AI 判断结果 "
f"(raw={raw_id}, recoverable={verdict}, detail={detail or '-'})"
)
except Exception as e:
verdict, detail = None, f"api_error:{e}"
logger.warning(f"[TabRecovery] AI 判断失败 (raw={raw_id}): {e}")
# 6. 决策
if verdict is False:
self._resolve(
entry,
release=False,
reason=f"ai_judged_unrecoverable:{detail or '-'}",
)
return
if verdict is None:
if not AppConfig.is_tab_recovery_refresh_on_unknown():
self._resolve(
entry,
release=False,
reason=f"judgement_unknown:{detail or '-'}",
)
return
refresh_reason = "unknown_refresh"
else:
refresh_reason = "ai_judged_recoverable"
# 7. 刷新与解除
self._refresh_and_release(entry, refresh_reason)
# ================= 步骤实现 =================
def _wait_for_worker_exit(self, entry: TabQuarantineEntry) -> str:
"""轮询旧 worker 是否退出。返回 'exited' / 'timeout' / 'aborted'。"""
max_wait = AppConfig.get_tab_recovery_worker_exit_wait_sec()
deadline = time.monotonic() + max(0.0, float(max_wait))
while True:
if self._shutdown:
return "aborted"
if self._is_worker_exited(entry):
return "exited"
remaining = deadline - time.monotonic()
if remaining <= 0:
return "timeout"
time.sleep(min(self.WORKER_POLL_INTERVAL_SEC, max(0.1, remaining)))
def _is_worker_exited(self, entry: TabQuarantineEntry) -> bool:
task_id = str(entry.task_id or "").strip()
if not task_id:
# 隔离时没有绑定任务,无 worker 可等
return True
ctx = None
try:
from app.services.request_manager import request_manager
ctx = request_manager.get_request(task_id)
except Exception as e:
logger.debug(
f"[TabRecovery] 查询请求上下文失败 (task={task_id}): {e}"
)
thread = getattr(ctx, "worker_thread", None) if ctx is not None else None
if thread is not None and hasattr(thread, "is_alive"):
try:
return not thread.is_alive()
except Exception as e:
logger.debug(
f"[TabRecovery] 读取 worker 线程状态失败 (task={task_id}): {e}"
)
# 拿不到线程引用(非流式请求 worker 在 request_lifecycle 内创建、
# 或请求上下文已被清理):回退时间宽限,隔离时刻起 180s 视为已退出。
elapsed = time.monotonic() - float(entry.since or 0.0)
return elapsed >= self.WORKER_EXIT_FALLBACK_GRACE_SEC
def _capture_screenshot(self, entry: TabQuarantineEntry) -> str:
tab = entry.tab
if tab is None:
raise RuntimeError("tab reference unavailable")
# DrissionPage 4.x:get_screenshot(as_base64=True) 返回 PNG 的 base64 字符串
data = self._call_with_timeout(
lambda: tab.get_screenshot(as_base64=True),
self.SCREENSHOT_TIMEOUT_SEC,
"screenshot",
)
if isinstance(data, (bytes, bytearray)):
data = base64.b64encode(bytes(data)).decode("ascii")
text = str(data or "").strip()
if not text:
raise RuntimeError("empty screenshot payload")
return text
def _judge_recoverable(
self,
entry: TabQuarantineEntry,
image_b64: str,
) -> Tuple[Optional[bool], str]:
"""调用 OpenAI 兼容接口判断刷新能否恢复。返回 (verdict, detail)。"""
api_url = AppConfig.get_tab_recovery_api_url()
if api_url:
target_url = api_url
api_key = AppConfig.get_tab_recovery_api_key()
local_mode = False
else:
# 本地模式:走本服务自身的 /v1/chat/completions,
# 由其他空闲标签页完成实际判断
target_url = (
f"http://127.0.0.1:{AppConfig.get_port()}/v1/chat/completions"
)
api_key = (
AppConfig.get_auth_token() if AppConfig.is_auth_enabled() else ""
)
local_mode = True
site_url = str(entry.url or "").strip() or "未知"
prompt = (
"这是反代服务的一个疑似卡死的网页标签页截图,"
f"站点为 {site_url},错误原因 {entry.reason or '未知'}。"
"请判断刷新页面是否可能恢复正常使用。只输出 JSON:"
"{\"recoverable\": true/false, \"reason\": \"...\"}。"
"登录失效、封号提示、人机验证页输出 false;"
"加载中、白屏、内容正常、临时错误提示输出 true。"
)
payload = {
"model": AppConfig.get_tab_recovery_model(),
"stream": False,
"messages": [
{
"role": "user",
"content": [
{"type": "text", "text": prompt},
{
"type": "image_url",
"image_url": {
"url": f"data:image/png;base64,{image_b64}",
},
},
],
}
],
}
headers = {"Content-Type": "application/json"}
if api_key:
headers["Authorization"] = f"Bearer {api_key}"
timeout_sec = AppConfig.get_tab_recovery_timeout_sec()
request = urllib.request.Request(
target_url,
data=json.dumps(payload).encode("utf-8"),
headers=headers,
method="POST",
)
logger.debug(
f"[TabRecovery] 调用 AI 判断 "
f"(raw={entry.raw_tab_id}, url={target_url}, "
f"local_mode={local_mode}, timeout={timeout_sec}s)"
)
if local_mode:
# 本地回环地址不应经过环境代理
opener = urllib.request.build_opener(urllib.request.ProxyHandler({}))
response = opener.open(request, timeout=timeout_sec)
else:
response = urllib.request.urlopen(request, timeout=timeout_sec)
with response:
body = response.read().decode("utf-8", "replace")
data = json.loads(body)
choices = data.get("choices") or []
if not choices:
return None, "empty_choices"
message = choices[0].get("message") or {}
content = message.get("content")
text = self._flatten_message_content(content)
parsed = self._extract_first_json_object(text)
if not isinstance(parsed, dict):
return None, f"parse_failed:{text[:120]!r}"
verdict = self._coerce_bool(parsed.get("recoverable"))
detail = str(parsed.get("reason") or "").strip()
if verdict is None:
return None, f"invalid_recoverable_field:{parsed.get('recoverable')!r}"
return verdict, detail
def _refresh_and_release(
self,
entry: TabQuarantineEntry,
refresh_reason: str,
) -> None:
raw_id = entry.raw_tab_id
try:
tab = entry.tab
if tab is None:
raise RuntimeError("tab reference unavailable")
self._call_with_timeout(
lambda: tab.refresh(),
self.REFRESH_TIMEOUT_SEC,
"refresh",
)
time.sleep(max(0.0, float(self.POST_REFRESH_SETTLE_SEC)))
current_url = self._call_with_timeout(
lambda: str(getattr(tab, "url", "") or ""),
self.REFRESH_TIMEOUT_SEC,
"health-check",
)
if not str(current_url or "").strip():
raise RuntimeError("empty url after refresh")
except Exception as e:
logger.warning(f"[TabRecovery] 刷新/健康检查失败 (raw={raw_id}): {e}")
self._resolve(entry, release=False, reason=f"refresh_failed:{e}")
return
with self._attempts_lock:
self._attempts[raw_id] = int(self._attempts.get(raw_id, 0)) + 1
self._attempts.move_to_end(raw_id)
entry.attempts = self._attempts[raw_id]
while len(self._attempts) > self.MAX_ATTEMPT_CACHE_ENTRIES:
self._attempts.popitem(last=False)
self._resolve(entry, release=True, reason=refresh_reason)
logger.info(
f"[TabRecovery] 标签页已恢复并重新入池 "
f"(raw={raw_id}, idx=#{entry.persistent_index or '-'}, "
f"url={str(current_url)[:80]}, attempts={entry.attempts}, "
f"reason={refresh_reason})"
)
# ================= 辅助 =================
def _resolve(
self,
entry: TabQuarantineEntry,
*,
release: bool,
reason: str,
) -> None:
entry.reason = str(reason or "").strip() or entry.reason
try:
resolved = bool(self._resolve_quarantine(entry.raw_tab_id, release))
except Exception as e:
logger.warning(
f"[TabRecovery] 解除/永久隔离操作失败 "
f"(raw={entry.raw_tab_id}, release={release}): {e}"
)
return
if release:
logger.info(
f"[TabRecovery] 解除隔离,标签页将由下次扫描重新入池 "
f"(raw={entry.raw_tab_id}, reason={reason}, resolved={resolved})"
)
else:
logger.info(
f"[TabRecovery] 永久隔离标签页 "
f"(raw={entry.raw_tab_id}, reason={reason}, resolved={resolved})"
)
@staticmethod
def _call_with_timeout(fn: Callable[[], Any], timeout_sec: float, label: str) -> Any:
"""在守护线程中执行阻塞调用并限时等待,防止 CDP 调用无限挂起。"""
wait_timeout = max(0.1, float(timeout_sec))
if not _RECOVERY_CALL_SLOTS.acquire(timeout=wait_timeout):
raise TimeoutError(f"{label} call slots exhausted")
result: Dict[str, Any] = {}
done = threading.Event()
def _runner():
try:
result["value"] = fn()
except BaseException as e: # noqa: BLE001 - 结果原样转抛给调用方
result["error"] = e
finally:
done.set()
_RECOVERY_CALL_SLOTS.release()
thread = threading.Thread(
target=_runner,
name=f"tab-recovery-{label}",
daemon=True,
)
try:
thread.start()
except BaseException:
_RECOVERY_CALL_SLOTS.release()
raise
if not done.wait(wait_timeout):
raise TimeoutError(f"{label} timed out after {timeout_sec}s")
if "error" in result:
raise result["error"]
return result.get("value")
@staticmethod
def _flatten_message_content(content: Any) -> str:
if isinstance(content, str):
return content
if isinstance(content, list):
parts = []
for item in content:
if isinstance(item, dict):
text = item.get("text")
if isinstance(text, str):
parts.append(text)
elif isinstance(item, str):
parts.append(item)
return "\n".join(parts)
return str(content or "")
@staticmethod
def _extract_first_json_object(text: str) -> Optional[Dict[str, Any]]:
"""宽松提取文本中第一个平衡的 {...} 并解析为 dict。"""
source = str(text or "")
start = source.find("{")
while start != -1:
depth = 0
in_string = False
escaped = False
for index in range(start, len(source)):
char = source[index]
if in_string:
if escaped:
escaped = False
elif char == "\\":
escaped = True
elif char == '"':
in_string = False
continue
if char == '"':
in_string = True
elif char == "{":
depth += 1
elif char == "}":
depth -= 1
if depth == 0:
candidate = source[start:index + 1]
try:
parsed = json.loads(candidate)
if isinstance(parsed, dict):
return parsed
except Exception:
pass
break
start = source.find("{", start + 1)
return None
@staticmethod
def _coerce_bool(value: Any) -> Optional[bool]:
if isinstance(value, bool):
return value
text = str(value or "").strip().lower()
if text in {"true", "1", "yes", "y"}:
return True
if text in {"false", "0", "no", "n"}:
return False
return None