531 lines
20 KiB
Python
531 lines
20 KiB
Python
"""
|
||
app/core/tab_pool_parts/recovery.py - 错误标签页隔离与 AI 恢复服务
|
||
|
||
职责:
|
||
- TabQuarantineEntry:错误标签页的隔离记录(由 TabPoolManager 在池锁内维护)
|
||
- TabRecoveryService:单线程后台服务,等旧 worker 退出后按配置决定
|
||
截图 → AI 判断 → 刷新 → 解除隔离 / 永久隔离
|
||
|
||
锁纪律:
|
||
- 本模块所有阻塞操作(等待旧 worker、截图、AI HTTP 调用、刷新)都发生在
|
||
自持的 ThreadPoolExecutor(max_workers=1) 线程里,绝不在事件循环线程执行,
|
||
也绝不持有 TabPoolManager 的池锁。
|
||
- 与 TabPoolManager 的交互只通过构造时注入的两个回调
|
||
(resolve_quarantine / get_quarantine_entry),本模块不 import manager,
|
||
避免循环导入;request_manager 的 import 放在函数内。
|
||
"""
|
||
import base64
|
||
import json
|
||
import threading
|
||
import time
|
||
import urllib.request
|
||
from collections import OrderedDict
|
||
from concurrent.futures import ThreadPoolExecutor
|
||
from dataclasses import dataclass, field
|
||
from typing import Any, Callable, Dict, Optional, Tuple
|
||
|
||
from app.core.config import AppConfig, logger
|
||
|
||
|
||
# DrissionPage/CDP 调用可能在底层连接断开时永久阻塞。超时后 daemon
|
||
# 线程无法被强制终止,因此限制同时存活的超时调用数量,避免每次异常
|
||
# 都累积一个永不退出的线程。
|
||
_RECOVERY_CALL_SLOTS = threading.BoundedSemaphore(4)
|
||
|
||
|
||
@dataclass
|
||
class TabQuarantineEntry:
|
||
"""错误标签页的隔离记录(在 TabPoolManager 的池锁保护下读写)"""
|
||
raw_tab_id: str
|
||
persistent_index: int = 0
|
||
task_id: str = ""
|
||
tab: Any = None
|
||
reason: str = ""
|
||
since: float = field(default_factory=time.monotonic)
|
||
permanent: bool = False
|
||
attempts: int = 0
|
||
session_id: str = ""
|
||
url: str = ""
|
||
|
||
|
||
class TabRecoveryService:
|
||
"""错误标签页 AI 恢复服务(由 TabPoolManager 懒创建并持有,单例风格)"""
|
||
|
||
WORKER_POLL_INTERVAL_SEC = 5.0
|
||
# 拿不到旧 worker 线程引用时的时间宽限:隔离时刻起 180s 视为已退出
|
||
WORKER_EXIT_FALLBACK_GRACE_SEC = 180.0
|
||
SCREENSHOT_TIMEOUT_SEC = 30.0
|
||
REFRESH_TIMEOUT_SEC = 30.0
|
||
POST_REFRESH_SETTLE_SEC = 3.0
|
||
MAX_ATTEMPT_CACHE_ENTRIES = 4096
|
||
|
||
def __init__(
|
||
self,
|
||
*,
|
||
resolve_quarantine: Callable[[str, bool], bool],
|
||
get_quarantine_entry: Callable[[str], Optional[TabQuarantineEntry]],
|
||
):
|
||
self._resolve_quarantine = resolve_quarantine
|
||
self._get_quarantine_entry = get_quarantine_entry
|
||
self._attempts: OrderedDict[str, int] = OrderedDict()
|
||
self._attempts_lock = threading.Lock()
|
||
self._shutdown = False
|
||
self._executor = ThreadPoolExecutor(
|
||
max_workers=1,
|
||
thread_name_prefix="tab-recovery",
|
||
)
|
||
|
||
# ================= 对外接口 =================
|
||
|
||
def submit(self, entry: TabQuarantineEntry) -> bool:
|
||
"""受理一个隔离标签页;仅入队,不阻塞调用线程。"""
|
||
if entry is None or not str(getattr(entry, "raw_tab_id", "") or "").strip():
|
||
return False
|
||
if self._shutdown:
|
||
return False
|
||
try:
|
||
self._executor.submit(self._process_safe, entry)
|
||
logger.info(
|
||
f"[TabRecovery] 已受理隔离标签页,排队处理 "
|
||
f"(raw={entry.raw_tab_id}, tab={entry.session_id or '-'}, "
|
||
f"idx=#{entry.persistent_index or '-'}, task={entry.task_id or '-'}, "
|
||
f"reason={entry.reason or '-'})"
|
||
)
|
||
return True
|
||
except RuntimeError as e:
|
||
logger.debug(f"[TabRecovery] submit failed ({entry.raw_tab_id}): {e}")
|
||
return False
|
||
|
||
def shutdown(self) -> None:
|
||
self._shutdown = True
|
||
with self._attempts_lock:
|
||
self._attempts.clear()
|
||
try:
|
||
self._executor.shutdown(wait=False, cancel_futures=True)
|
||
except Exception as e:
|
||
logger.debug(f"[TabRecovery] executor shutdown failed: {e}")
|
||
|
||
# ================= 主流程 =================
|
||
|
||
def _process_safe(self, entry: TabQuarantineEntry) -> None:
|
||
"""线程池任务入口:吃掉所有异常,绝不向外抛。"""
|
||
try:
|
||
self._process(entry)
|
||
except Exception as e:
|
||
logger.warning(
|
||
f"[TabRecovery] 恢复流程异常 (raw={entry.raw_tab_id}): {e}"
|
||
)
|
||
# 未知内部错误时保守处理:永久隔离,避免把状态不明的标签页重新入池。
|
||
try:
|
||
self._resolve(entry, release=False, reason=f"internal_error:{e}")
|
||
except Exception:
|
||
pass
|
||
|
||
def _process(self, entry: TabQuarantineEntry) -> None:
|
||
raw_id = entry.raw_tab_id
|
||
logger.info(
|
||
f"[TabRecovery] 开始处理隔离标签页 "
|
||
f"(raw={raw_id}, tab={entry.session_id or '-'}, "
|
||
f"task={entry.task_id or '-'}, reason={entry.reason or '-'})"
|
||
)
|
||
|
||
# 1. 等旧 worker 退出:串扰的根源是旧请求的 worker 线程可能仍阻塞在
|
||
# 对该 tab 的同步 DrissionPage 调用里。
|
||
wait_result = self._wait_for_worker_exit(entry)
|
||
if wait_result == "aborted":
|
||
logger.debug(f"[TabRecovery] 服务关闭,放弃处理 (raw={raw_id})")
|
||
return
|
||
if wait_result == "timeout":
|
||
self._resolve(entry, release=False, reason="worker_never_exited")
|
||
return
|
||
logger.debug(f"[TabRecovery] 旧 worker 已确认退出 (raw={raw_id})")
|
||
|
||
# 2. 次数检查:解除隔离后再次错误 → 不再调 API,直接永久隔离。
|
||
with self._attempts_lock:
|
||
attempts = int(self._attempts.get(raw_id, 0))
|
||
if raw_id in self._attempts:
|
||
# 最近使用的 raw id 保留在缓存尾部,便于后续按上限淘汰。
|
||
self._attempts.move_to_end(raw_id)
|
||
entry.attempts = attempts
|
||
max_attempts = AppConfig.get_tab_recovery_max_attempts()
|
||
if attempts >= max_attempts:
|
||
logger.info(
|
||
f"[TabRecovery] 恢复次数已用尽,不再调用 AI "
|
||
f"(raw={raw_id}, attempts={attempts}, max={max_attempts})"
|
||
)
|
||
self._resolve(entry, release=False, reason="attempts_exhausted")
|
||
return
|
||
|
||
# 3. 功能开关:未启用时不截图不调 API,直接解除隔离——保持既有的
|
||
# 自动重入池行为,但此时旧 worker 已确认退出,串扰已消除。
|
||
if not AppConfig.is_tab_recovery_enabled():
|
||
self._resolve(entry, release=True, reason="recovery_disabled")
|
||
return
|
||
|
||
# 4/5. 截图 + AI 判断;任一失败 → verdict=None(未知)
|
||
verdict: Optional[bool] = None
|
||
detail = ""
|
||
image_b64 = ""
|
||
try:
|
||
image_b64 = self._capture_screenshot(entry)
|
||
except Exception as e:
|
||
detail = f"screenshot_failed:{e}"
|
||
logger.warning(f"[TabRecovery] 截图失败 (raw={raw_id}): {e}")
|
||
|
||
if image_b64:
|
||
try:
|
||
verdict, detail = self._judge_recoverable(entry, image_b64)
|
||
logger.info(
|
||
f"[TabRecovery] AI 判断结果 "
|
||
f"(raw={raw_id}, recoverable={verdict}, detail={detail or '-'})"
|
||
)
|
||
except Exception as e:
|
||
verdict, detail = None, f"api_error:{e}"
|
||
logger.warning(f"[TabRecovery] AI 判断失败 (raw={raw_id}): {e}")
|
||
|
||
# 6. 决策
|
||
if verdict is False:
|
||
self._resolve(
|
||
entry,
|
||
release=False,
|
||
reason=f"ai_judged_unrecoverable:{detail or '-'}",
|
||
)
|
||
return
|
||
if verdict is None:
|
||
if not AppConfig.is_tab_recovery_refresh_on_unknown():
|
||
self._resolve(
|
||
entry,
|
||
release=False,
|
||
reason=f"judgement_unknown:{detail or '-'}",
|
||
)
|
||
return
|
||
refresh_reason = "unknown_refresh"
|
||
else:
|
||
refresh_reason = "ai_judged_recoverable"
|
||
|
||
# 7. 刷新与解除
|
||
self._refresh_and_release(entry, refresh_reason)
|
||
|
||
# ================= 步骤实现 =================
|
||
|
||
def _wait_for_worker_exit(self, entry: TabQuarantineEntry) -> str:
|
||
"""轮询旧 worker 是否退出。返回 'exited' / 'timeout' / 'aborted'。"""
|
||
max_wait = AppConfig.get_tab_recovery_worker_exit_wait_sec()
|
||
deadline = time.monotonic() + max(0.0, float(max_wait))
|
||
while True:
|
||
if self._shutdown:
|
||
return "aborted"
|
||
if self._is_worker_exited(entry):
|
||
return "exited"
|
||
remaining = deadline - time.monotonic()
|
||
if remaining <= 0:
|
||
return "timeout"
|
||
time.sleep(min(self.WORKER_POLL_INTERVAL_SEC, max(0.1, remaining)))
|
||
|
||
def _is_worker_exited(self, entry: TabQuarantineEntry) -> bool:
|
||
task_id = str(entry.task_id or "").strip()
|
||
if not task_id:
|
||
# 隔离时没有绑定任务,无 worker 可等
|
||
return True
|
||
|
||
ctx = None
|
||
try:
|
||
from app.services.request_manager import request_manager
|
||
|
||
ctx = request_manager.get_request(task_id)
|
||
except Exception as e:
|
||
logger.debug(
|
||
f"[TabRecovery] 查询请求上下文失败 (task={task_id}): {e}"
|
||
)
|
||
|
||
thread = getattr(ctx, "worker_thread", None) if ctx is not None else None
|
||
if thread is not None and hasattr(thread, "is_alive"):
|
||
try:
|
||
return not thread.is_alive()
|
||
except Exception as e:
|
||
logger.debug(
|
||
f"[TabRecovery] 读取 worker 线程状态失败 (task={task_id}): {e}"
|
||
)
|
||
|
||
# 拿不到线程引用(非流式请求 worker 在 request_lifecycle 内创建、
|
||
# 或请求上下文已被清理):回退时间宽限,隔离时刻起 180s 视为已退出。
|
||
elapsed = time.monotonic() - float(entry.since or 0.0)
|
||
return elapsed >= self.WORKER_EXIT_FALLBACK_GRACE_SEC
|
||
|
||
def _capture_screenshot(self, entry: TabQuarantineEntry) -> str:
|
||
tab = entry.tab
|
||
if tab is None:
|
||
raise RuntimeError("tab reference unavailable")
|
||
# DrissionPage 4.x:get_screenshot(as_base64=True) 返回 PNG 的 base64 字符串
|
||
data = self._call_with_timeout(
|
||
lambda: tab.get_screenshot(as_base64=True),
|
||
self.SCREENSHOT_TIMEOUT_SEC,
|
||
"screenshot",
|
||
)
|
||
if isinstance(data, (bytes, bytearray)):
|
||
data = base64.b64encode(bytes(data)).decode("ascii")
|
||
text = str(data or "").strip()
|
||
if not text:
|
||
raise RuntimeError("empty screenshot payload")
|
||
return text
|
||
|
||
def _judge_recoverable(
|
||
self,
|
||
entry: TabQuarantineEntry,
|
||
image_b64: str,
|
||
) -> Tuple[Optional[bool], str]:
|
||
"""调用 OpenAI 兼容接口判断刷新能否恢复。返回 (verdict, detail)。"""
|
||
api_url = AppConfig.get_tab_recovery_api_url()
|
||
if api_url:
|
||
target_url = api_url
|
||
api_key = AppConfig.get_tab_recovery_api_key()
|
||
local_mode = False
|
||
else:
|
||
# 本地模式:走本服务自身的 /v1/chat/completions,
|
||
# 由其他空闲标签页完成实际判断
|
||
target_url = (
|
||
f"http://127.0.0.1:{AppConfig.get_port()}/v1/chat/completions"
|
||
)
|
||
api_key = (
|
||
AppConfig.get_auth_token() if AppConfig.is_auth_enabled() else ""
|
||
)
|
||
local_mode = True
|
||
|
||
site_url = str(entry.url or "").strip() or "未知"
|
||
prompt = (
|
||
"这是反代服务的一个疑似卡死的网页标签页截图,"
|
||
f"站点为 {site_url},错误原因 {entry.reason or '未知'}。"
|
||
"请判断刷新页面是否可能恢复正常使用。只输出 JSON:"
|
||
"{\"recoverable\": true/false, \"reason\": \"...\"}。"
|
||
"登录失效、封号提示、人机验证页输出 false;"
|
||
"加载中、白屏、内容正常、临时错误提示输出 true。"
|
||
)
|
||
payload = {
|
||
"model": AppConfig.get_tab_recovery_model(),
|
||
"stream": False,
|
||
"messages": [
|
||
{
|
||
"role": "user",
|
||
"content": [
|
||
{"type": "text", "text": prompt},
|
||
{
|
||
"type": "image_url",
|
||
"image_url": {
|
||
"url": f"data:image/png;base64,{image_b64}",
|
||
},
|
||
},
|
||
],
|
||
}
|
||
],
|
||
}
|
||
headers = {"Content-Type": "application/json"}
|
||
if api_key:
|
||
headers["Authorization"] = f"Bearer {api_key}"
|
||
|
||
timeout_sec = AppConfig.get_tab_recovery_timeout_sec()
|
||
request = urllib.request.Request(
|
||
target_url,
|
||
data=json.dumps(payload).encode("utf-8"),
|
||
headers=headers,
|
||
method="POST",
|
||
)
|
||
logger.debug(
|
||
f"[TabRecovery] 调用 AI 判断 "
|
||
f"(raw={entry.raw_tab_id}, url={target_url}, "
|
||
f"local_mode={local_mode}, timeout={timeout_sec}s)"
|
||
)
|
||
if local_mode:
|
||
# 本地回环地址不应经过环境代理
|
||
opener = urllib.request.build_opener(urllib.request.ProxyHandler({}))
|
||
response = opener.open(request, timeout=timeout_sec)
|
||
else:
|
||
response = urllib.request.urlopen(request, timeout=timeout_sec)
|
||
with response:
|
||
body = response.read().decode("utf-8", "replace")
|
||
|
||
data = json.loads(body)
|
||
choices = data.get("choices") or []
|
||
if not choices:
|
||
return None, "empty_choices"
|
||
message = choices[0].get("message") or {}
|
||
content = message.get("content")
|
||
text = self._flatten_message_content(content)
|
||
parsed = self._extract_first_json_object(text)
|
||
if not isinstance(parsed, dict):
|
||
return None, f"parse_failed:{text[:120]!r}"
|
||
verdict = self._coerce_bool(parsed.get("recoverable"))
|
||
detail = str(parsed.get("reason") or "").strip()
|
||
if verdict is None:
|
||
return None, f"invalid_recoverable_field:{parsed.get('recoverable')!r}"
|
||
return verdict, detail
|
||
|
||
def _refresh_and_release(
|
||
self,
|
||
entry: TabQuarantineEntry,
|
||
refresh_reason: str,
|
||
) -> None:
|
||
raw_id = entry.raw_tab_id
|
||
try:
|
||
tab = entry.tab
|
||
if tab is None:
|
||
raise RuntimeError("tab reference unavailable")
|
||
self._call_with_timeout(
|
||
lambda: tab.refresh(),
|
||
self.REFRESH_TIMEOUT_SEC,
|
||
"refresh",
|
||
)
|
||
time.sleep(max(0.0, float(self.POST_REFRESH_SETTLE_SEC)))
|
||
current_url = self._call_with_timeout(
|
||
lambda: str(getattr(tab, "url", "") or ""),
|
||
self.REFRESH_TIMEOUT_SEC,
|
||
"health-check",
|
||
)
|
||
if not str(current_url or "").strip():
|
||
raise RuntimeError("empty url after refresh")
|
||
except Exception as e:
|
||
logger.warning(f"[TabRecovery] 刷新/健康检查失败 (raw={raw_id}): {e}")
|
||
self._resolve(entry, release=False, reason=f"refresh_failed:{e}")
|
||
return
|
||
|
||
with self._attempts_lock:
|
||
self._attempts[raw_id] = int(self._attempts.get(raw_id, 0)) + 1
|
||
self._attempts.move_to_end(raw_id)
|
||
entry.attempts = self._attempts[raw_id]
|
||
while len(self._attempts) > self.MAX_ATTEMPT_CACHE_ENTRIES:
|
||
self._attempts.popitem(last=False)
|
||
self._resolve(entry, release=True, reason=refresh_reason)
|
||
logger.info(
|
||
f"[TabRecovery] 标签页已恢复并重新入池 "
|
||
f"(raw={raw_id}, idx=#{entry.persistent_index or '-'}, "
|
||
f"url={str(current_url)[:80]}, attempts={entry.attempts}, "
|
||
f"reason={refresh_reason})"
|
||
)
|
||
|
||
# ================= 辅助 =================
|
||
|
||
def _resolve(
|
||
self,
|
||
entry: TabQuarantineEntry,
|
||
*,
|
||
release: bool,
|
||
reason: str,
|
||
) -> None:
|
||
entry.reason = str(reason or "").strip() or entry.reason
|
||
try:
|
||
resolved = bool(self._resolve_quarantine(entry.raw_tab_id, release))
|
||
except Exception as e:
|
||
logger.warning(
|
||
f"[TabRecovery] 解除/永久隔离操作失败 "
|
||
f"(raw={entry.raw_tab_id}, release={release}): {e}"
|
||
)
|
||
return
|
||
if release:
|
||
logger.info(
|
||
f"[TabRecovery] 解除隔离,标签页将由下次扫描重新入池 "
|
||
f"(raw={entry.raw_tab_id}, reason={reason}, resolved={resolved})"
|
||
)
|
||
else:
|
||
logger.info(
|
||
f"[TabRecovery] 永久隔离标签页 "
|
||
f"(raw={entry.raw_tab_id}, reason={reason}, resolved={resolved})"
|
||
)
|
||
|
||
@staticmethod
|
||
def _call_with_timeout(fn: Callable[[], Any], timeout_sec: float, label: str) -> Any:
|
||
"""在守护线程中执行阻塞调用并限时等待,防止 CDP 调用无限挂起。"""
|
||
wait_timeout = max(0.1, float(timeout_sec))
|
||
if not _RECOVERY_CALL_SLOTS.acquire(timeout=wait_timeout):
|
||
raise TimeoutError(f"{label} call slots exhausted")
|
||
|
||
result: Dict[str, Any] = {}
|
||
done = threading.Event()
|
||
|
||
def _runner():
|
||
try:
|
||
result["value"] = fn()
|
||
except BaseException as e: # noqa: BLE001 - 结果原样转抛给调用方
|
||
result["error"] = e
|
||
finally:
|
||
done.set()
|
||
_RECOVERY_CALL_SLOTS.release()
|
||
|
||
thread = threading.Thread(
|
||
target=_runner,
|
||
name=f"tab-recovery-{label}",
|
||
daemon=True,
|
||
)
|
||
try:
|
||
thread.start()
|
||
except BaseException:
|
||
_RECOVERY_CALL_SLOTS.release()
|
||
raise
|
||
if not done.wait(wait_timeout):
|
||
raise TimeoutError(f"{label} timed out after {timeout_sec}s")
|
||
if "error" in result:
|
||
raise result["error"]
|
||
return result.get("value")
|
||
|
||
@staticmethod
|
||
def _flatten_message_content(content: Any) -> str:
|
||
if isinstance(content, str):
|
||
return content
|
||
if isinstance(content, list):
|
||
parts = []
|
||
for item in content:
|
||
if isinstance(item, dict):
|
||
text = item.get("text")
|
||
if isinstance(text, str):
|
||
parts.append(text)
|
||
elif isinstance(item, str):
|
||
parts.append(item)
|
||
return "\n".join(parts)
|
||
return str(content or "")
|
||
|
||
@staticmethod
|
||
def _extract_first_json_object(text: str) -> Optional[Dict[str, Any]]:
|
||
"""宽松提取文本中第一个平衡的 {...} 并解析为 dict。"""
|
||
source = str(text or "")
|
||
start = source.find("{")
|
||
while start != -1:
|
||
depth = 0
|
||
in_string = False
|
||
escaped = False
|
||
for index in range(start, len(source)):
|
||
char = source[index]
|
||
if in_string:
|
||
if escaped:
|
||
escaped = False
|
||
elif char == "\\":
|
||
escaped = True
|
||
elif char == '"':
|
||
in_string = False
|
||
continue
|
||
if char == '"':
|
||
in_string = True
|
||
elif char == "{":
|
||
depth += 1
|
||
elif char == "}":
|
||
depth -= 1
|
||
if depth == 0:
|
||
candidate = source[start:index + 1]
|
||
try:
|
||
parsed = json.loads(candidate)
|
||
if isinstance(parsed, dict):
|
||
return parsed
|
||
except Exception:
|
||
pass
|
||
break
|
||
start = source.find("{", start + 1)
|
||
return None
|
||
|
||
@staticmethod
|
||
def _coerce_bool(value: Any) -> Optional[bool]:
|
||
if isinstance(value, bool):
|
||
return value
|
||
text = str(value or "").strip().lower()
|
||
if text in {"true", "1", "yes", "y"}:
|
||
return True
|
||
if text in {"false", "0", "no", "n"}:
|
||
return False
|
||
return None
|