Initial commit
This commit is contained in:
commit
57fc3525eb
7311 files changed
+1291398
No files matched your search
@@ -0,0 +1,22 @@
|
||||
"""
|
||||
app/services - 业务服务层
|
||||
|
||||
职责:
|
||||
- 请求管理
|
||||
- 配置引擎
|
||||
"""
|
||||
|
||||
from .request_manager import request_manager, RequestManager, RequestContext, RequestStatus
|
||||
from .config_engine import config_engine, ConfigEngine
|
||||
|
||||
__all__ = [
|
||||
# 请求管理
|
||||
'request_manager',
|
||||
'RequestManager',
|
||||
'RequestContext',
|
||||
'RequestStatus',
|
||||
|
||||
# 配置引擎
|
||||
'config_engine',
|
||||
'ConfigEngine',
|
||||
]
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -0,0 +1,147 @@
|
||||
"""R1-6:站点适配器健康巡检——只查询页面 DOM,绝不输入、点击或发送消息。
|
||||
|
||||
对站点当前预设里配置的每个选择器,在该站点已打开的空闲标签页上统计命中数量,给出:
|
||||
|
||||
- ``broken``:输入框找不到(常见原因:未登录、页面改版),或存在语法无效的选择器;
|
||||
- ``degraded``:其他必需元素找不到。发送按钮通常要输入内容后才出现,回复容器要有对话后才出现,
|
||||
所以这里只作提示,不算坏;
|
||||
- ``healthy``:输入框能找到,且没有无效选择器;
|
||||
- ``unreachable``:浏览器里没有该站点的空闲标签页(巡检不会自行打开页面)。
|
||||
|
||||
``evaluate_preset`` 是与浏览器无关的纯函数(便于单测);``DrissionProbe`` 把 DrissionPage 标签页适配成探针。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
from typing import Any, Dict, Iterable, List, Optional, Protocol
|
||||
|
||||
from app.core.config import get_logger
|
||||
from app.core.driver.locators import to_locator as _driver_to_locator
|
||||
from app.services.selector_quality import fallback_suggestions, lint_selector
|
||||
|
||||
logger = get_logger("ADAPTER_HEALTH")
|
||||
|
||||
ALWAYS_PRESENT_KEYS = ("input_box",)
|
||||
_HINTS = {
|
||||
"send_btn": "发送按钮通常在输入内容后才出现或可点击",
|
||||
"result_container": "回复容器通常在已有对话时才出现",
|
||||
}
|
||||
|
||||
|
||||
class Probe(Protocol):
|
||||
url: str
|
||||
|
||||
def count(self, selector: str) -> int:
|
||||
"""返回命中数量;选择器语法无效时返回 -1。"""
|
||||
|
||||
|
||||
# R2-1:定位规则统一到驱动层,这里保留旧名以兼容已有调用与测试
|
||||
to_locator = _driver_to_locator
|
||||
|
||||
|
||||
class DrissionProbe:
|
||||
"""把标签页适配成探针。R2-1 起经由统一驱动查询元素(参数既可以是 DrissionPage 标签页,也可以是驱动对象)。"""
|
||||
|
||||
def __init__(self, tab: Any, url: str = "", timeout: float = 0.3):
|
||||
from app.core.driver import driver_for_tab
|
||||
|
||||
self.driver = driver_for_tab(tab)
|
||||
self.tab = self.driver.raw
|
||||
self.url = url or self.driver.url
|
||||
self.timeout = timeout
|
||||
|
||||
def count(self, selector: str) -> int:
|
||||
from app.core.driver import DriverError
|
||||
|
||||
try:
|
||||
# 配置里的选择器默认是 CSS:先规范化,再交给驱动(驱动按 DrissionPage 原生语法原样传递)
|
||||
return len(self.driver.find_all(to_locator(selector), timeout=self.timeout))
|
||||
except DriverError as exc:
|
||||
logger.debug(f"选择器无效或查询失败: {selector!r}: {exc}")
|
||||
return -1
|
||||
|
||||
|
||||
def evaluate_preset(probe: Probe, preset: Dict[str, Any], definitions: Iterable[Dict[str, Any]]) -> Dict[str, Any]:
|
||||
required = {str(d.get("key")) for d in definitions or [] if isinstance(d, dict) and d.get("required")}
|
||||
rows: List[Dict[str, Any]] = []
|
||||
for key, selector in (preset.get("selectors") or {}).items():
|
||||
if not isinstance(selector, str) or not selector.strip():
|
||||
continue
|
||||
count = probe.count(selector)
|
||||
status = "invalid" if count < 0 else ("missing" if count == 0 else "ok")
|
||||
row = {"key": key, "selector": selector, "count": max(count, 0), "status": status, "required": key in required,
|
||||
"quality": lint_selector(selector)}
|
||||
if status == "missing" and key in _HINTS:
|
||||
row["hint"] = _HINTS[key]
|
||||
if status != "ok" and key in required:
|
||||
# R1-7:运行时找不到配置的元素时会尝试这些回退选择器;命中的可作为修复起点
|
||||
row["suggestions"] = fallback_suggestions(key, probe)
|
||||
rows.append(row)
|
||||
|
||||
by_key = {row["key"]: row for row in rows}
|
||||
problems = []
|
||||
if any(row["status"] == "invalid" for row in rows):
|
||||
problems.append("存在语法无效的选择器")
|
||||
for key in ALWAYS_PRESENT_KEYS:
|
||||
if key not in by_key:
|
||||
problems.append(f"预设没有配置 {key}")
|
||||
elif by_key[key]["status"] != "ok":
|
||||
problems.append(f"找不到 {key}(可能未登录或页面已改版)")
|
||||
if problems:
|
||||
overall = "broken"
|
||||
elif any(row["required"] and row["status"] != "ok" for row in rows):
|
||||
overall = "degraded"
|
||||
else:
|
||||
overall = "healthy"
|
||||
return {"status": overall, "url": probe.url, "problems": problems, "selectors": rows}
|
||||
|
||||
|
||||
def check_site_health(domain: str, preset_name: Optional[str] = None, *, timeout: float = 0.3) -> Dict[str, Any]:
|
||||
"""在浏览器里该站点的空闲标签页上巡检(与 /api/debug/test-selector 相同的借用/归还方式)。"""
|
||||
from app.core.browser import get_browser
|
||||
from app.services.config_engine import config_engine
|
||||
from app.utils.site_url import extract_remote_site_domain, route_domain_matches
|
||||
|
||||
site = config_engine.sites.get(domain)
|
||||
if not isinstance(site, dict):
|
||||
raise KeyError(f"未知站点: {domain}")
|
||||
presets = site.get("presets") or {}
|
||||
name = preset_name or site.get("default_preset") or next(iter(presets), "")
|
||||
if name not in presets:
|
||||
raise KeyError(f"站点 {domain} 没有预设 {name!r}")
|
||||
definitions = config_engine.global_config.to_dict().get("selector_definitions") or []
|
||||
|
||||
browser = get_browser()
|
||||
pool = browser.tab_pool
|
||||
pool.refresh_tabs()
|
||||
reports, busy = [], 0
|
||||
for session in pool.get_sessions_snapshot():
|
||||
cached = str(getattr(session, "current_domain", "") or "")
|
||||
if cached and not route_domain_matches(domain, cached):
|
||||
continue
|
||||
if str(getattr(getattr(session, "status", None), "value", "")) != "idle":
|
||||
busy += 1 if cached else 0
|
||||
continue
|
||||
raw_tab_id = str(getattr(getattr(session, "tab", None), "tab_id", "") or "")
|
||||
if not raw_tab_id:
|
||||
continue
|
||||
task_id = f"adapter_health_{time.time_ns()}"
|
||||
acquired = pool.acquire_by_raw_tab_id(raw_tab_id, task_id, timeout=1.0, count_request=False, activate=False)
|
||||
if acquired is None:
|
||||
continue
|
||||
try:
|
||||
url = str(getattr(acquired.tab, "url", "") or "")
|
||||
if not route_domain_matches(domain, extract_remote_site_domain(url) or ""):
|
||||
continue
|
||||
reports.append(evaluate_preset(DrissionProbe(acquired.tab, url, timeout), presets[name], definitions))
|
||||
finally:
|
||||
pool.release(acquired.id, check_triggers=False, expected_task_id=task_id)
|
||||
|
||||
if not reports:
|
||||
return {"site": domain, "preset": name, "status": "unreachable", "busy_tabs": busy,
|
||||
"problems": [f"浏览器里没有 {domain} 的空闲标签页;巡检不会自行打开页面"], "tabs": []}
|
||||
order = {"healthy": 0, "degraded": 1, "broken": 2}
|
||||
best = min(reports, key=lambda report: order[report["status"]])
|
||||
return {"site": domain, "preset": name, "status": best["status"], "busy_tabs": busy,
|
||||
"problems": best["problems"], "tabs": reports}
|
||||
@@ -0,0 +1,231 @@
|
||||
"""R1-3:站点适配器在线更新(不发整包也能热更新站点配置)。
|
||||
|
||||
依据三份信息逐站点比较:
|
||||
|
||||
- 官方 ``config/sites/index.json``:官方当前版本的规范化摘要(config_sha256)、文件摘要、版本号;
|
||||
- 本地 ``config/sites/index.json``:本地**安装时**附带的清单,代表“用户拿到的原始版本”;
|
||||
- 本地站点文件的当前内容:与安装清单不同,说明用户改过。
|
||||
|
||||
状态:
|
||||
- ``up_to_date``:本地内容与官方一致;
|
||||
- ``update_available``:官方有变化,且用户没改过本地文件,可以安全自动应用;
|
||||
- ``conflict``:官方有变化,且用户也改过(或来源未知),需要显式确认,按“本地优先”合并;
|
||||
- ``new_site``:官方新增、本地没有;
|
||||
- ``local_only``:只在本地存在;
|
||||
- ``requires_newer_app``:官方版本要求更高的 min_app_version。
|
||||
|
||||
应用更新时逐个下载并按 ``file_sha256`` 校验、Schema 校验,写入后更新本地安装清单中对应条目,最后热重载。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
from pathlib import Path
|
||||
from typing import Any, Callable, Dict, Iterable, List, Optional
|
||||
|
||||
from app.core.config import get_logger
|
||||
from app.services.config.site_schema import site_file_errors
|
||||
from app.services.config.site_store import SiteStore, canonical_config_text
|
||||
|
||||
logger = get_logger("ADAPTER_UPD")
|
||||
|
||||
INDEX_NAME = "index.json"
|
||||
SAFE_STATUSES = ("update_available", "new_site")
|
||||
_FILE_PATTERN = re.compile(r"^[A-Za-z0-9._%~-]+\.json$")
|
||||
_SHA_PATTERN = re.compile(r"^[0-9a-f]{64}$")
|
||||
|
||||
FetchBytes = Callable[[str], bytes]
|
||||
|
||||
|
||||
def config_sha256(config: Any) -> str:
|
||||
return hashlib.sha256(canonical_config_text(config).encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def _version_tuple(value: str) -> tuple:
|
||||
return tuple(int(part) for part in re.findall(r"\d+", str(value or ""))[:4]) or (0,)
|
||||
|
||||
|
||||
def _atomic_write(path: Path, text: str) -> None:
|
||||
tmp = path.with_name(path.name + ".tmp")
|
||||
with open(tmp, "w", encoding="utf-8", newline="\n") as handle:
|
||||
handle.write(text)
|
||||
handle.flush()
|
||||
os.fsync(handle.fileno())
|
||||
os.replace(tmp, path)
|
||||
|
||||
|
||||
class AdapterUpdater:
|
||||
def __init__(
|
||||
self,
|
||||
sites_dir: os.PathLike | str,
|
||||
fetch_bytes: FetchBytes,
|
||||
*,
|
||||
app_version: str,
|
||||
reload: Optional[Callable[[], None]] = None,
|
||||
lock: Any = None,
|
||||
):
|
||||
"""``fetch_bytes(relative_path)`` 返回官方仓库中该路径的原始字节(由调用方负责安全抓取与大小限制)。"""
|
||||
self.sites_dir = Path(sites_dir)
|
||||
self.fetch_bytes = fetch_bytes
|
||||
self.app_version = str(app_version or "0")
|
||||
self.reload = reload
|
||||
self.lock = lock
|
||||
|
||||
# ------------------------------------------------------------------ 读取
|
||||
def _local_index(self) -> Dict[str, Any]:
|
||||
try:
|
||||
payload = json.loads((self.sites_dir / INDEX_NAME).read_text(encoding="utf-8-sig"))
|
||||
return payload if isinstance(payload, dict) else {}
|
||||
except (OSError, ValueError):
|
||||
return {}
|
||||
|
||||
def _local_envelopes(self) -> Dict[str, Dict[str, Any]]:
|
||||
store = SiteStore(self.sites_dir, self.sites_dir / "__no_legacy__.json")
|
||||
result = {}
|
||||
for path in store.site_files():
|
||||
try:
|
||||
payload = store.read_envelope(path)
|
||||
except (OSError, ValueError):
|
||||
continue
|
||||
result[payload["site"]] = {"path": path, "payload": payload}
|
||||
return result
|
||||
|
||||
def _official_index(self) -> Dict[str, Any]:
|
||||
raw = self.fetch_bytes(f"config/sites/{INDEX_NAME}")
|
||||
index = json.loads(raw.decode("utf-8-sig"))
|
||||
entries = index.get("sites") if isinstance(index, dict) else None
|
||||
if not isinstance(entries, dict):
|
||||
raise ValueError("官方 index.json 缺少 sites 清单")
|
||||
for site, entry in entries.items():
|
||||
if not isinstance(entry, dict) or not _FILE_PATTERN.fullmatch(str(entry.get("file") or "")) \
|
||||
or not _SHA_PATTERN.fullmatch(str(entry.get("file_sha256") or "")) \
|
||||
or not _SHA_PATTERN.fullmatch(str(entry.get("config_sha256") or "")):
|
||||
raise ValueError(f"官方 index.json 中站点 {site} 的条目无效")
|
||||
return index
|
||||
|
||||
# ------------------------------------------------------------------ 检查
|
||||
def check(self) -> Dict[str, Any]:
|
||||
official = self._official_index()["sites"]
|
||||
installed = (self._local_index().get("sites") or {})
|
||||
local = self._local_envelopes()
|
||||
rows: List[Dict[str, Any]] = []
|
||||
for site in sorted(set(official) | set(local)):
|
||||
official_entry = official.get(site)
|
||||
local_item = local.get(site)
|
||||
row = {
|
||||
"site": site,
|
||||
"local_version": (local_item or {}).get("payload", {}).get("adapter_version", ""),
|
||||
"official_version": (official_entry or {}).get("adapter_version", ""),
|
||||
"min_app_version": (official_entry or {}).get("min_app_version", ""),
|
||||
}
|
||||
if official_entry is None:
|
||||
row["status"] = "local_only"
|
||||
else:
|
||||
too_new = _version_tuple(official_entry.get("min_app_version")) > _version_tuple(self.app_version)
|
||||
if local_item is None:
|
||||
row["status"] = "requires_newer_app" if too_new else "new_site"
|
||||
else:
|
||||
local_hash = config_sha256(local_item["payload"]["config"])
|
||||
installed_hash = (installed.get(site) or {}).get("config_sha256")
|
||||
if local_hash == official_entry["config_sha256"]:
|
||||
row["status"] = "up_to_date"
|
||||
elif too_new:
|
||||
row["status"] = "requires_newer_app"
|
||||
elif installed_hash and local_hash == installed_hash:
|
||||
row["status"] = "update_available"
|
||||
else:
|
||||
row["status"] = "conflict"
|
||||
row["locally_modified"] = not (installed_hash and local_hash == installed_hash)
|
||||
rows.append(row)
|
||||
summary: Dict[str, int] = {}
|
||||
for row in rows:
|
||||
summary[row["status"]] = summary.get(row["status"], 0) + 1
|
||||
return {"app_version": self.app_version, "sites": rows, "summary": summary}
|
||||
|
||||
# ------------------------------------------------------------------ 应用
|
||||
def apply(self, sites: Optional[Iterable[str]] = None, *, include_conflicts: bool = False) -> Dict[str, Any]:
|
||||
"""应用更新。``sites`` 为空时应用全部安全更新(update_available / new_site)。"""
|
||||
from updater import merge_site_records # 与更新器、迁移使用同一合并语义
|
||||
|
||||
official_index = self._official_index()
|
||||
report = self.check()
|
||||
status_by_site = {row["site"]: row["status"] for row in report["sites"]}
|
||||
allowed = set(SAFE_STATUSES) | ({"conflict"} if include_conflicts else set())
|
||||
targets = list(sites) if sites else [s for s, st in status_by_site.items() if st in SAFE_STATUSES]
|
||||
|
||||
applied, skipped = [], []
|
||||
pending_writes: List[tuple] = []
|
||||
local = self._local_envelopes()
|
||||
for site in targets:
|
||||
status = status_by_site.get(site)
|
||||
if status not in allowed:
|
||||
skipped.append({"site": site, "status": status or "unknown"})
|
||||
continue
|
||||
entry = official_index["sites"][site]
|
||||
raw = self.fetch_bytes(f"config/sites/{entry['file']}")
|
||||
if hashlib.sha256(raw).hexdigest() != entry["file_sha256"]:
|
||||
raise ValueError(f"官方站点文件 {entry['file']} 的 sha256 与 index.json 不一致,已中止")
|
||||
payload = json.loads(raw.decode("utf-8-sig"))
|
||||
errors = site_file_errors(payload)
|
||||
if errors or payload.get("site") != site:
|
||||
raise ValueError(f"官方站点文件 {entry['file']} 未通过校验: {errors[:3] or 'site 字段不符'}")
|
||||
if status == "conflict":
|
||||
merged = dict(payload)
|
||||
merged["config"] = merge_site_records(local[site]["payload"]["config"], payload["config"])
|
||||
text = json.dumps(merged, indent=2, ensure_ascii=False) + "\n"
|
||||
else:
|
||||
text = raw.decode("utf-8-sig")
|
||||
target = local[site]["path"] if site in local else self.sites_dir / entry["file"]
|
||||
pending_writes.append((site, target, text, entry, status))
|
||||
|
||||
def write_all():
|
||||
if not pending_writes:
|
||||
return
|
||||
local_index = self._local_index() or {"schema_version": 1, "sites": {}}
|
||||
local_index.setdefault("sites", {})
|
||||
for site, target, text, entry, status in pending_writes:
|
||||
_atomic_write(target, text)
|
||||
local_index["sites"][site] = dict(entry)
|
||||
applied.append({"site": site, "status": status, "version": entry.get("adapter_version", "")})
|
||||
local_index["sites"] = dict(sorted(local_index["sites"].items()))
|
||||
_atomic_write(self.sites_dir / INDEX_NAME, json.dumps(local_index, indent=2, ensure_ascii=False) + "\n")
|
||||
|
||||
if self.lock is not None:
|
||||
with self.lock:
|
||||
write_all()
|
||||
else:
|
||||
write_all()
|
||||
if applied and self.reload is not None:
|
||||
self.reload()
|
||||
logger.info(f"站点适配器更新:应用 {len(applied)} 个,跳过 {len(skipped)} 个")
|
||||
return {"applied": applied, "skipped": skipped}
|
||||
|
||||
|
||||
def default_updater(branch: str = "main") -> AdapterUpdater:
|
||||
"""使用全局 config_engine 与官方仓库(GITHUB_REPO)的更新器。"""
|
||||
from app import __version__ as app_version
|
||||
from app.api import config_compare_support as support
|
||||
from app.services.config_engine import config_engine
|
||||
from app.utils.remote_resource import get_public_remote_resource
|
||||
|
||||
repo = support._resolve_official_repo()
|
||||
|
||||
def fetch_bytes(relative_path: str) -> bytes:
|
||||
url = support._official_config_url(repo, branch, relative_path)
|
||||
response = get_public_remote_resource(
|
||||
url, headers={"Accept": "application/json", "User-Agent": "Universal-Web-API-Adapter-Update/1.0"},
|
||||
timeout=(5, 15), stream=True,
|
||||
)
|
||||
try:
|
||||
response.raise_for_status()
|
||||
return support._read_limited_response(response)
|
||||
finally:
|
||||
response.close()
|
||||
|
||||
return AdapterUpdater(
|
||||
config_engine.sites_dir, fetch_bytes, app_version=app_version,
|
||||
reload=config_engine.reload_config, lock=config_engine._io_lock,
|
||||
)
|
||||
@@ -0,0 +1,105 @@
|
||||
"""
|
||||
app/services/arena_cf_solver.py - Arena Cloudflare Turnstile / 5s 盾自动过盾服务(兼容适配层)
|
||||
|
||||
说明:
|
||||
本模块已重构并委托底层通用过盾引擎 `app.services.cf_turnstile_solver`,
|
||||
在此处保留 Arena 专有的 URL 校验规则与兼容别名。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from typing import Any, Callable, Dict, Optional, Tuple
|
||||
|
||||
from app.core.config import logger as default_logger
|
||||
from app.utils.human_mouse import cdp_precise_click, smooth_move_mouse
|
||||
from app.services.cf_turnstile_solver import (
|
||||
DEFAULT_FIND_CLICK_POINT_JS as _FIND_CLICK_POINT_JS,
|
||||
DEFAULT_PAGE_READY_PROBE_JS as _PAGE_READY_PROBE_JS,
|
||||
_interruptible_sleep,
|
||||
_urls_match,
|
||||
_wait_for_challenge_resolved,
|
||||
_wait_for_clickable_point,
|
||||
solve_turnstile_challenge,
|
||||
)
|
||||
|
||||
_DEFAULT_MAX_ATTEMPTS = 3
|
||||
|
||||
|
||||
def _is_valid_arena_url(url: Optional[str]) -> bool:
|
||||
if not url or not isinstance(url, str):
|
||||
return False
|
||||
u = url.lower().strip()
|
||||
if not (u.startswith("http://") or u.startswith("https://")):
|
||||
return False
|
||||
if "challenges.cloudflare.com" in u or "challenge-platform" in u:
|
||||
return False
|
||||
if "arena.ai" in u or "lmarena.ai" in u:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _get_occupied_url(tab: Any, session: Any = None) -> str:
|
||||
"""提取本次请求在过盾前绑定的实际占用 URL。"""
|
||||
if session is not None:
|
||||
occupied = getattr(session, "_request_occupied_url", None)
|
||||
if occupied and _is_valid_arena_url(occupied):
|
||||
return str(occupied).strip()
|
||||
last_known = getattr(session, "last_known_url", None)
|
||||
if last_known and _is_valid_arena_url(last_known):
|
||||
return str(last_known).strip()
|
||||
try:
|
||||
tab_url = str(getattr(tab, "url", "") or "").strip()
|
||||
if tab_url and _is_valid_arena_url(tab_url):
|
||||
return tab_url
|
||||
except Exception:
|
||||
pass
|
||||
return "https://arena.ai/code"
|
||||
|
||||
|
||||
def solve_arena_turnstile_challenge(
|
||||
tab: Any,
|
||||
session: Any = None,
|
||||
logger: Any = None,
|
||||
max_attempts: int = _DEFAULT_MAX_ATTEMPTS,
|
||||
raise_if_cancelled: Optional[Callable[[], None]] = None,
|
||||
fallback_coords: Optional[Tuple[int, int]] = None,
|
||||
click_timeout_sec: float = 6.0,
|
||||
resolve_timeout_sec: float = 8.0,
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
自动处理 Arena 页面 Cloudflare Turnstile / 5s 盾的核心流程。
|
||||
"""
|
||||
# Fetch module-level smooth_move_mouse and cdp_precise_click in case patched by tests
|
||||
current_module = sys.modules.get(__name__)
|
||||
cur_smooth_move = getattr(current_module, "smooth_move_mouse", smooth_move_mouse)
|
||||
cur_cdp_click = getattr(current_module, "cdp_precise_click", cdp_precise_click)
|
||||
|
||||
return solve_turnstile_challenge(
|
||||
tab=tab,
|
||||
session=session,
|
||||
is_valid_url_fn=_is_valid_arena_url,
|
||||
max_attempts=max_attempts,
|
||||
raise_if_cancelled=raise_if_cancelled,
|
||||
fallback_coords=fallback_coords,
|
||||
click_timeout_sec=click_timeout_sec,
|
||||
resolve_timeout_sec=resolve_timeout_sec,
|
||||
smooth_move_fn=cur_smooth_move,
|
||||
cdp_click_fn=cur_cdp_click,
|
||||
logger=logger or default_logger,
|
||||
)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"_FIND_CLICK_POINT_JS",
|
||||
"_PAGE_READY_PROBE_JS",
|
||||
"_get_occupied_url",
|
||||
"_interruptible_sleep",
|
||||
"_is_valid_arena_url",
|
||||
"_urls_match",
|
||||
"_wait_for_challenge_resolved",
|
||||
"_wait_for_clickable_point",
|
||||
"cdp_precise_click",
|
||||
"smooth_move_mouse",
|
||||
"solve_arena_turnstile_challenge",
|
||||
]
|
||||
@@ -0,0 +1,816 @@
|
||||
"""Read Arena Direct's model catalog from the already-loaded page state."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import copy
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import threading
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from app.core.config import logger
|
||||
from app.utils.site_url import extract_remote_site_domain, route_domain_matches
|
||||
from app.core.driver import driver_for_tab
|
||||
|
||||
|
||||
ARENA_DIRECT_MODEL_PREFIX = "arena.ai/direct/"
|
||||
ARENA_DIRECT_MODEL_CACHE_TTL = 300.0
|
||||
MODEL_CATALOG_SOURCE = "arena_direct"
|
||||
ARENA_MODEL_ALIAS_OVERRIDES_PATH = Path(
|
||||
os.getenv(
|
||||
"ARENA_MODEL_ALIAS_OVERRIDES_PATH",
|
||||
str(Path(__file__).resolve().parents[2] / "config" / "arena_model_aliases.local.json"),
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
_ARENA_DIRECT_MODEL_EXTRACT_JS = r"""
|
||||
return (() => {
|
||||
const pathname = (typeof window !== 'undefined' && window.location && window.location.pathname) ? String(window.location.pathname).toLowerCase() : '';
|
||||
const isImagePage = pathname.includes('/image');
|
||||
const isCodePage = pathname.includes('/code');
|
||||
const defaultPageModality = isImagePage ? 'image' : (isCodePage ? 'code' : 'text');
|
||||
|
||||
// 1. 优先从 React Fiber 提取前端真实加载的可用模型
|
||||
const all = Array.from(document.querySelectorAll('*'));
|
||||
let fiberModels = null;
|
||||
for (const el of all) {
|
||||
const fiberKey = Object.keys(el).find(k => k.startsWith('__reactFiber') || k.startsWith('__reactInternalInstance'));
|
||||
if (!fiberKey) continue;
|
||||
let cur = el[fiberKey];
|
||||
let d = 0;
|
||||
while (cur && d < 25) {
|
||||
if (cur.memoizedProps) {
|
||||
for (const [k, v] of Object.entries(cur.memoizedProps)) {
|
||||
if (Array.isArray(v) && v.length >= 30 && v.length <= 300 && v[0] && (v[0].id && (v[0].displayName || v[0].name || v[0].publicName))) {
|
||||
fiberModels = v;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (fiberModels) break;
|
||||
cur = cur.return;
|
||||
d++;
|
||||
}
|
||||
if (fiberModels) break;
|
||||
}
|
||||
|
||||
const seenIds = new Set();
|
||||
const results = [];
|
||||
|
||||
const appendModel = (model, defaultModality) => {
|
||||
if (!model || !model.id) return;
|
||||
const idStr = String(model.id);
|
||||
const idLower = idStr.toLowerCase();
|
||||
if (seenIds.has(idLower)) return;
|
||||
seenIds.add(idLower);
|
||||
|
||||
const caps = model.capabilities || {};
|
||||
const outCaps = caps.outputCapabilities || model.outputCapabilities || {};
|
||||
const ranks = model.rankByModality || {};
|
||||
const rankIsFinite = (key) => (
|
||||
typeof ranks[key] === 'number' &&
|
||||
Number.isFinite(ranks[key]) &&
|
||||
ranks[key] < Number.MAX_SAFE_INTEGER
|
||||
);
|
||||
const hasText = Boolean(outCaps.text);
|
||||
const hasImage = Boolean(outCaps.image) || rankIsFinite('image');
|
||||
const hasVideo = Boolean(outCaps.video) || rankIsFinite('video');
|
||||
// `outputCapabilities.web` means the model can browse/use web data;
|
||||
// it is not the WebDev/code modality. Only the modality rank marks
|
||||
// a model as code-capable for catalog filtering.
|
||||
const hasWebDev = rankIsFinite('webdev') || Boolean(outCaps.web);
|
||||
const hasSearch = Boolean(outCaps.search) || rankIsFinite('search');
|
||||
|
||||
const dispName = String(model.displayName || model.publicName || model.name || idStr).trim();
|
||||
const internalName = String(model.name || model.id || dispName).trim();
|
||||
const pubName = String(model.publicName || dispName || internalName).trim();
|
||||
const nameSearch = `${internalName} ${dispName} ${pubName} ${idStr}`.toLowerCase();
|
||||
|
||||
const isKnownImageModel = nameSearch.includes('gpt-image') ||
|
||||
nameSearch.includes('mona-lisa') ||
|
||||
nameSearch.includes('luna-lisa') ||
|
||||
nameSearch.includes('lina-alpha') ||
|
||||
nameSearch.includes('lina-f-alpha') ||
|
||||
nameSearch.includes('silver_halide') ||
|
||||
nameSearch.includes('flux') ||
|
||||
nameSearch.includes('imagen') ||
|
||||
nameSearch.includes('seedream') ||
|
||||
nameSearch.includes('seededit') ||
|
||||
nameSearch.includes('z-image') ||
|
||||
nameSearch.includes('midjourney') ||
|
||||
nameSearch.includes('dall-e') ||
|
||||
nameSearch.includes('recraft') ||
|
||||
nameSearch.includes('krea') ||
|
||||
(nameSearch.includes('grok') && nameSearch.includes('image')) ||
|
||||
(nameSearch.includes('imagine') && !nameSearch.includes('video'));
|
||||
|
||||
let modality = defaultModality === 'image' ? 'image' : (defaultModality === 'code' ? 'code' : 'text');
|
||||
if (hasVideo && !hasText && !isKnownImageModel) {
|
||||
modality = 'video';
|
||||
} else if (isKnownImageModel || (hasImage && defaultModality === 'image') || (hasImage && !hasText)) {
|
||||
modality = 'image';
|
||||
} else if (defaultModality === 'code' || (hasWebDev && !hasText)) {
|
||||
modality = 'code';
|
||||
} else if (hasText) {
|
||||
modality = 'text';
|
||||
} else if (hasSearch) {
|
||||
modality = 'search';
|
||||
} else if (hasWebDev) {
|
||||
modality = 'code';
|
||||
} else if (hasImage) {
|
||||
modality = 'image';
|
||||
} else if (hasVideo) {
|
||||
modality = 'video';
|
||||
}
|
||||
|
||||
results.push({
|
||||
arena_model_id: idStr,
|
||||
name: internalName,
|
||||
public_name: dispName,
|
||||
display_name: dispName,
|
||||
provider: String(model.provider || ''),
|
||||
organization: String(model.organization || model.provider || 'arena.ai'),
|
||||
modality
|
||||
});
|
||||
};
|
||||
|
||||
const isModelAvailable = (model) => {
|
||||
if (!model || model.userSelectable === false) return false;
|
||||
const ranks = model.rankByModality || {};
|
||||
const rankValues = Object.values(ranks);
|
||||
const hasVisibleRank = rankValues.some((rank) => (
|
||||
typeof rank === 'number' &&
|
||||
Number.isFinite(rank) &&
|
||||
rank < Number.MAX_SAFE_INTEGER
|
||||
));
|
||||
const hasPublicRank = typeof model.rank === 'number' &&
|
||||
Number.isFinite(model.rank) && model.rank < 1000;
|
||||
const hasNoAvailabilityMetadata = rankValues.length === 0 &&
|
||||
typeof model.rank === 'undefined';
|
||||
// Arena can keep a user-selectable model at MAX_SAFE_INTEGER for a
|
||||
// modality while still exposing that modality as a valid capability.
|
||||
// userSelectable is the authoritative visibility signal in that case.
|
||||
const hasCapabilityMetadata = Object.values(
|
||||
model.capabilities?.outputCapabilities || model.outputCapabilities || {}
|
||||
).some(Boolean);
|
||||
return hasVisibleRank || hasPublicRank || hasNoAvailabilityMetadata ||
|
||||
(model.userSelectable === true && hasCapabilityMetadata);
|
||||
};
|
||||
|
||||
if (fiberModels && fiberModels.length > 0) {
|
||||
for (const m of fiberModels) {
|
||||
if (isModelAvailable(m)) appendModel(m, defaultPageModality);
|
||||
}
|
||||
} else {
|
||||
// 2. 仅在未挂载 React Fiber 时才从 SSR initialModels 提取保底
|
||||
const readArray = (text, marker) => {
|
||||
const markerIndex = text.indexOf(marker);
|
||||
if (markerIndex < 0) return null;
|
||||
const start = text.indexOf('[', markerIndex + marker.length);
|
||||
if (start < 0) return null;
|
||||
let depth = 0;
|
||||
let quoted = false;
|
||||
let escaped = false;
|
||||
for (let index = start; index < text.length; index += 1) {
|
||||
const char = text[index];
|
||||
if (quoted) {
|
||||
if (escaped) escaped = false;
|
||||
else if (char === '\\') escaped = true;
|
||||
else if (char === '"') quoted = false;
|
||||
continue;
|
||||
}
|
||||
if (char === '"') quoted = true;
|
||||
else if (char === '[') depth += 1;
|
||||
else if (char === ']') {
|
||||
depth -= 1;
|
||||
if (depth === 0) return text.slice(start, index + 1);
|
||||
}
|
||||
}
|
||||
return null;
|
||||
};
|
||||
|
||||
const payloadTexts = [];
|
||||
for (const script of document.scripts) {
|
||||
const source = String(script.textContent || '').trim();
|
||||
const cleanSource = source.replace(/;\s*$/, '');
|
||||
const prefix = 'self.__next_f.push(';
|
||||
if (!cleanSource.startsWith(prefix) || !cleanSource.endsWith(')')) continue;
|
||||
try {
|
||||
const payload = JSON.parse(cleanSource.slice(prefix.length, -1));
|
||||
if (Array.isArray(payload) && typeof payload[1] === 'string') {
|
||||
payloadTexts.push(payload[1]);
|
||||
}
|
||||
} catch (_) {}
|
||||
}
|
||||
|
||||
for (const text of payloadTexts) {
|
||||
const rawModels = readArray(text, '"initialModels":');
|
||||
if (!rawModels) continue;
|
||||
try {
|
||||
const models = JSON.parse(rawModels);
|
||||
for (const model of models) {
|
||||
if (!model) continue;
|
||||
const effectiveName = String(model.displayName || model.publicName || model.name || '').trim();
|
||||
const nameLower = effectiveName.toLowerCase();
|
||||
const isUserSpecifiedImage = nameLower.includes('gpt-image') ||
|
||||
nameLower.includes('mona-lisa') ||
|
||||
nameLower.includes('luna-lisa') ||
|
||||
nameLower.includes('lina-alpha') ||
|
||||
nameLower.includes('lina-f-alpha') ||
|
||||
nameLower.includes('silver_halide') ||
|
||||
nameLower.includes('flux') ||
|
||||
nameLower.includes('imagen') ||
|
||||
nameLower.includes('seedream') ||
|
||||
nameLower.includes('seededit') ||
|
||||
nameLower.includes('z-image') ||
|
||||
nameLower.includes('midjourney') ||
|
||||
nameLower.includes('dall-e') ||
|
||||
nameLower.includes('recraft') ||
|
||||
nameLower.includes('krea') ||
|
||||
nameLower.includes('grok-imagine') ||
|
||||
(nameLower.includes('grok') && nameLower.includes('image'));
|
||||
if (isUserSpecifiedImage && isModelAvailable(model)) {
|
||||
appendModel(model, 'image');
|
||||
} else if (isModelAvailable(model)) {
|
||||
appendModel(model, defaultPageModality);
|
||||
}
|
||||
}
|
||||
} catch (_) {}
|
||||
}
|
||||
}
|
||||
|
||||
return results;
|
||||
})();
|
||||
"""
|
||||
|
||||
|
||||
_cache_lock = threading.RLock()
|
||||
_refresh_lock = threading.Lock()
|
||||
_cached_at = 0.0
|
||||
_cached_models: List[Dict[str, Any]] = []
|
||||
|
||||
|
||||
def _canonical_modality(value: Any) -> str:
|
||||
"""Normalize Arena's page/API terminology to the catalog terminology."""
|
||||
modality = str(value or "").strip().lower()
|
||||
return {
|
||||
"webdev": "code",
|
||||
"web": "code",
|
||||
"chat": "text",
|
||||
}.get(modality, modality)
|
||||
|
||||
|
||||
def build_arena_direct_model_id(arena_model_id: Any) -> str:
|
||||
clean_id = str(arena_model_id or "").strip()
|
||||
return f"{ARENA_DIRECT_MODEL_PREFIX}{clean_id}" if clean_id else ""
|
||||
|
||||
|
||||
def parse_arena_direct_model_id(model_id: Any) -> str:
|
||||
value = str(model_id or "").strip()
|
||||
if not value.lower().startswith(ARENA_DIRECT_MODEL_PREFIX):
|
||||
return ""
|
||||
return value[len(ARENA_DIRECT_MODEL_PREFIX):].strip()
|
||||
|
||||
|
||||
def is_arena_direct_model_id(model_id: Any) -> bool:
|
||||
return bool(parse_arena_direct_model_id(model_id))
|
||||
|
||||
|
||||
def get_arena_direct_model_public_id(model: Any) -> str:
|
||||
if not isinstance(model, dict):
|
||||
return ""
|
||||
return str(
|
||||
model.get("display_name")
|
||||
or model.get("public_name")
|
||||
or model.get("search_name")
|
||||
or model.get("name")
|
||||
or ""
|
||||
).strip()
|
||||
|
||||
|
||||
def match_arena_direct_model(
|
||||
models: Any,
|
||||
requested_model: Any,
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
if not isinstance(models, (list, tuple, set)):
|
||||
return None
|
||||
requested_value = str(requested_model or "").strip()
|
||||
if not requested_value:
|
||||
return None
|
||||
parsed_id = parse_arena_direct_model_id(requested_value)
|
||||
candidates = {requested_value.casefold()}
|
||||
if parsed_id:
|
||||
candidates.add(parsed_id.casefold())
|
||||
|
||||
# 1. 优先匹配 UUID (arena_model_id)
|
||||
for model in models or []:
|
||||
if not isinstance(model, dict):
|
||||
continue
|
||||
mid = str(model.get("arena_model_id") or "").strip().casefold()
|
||||
if mid and mid in candidates:
|
||||
return model
|
||||
|
||||
# 2. 匹配标准名称 / 展示名称 / 搜索名称
|
||||
for model in models or []:
|
||||
if not isinstance(model, dict):
|
||||
continue
|
||||
for key in ("name", "public_name", "display_name", "search_name"):
|
||||
val = str(model.get(key) or "").strip().casefold()
|
||||
if val and val in candidates:
|
||||
return model
|
||||
|
||||
# 3. 匹配别名
|
||||
for model in models or []:
|
||||
if not isinstance(model, dict):
|
||||
continue
|
||||
for alias in (model.get("aliases") or []):
|
||||
alias_val = str(alias or "").strip().casefold()
|
||||
if alias_val and alias_val in candidates:
|
||||
return model
|
||||
|
||||
# 4. 匹配去除括号后缀的基础名 (例如 gpt-image-2 匹配 gpt-image-2 (medium)),以及 -high、-preview、尾部数字等基础名
|
||||
for model in models or []:
|
||||
if not isinstance(model, dict):
|
||||
continue
|
||||
for key in ("name", "public_name", "display_name", "search_name"):
|
||||
raw_val = str(model.get(key) or "").strip().casefold()
|
||||
if not raw_val:
|
||||
continue
|
||||
base_val = raw_val.split("(")[0].strip()
|
||||
if base_val and base_val in candidates:
|
||||
return model
|
||||
if raw_val.endswith("-high"):
|
||||
high_base = raw_val[:-5].strip()
|
||||
if high_base and high_base in candidates:
|
||||
return model
|
||||
if raw_val.endswith("-preview"):
|
||||
prev_base = raw_val[:-8].strip()
|
||||
if prev_base and prev_base in candidates:
|
||||
return model
|
||||
if "-" in raw_val:
|
||||
dash_base = raw_val.rsplit("-", 1)[0].strip()
|
||||
if dash_base and dash_base in candidates:
|
||||
return model
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def normalize_model_catalog_config(value: Any) -> Dict[str, Any]:
|
||||
from app.services.arena_model_catalog import normalize_arena_model_catalog_config
|
||||
return normalize_arena_model_catalog_config(value)
|
||||
|
||||
|
||||
def get_model_catalog_preset(config_engine: Any, domain: Any) -> Optional[Dict[str, Any]]:
|
||||
from app.services.arena_model_catalog import get_model_catalog_preset as _get_preset
|
||||
return _get_preset(config_engine, domain)
|
||||
|
||||
|
||||
def get_arena_direct_catalog_for_tab(
|
||||
config_engine: Any,
|
||||
tab: Any,
|
||||
*,
|
||||
preset_name: Any = None,
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
from app.services.arena_model_catalog import get_arena_direct_catalog_for_tab as _get_tab_catalog
|
||||
return _get_tab_catalog(config_engine, tab, preset_name=preset_name)
|
||||
|
||||
|
||||
def _is_arena_direct_url(
|
||||
value: Any,
|
||||
catalog_preset: Optional[Dict[str, Any]] = None,
|
||||
catalog_config: Optional[Dict[str, Any]] = None,
|
||||
) -> bool:
|
||||
current_url = str(value or "").strip()
|
||||
if not current_url or current_url.lower() in {"about:blank", "javascript:void(0)"}:
|
||||
return False
|
||||
|
||||
try:
|
||||
actual_domain = extract_remote_site_domain(current_url) or ""
|
||||
if not route_domain_matches("arena.ai", actual_domain):
|
||||
return False
|
||||
path = str(urlparse(current_url).path or "").rstrip("/").lower()
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
if path in {"/direct", "/text/direct", "/image/direct", "/code/direct"} or path.startswith((
|
||||
"/direct/",
|
||||
"/text/direct/",
|
||||
"/image/direct/",
|
||||
"/code/direct/",
|
||||
)) or path.endswith("/direct"):
|
||||
return True
|
||||
|
||||
if path == "/c" or path.startswith("/c/"):
|
||||
catalog = catalog_config
|
||||
if catalog is None and catalog_preset is not None:
|
||||
catalog = normalize_model_catalog_config(catalog_preset.get("model_catalog"))
|
||||
if catalog is not None:
|
||||
return bool(catalog.get("enabled") and catalog.get("source") == MODEL_CATALOG_SOURCE)
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
|
||||
def _filter_models(
|
||||
models: List[Dict[str, Any]],
|
||||
catalog_config: Any,
|
||||
) -> List[Dict[str, Any]]:
|
||||
from app.services.arena_model_catalog import filter_arena_catalog_models
|
||||
return filter_arena_catalog_models(models, catalog_config)
|
||||
|
||||
|
||||
def _natural_sort_key(text: Any) -> list:
|
||||
value = str(text or "").strip()
|
||||
return [int(part) if part.isdecimal() else part.casefold() for part in re.split(r"(\d+)", value)]
|
||||
|
||||
|
||||
def _normalize_models(raw_models: Any) -> List[Dict[str, Any]]:
|
||||
if not isinstance(raw_models, list):
|
||||
return []
|
||||
|
||||
normalized: List[Dict[str, Any]] = []
|
||||
alias_overrides = _load_alias_overrides()
|
||||
seen_ids = set()
|
||||
seen_names = set()
|
||||
for raw in raw_models:
|
||||
if not isinstance(raw, dict):
|
||||
continue
|
||||
arena_model_id = str(raw.get("arena_model_id") or "").strip()
|
||||
name = str(raw.get("name") or raw.get("display_name") or raw.get("public_name") or "").strip()
|
||||
if not arena_model_id or not name:
|
||||
continue
|
||||
id_key = arena_model_id.lower()
|
||||
name_key = name.lower()
|
||||
if id_key in seen_ids or name_key in seen_names:
|
||||
continue
|
||||
seen_ids.add(id_key)
|
||||
seen_names.add(name_key)
|
||||
public_name = str(raw.get("public_name") or raw.get("display_name") or name).strip()
|
||||
display_name = str(raw.get("display_name") or public_name or name).strip()
|
||||
override = (
|
||||
alias_overrides.get(name.casefold())
|
||||
or alias_overrides.get(display_name.casefold())
|
||||
or alias_overrides.get(public_name.casefold())
|
||||
)
|
||||
if not isinstance(override, dict):
|
||||
override = {}
|
||||
search_name = str(
|
||||
override.get("search_name") or display_name or public_name or name
|
||||
).strip()
|
||||
aliases = []
|
||||
for alias in (
|
||||
name,
|
||||
public_name,
|
||||
display_name,
|
||||
search_name,
|
||||
*(override.get("aliases") or [] if isinstance(override.get("aliases"), list) else []),
|
||||
):
|
||||
alias_text = str(alias or "").strip()
|
||||
if alias_text and alias_text.casefold() not in {item.casefold() for item in aliases}:
|
||||
aliases.append(alias_text)
|
||||
|
||||
for candidate_name in (display_name, public_name, search_name, name):
|
||||
if not candidate_name:
|
||||
continue
|
||||
if "(" in candidate_name:
|
||||
base_no_paren = candidate_name.split("(")[0].strip()
|
||||
if base_no_paren and base_no_paren.casefold() not in {item.casefold() for item in aliases}:
|
||||
aliases.append(base_no_paren)
|
||||
if candidate_name.endswith("-high"):
|
||||
base_no_high = candidate_name[:-5].strip()
|
||||
if base_no_high and base_no_high.casefold() not in {item.casefold() for item in aliases}:
|
||||
aliases.append(base_no_high)
|
||||
if candidate_name.endswith("-preview"):
|
||||
base_no_prev = candidate_name[:-8].strip()
|
||||
if base_no_prev and base_no_prev.casefold() not in {item.casefold() for item in aliases}:
|
||||
aliases.append(base_no_prev)
|
||||
raw_modality = _canonical_modality(raw.get("modality"))
|
||||
name_check = f"{name} {display_name} {public_name} {arena_model_id}".lower()
|
||||
if not raw_modality and any(
|
||||
k in name_check
|
||||
for k in (
|
||||
"gpt-image",
|
||||
"mona-lisa",
|
||||
"luna-lisa",
|
||||
"lina-alpha",
|
||||
"lina-f-alpha",
|
||||
"silver_halide",
|
||||
"flux",
|
||||
"seedream",
|
||||
"seededit",
|
||||
"imagen",
|
||||
"z-image",
|
||||
"midjourney",
|
||||
"dall-e",
|
||||
"recraft",
|
||||
"krea",
|
||||
"grok-imagine",
|
||||
)
|
||||
):
|
||||
raw_modality = "image"
|
||||
normalized.append(
|
||||
{
|
||||
"arena_model_id": arena_model_id,
|
||||
"name": name,
|
||||
"public_name": public_name or name,
|
||||
"display_name": display_name or public_name or name,
|
||||
"search_name": search_name or display_name or public_name or name,
|
||||
"aliases": aliases,
|
||||
"provider": str(raw.get("provider") or "").strip(),
|
||||
"organization": str(raw.get("organization") or raw.get("provider") or "arena.ai").strip(),
|
||||
"modality": raw_modality or "text",
|
||||
}
|
||||
)
|
||||
return normalized
|
||||
|
||||
|
||||
def _load_alias_overrides() -> Dict[str, Dict[str, Any]]:
|
||||
try:
|
||||
payload = json.loads(ARENA_MODEL_ALIAS_OVERRIDES_PATH.read_text(encoding="utf-8"))
|
||||
except (FileNotFoundError, OSError, ValueError, TypeError):
|
||||
return {}
|
||||
models = payload.get("models") if isinstance(payload, dict) else None
|
||||
return models if isinstance(models, dict) else {}
|
||||
|
||||
|
||||
def read_arena_direct_models_from_tab(tab: Any) -> List[Dict[str, Any]]:
|
||||
if tab is None:
|
||||
return []
|
||||
try:
|
||||
try:
|
||||
return _normalize_models(driver_for_tab(tab).run_js(_ARENA_DIRECT_MODEL_EXTRACT_JS, timeout=3.0))
|
||||
except TypeError:
|
||||
return _normalize_models(driver_for_tab(tab).run_js(_ARENA_DIRECT_MODEL_EXTRACT_JS))
|
||||
except Exception as e:
|
||||
logger.debug(f"从标签页读取 Arena 直连模型失败(已优雅降级): {e}")
|
||||
return []
|
||||
|
||||
|
||||
_session_cached_models: Dict[str, tuple[float, List[Dict[str, Any]]]] = {}
|
||||
|
||||
|
||||
def _get_session_cache_key(session_or_tab: Any) -> str:
|
||||
if session_or_tab is None:
|
||||
return ""
|
||||
if isinstance(session_or_tab, dict):
|
||||
idx = session_or_tab.get("persistent_index")
|
||||
tid = session_or_tab.get("tab_id")
|
||||
return f"idx_{idx}" if idx is not None else (f"id_{tid}" if tid else "")
|
||||
idx = getattr(session_or_tab, "persistent_index", None)
|
||||
sid = getattr(session_or_tab, "id", None)
|
||||
return f"idx_{idx}" if idx is not None else (f"id_{sid}" if sid else "")
|
||||
|
||||
|
||||
def _session_cache_snapshot(key: str) -> tuple[float, List[Dict[str, Any]]]:
|
||||
if not key:
|
||||
return 0.0, []
|
||||
with _cache_lock:
|
||||
if key in _session_cached_models:
|
||||
ts, models = _session_cached_models[key]
|
||||
return ts, copy.deepcopy(models)
|
||||
return 0.0, []
|
||||
|
||||
|
||||
def _replace_session_cache(key: str, models: Any) -> List[Dict[str, Any]]:
|
||||
normalized = _normalize_models(models)
|
||||
if not key:
|
||||
return normalized
|
||||
with _cache_lock:
|
||||
_session_cached_models[key] = (time.monotonic(), copy.deepcopy(normalized))
|
||||
return copy.deepcopy(normalized)
|
||||
|
||||
|
||||
def _cache_snapshot() -> tuple[float, List[Dict[str, Any]]]:
|
||||
with _cache_lock:
|
||||
return _cached_at, copy.deepcopy(_cached_models)
|
||||
|
||||
|
||||
def _replace_cache(models: Any) -> List[Dict[str, Any]]:
|
||||
global _cached_at, _cached_models
|
||||
normalized = _normalize_models(models)
|
||||
if not normalized:
|
||||
return []
|
||||
with _cache_lock:
|
||||
_cached_at = time.monotonic()
|
||||
_cached_models = copy.deepcopy(normalized)
|
||||
return copy.deepcopy(normalized)
|
||||
|
||||
|
||||
def _session_is_idle(session: Any) -> bool:
|
||||
raw_status = getattr(session, "status", None)
|
||||
status = str(getattr(raw_status, "value", None) or raw_status or "").strip().lower()
|
||||
return status in {"", "idle"}
|
||||
|
||||
|
||||
def _arena_sessions(browser: Any) -> List[Any]:
|
||||
try:
|
||||
sessions = browser.tab_pool.get_sessions_snapshot()
|
||||
except Exception:
|
||||
return []
|
||||
|
||||
from app.services.config_engine import config_engine
|
||||
|
||||
result = []
|
||||
for session in sessions or []:
|
||||
raw_status = getattr(session, "status", None)
|
||||
status = str(getattr(raw_status, "value", None) or raw_status or "").strip().lower()
|
||||
if status not in {"idle", "busy"}:
|
||||
continue
|
||||
try:
|
||||
current_url, _domain = session.get_cached_route_snapshot()
|
||||
except Exception:
|
||||
current_url = str(getattr(getattr(session, "tab", None), "url", "") or "")
|
||||
|
||||
tab_dict = {
|
||||
"status": status,
|
||||
"url": current_url,
|
||||
"preset_name": getattr(session, "preset_name", None),
|
||||
"terminating": False,
|
||||
}
|
||||
if not get_arena_direct_catalog_for_tab(config_engine, tab_dict):
|
||||
continue
|
||||
result.append(session)
|
||||
result.sort(key=lambda item: (not _session_is_idle(item), int(getattr(item, "persistent_index", 0) or 0)))
|
||||
return result
|
||||
|
||||
|
||||
def list_arena_direct_models(
|
||||
browser: Any,
|
||||
*,
|
||||
force: bool = False,
|
||||
catalog_config: Any = None,
|
||||
tab: Any = None,
|
||||
) -> List[Dict[str, Any]]:
|
||||
# 指定单标签页时执行严格标签页隔离读取与缓存
|
||||
if tab is not None:
|
||||
target_session = None
|
||||
target_tab_obj = None
|
||||
if hasattr(tab, "tab"):
|
||||
target_session = tab
|
||||
target_tab_obj = getattr(tab, "tab", None)
|
||||
elif hasattr(tab, "run_js"):
|
||||
target_tab_obj = tab
|
||||
elif isinstance(tab, dict):
|
||||
target_tab_obj = tab.get("tab")
|
||||
if browser and hasattr(browser, "tab_pool"):
|
||||
try:
|
||||
p_idx = tab.get("persistent_index")
|
||||
t_id = tab.get("tab_id")
|
||||
sessions = browser.tab_pool.get_sessions_snapshot()
|
||||
for s in sessions or []:
|
||||
if p_idx is not None and getattr(s, "persistent_index", None) == p_idx:
|
||||
target_session = s
|
||||
target_tab_obj = getattr(s, "tab", None)
|
||||
break
|
||||
if t_id and getattr(s, "id", None) == t_id:
|
||||
target_session = s
|
||||
target_tab_obj = getattr(s, "tab", None)
|
||||
break
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
key = _get_session_cache_key(target_session or tab)
|
||||
cached_at, cached = _session_cache_snapshot(key)
|
||||
if cached and not force and time.monotonic() - cached_at < ARENA_DIRECT_MODEL_CACHE_TTL:
|
||||
filtered = _filter_models(cached, catalog_config)
|
||||
if filtered:
|
||||
return filtered
|
||||
|
||||
if target_tab_obj is not None:
|
||||
if target_session is None or _session_is_idle(target_session):
|
||||
try:
|
||||
models = read_arena_direct_models_from_tab(target_tab_obj)
|
||||
if models:
|
||||
_replace_session_cache(key, models)
|
||||
return _filter_models(models, catalog_config)
|
||||
except Exception as exc:
|
||||
logger.debug(f"从目标标签页读取 Arena 直连模型失败: {exc}")
|
||||
|
||||
if cached:
|
||||
return _filter_models(cached, catalog_config)
|
||||
return []
|
||||
|
||||
sessions = _arena_sessions(browser)
|
||||
if not sessions:
|
||||
return []
|
||||
|
||||
cached_at, cached = _cache_snapshot()
|
||||
if cached and not force and time.monotonic() - cached_at < ARENA_DIRECT_MODEL_CACHE_TTL:
|
||||
filtered = _filter_models(cached, catalog_config)
|
||||
if filtered:
|
||||
return filtered
|
||||
|
||||
if not _refresh_lock.acquire(blocking=False):
|
||||
return _filter_models(cached, catalog_config)
|
||||
try:
|
||||
cached_at, cached = _cache_snapshot()
|
||||
if cached and not force and time.monotonic() - cached_at < ARENA_DIRECT_MODEL_CACHE_TTL:
|
||||
filtered = _filter_models(cached, catalog_config)
|
||||
if filtered:
|
||||
return filtered
|
||||
|
||||
all_extracted_models: List[Dict[str, Any]] = []
|
||||
for session in sessions:
|
||||
if not _session_is_idle(session):
|
||||
continue
|
||||
try:
|
||||
models = read_arena_direct_models_from_tab(session.tab)
|
||||
if models:
|
||||
s_key = _get_session_cache_key(session)
|
||||
_replace_session_cache(s_key, models)
|
||||
all_extracted_models.extend(models)
|
||||
except Exception as exc:
|
||||
logger.debug(f"Arena Direct 模型目录读取失败(尝试下一标签页): {exc}")
|
||||
continue
|
||||
|
||||
if all_extracted_models:
|
||||
merged_models = list(cached) + all_extracted_models
|
||||
updated = _replace_cache(merged_models)
|
||||
logger.info(f"Arena Direct 模型目录已刷新: {len(updated)} 个模型")
|
||||
return _filter_models(updated, catalog_config)
|
||||
return _filter_models(cached, catalog_config)
|
||||
finally:
|
||||
_refresh_lock.release()
|
||||
|
||||
|
||||
def resolve_arena_direct_model(
|
||||
tab: Any,
|
||||
requested_model: Any,
|
||||
*,
|
||||
catalog_config: Any = None,
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
requested_value = str(requested_model or "").strip()
|
||||
if not requested_value:
|
||||
return None
|
||||
|
||||
_cached_at_value, cached = _cache_snapshot()
|
||||
matched = match_arena_direct_model(_filter_models(cached, catalog_config), requested_value)
|
||||
if matched:
|
||||
return matched
|
||||
|
||||
if tab is None:
|
||||
try:
|
||||
from app.core.browser import get_browser
|
||||
browser = get_browser(auto_connect=False)
|
||||
if browser:
|
||||
models = list_arena_direct_models(browser, catalog_config=catalog_config)
|
||||
if models:
|
||||
return match_arena_direct_model(_filter_models(models, catalog_config), requested_value)
|
||||
except Exception:
|
||||
pass
|
||||
else:
|
||||
models = read_arena_direct_models_from_tab(tab)
|
||||
if models:
|
||||
_cached_at_val, current_cached = _cache_snapshot()
|
||||
merged = list(current_cached) + list(models)
|
||||
updated = _replace_cache(merged)
|
||||
return match_arena_direct_model(_filter_models(updated, catalog_config), requested_value)
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def build_openai_model_entries(models: List[Dict[str, Any]], *, created: int) -> List[Dict[str, Any]]:
|
||||
entries = []
|
||||
seen_ids = set()
|
||||
for model in models or []:
|
||||
if not isinstance(model, dict):
|
||||
continue
|
||||
model_id = get_arena_direct_model_public_id(model)
|
||||
normalized_id = model_id.casefold()
|
||||
if not model_id or normalized_id in seen_ids:
|
||||
continue
|
||||
seen_ids.add(normalized_id)
|
||||
entries.append(
|
||||
{
|
||||
"id": model_id,
|
||||
"object": "model",
|
||||
"type": "model",
|
||||
"created": created,
|
||||
"owned_by": model.get("organization") or model.get("provider") or "arena.ai",
|
||||
"display_name": model.get("display_name") or model.get("public_name") or model_id,
|
||||
}
|
||||
)
|
||||
entries.sort(key=lambda e: _natural_sort_key(str(e.get("id") or "")))
|
||||
return entries
|
||||
|
||||
|
||||
__all__ = [
|
||||
"ARENA_DIRECT_MODEL_PREFIX",
|
||||
"build_arena_direct_model_id",
|
||||
"build_openai_model_entries",
|
||||
"get_arena_direct_catalog_for_tab",
|
||||
"get_model_catalog_preset",
|
||||
"get_arena_direct_model_public_id",
|
||||
"is_arena_direct_model_id",
|
||||
"list_arena_direct_models",
|
||||
"match_arena_direct_model",
|
||||
"normalize_model_catalog_config",
|
||||
"parse_arena_direct_model_id",
|
||||
"read_arena_direct_models_from_tab",
|
||||
"resolve_arena_direct_model",
|
||||
]
|
||||
@@ -0,0 +1,891 @@
|
||||
"""Dedicated Arena blind-test runner for GPT Image 2 C2PA detection."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import random
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
from app.services.arena_image_generation import (
|
||||
image_signatures as _image_signatures,
|
||||
read_image_bytes as _read_image_bytes,
|
||||
same_image as _same_image,
|
||||
)
|
||||
from app.core.driver import as_element, driver_for_tab
|
||||
|
||||
|
||||
GPT_IMAGE_MARKERS = (
|
||||
b"gpt-image",
|
||||
b"OpenAI Media Service API",
|
||||
b"OpenAI OpCo, LLC",
|
||||
)
|
||||
|
||||
ARENA_RESULT_IMAGE_SELECTOR = (
|
||||
"img.transition-opacity.duration-500.opacity-100.aspect-square.object-cover"
|
||||
)
|
||||
ARENA_AGENT_PAGE_PATH = "/agent"
|
||||
|
||||
|
||||
def inspect_gpt_image2_png(payload: bytes) -> dict[str, Any]:
|
||||
"""Return strong GPT Image 2 evidence found in a PNG C2PA caBX chunk."""
|
||||
data = bytes(payload or b"")
|
||||
if not data.startswith(b"\x89PNG\r\n\x1a\n"):
|
||||
return {"matched": False, "reason": "not_png", "markers": []}
|
||||
|
||||
offset = 8
|
||||
cabx_chunks: list[bytes] = []
|
||||
while offset + 12 <= len(data):
|
||||
length = int.from_bytes(data[offset : offset + 4], "big")
|
||||
chunk_type = data[offset + 4 : offset + 8]
|
||||
chunk_end = offset + 12 + length
|
||||
if chunk_end > len(data):
|
||||
return {"matched": False, "reason": "invalid_png", "markers": []}
|
||||
if chunk_type == b"caBX":
|
||||
cabx_chunks.append(data[offset + 8 : offset + 8 + length])
|
||||
offset = chunk_end
|
||||
if chunk_type == b"IEND":
|
||||
break
|
||||
|
||||
if not cabx_chunks:
|
||||
return {"matched": False, "reason": "c2pa_missing", "markers": []}
|
||||
|
||||
manifest = b"".join(cabx_chunks)
|
||||
marker_hits = [marker.decode("ascii") for marker in GPT_IMAGE_MARKERS if marker in manifest]
|
||||
# CBOR text values use one-byte length prefixes here: i=9 for gpt-image,
|
||||
# c=3 for 2.0. Requiring the adjacent fields prevents a stray version
|
||||
# elsewhere in the signed manifest from producing a false positive.
|
||||
has_model = b"dnameigpt-imagegversionc2.0" in manifest
|
||||
has_openai_claim = b"OpenAI Media Service API" in manifest
|
||||
matched = has_model and has_openai_claim
|
||||
return {
|
||||
"matched": matched,
|
||||
"reason": "gpt_image_2_c2pa" if matched else "c2pa_not_gpt_image_2",
|
||||
"markers": marker_hits,
|
||||
"caBX_bytes": len(manifest),
|
||||
}
|
||||
|
||||
|
||||
def _value(values: dict[str, Any], key: str, default: Any = "") -> Any:
|
||||
value = values.get(key, default)
|
||||
return default if value is None else value
|
||||
|
||||
|
||||
def _sleep(ctx: dict[str, Any], seconds: float, step: float = 0.2) -> None:
|
||||
deadline = time.time() + max(0.0, float(seconds))
|
||||
while time.time() < deadline:
|
||||
ctx["raise_if_cancelled"]()
|
||||
ctx["raise_if_command_loop_cancelled"]()
|
||||
agent_guard = ctx.get("_arena_agent_guard")
|
||||
if agent_guard and agent_guard():
|
||||
raise RuntimeError("arena_agent_redirected")
|
||||
ctx["reset_timeout"]()
|
||||
time.sleep(min(step, max(0.0, deadline - time.time())))
|
||||
|
||||
|
||||
def _current_url(tab: Any) -> str:
|
||||
try:
|
||||
return str(driver_for_tab(tab).run_js("return location.href") or "").strip()
|
||||
except Exception:
|
||||
return str(getattr(tab, "url", "") or "").strip()
|
||||
|
||||
|
||||
def _is_agent_page_url(url: str) -> bool:
|
||||
"""Return whether Arena has landed on its agent page, which is abnormal for this runner."""
|
||||
try:
|
||||
parsed = urlsplit(str(url or "").strip())
|
||||
return (
|
||||
parsed.scheme.lower() == "https"
|
||||
and parsed.netloc.lower() == "arena.ai"
|
||||
and parsed.path.rstrip("/") == ARENA_AGENT_PAGE_PATH
|
||||
)
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def _redirect_if_agent_page(ctx: dict[str, Any], redirect_url: str) -> bool:
|
||||
"""Redirect an unexpected /agent page and report whether the guard fired."""
|
||||
tab = ctx["tab"]
|
||||
current_url = _current_url(tab)
|
||||
if not _is_agent_page_url(current_url):
|
||||
return False
|
||||
|
||||
logger = ctx["logger"]
|
||||
logger.warning(
|
||||
f"[GPT-IMAGE-2] unexpected Arena agent page: {current_url}; "
|
||||
f"navigating to {redirect_url}"
|
||||
)
|
||||
try:
|
||||
tab.get(redirect_url)
|
||||
except Exception as navigate_error:
|
||||
logger.warning(
|
||||
"[GPT-IMAGE-2] agent-page redirect tab.get() failed, "
|
||||
f"falling back to location.replace(): {navigate_error}"
|
||||
)
|
||||
driver_for_tab(tab).run_js("location.replace(arguments[0])", redirect_url)
|
||||
return True
|
||||
|
||||
|
||||
def _same_page_url(current_url: str, target_url: str) -> bool:
|
||||
try:
|
||||
current = urlsplit(str(current_url or "").strip())
|
||||
target = urlsplit(str(target_url or "").strip())
|
||||
return (
|
||||
current.scheme.lower() == target.scheme.lower()
|
||||
and current.netloc.lower() == target.netloc.lower()
|
||||
and current.path.rstrip("/") == target.path.rstrip("/")
|
||||
)
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def _new_image_chat_ready(tab: Any, redirect_url: str) -> bool:
|
||||
if not _same_page_url(_current_url(tab), redirect_url):
|
||||
return False
|
||||
input_box = _find(tab, "css:textarea[name='message']", 1.0)
|
||||
if not input_box:
|
||||
return False
|
||||
try:
|
||||
return int(as_element(input_box).run_js("return String(this.value || '').length")) == 0
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def _visible_image_sources(tab: Any) -> list[str]:
|
||||
script = f"""
|
||||
const collect = (images) => Array.from(images)
|
||||
.filter((img) => {{
|
||||
const r = img.getBoundingClientRect();
|
||||
const s = getComputedStyle(img);
|
||||
return img.isConnected && img.complete && img.naturalWidth >= 256
|
||||
&& img.naturalHeight >= 256 && r.width >= 120 && r.height >= 120
|
||||
&& s.display !== 'none' && s.visibility !== 'hidden';
|
||||
}});
|
||||
const resultImages = collect(document.querySelectorAll(
|
||||
{ARENA_RESULT_IMAGE_SELECTOR!r}
|
||||
));
|
||||
const images = resultImages.length ? resultImages : collect(document.images);
|
||||
return images
|
||||
.sort((a, b) => a.getBoundingClientRect().left - b.getBoundingClientRect().left)
|
||||
.map((img) => img.currentSrc || img.src)
|
||||
.filter(Boolean);
|
||||
"""
|
||||
try:
|
||||
return list(dict.fromkeys(str(item) for item in (driver_for_tab(tab).run_js(script) or []) if item))
|
||||
except Exception:
|
||||
return []
|
||||
|
||||
|
||||
def _log_image_url(url: str) -> str:
|
||||
try:
|
||||
parsed = urlsplit(str(url or ""))
|
||||
return f"{parsed.scheme}://{parsed.netloc}{parsed.path}"[:200]
|
||||
except Exception:
|
||||
return str(url or "").split("?", 1)[0][:200]
|
||||
|
||||
|
||||
def _find(tab: Any, selector: str, timeout: float = 1.0) -> Any:
|
||||
try:
|
||||
return driver_for_tab(tab).find(selector, timeout=timeout)
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def _upload_reference_image(ctx: dict[str, Any], path: str) -> bool:
|
||||
tab = ctx["tab"]
|
||||
logger = ctx["logger"]
|
||||
image_path = Path(os.path.expandvars(os.path.expanduser(path))).resolve()
|
||||
if not image_path.is_file():
|
||||
raise RuntimeError(f"reference_image_missing:{image_path}")
|
||||
|
||||
inputs = []
|
||||
try:
|
||||
inputs = list(driver_for_tab(tab).find_all('css:input[type="file"]', timeout=1.5) or [])
|
||||
except Exception:
|
||||
pass
|
||||
if not inputs:
|
||||
for selector in (
|
||||
"xpath://button[contains(., 'Upload') or contains(., '上传')]",
|
||||
"css:button[aria-label*='upload' i]",
|
||||
"css:button[aria-label*='image' i]",
|
||||
):
|
||||
button = _find(tab, selector, 0.5)
|
||||
if button:
|
||||
try:
|
||||
button.click()
|
||||
_sleep(ctx, 0.5)
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
inputs = list(driver_for_tab(tab).find_all('css:input[type="file"]', timeout=1.0) or [])
|
||||
except Exception:
|
||||
inputs = []
|
||||
if inputs:
|
||||
break
|
||||
|
||||
for file_input in inputs:
|
||||
try:
|
||||
file_input.input(str(image_path))
|
||||
logger.info(f"[GPT-IMAGE-2] reference image uploaded: {image_path.name}")
|
||||
_sleep(ctx, 2.0)
|
||||
return True
|
||||
except Exception as error:
|
||||
logger.debug(f"[GPT-IMAGE-2] file input upload failed: {error}")
|
||||
return False
|
||||
|
||||
|
||||
def _refresh_page(ctx: dict[str, Any]) -> None:
|
||||
tab = ctx["tab"]
|
||||
try:
|
||||
tab.refresh()
|
||||
except Exception:
|
||||
driver_for_tab(tab).run_js("location.reload()")
|
||||
_sleep(ctx, 2.0)
|
||||
|
||||
|
||||
def _is_generating(tab: Any) -> bool:
|
||||
try:
|
||||
result = driver_for_tab(tab).run_js(
|
||||
"""
|
||||
return (() => {
|
||||
const visible = (element) => {
|
||||
if (!(element instanceof HTMLElement) || !element.isConnected) return false;
|
||||
const rect = element.getBoundingClientRect();
|
||||
const style = getComputedStyle(element);
|
||||
return rect.width >= 8 && rect.height >= 8
|
||||
&& style.display !== 'none' && style.visibility !== 'hidden';
|
||||
};
|
||||
const hasStopButton = Array.from(document.querySelectorAll(
|
||||
'button[aria-label="Stop generation"], button[aria-label*="Stop" i], button[data-testid*="stop" i]'
|
||||
)).some(visible);
|
||||
if (hasStopButton) return true;
|
||||
return Array.from(document.querySelectorAll('main *, [role="main"] *'))
|
||||
.filter(visible)
|
||||
.some((element) => /^generating\\s+image(?:\\.\\.\\.)?$/i.test(
|
||||
String(element.textContent || '').trim()
|
||||
));
|
||||
})();
|
||||
"""
|
||||
)
|
||||
return result is True
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def _is_generation_stopped(tab: Any) -> bool:
|
||||
try:
|
||||
result = driver_for_tab(tab).run_js(
|
||||
"""
|
||||
return (() => {
|
||||
const visible = (element) => {
|
||||
if (!(element instanceof HTMLElement) || !element.isConnected) return false;
|
||||
const rect = element.getBoundingClientRect();
|
||||
const style = getComputedStyle(element);
|
||||
return rect.width >= 8 && rect.height >= 8
|
||||
&& rect.bottom > 0 && rect.right > 0
|
||||
&& rect.top < innerHeight && rect.left < innerWidth
|
||||
&& style.display !== 'none' && style.visibility !== 'hidden';
|
||||
};
|
||||
return Array.from(document.querySelectorAll('main *, [role="main"] *'))
|
||||
.filter(visible)
|
||||
.some((element) => /^generation\\s+stopped$/i.test(
|
||||
String(element.textContent || '').trim()
|
||||
));
|
||||
})();
|
||||
"""
|
||||
)
|
||||
return result is True
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def _stop_generation(ctx: dict[str, Any]) -> bool:
|
||||
tab = ctx["tab"]
|
||||
logger = ctx["logger"]
|
||||
for selector in (
|
||||
'css:button[aria-label="Stop generation"]',
|
||||
"css:button[aria-label*='Stop' i]",
|
||||
"css:button[data-testid*='stop' i]",
|
||||
):
|
||||
button = _find(tab, selector, 0.8)
|
||||
if not button:
|
||||
continue
|
||||
try:
|
||||
button.click()
|
||||
logger.warning("[GPT-IMAGE-2] generation still active after recovery minute; clicked stop")
|
||||
_sleep(ctx, 2.0)
|
||||
return True
|
||||
except Exception as error:
|
||||
logger.warning(f"[GPT-IMAGE-2] stop generation click failed: {error}")
|
||||
logger.error("[GPT-IMAGE-2] generation is active but stop button was not found")
|
||||
return False
|
||||
|
||||
|
||||
def _stop_generation_and_confirm(ctx: dict[str, Any]) -> bool:
|
||||
if not _stop_generation(ctx):
|
||||
return False
|
||||
deadline = time.time() + 5.0
|
||||
while time.time() < deadline:
|
||||
if _is_generation_stopped(ctx["tab"]) or not _is_generating(ctx["tab"]):
|
||||
return True
|
||||
_sleep(ctx, 0.25, 0.05)
|
||||
return False
|
||||
|
||||
|
||||
def _vote_a_better(ctx: dict[str, Any]) -> bool:
|
||||
tab = ctx["tab"]
|
||||
logger = ctx["logger"]
|
||||
script = r"""
|
||||
return (() => {
|
||||
const norm = (value) => String(value || '').replace(/\s+/g, ' ').trim().toLowerCase();
|
||||
const visible = (element) => {
|
||||
if (!(element instanceof HTMLElement) || !element.isConnected) return false;
|
||||
const rect = element.getBoundingClientRect();
|
||||
const style = getComputedStyle(element);
|
||||
return rect.width >= 8 && rect.height >= 8 && rect.bottom > 0 && rect.right > 0
|
||||
&& rect.top < innerHeight && rect.left < innerWidth
|
||||
&& style.display !== 'none' && style.visibility !== 'hidden'
|
||||
&& style.pointerEvents !== 'none' && Number(style.opacity || '1') > 0.2;
|
||||
};
|
||||
const matches = (text) => {
|
||||
const value = norm(text);
|
||||
return value.includes('a 更好') || value.includes('a is better')
|
||||
|| value === 'a better' || value.includes('← a');
|
||||
};
|
||||
const elements = Array.from(document.querySelectorAll('button, [role="button"]'));
|
||||
const button = elements.find((element) => {
|
||||
const text = element.innerText || element.textContent || element.getAttribute('aria-label');
|
||||
return visible(element) && !element.disabled
|
||||
&& element.getAttribute('aria-disabled') !== 'true' && matches(text);
|
||||
});
|
||||
if (!button) return false;
|
||||
button.scrollIntoView({block: 'center', inline: 'center'});
|
||||
button.click();
|
||||
return true;
|
||||
})();
|
||||
"""
|
||||
try:
|
||||
clicked = bool(driver_for_tab(tab).run_js(script))
|
||||
except Exception as error:
|
||||
logger.warning(f"[GPT-IMAGE-2] failed to click A better: {error}")
|
||||
return False
|
||||
if clicked:
|
||||
logger.info("[GPT-IMAGE-2] non-target result; selected A better")
|
||||
_sleep(ctx, 1.0)
|
||||
else:
|
||||
logger.warning("[GPT-IMAGE-2] non-target result; A better button was not found")
|
||||
return clicked
|
||||
|
||||
|
||||
def _start_new_image_chat(ctx: dict[str, Any], redirect_url: str) -> bool:
|
||||
tab = ctx["tab"]
|
||||
logger = ctx["logger"]
|
||||
selectors = (
|
||||
"css:a[href='/image']",
|
||||
"xpath://a[contains(., 'New Chat') or contains(., 'New chat')]",
|
||||
"xpath://button[contains(., 'New Chat') or contains(., 'New chat')]",
|
||||
)
|
||||
if _new_image_chat_ready(tab, redirect_url):
|
||||
logger.stream(f"[GPT-IMAGE-2] new image chat ready: url={_current_url(tab)}")
|
||||
return True
|
||||
|
||||
for selector in selectors:
|
||||
element = _find(tab, selector, 0.8)
|
||||
if not element:
|
||||
continue
|
||||
try:
|
||||
element.click()
|
||||
_sleep(ctx, 1.2)
|
||||
if _new_image_chat_ready(tab, redirect_url):
|
||||
logger.stream(f"[GPT-IMAGE-2] new image chat confirmed: url={_current_url(tab)}")
|
||||
return True
|
||||
except Exception:
|
||||
continue
|
||||
try:
|
||||
logger.stream(f"[GPT-IMAGE-2] New Chat not confirmed; navigating to {redirect_url}")
|
||||
tab.get(redirect_url)
|
||||
_sleep(ctx, 2.0)
|
||||
return _new_image_chat_ready(tab, redirect_url)
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def _send_prompt(
|
||||
ctx: dict[str, Any], prompt: str, *, require_clear_confirmation: bool = False
|
||||
) -> bool:
|
||||
tab = ctx["tab"]
|
||||
logger = ctx["logger"]
|
||||
input_box = _find(tab, "css:textarea[name='message']", 5.0)
|
||||
if not input_box:
|
||||
return False
|
||||
if require_clear_confirmation:
|
||||
try:
|
||||
replaced = as_element(input_box).run_js(
|
||||
"""
|
||||
const value = String(arguments[0] || '');
|
||||
const setter = Object.getOwnPropertyDescriptor(
|
||||
HTMLTextAreaElement.prototype, 'value'
|
||||
).set;
|
||||
this.focus();
|
||||
setter.call(this, value);
|
||||
this.dispatchEvent(new InputEvent('input', {
|
||||
bubbles: true,
|
||||
data: value,
|
||||
inputType: 'insertFromPaste'
|
||||
}));
|
||||
this.dispatchEvent(new Event('change', {bubbles: true}));
|
||||
this.setSelectionRange(value.length, value.length);
|
||||
return String(this.value || '');
|
||||
""",
|
||||
prompt,
|
||||
)
|
||||
except Exception as error:
|
||||
logger.error(f"[GPT-IMAGE-2] failed to replace composer text: {error}")
|
||||
return False
|
||||
if str(replaced or "") != prompt:
|
||||
logger.error("[GPT-IMAGE-2] composer replacement did not match the prompt")
|
||||
return False
|
||||
|
||||
stable_checks = 0
|
||||
deadline = time.time() + 1.5
|
||||
while time.time() < deadline:
|
||||
current_input = _find(tab, "css:textarea[name='message']", 0.3)
|
||||
try:
|
||||
current_value = str(
|
||||
as_element(current_input).run_js("return String(this.value || '')") if current_input else ""
|
||||
)
|
||||
except Exception:
|
||||
current_value = ""
|
||||
if current_value == prompt:
|
||||
stable_checks += 1
|
||||
if stable_checks >= 3:
|
||||
break
|
||||
else:
|
||||
stable_checks = 0
|
||||
_sleep(ctx, 0.15, 0.05)
|
||||
if stable_checks < 3:
|
||||
logger.error("[GPT-IMAGE-2] controlled composer restored or changed the prompt")
|
||||
return False
|
||||
|
||||
def composer_is_empty() -> bool:
|
||||
current_input = _find(tab, "css:textarea[name='message']", 0.3)
|
||||
if not current_input:
|
||||
return False
|
||||
try:
|
||||
return int(as_element(current_input).run_js("return String(this.value || '').length")) == 0
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
def wait_until_sent(timeout: float = 3.0) -> bool:
|
||||
deadline = time.time() + timeout
|
||||
while time.time() < deadline:
|
||||
if composer_is_empty():
|
||||
return True
|
||||
_sleep(ctx, 0.15, 0.05)
|
||||
return False
|
||||
|
||||
button = _find(
|
||||
tab,
|
||||
"css:button[aria-label='Send message'][type='submit']:not(:disabled), "
|
||||
"button[type='submit']:not([aria-label='Stop generation']):not(:disabled)",
|
||||
1.0,
|
||||
)
|
||||
if button:
|
||||
try:
|
||||
button.click()
|
||||
except Exception as error:
|
||||
logger.warning(f"[GPT-IMAGE-2] send button click failed: {error}")
|
||||
else:
|
||||
logger.warning("[GPT-IMAGE-2] send button not found")
|
||||
if wait_until_sent():
|
||||
logger.stream("[GPT-IMAGE-2] send confirmed after button click")
|
||||
return True
|
||||
|
||||
for enter_attempt in range(2):
|
||||
current_input = _find(tab, "css:textarea[name='message']", 0.5)
|
||||
if not current_input:
|
||||
logger.warning("[GPT-IMAGE-2] composer missing before Enter fallback")
|
||||
else:
|
||||
try:
|
||||
current_input.input("\n")
|
||||
except Exception as error:
|
||||
logger.warning(
|
||||
f"[GPT-IMAGE-2] Enter fallback failed: "
|
||||
f"attempt={enter_attempt + 1}/2 error={error}"
|
||||
)
|
||||
if wait_until_sent():
|
||||
logger.stream(
|
||||
f"[GPT-IMAGE-2] send confirmed after Enter: attempt={enter_attempt + 1}/2"
|
||||
)
|
||||
return True
|
||||
logger.warning(
|
||||
f"[GPT-IMAGE-2] composer still contains text after Enter: "
|
||||
f"attempt={enter_attempt + 1}/2"
|
||||
)
|
||||
return False
|
||||
|
||||
try:
|
||||
input_box.clear()
|
||||
except Exception:
|
||||
pass
|
||||
input_box.input(prompt)
|
||||
_sleep(ctx, 0.3)
|
||||
before_url = _current_url(tab)
|
||||
|
||||
def submission_confirmed(timeout: float = 2.0) -> bool:
|
||||
deadline = time.time() + timeout
|
||||
while time.time() < deadline:
|
||||
current_url = _current_url(tab)
|
||||
if current_url and current_url != before_url:
|
||||
return True
|
||||
try:
|
||||
if int(as_element(input_box).run_js("return String(this.value || '').length")) == 0:
|
||||
return True
|
||||
except Exception:
|
||||
pass
|
||||
for selector in (
|
||||
"css:button[aria-label*='Stop' i]",
|
||||
"css:button[data-testid*='stop' i]",
|
||||
"css:[aria-busy='true']",
|
||||
):
|
||||
if _find(tab, selector, 0.15):
|
||||
return True
|
||||
_sleep(ctx, 0.15, 0.05)
|
||||
return False
|
||||
|
||||
for attempt in range(2):
|
||||
try:
|
||||
input_box.input("\n")
|
||||
except Exception as error:
|
||||
if isinstance(error, RuntimeError) and "cancelled" in str(error).lower():
|
||||
raise
|
||||
logger.warning(
|
||||
f"[GPT-IMAGE-2] Enter submit failed: attempt={attempt + 1}/2 error={error}"
|
||||
)
|
||||
if submission_confirmed(2.5):
|
||||
logger.stream(f"[GPT-IMAGE-2] Enter submit confirmed: attempt={attempt + 1}/2")
|
||||
return True
|
||||
if attempt == 0:
|
||||
logger.stream("[GPT-IMAGE-2] Enter submit not confirmed; retrying Enter")
|
||||
_sleep(ctx, 0.5)
|
||||
|
||||
button = _find(tab, "css:button[type='submit']", 1.0)
|
||||
if button:
|
||||
try:
|
||||
logger.stream("[GPT-IMAGE-2] Enter submit not confirmed; trying send button")
|
||||
button.click()
|
||||
if submission_confirmed(2.5):
|
||||
logger.stream("[GPT-IMAGE-2] send button submit confirmed")
|
||||
return True
|
||||
except Exception as error:
|
||||
logger.warning(f"[GPT-IMAGE-2] send button fallback failed: {error}")
|
||||
return False
|
||||
|
||||
|
||||
def _wait_for_gpt_image2(
|
||||
ctx: dict[str, Any],
|
||||
baseline: set[str],
|
||||
timeout: float,
|
||||
reference_signature: dict[str, Any] | None = None,
|
||||
) -> tuple[list[dict[str, Any]], list[str]]:
|
||||
tab = ctx["tab"]
|
||||
logger = ctx["logger"]
|
||||
deadline = time.time() + timeout
|
||||
inspected: dict[str, str] = {}
|
||||
matches: list[dict[str, Any]] = []
|
||||
candidates: list[str] = []
|
||||
last_new_image_at = 0.0
|
||||
|
||||
while time.time() < deadline:
|
||||
_sleep(ctx, 0.5)
|
||||
sources = [url for url in _visible_image_sources(tab) if url not in baseline]
|
||||
if sources:
|
||||
last_new_image_at = last_new_image_at or time.time()
|
||||
for side_index, url in enumerate(sources):
|
||||
if url in inspected:
|
||||
continue
|
||||
payload = _read_image_bytes(tab, url)
|
||||
if not payload:
|
||||
continue
|
||||
signatures = _image_signatures(payload)
|
||||
inspected[url] = signatures["sha256"]
|
||||
if _same_image(signatures, reference_signature):
|
||||
logger.info(
|
||||
f"[GPT-IMAGE-2] skipped submitted reference image: "
|
||||
f"side={'A' if side_index == 0 else 'B'} "
|
||||
f"url={_log_image_url(url)}"
|
||||
)
|
||||
continue
|
||||
if url not in candidates:
|
||||
candidates.append(url)
|
||||
evidence = inspect_gpt_image2_png(payload)
|
||||
logger.info(
|
||||
f"[GPT-IMAGE-2][C2PA] side={'A' if side_index == 0 else 'B'} "
|
||||
f"matched={evidence['matched']} reason={evidence['reason']} "
|
||||
f"bytes={len(payload)} url={_log_image_url(url)}"
|
||||
)
|
||||
if evidence["matched"]:
|
||||
matches.append({"url": url, "side": "A" if side_index == 0 else "B", **evidence})
|
||||
generation_done = _is_generation_stopped(tab) or not _is_generating(tab)
|
||||
if (
|
||||
last_new_image_at
|
||||
and time.time() - last_new_image_at >= 4.0
|
||||
and generation_done
|
||||
):
|
||||
# Arena can reuse a URL while replacing an early preview with the
|
||||
# final PNG, so force one content-aware pass after generation ends.
|
||||
final_sources = [
|
||||
url for url in _visible_image_sources(tab) if url not in baseline
|
||||
]
|
||||
for side_index, url in enumerate(final_sources):
|
||||
payload = _read_image_bytes(tab, url)
|
||||
if not payload:
|
||||
continue
|
||||
signatures = _image_signatures(payload)
|
||||
final_side = "A" if side_index == 0 else "B"
|
||||
if inspected.get(url) == signatures["sha256"]:
|
||||
for match in matches:
|
||||
if match.get("url") == url:
|
||||
match["side"] = final_side
|
||||
continue
|
||||
inspected[url] = signatures["sha256"]
|
||||
if _same_image(signatures, reference_signature):
|
||||
continue
|
||||
if url not in candidates:
|
||||
candidates.append(url)
|
||||
evidence = inspect_gpt_image2_png(payload)
|
||||
logger.info(
|
||||
f"[GPT-IMAGE-2][C2PA][FINAL] "
|
||||
f"side={final_side} "
|
||||
f"matched={evidence['matched']} reason={evidence['reason']} "
|
||||
f"bytes={len(payload)} url={_log_image_url(url)}"
|
||||
)
|
||||
if evidence["matched"]:
|
||||
matches.append(
|
||||
{
|
||||
"url": url,
|
||||
"side": final_side,
|
||||
**evidence,
|
||||
}
|
||||
)
|
||||
break
|
||||
unique_matches = list(
|
||||
{item.get("url", ""): item for item in matches if item.get("url")}.values()
|
||||
)
|
||||
return unique_matches, candidates
|
||||
|
||||
|
||||
def _recover_timed_out_generation(
|
||||
ctx: dict[str, Any],
|
||||
initial_matches: list[dict[str, Any]],
|
||||
initial_candidates: list[str],
|
||||
reference_signature: dict[str, Any] | None = None,
|
||||
) -> tuple[list[dict[str, Any]], list[str], bool]:
|
||||
tab = ctx["tab"]
|
||||
logger = ctx["logger"]
|
||||
matches = list(initial_matches)
|
||||
candidates = list(initial_candidates)
|
||||
|
||||
_refresh_page(ctx)
|
||||
if _is_generating(tab):
|
||||
logger.warning("[GPT-IMAGE-2] generation still active after refresh; stopping immediately")
|
||||
if not _stop_generation_and_confirm(ctx):
|
||||
raise _StopGenerationFailure(
|
||||
"generation_still_active_after_stop; command loop terminated"
|
||||
)
|
||||
logger.info("[GPT-IMAGE-2] timed-out generation stopped; continuing next round")
|
||||
return matches, candidates, True
|
||||
else:
|
||||
logger.info("[GPT-IMAGE-2] generation completed after refresh; inspecting current images")
|
||||
recovered_matches, recovered_candidates = _wait_for_gpt_image2(
|
||||
ctx, set(), 5.0, reference_signature
|
||||
)
|
||||
matches.extend(recovered_matches)
|
||||
candidates.extend(recovered_candidates)
|
||||
|
||||
unique_matches = list({item.get("url", ""): item for item in matches if item.get("url")}.values())
|
||||
unique_candidates = list(dict.fromkeys(url for url in candidates if url))
|
||||
return unique_matches, unique_candidates, False
|
||||
|
||||
|
||||
class _StopGenerationFailure(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
def run(ctx: dict[str, Any], *, single_mode: bool = False) -> str:
|
||||
tab = ctx["tab"]
|
||||
logger = ctx["logger"]
|
||||
session = ctx.get("session")
|
||||
current_command_id = str(getattr(session, "_current_command_id", "") or "").strip()
|
||||
current_command_name = str(getattr(session, "current_command_name", "") or "").strip()
|
||||
single_mode = bool(
|
||||
single_mode
|
||||
or current_command_id == "cmd_15d91034"
|
||||
or current_command_name == "ARENA自动盲测不翻牌 -gpt image 2 - 单"
|
||||
)
|
||||
ui = ctx.get("command_ui") if isinstance(ctx.get("command_ui"), dict) else {}
|
||||
values = ui.get("values") if isinstance(ui.get("values"), dict) else ui
|
||||
values = values if isinstance(values, dict) else {}
|
||||
|
||||
total_runs = max(1, int(_value(values, "total_runs", 20)))
|
||||
prompt = str(_value(values, "prompt_text", "Create an image.")).strip()
|
||||
random_chars = str(_value(values, "random_insert_chars", "abcdefghijklmnopqrstuvwxyz"))
|
||||
reference_image = str(_value(values, "reference_image", "")).strip()
|
||||
reference_signature = None
|
||||
if reference_image:
|
||||
reference_path = Path(
|
||||
os.path.expandvars(os.path.expanduser(reference_image))
|
||||
).resolve()
|
||||
if reference_path.is_file():
|
||||
reference_signature = _image_signatures(reference_path.read_bytes())
|
||||
generation_mode = "image-to-image" if reference_image else "text-to-image"
|
||||
timeout = 150.0 if single_mode else min(
|
||||
100.0,
|
||||
max(1.0, float(_value(values, "wait_reply_timeout_sec", 100))),
|
||||
)
|
||||
redirect_url = str(_value(values, "redirect_url", "https://arena.ai/image")).strip()
|
||||
ctx["_arena_agent_guard"] = lambda: _redirect_if_agent_page(ctx, redirect_url)
|
||||
hit_urls: list[str] = []
|
||||
consecutive_send_failures = 0
|
||||
|
||||
logger.stream(
|
||||
f"[GPT-IMAGE-2] starting {total_runs} rounds; "
|
||||
f"mode={generation_mode}; timeout={timeout:g}s"
|
||||
)
|
||||
for index in range(total_runs):
|
||||
status = "completed"
|
||||
ctx["begin_command_loop"](index + 1, total_runs, "Arena GPT Image 2")
|
||||
try:
|
||||
if _redirect_if_agent_page(ctx, redirect_url):
|
||||
raise RuntimeError("arena_agent_redirected")
|
||||
if not single_mode:
|
||||
logger.stream(
|
||||
f"[GPT-IMAGE-2] round {index + 1}/{total_runs}: creating a new image chat"
|
||||
)
|
||||
if not _start_new_image_chat(ctx, redirect_url):
|
||||
raise RuntimeError("new_image_chat_failed")
|
||||
if reference_image and not _upload_reference_image(ctx, reference_image):
|
||||
raise RuntimeError("reference_image_upload_failed")
|
||||
|
||||
baseline = set(_visible_image_sources(tab))
|
||||
run_prompt = prompt
|
||||
if random_chars:
|
||||
position = random.randint(0, len(run_prompt))
|
||||
run_prompt = run_prompt[:position] + random.choice(random_chars) + run_prompt[position:]
|
||||
logger.stream(f"[GPT-IMAGE-2] round {index + 1}/{total_runs}: sending prompt")
|
||||
if not _send_prompt(
|
||||
ctx,
|
||||
run_prompt,
|
||||
require_clear_confirmation=single_mode,
|
||||
):
|
||||
if not single_mode:
|
||||
raise RuntimeError("send_failed")
|
||||
consecutive_send_failures += 1
|
||||
status = "failed"
|
||||
logger.error(
|
||||
f"[GPT-IMAGE-2] send failed {consecutive_send_failures} consecutive rounds; "
|
||||
"recovering the stuck page before the next round"
|
||||
)
|
||||
_refresh_page(ctx)
|
||||
if _is_generating(tab):
|
||||
logger.warning(
|
||||
"[GPT-IMAGE-2] stale generation remained after failed send; stopping it"
|
||||
)
|
||||
if not _stop_generation_and_confirm(ctx):
|
||||
raise _StopGenerationFailure(
|
||||
"generation_still_active_after_stop; command loop terminated"
|
||||
)
|
||||
continue
|
||||
consecutive_send_failures = 0
|
||||
|
||||
matches, candidates = _wait_for_gpt_image2(
|
||||
ctx, baseline, timeout, reference_signature
|
||||
)
|
||||
if single_mode and not matches and _is_generation_stopped(tab):
|
||||
logger.info(
|
||||
f"[GPT-IMAGE-2] round {index + 1}/{total_runs} generation stopped "
|
||||
"without a target match; continuing next round without voting"
|
||||
)
|
||||
continue
|
||||
if single_mode and (_is_generating(tab) or not candidates):
|
||||
logger.warning(
|
||||
f"[GPT-IMAGE-2] round {index + 1}/{total_runs} reached {timeout:g}s "
|
||||
"without a completed image response; refreshing within the same round"
|
||||
)
|
||||
matches, candidates, interrupted = _recover_timed_out_generation(
|
||||
ctx, matches, candidates, reference_signature
|
||||
)
|
||||
if interrupted:
|
||||
continue
|
||||
page_url = _current_url(tab)
|
||||
if matches:
|
||||
info = {
|
||||
"url": page_url,
|
||||
"response_sides": [
|
||||
f"gpt-image 2.0 OpenAI Media Service API mode:{generation_mode}"
|
||||
if item["side"] == side else "other image model"
|
||||
for side in ("A", "B")
|
||||
for item in matches[:1]
|
||||
],
|
||||
}
|
||||
record_result = ctx["record_arena_rule_candidates"](
|
||||
info,
|
||||
run_prompt,
|
||||
source="image-c2pa/gpt-image-2",
|
||||
)
|
||||
if page_url and page_url not in hit_urls:
|
||||
hit_urls.append(page_url)
|
||||
logger.info(
|
||||
f"[GPT-IMAGE-2][HIT] sides={','.join(item['side'] for item in matches)} "
|
||||
f"url={page_url} drawer_matches={record_result.get('matched', 0)}"
|
||||
)
|
||||
if single_mode:
|
||||
break
|
||||
elif single_mode:
|
||||
logger.info(
|
||||
f"[GPT-IMAGE-2] no GPT Image 2 C2PA match; images={len(candidates)} url={page_url}"
|
||||
)
|
||||
if candidates:
|
||||
_vote_a_better(ctx)
|
||||
else:
|
||||
logger.info(
|
||||
f"[GPT-IMAGE-2] no GPT Image 2 C2PA match; images={len(candidates)} url={page_url}"
|
||||
)
|
||||
except _StopGenerationFailure:
|
||||
status = "failed"
|
||||
raise
|
||||
except RuntimeError as error:
|
||||
if str(error) in {"python_script_cancelled", "python_script_loop_cancelled"}:
|
||||
status = "cancelled"
|
||||
if str(error) == "python_script_cancelled":
|
||||
break
|
||||
elif str(error) == "arena_agent_redirected":
|
||||
status = "failed"
|
||||
logger.warning(
|
||||
f"[GPT-IMAGE-2] round {index + 1} treated as abnormal because the page was /agent"
|
||||
)
|
||||
else:
|
||||
status = "failed"
|
||||
logger.error(f"[GPT-IMAGE-2] round {index + 1} failed: {error}")
|
||||
if not single_mode:
|
||||
try:
|
||||
tab.get(redirect_url)
|
||||
_sleep(ctx, 2.0)
|
||||
except Exception:
|
||||
pass
|
||||
except Exception as error:
|
||||
status = "failed"
|
||||
logger.error(f"[GPT-IMAGE-2] round {index + 1} failed: {error}")
|
||||
finally:
|
||||
ctx["end_command_loop"](status)
|
||||
|
||||
if hit_urls:
|
||||
return "GPT Image 2 C2PA hit URLs:\n" + "\n".join(
|
||||
f"{index}. {url}" for index, url in enumerate(hit_urls, 1)
|
||||
)
|
||||
return "GPT Image 2 C2PA hits: 0"
|
||||
|
||||
|
||||
__all__ = ["GPT_IMAGE_MARKERS", "inspect_gpt_image2_png", "run"]
|
||||
@@ -0,0 +1,968 @@
|
||||
"""Arena-only guards for browser image generation.
|
||||
|
||||
This module deliberately owns Arena's DOM error handling and image-to-image
|
||||
validation. Callers outside Arena should never enable these rules.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import hashlib
|
||||
import json
|
||||
import uuid
|
||||
from dataclasses import dataclass
|
||||
from io import BytesIO
|
||||
from pathlib import Path
|
||||
from typing import Any, Iterable, Optional
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
from app.core.config import logger
|
||||
from app.utils.image_validation import (
|
||||
extract_media_item_bytes as _common_extract_media_item_bytes,
|
||||
get_current_page_url as current_page_url,
|
||||
image_signatures,
|
||||
read_image_bytes,
|
||||
read_local_image_bytes,
|
||||
read_uploaded_image_bytes as _common_read_uploaded_image_bytes,
|
||||
same_image,
|
||||
validate_generated_images as _common_validate_generated_images,
|
||||
)
|
||||
from app.core.driver import driver_for_tab
|
||||
|
||||
|
||||
ARENA_NATIVE_STOP_SELECTOR = 'css:button[aria-label="Stop generation"]'
|
||||
ARENA_PROMPT_REJECTED_CODE = "arena_prompt_rejected"
|
||||
ARENA_IMAGE_GENERATION_FAILED_CODE = "arena_image_generation_failed"
|
||||
ARENA_IMAGE_UNCHANGED_CODE = "arena_image_unchanged"
|
||||
ARENA_RESULT_BASELINE_PROPERTY = "__universalProxyArenaResultBaseline"
|
||||
ARENA_NON_RETRYABLE_CODES = frozenset(
|
||||
{
|
||||
ARENA_PROMPT_REJECTED_CODE,
|
||||
ARENA_IMAGE_GENERATION_FAILED_CODE,
|
||||
ARENA_IMAGE_UNCHANGED_CODE,
|
||||
}
|
||||
)
|
||||
|
||||
_IMAGE_PROMPT_MARKERS = (
|
||||
"生成图片",
|
||||
"生成图像",
|
||||
"生成一张图",
|
||||
"生成一张图片",
|
||||
"画一张",
|
||||
"画一幅",
|
||||
"帮我画",
|
||||
"请画",
|
||||
"出图",
|
||||
"做图",
|
||||
"文生图",
|
||||
"以图生图",
|
||||
"image generation",
|
||||
"generate image",
|
||||
"generate an image",
|
||||
"create image",
|
||||
"create an image",
|
||||
"draw an image",
|
||||
"draw me",
|
||||
"make an image",
|
||||
"render an image",
|
||||
"render image",
|
||||
)
|
||||
|
||||
_INTERRUPTED_STREAM_MARKERS = (
|
||||
"未完整结束",
|
||||
"提前关闭",
|
||||
"连接已关闭",
|
||||
"未收到完成标志",
|
||||
"缺少明确结束事件",
|
||||
"未产出有效正文",
|
||||
"响应正文未就绪",
|
||||
"network listener timeout",
|
||||
"stream ended without completion",
|
||||
"connection closed",
|
||||
)
|
||||
|
||||
|
||||
class ArenaImageGenerationError(RuntimeError):
|
||||
"""A terminal Arena image-generation error that must not be retried."""
|
||||
|
||||
status_code = 422
|
||||
retryable = False
|
||||
|
||||
def __init__(self, code: str, message: str):
|
||||
super().__init__(message)
|
||||
self.code = str(code)
|
||||
self.message = str(message)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ArenaImageGenerationObservation:
|
||||
"""The state used to decide whether an Arena image response is final."""
|
||||
|
||||
stop_present: bool
|
||||
has_new_image: bool
|
||||
terminal_error: Optional[ArenaImageGenerationError] = None
|
||||
has_stop: Optional[bool] = None
|
||||
has_generating_text: bool = False
|
||||
has_spin_canvas: bool = False
|
||||
still_generating: Optional[bool] = None
|
||||
|
||||
def __post_init__(self):
|
||||
if self.has_stop is None:
|
||||
object.__setattr__(self, "has_stop", bool(self.stop_present))
|
||||
if self.still_generating is None:
|
||||
object.__setattr__(
|
||||
self,
|
||||
"still_generating",
|
||||
bool(self.stop_present or (self.has_generating_text and self.has_spin_canvas)),
|
||||
)
|
||||
|
||||
@property
|
||||
def is_complete(self) -> bool:
|
||||
if self.terminal_error is not None:
|
||||
return True
|
||||
return bool(not self.still_generating and self.has_new_image)
|
||||
|
||||
|
||||
def is_arena_page_url(page_url: str) -> bool:
|
||||
"""Return true only for Arena hosts, not lookalike host names."""
|
||||
try:
|
||||
hostname = str(urlsplit(str(page_url or "")).hostname or "").lower().rstrip(".")
|
||||
except Exception:
|
||||
return False
|
||||
return hostname in {"arena.ai", "lmarena.ai"} or hostname.endswith(
|
||||
(".arena.ai", ".lmarena.ai")
|
||||
)
|
||||
|
||||
|
||||
def looks_like_image_generation_request(text: str) -> bool:
|
||||
"""Recognize an image generation request without binding it to a site."""
|
||||
lowered = str(text or "").strip().lower()
|
||||
if not lowered:
|
||||
return False
|
||||
if any(marker in lowered for marker in _IMAGE_PROMPT_MARKERS):
|
||||
return True
|
||||
|
||||
english_actions = ("generate", "create", "draw", "make", "render", "design", "produce")
|
||||
english_objects = (
|
||||
"image",
|
||||
"images",
|
||||
"picture",
|
||||
"pictures",
|
||||
"photo",
|
||||
"photos",
|
||||
"illustration",
|
||||
"artwork",
|
||||
"poster",
|
||||
"logo",
|
||||
"icon",
|
||||
"banner",
|
||||
"wallpaper",
|
||||
"portrait",
|
||||
)
|
||||
if any(action in lowered for action in english_actions) and any(
|
||||
item in lowered for item in english_objects
|
||||
):
|
||||
return True
|
||||
|
||||
chinese_actions = ("画", "绘制", "生成", "创作", "设计", "修改", "改成", "编辑")
|
||||
chinese_objects = ("图片", "图像", "照片", "插画", "海报", "logo", "图标", "头像", "封面", "壁纸")
|
||||
return any(action in lowered for action in chinese_actions) and any(
|
||||
item in lowered for item in chinese_objects
|
||||
)
|
||||
|
||||
|
||||
def _image_modality_enabled(image_config: Optional[dict[str, Any]]) -> bool:
|
||||
config = image_config or {}
|
||||
if bool(config.get("enabled", False)):
|
||||
return True
|
||||
image_setting = (config.get("modalities") or {}).get("image")
|
||||
if isinstance(image_setting, dict):
|
||||
return bool(image_setting.get("enabled", False))
|
||||
return bool(image_setting)
|
||||
|
||||
|
||||
def is_arena_image_generation_request(
|
||||
page_url: str,
|
||||
prompt: str,
|
||||
image_config: Optional[dict[str, Any]] = None,
|
||||
uploaded_images: Optional[Iterable[Any]] = None,
|
||||
*,
|
||||
preset_name: str = "",
|
||||
) -> bool:
|
||||
"""Enable Arena image-generation rules only for marked templates."""
|
||||
config = image_config or {}
|
||||
if not is_arena_page_url(page_url) or not _image_modality_enabled(config):
|
||||
return False
|
||||
marker = config.get("_arena_image_generation_active")
|
||||
if marker is None:
|
||||
marker = config.get("arena_image_generation", False)
|
||||
return bool(marker)
|
||||
|
||||
|
||||
def is_interrupted_stream_reason(reason: str) -> bool:
|
||||
normalized = str(reason or "").strip().lower()
|
||||
return bool(normalized) and any(marker in normalized for marker in _INTERRUPTED_STREAM_MARKERS)
|
||||
|
||||
|
||||
def auto_skip_arena_direct_comparison(tab: Any) -> bool:
|
||||
"""Detect and click the Skip button in Arena Direct mode's occasional image comparison prompt with debouncing."""
|
||||
script = r"""
|
||||
try {
|
||||
const evalBars = Array.from(document.querySelectorAll('div')).filter(d => {
|
||||
const text = (d.innerText || '');
|
||||
return (text.includes('继续使用 A') || text.includes('Continue with A') || text.includes('Use A'))
|
||||
&& (text.includes('跳过') || text.includes('Skip'));
|
||||
});
|
||||
const searchRoots = evalBars.length > 0 ? evalBars : [document.body];
|
||||
for (const root of searchRoots) {
|
||||
const buttons = Array.from(root.querySelectorAll('button:not([data-skip-triggered="true"])'));
|
||||
const skipBtn = buttons.find(b => {
|
||||
const text = (b.innerText || '').trim();
|
||||
const hasSkipSvg = !!b.querySelector('svg path[d*="M18 7V17"]') || !!b.querySelector('svg path[d*="M18 6V18"]');
|
||||
const isVisible = b.offsetWidth > 0 && b.offsetHeight > 0;
|
||||
return isVisible && (text === '跳过' || text === 'Skip' || hasSkipSvg);
|
||||
});
|
||||
if (skipBtn) {
|
||||
skipBtn.setAttribute('data-skip-triggered', 'true');
|
||||
skipBtn.click();
|
||||
return true;
|
||||
}
|
||||
}
|
||||
} catch (_) {}
|
||||
return false;
|
||||
"""
|
||||
try:
|
||||
return bool(driver_for_tab(tab).run_js(script))
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def detect_arena_render_crash(tab: Any) -> bool:
|
||||
"""Detect unstyled / FOUC layout crash where Tailwind styles failed to load."""
|
||||
script = r"""
|
||||
try {
|
||||
const ol = document.querySelector('ol');
|
||||
if (ol) {
|
||||
const display = window.getComputedStyle(ol)?.display;
|
||||
if (display === 'block') return true;
|
||||
}
|
||||
if (document.styleSheets.length === 0) return true;
|
||||
} catch (_) {}
|
||||
return false;
|
||||
"""
|
||||
try:
|
||||
return bool(driver_for_tab(tab).run_js(script))
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def evaluate_arena_direct_generation_state(
|
||||
tab: Any,
|
||||
baseline_depth: int = 0,
|
||||
current_prompt: str = "",
|
||||
stop_selector: str = ARENA_NATIVE_STOP_SELECTOR,
|
||||
) -> Optional[dict[str, Any]]:
|
||||
"""Execute node-scoped five-state generation evaluation for Arena Direct image requests."""
|
||||
script = r"""
|
||||
return ((baselineDepth, currentPrompt, stopSelector) => {
|
||||
const result = {
|
||||
status: 'IDLE',
|
||||
still_generating: false,
|
||||
is_complete: false,
|
||||
image_urls: [],
|
||||
error_code: null,
|
||||
error_msg: '',
|
||||
skipped_comparison: false
|
||||
};
|
||||
|
||||
// 0. Auto-skip occasional comparison bar if present (scoped & debounced)
|
||||
try {
|
||||
const evalBars = Array.from(document.querySelectorAll('div')).filter(d => {
|
||||
const text = (d.innerText || '');
|
||||
return (text.includes('继续使用 A') || text.includes('Continue with A') || text.includes('Use A'))
|
||||
&& (text.includes('跳过') || text.includes('Skip'));
|
||||
});
|
||||
const searchRoots = evalBars.length > 0 ? evalBars : [document.body];
|
||||
for (const root of searchRoots) {
|
||||
const buttons = Array.from(root.querySelectorAll('button:not([data-skip-triggered="true"])'));
|
||||
const skipBtn = buttons.find(b => {
|
||||
const text = (b.innerText || '').trim();
|
||||
const hasSkipSvg = !!b.querySelector('svg path[d*="M18 7V17"]') || !!b.querySelector('svg path[d*="M18 6V18"]');
|
||||
return b.offsetWidth > 0 && (text === '跳过' || text === 'Skip' || hasSkipSvg);
|
||||
});
|
||||
if (skipBtn) {
|
||||
skipBtn.setAttribute('data-skip-triggered', 'true');
|
||||
skipBtn.click();
|
||||
result.skipped_comparison = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
} catch (_) {}
|
||||
|
||||
const ol = document.querySelector('ol.flex-col-reverse') || document.querySelector('ol');
|
||||
const textarea = document.querySelector('textarea, [contenteditable="true"]');
|
||||
if (!ol) {
|
||||
result.status = 'WAITING_HYDRATION';
|
||||
result.still_generating = true;
|
||||
return result;
|
||||
}
|
||||
|
||||
const rawChildren = Array.from(ol.children);
|
||||
const validChildren = rawChildren.filter(c => c.tagName === 'DIV' && !c.classList.contains('h-0'));
|
||||
|
||||
if (validChildren.length === 0) {
|
||||
result.status = 'WAITING_HYDRATION';
|
||||
result.still_generating = true;
|
||||
return result;
|
||||
}
|
||||
|
||||
// 1. Blocker Fix: Assistant-Only Depth Guard
|
||||
const assistantNodes = validChildren.filter(c => !c.classList.contains('justify-end') && !c.querySelector('[data-role="user"]'));
|
||||
if (baselineDepth > 0 && assistantNodes.length < baselineDepth + 1) {
|
||||
result.status = 'WAITING_HYDRATION';
|
||||
result.still_generating = true;
|
||||
return result;
|
||||
}
|
||||
|
||||
// 2. Blocker Fix: Physical Top Child Isolation
|
||||
// In flex-col-reverse, index 0 is visually at the very bottom.
|
||||
// If index 0 is a User node, the new Assistant node has NOT mounted yet!
|
||||
const firstChild = validChildren[0];
|
||||
const isFirstChildUser = firstChild.classList.contains('justify-end') || !!firstChild.querySelector('[data-role="user"]');
|
||||
if (isFirstChildUser) {
|
||||
result.status = 'WAITING_HYDRATION';
|
||||
result.still_generating = true;
|
||||
return result;
|
||||
}
|
||||
const latestAssistant = firstChild;
|
||||
|
||||
// 3. Prompt Echo Guard on the latest User node. Do not return
|
||||
// early: active generation and a newly rendered image are stronger
|
||||
// evidence than a UI-normalized prompt prefix mismatch.
|
||||
let promptEchoMismatch = false;
|
||||
if (currentPrompt && String(currentPrompt).trim()) {
|
||||
const latestUser = validChildren.find(c => c.classList.contains('justify-end') || !!c.querySelector('[data-role="user"]'));
|
||||
if (latestUser) {
|
||||
const userText = (latestUser.innerText || '').replace(/\s+/g, ' ').trim();
|
||||
const promptSnippet = String(currentPrompt).trim().slice(0, 15);
|
||||
if (promptSnippet && !userText.includes(promptSnippet)) {
|
||||
promptEchoMismatch = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 4. Stop button & active generating status
|
||||
let hasStop = false;
|
||||
try {
|
||||
if (window.__arenaHardStop && typeof window.__arenaHardStop.status === 'function') {
|
||||
const st = window.__arenaHardStop.status();
|
||||
if (st && (st.hasNativeStopButton || st.hasOverlayStopButton)) hasStop = true;
|
||||
}
|
||||
} catch (_) {}
|
||||
|
||||
if (!hasStop && stopSelector) {
|
||||
try {
|
||||
const sel = String(stopSelector).trim().replace(/^css:/i, '');
|
||||
const stopEl = document.querySelector(sel);
|
||||
if (stopEl && stopEl.offsetWidth > 0 && stopEl.offsetHeight > 0) hasStop = true;
|
||||
} catch (_) {}
|
||||
}
|
||||
|
||||
if (!hasStop) {
|
||||
const stopOverlays = ['button[aria-label="Stop generation"]', '[data-arena-hard-stop-overlay="true"]'];
|
||||
for (const s of stopOverlays) {
|
||||
const el = document.querySelector(s);
|
||||
if (el && el.offsetWidth > 0 && el.offsetHeight > 0) {
|
||||
hasStop = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const hasSpinner = !!latestAssistant.querySelector('.lucide-loader.animate-spin, .animate-spin, [class*="animate-spin"]');
|
||||
const isTextareaBusy = textarea ? (textarea.disabled || textarea.getAttribute('aria-busy') === 'true') : false;
|
||||
|
||||
// Priority 1: Still generating
|
||||
if (hasStop || hasSpinner || isTextareaBusy) {
|
||||
result.status = 'GENERATING';
|
||||
result.still_generating = true;
|
||||
return result;
|
||||
}
|
||||
|
||||
// Priority 2: Error detection inside latest Assistant container
|
||||
const assistantText = (latestAssistant.innerText || '').toLowerCase();
|
||||
if (assistantText.includes('something went wrong with this response, please try again') ||
|
||||
assistantText.includes('an error occurred while generating')) {
|
||||
result.status = 'ERROR';
|
||||
result.error_code = 'arena_image_generation_failed';
|
||||
result.error_msg = 'Arena 图片生成失败:Something went wrong with this response, please try again.';
|
||||
result.is_complete = true;
|
||||
return result;
|
||||
}
|
||||
|
||||
// Priority 3: Safety / Terms of Use violation inside latest Assistant container
|
||||
if (assistantText.includes('this content violates our terms of use')) {
|
||||
result.status = 'SAFETY_VIOLATION';
|
||||
result.error_code = 'arena_prompt_rejected';
|
||||
result.error_msg = 'Arena 拒绝了该图片请求:内容违反 Terms of Use';
|
||||
result.is_complete = true;
|
||||
return result;
|
||||
}
|
||||
|
||||
// Priority 4: Success - image URLs resolution inside latest Assistant container
|
||||
const imgs = Array.from(latestAssistant.querySelectorAll('img')).filter(img => {
|
||||
const src = String(img.src || img.currentSrc || '').trim();
|
||||
const isAvatar = img.closest('[class*="avatar"], .size-6, .rounded-full') || (img.naturalWidth > 0 && img.naturalWidth <= 96);
|
||||
return src && !isAvatar && (src.includes('r2.cloudflarestorage.com') || src.startsWith('blob:') || src.startsWith('http'));
|
||||
});
|
||||
|
||||
if (imgs.length > 0) {
|
||||
result.status = 'SUCCESS';
|
||||
result.is_complete = true;
|
||||
result.still_generating = false;
|
||||
result.image_urls = imgs.map(img => img.src || img.currentSrc);
|
||||
return result;
|
||||
}
|
||||
|
||||
if (promptEchoMismatch) {
|
||||
result.status = 'WAITING_HYDRATION';
|
||||
result.still_generating = true;
|
||||
return result;
|
||||
}
|
||||
|
||||
// Priority 5: Empty abort fallback (Stop button gone, action bar mounted, but no image or error)
|
||||
const hasActionBar = !!latestAssistant.querySelector('button[aria-label*="Like" i], button[aria-label*="Dislike" i]') ||
|
||||
Array.from(latestAssistant.querySelectorAll('button')).some(b => b.innerText.includes('Edit'));
|
||||
if (hasActionBar) {
|
||||
result.status = 'EMPTY_ABORT';
|
||||
result.error_code = 'arena_image_empty_response';
|
||||
result.error_msg = 'Arena 图片生成结束但未产出有效图片或错误';
|
||||
result.is_complete = true;
|
||||
return result;
|
||||
}
|
||||
|
||||
result.status = 'GENERATING';
|
||||
result.still_generating = true;
|
||||
return result;
|
||||
})(arguments[0], arguments[1], arguments[2]);
|
||||
"""
|
||||
try:
|
||||
data = _decode_js_result(
|
||||
driver_for_tab(tab).run_js(_json_js(script), baseline_depth, current_prompt, stop_selector)
|
||||
)
|
||||
if isinstance(data, dict) and data.get("status"):
|
||||
return data
|
||||
except Exception as exc:
|
||||
logger.debug(f"evaluate_arena_direct_generation_state 异常: {exc}")
|
||||
return None
|
||||
|
||||
|
||||
def _json_js(script: str) -> str:
|
||||
"""P0-2: stringify a polled script's result in-page (no RemoteObject leak)."""
|
||||
from app.core.cdp_hygiene import wrap_js_json
|
||||
|
||||
return wrap_js_json(script)
|
||||
|
||||
|
||||
def _decode_js_result(raw: Any) -> Any:
|
||||
"""Decode a JSON string from :func:`_json_js`; keep non-JSON values as-is."""
|
||||
if isinstance(raw, str):
|
||||
text = raw.strip()
|
||||
if text[:1] in ("{", "[", '"') or text in ("null", "true", "false"):
|
||||
try:
|
||||
return json.loads(text)
|
||||
except ValueError:
|
||||
return raw
|
||||
return raw
|
||||
|
||||
|
||||
def get_arena_generation_status(
|
||||
tab: Any,
|
||||
selector: str = ARENA_NATIVE_STOP_SELECTOR,
|
||||
) -> dict[str, bool]:
|
||||
"""Return structured multi-modal generation indicators and fault-tolerant generation state."""
|
||||
script = r"""
|
||||
const selector = String(arguments[0] || '').trim().replace(/^css:/i, '');
|
||||
const visible = (element) => {
|
||||
if (!(element instanceof Element) || !element.isConnected) return false;
|
||||
if (element.disabled || element.getAttribute('aria-disabled') === 'true' || element.getAttribute('data-disabled') === 'true') {
|
||||
return false;
|
||||
}
|
||||
if (element.hidden || element.closest('[hidden], [inert], [aria-hidden="true"]')) {
|
||||
return false;
|
||||
}
|
||||
if (element.closest('[data-message-author="user"], [data-role="user"]')) {
|
||||
return false;
|
||||
}
|
||||
const style = getComputedStyle(element);
|
||||
if (style.display === 'none' || style.visibility === 'hidden'
|
||||
|| style.visibility === 'collapse' || Number(style.opacity) === 0) {
|
||||
return false;
|
||||
}
|
||||
const rect = element.getBoundingClientRect();
|
||||
return rect.width > 0 && rect.height > 0;
|
||||
};
|
||||
|
||||
// 1. Indicator 1: Stop button & __arenaHardStop
|
||||
let has_stop = false;
|
||||
try {
|
||||
if (window.__arenaHardStop && typeof window.__arenaHardStop.status === 'function') {
|
||||
const st = window.__arenaHardStop.status();
|
||||
if (st && (st.hasNativeStopButton || st.hasOverlayStopButton)) {
|
||||
has_stop = true;
|
||||
}
|
||||
}
|
||||
} catch (_) {}
|
||||
|
||||
if (!has_stop && selector) {
|
||||
try {
|
||||
if (Array.from(document.querySelectorAll(selector)).some(visible)) {
|
||||
has_stop = true;
|
||||
}
|
||||
} catch (_) {}
|
||||
}
|
||||
|
||||
if (!has_stop) {
|
||||
try {
|
||||
const overlaySelectors = [
|
||||
'button[aria-label="Stop generation"]',
|
||||
'[data-arena-hard-stop-overlay="true"]',
|
||||
'button[aria-label="Hard stop Arena stream"]',
|
||||
];
|
||||
for (const sel of overlaySelectors) {
|
||||
if (Array.from(document.querySelectorAll(sel)).some(visible)) {
|
||||
has_stop = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
} catch (_) {}
|
||||
}
|
||||
|
||||
// 2. Indicator 2: "Generating image..." text placeholder
|
||||
let has_generating_text = false;
|
||||
try {
|
||||
const textPlaceholders = [
|
||||
'generating image',
|
||||
'generating images',
|
||||
'creating image',
|
||||
'creating your image',
|
||||
'image is being generated',
|
||||
'images are being generated',
|
||||
'正在生成图片',
|
||||
'正在生成图像',
|
||||
'正在创建您的图片',
|
||||
'图片正在生成',
|
||||
'图像正在生成',
|
||||
];
|
||||
const textCandidates = Array.from(document.querySelectorAll(
|
||||
'.text-shimmer, [class*="text-shimmer"], [class*="shimmer"], [data-testid*="generating"], div[role="status"]'
|
||||
));
|
||||
for (const el of textCandidates) {
|
||||
if (el.closest('[data-message-author="user"], [data-role="user"]')) {
|
||||
continue;
|
||||
}
|
||||
const text = (el.innerText || el.textContent || '').replace(/\s+/g, ' ').trim().toLowerCase();
|
||||
if (text && text.length <= 160) {
|
||||
if (textPlaceholders.some(p => text.includes(p))) {
|
||||
if (visible(el)) {
|
||||
has_generating_text = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (_) {}
|
||||
|
||||
// 3. Indicator 3: animate-spin canvas loading animation
|
||||
let has_spin_canvas = false;
|
||||
try {
|
||||
const spinCanvases = Array.from(document.querySelectorAll(
|
||||
'.animate-spin canvas, [class*="animate-spin"] canvas, canvas.animate-spin, canvas[class*="animate-spin"]'
|
||||
));
|
||||
for (const el of spinCanvases) {
|
||||
if (visible(el)) {
|
||||
has_spin_canvas = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
} catch (_) {}
|
||||
|
||||
const still_generating = Boolean(has_stop || (has_generating_text && has_spin_canvas));
|
||||
|
||||
return JSON.stringify({
|
||||
has_stop: Boolean(has_stop),
|
||||
has_generating_text: Boolean(has_generating_text),
|
||||
has_spin_canvas: Boolean(has_spin_canvas),
|
||||
still_generating: Boolean(still_generating),
|
||||
});
|
||||
"""
|
||||
try:
|
||||
result = driver_for_tab(tab).run_js(script, selector)
|
||||
if isinstance(result, str):
|
||||
# P0-2:页面内 JSON.stringify,避免 RemoteObject 泄漏
|
||||
try:
|
||||
result = json.loads(result)
|
||||
except ValueError:
|
||||
result = None
|
||||
except Exception:
|
||||
return {
|
||||
"has_stop": False,
|
||||
"has_generating_text": False,
|
||||
"has_spin_canvas": False,
|
||||
"still_generating": False,
|
||||
}
|
||||
|
||||
if isinstance(result, dict):
|
||||
has_stop = bool(result.get("has_stop", False))
|
||||
has_generating_text = bool(result.get("has_generating_text", False))
|
||||
has_spin_canvas = bool(result.get("has_spin_canvas", False))
|
||||
still_generating = bool(
|
||||
result.get(
|
||||
"still_generating",
|
||||
has_stop or (has_generating_text and has_spin_canvas),
|
||||
)
|
||||
)
|
||||
return {
|
||||
"has_stop": has_stop,
|
||||
"has_generating_text": has_generating_text,
|
||||
"has_spin_canvas": has_spin_canvas,
|
||||
"still_generating": still_generating,
|
||||
}
|
||||
elif isinstance(result, bool):
|
||||
return {
|
||||
"has_stop": bool(result),
|
||||
"has_generating_text": False,
|
||||
"has_spin_canvas": False,
|
||||
"still_generating": bool(result),
|
||||
}
|
||||
return {
|
||||
"has_stop": False,
|
||||
"has_generating_text": False,
|
||||
"has_spin_canvas": False,
|
||||
"still_generating": False,
|
||||
}
|
||||
|
||||
|
||||
def is_visible_arena_stop(tab: Any, selector: str = ARENA_NATIVE_STOP_SELECTOR) -> bool:
|
||||
"""Return whether the Arena stop button / overlay is visible."""
|
||||
status = get_arena_generation_status(tab, selector)
|
||||
return bool(status.get("has_stop", False))
|
||||
|
||||
|
||||
def capture_arena_result_baseline(
|
||||
tab: Any,
|
||||
result_selector: str,
|
||||
) -> Optional[dict[str, Any]]:
|
||||
"""Mark response nodes that existed immediately before request submission."""
|
||||
token = uuid.uuid4().hex
|
||||
script = r"""
|
||||
return ((resultSelector, baselineToken, baselineProperty) => {
|
||||
const selector = String(resultSelector || '').trim().replace(/^css:/i, '');
|
||||
const normalize = (value) => String(value || '').replace(/\s+/g, ' ').trim();
|
||||
let candidates = [];
|
||||
if (selector) {
|
||||
try {
|
||||
candidates = Array.from(document.querySelectorAll(selector));
|
||||
} catch (_) {
|
||||
candidates = [];
|
||||
}
|
||||
}
|
||||
if (!candidates.length) {
|
||||
candidates = Array.from(document.querySelectorAll(
|
||||
'[role="alert"], [role="status"]'
|
||||
));
|
||||
}
|
||||
let markedCount = 0;
|
||||
for (const element of candidates) {
|
||||
const marker = {
|
||||
token: String(baselineToken || ''),
|
||||
text: normalize(element.innerText || element.textContent || ''),
|
||||
};
|
||||
try {
|
||||
Object.defineProperty(element, baselineProperty, {
|
||||
configurable: true,
|
||||
writable: true,
|
||||
value: marker,
|
||||
});
|
||||
markedCount += 1;
|
||||
} catch (_) {
|
||||
try {
|
||||
element[baselineProperty] = marker;
|
||||
markedCount += 1;
|
||||
} catch (_) {}
|
||||
}
|
||||
}
|
||||
return {ok: true, node_count: candidates.length, marked_count: markedCount};
|
||||
})(arguments[0], arguments[1], arguments[2]);
|
||||
"""
|
||||
try:
|
||||
result = _decode_js_result(driver_for_tab(tab).run_js(
|
||||
_json_js(script),
|
||||
result_selector,
|
||||
token,
|
||||
ARENA_RESULT_BASELINE_PROPERTY,
|
||||
)) or {}
|
||||
except Exception:
|
||||
return None
|
||||
if not isinstance(result, dict) or not bool(result.get("ok")):
|
||||
return None
|
||||
return {
|
||||
"token": token,
|
||||
"property": ARENA_RESULT_BASELINE_PROPERTY,
|
||||
"node_count": int(result.get("node_count") or 0),
|
||||
"marked_count": int(result.get("marked_count") or 0),
|
||||
}
|
||||
|
||||
|
||||
class ArenaImageGenerationGuard:
|
||||
"""Arena-specific terminal-state and interrupted-stream recovery policy."""
|
||||
|
||||
DEFAULT_REFRESH_INTERVAL_SECONDS = 20.0
|
||||
DEFAULT_MAX_REFRESHES = 16
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
tab: Any,
|
||||
*,
|
||||
result_selector: str = "",
|
||||
stop_selector: str = ARENA_NATIVE_STOP_SELECTOR,
|
||||
baseline_token: str = "",
|
||||
baseline_property: str = "",
|
||||
):
|
||||
self.tab = tab
|
||||
self.result_selector = str(result_selector or "")
|
||||
self.stop_selector = str(stop_selector or ARENA_NATIVE_STOP_SELECTOR)
|
||||
self.baseline_token = str(baseline_token or "")
|
||||
self.baseline_property = str(baseline_property or "")
|
||||
|
||||
def generation_status(self) -> dict[str, bool]:
|
||||
return get_arena_generation_status(
|
||||
self.tab,
|
||||
self.stop_selector,
|
||||
)
|
||||
|
||||
def native_stop_present(self) -> bool:
|
||||
return bool(self.generation_status().get("has_stop", False))
|
||||
|
||||
def detect_terminal_error(self) -> Optional[ArenaImageGenerationError]:
|
||||
"""Map the current Arena result/error node to a terminal failure.
|
||||
|
||||
Do not scan the whole conversation: an older turn can legitimately
|
||||
contain the same error text and must not poison the current request.
|
||||
"""
|
||||
script = r"""
|
||||
return ((resultSelector, baselineToken, baselineProperty) => {
|
||||
const normalize = (value) => String(value || '')
|
||||
.replace(/\s+/g, ' ').trim();
|
||||
const visible = (element) => {
|
||||
if (!(element instanceof Element) || !element.isConnected) return false;
|
||||
const rect = element.getBoundingClientRect();
|
||||
const style = getComputedStyle(element);
|
||||
return rect.width > 0 && rect.height > 0
|
||||
&& style.display !== 'none' && style.visibility !== 'hidden';
|
||||
};
|
||||
const selector = String(resultSelector || '').trim().replace(/^css:/i, '');
|
||||
let candidates = [];
|
||||
if (selector) {
|
||||
try {
|
||||
candidates = Array.from(document.querySelectorAll(selector)).filter(visible);
|
||||
} catch (_) {
|
||||
candidates = [];
|
||||
}
|
||||
}
|
||||
if (!candidates.length) {
|
||||
candidates = Array.from(document.querySelectorAll(
|
||||
'[role="alert"], [role="status"]'
|
||||
)).filter(visible);
|
||||
}
|
||||
if (baselineToken && baselineProperty) {
|
||||
candidates = candidates.filter((element) => {
|
||||
const marker = element[baselineProperty];
|
||||
if (!marker || marker.token !== baselineToken) return true;
|
||||
return normalize(element.innerText || element.textContent || '')
|
||||
!== normalize(marker.text);
|
||||
});
|
||||
}
|
||||
// Arena renders its conversation with flex-col-reverse, so the
|
||||
// newest response is first in DOM order. Select by visual
|
||||
// position instead: the current response is closest to the
|
||||
// composer at the bottom of the conversation.
|
||||
const current = candidates.reduce((lowest, element) => {
|
||||
if (!lowest) return element;
|
||||
return element.getBoundingClientRect().bottom
|
||||
> lowest.getBoundingClientRect().bottom
|
||||
? element : lowest;
|
||||
}, null);
|
||||
const text = current
|
||||
? (current.innerText || current.textContent || '')
|
||||
.replace(/\s+/g, ' ').trim()
|
||||
: '';
|
||||
const normalized = text.toLowerCase();
|
||||
if (normalized.includes('this content violates our terms of use')) {
|
||||
return {code: 'arena_prompt_rejected', text};
|
||||
}
|
||||
if (normalized.includes('something went wrong with this response, please try again')) {
|
||||
return {code: 'arena_image_generation_failed', text};
|
||||
}
|
||||
return null;
|
||||
})(arguments[0], arguments[1], arguments[2]);
|
||||
"""
|
||||
try:
|
||||
if self.result_selector or self.baseline_token or self.baseline_property:
|
||||
result = driver_for_tab(self.tab).run_js(
|
||||
_json_js(script),
|
||||
self.result_selector,
|
||||
self.baseline_token,
|
||||
self.baseline_property,
|
||||
)
|
||||
else:
|
||||
# Preserve compatibility with lightweight tab doubles and
|
||||
# older wrappers whose run_js accepts only the script.
|
||||
result = driver_for_tab(self.tab).run_js(_json_js(script))
|
||||
except Exception:
|
||||
return None
|
||||
result = _decode_js_result(result)
|
||||
if isinstance(result, dict):
|
||||
code = str(result.get("code") or "").strip()
|
||||
text = str(result.get("text") or "").strip()
|
||||
else:
|
||||
code = ""
|
||||
text = str(result or "").strip()
|
||||
normalized = text.lower()
|
||||
if code == ARENA_PROMPT_REJECTED_CODE or "this content violates our terms of use" in normalized:
|
||||
return ArenaImageGenerationError(
|
||||
ARENA_PROMPT_REJECTED_CODE,
|
||||
"Arena 拒绝了该图片请求:内容违反 Terms of Use",
|
||||
)
|
||||
if code == ARENA_IMAGE_GENERATION_FAILED_CODE or (
|
||||
"something went wrong with this response, please try again" in normalized
|
||||
):
|
||||
return ArenaImageGenerationError(
|
||||
ARENA_IMAGE_GENERATION_FAILED_CODE,
|
||||
"Arena 图片生成失败:Something went wrong with this response, please try again.",
|
||||
)
|
||||
return None
|
||||
|
||||
def observe(self, has_new_image: bool) -> ArenaImageGenerationObservation:
|
||||
status = self.generation_status()
|
||||
obs = ArenaImageGenerationObservation(
|
||||
stop_present=bool(status.get("has_stop", False)),
|
||||
has_new_image=bool(has_new_image),
|
||||
terminal_error=self.detect_terminal_error(),
|
||||
has_stop=bool(status.get("has_stop", False)),
|
||||
has_generating_text=bool(status.get("has_generating_text", False)),
|
||||
has_spin_canvas=bool(status.get("has_spin_canvas", False)),
|
||||
still_generating=bool(status.get("still_generating", False)),
|
||||
)
|
||||
current_state = (
|
||||
obs.still_generating,
|
||||
obs.has_new_image,
|
||||
obs.is_complete,
|
||||
bool(obs.terminal_error),
|
||||
)
|
||||
if getattr(self, "_last_logged_state", None) != current_state:
|
||||
self._last_logged_state = current_state
|
||||
logger.debug(
|
||||
f"[Arena Guard] 生图状态变化: still_generating={obs.still_generating}, "
|
||||
f"has_new_image={obs.has_new_image}, complete={obs.is_complete}, "
|
||||
f"stop={obs.has_stop}, text={obs.has_generating_text}, spin={obs.has_spin_canvas}, "
|
||||
f"terminal_error={bool(obs.terminal_error)}"
|
||||
)
|
||||
return obs
|
||||
|
||||
@classmethod
|
||||
def refresh_interval_seconds(cls, image_config: Optional[dict[str, Any]]) -> float:
|
||||
config = image_config or {}
|
||||
value = config.get("arena_image_refresh_interval_seconds")
|
||||
# Do not inherit the old generic DOM recovery interval here. Arena image
|
||||
# recovery has a separate contract: absent an explicit Arena override,
|
||||
# its first and subsequent reloads happen every ten seconds.
|
||||
try:
|
||||
return max(1.0, float(value)) if value is not None else cls.DEFAULT_REFRESH_INTERVAL_SECONDS
|
||||
except (TypeError, ValueError):
|
||||
return cls.DEFAULT_REFRESH_INTERVAL_SECONDS
|
||||
|
||||
@classmethod
|
||||
def max_refreshes(
|
||||
cls,
|
||||
image_config: Optional[dict[str, Any]],
|
||||
hard_timeout_seconds: float,
|
||||
) -> int:
|
||||
config = image_config or {}
|
||||
value = config.get("arena_image_max_refreshes")
|
||||
if value is None:
|
||||
value = config.get("dom_interrupted_max_refreshes")
|
||||
if value is not None:
|
||||
try:
|
||||
return max(1, int(value))
|
||||
except (TypeError, ValueError):
|
||||
pass
|
||||
interval = cls.refresh_interval_seconds(config)
|
||||
return min(
|
||||
cls.DEFAULT_MAX_REFRESHES,
|
||||
max(12, int(max(1.0, float(hard_timeout_seconds)) / interval) + 1),
|
||||
)
|
||||
|
||||
|
||||
_read_local_image = read_local_image_bytes
|
||||
|
||||
|
||||
def _read_uploaded_image(value: Any) -> bytes:
|
||||
return _common_read_uploaded_image_bytes(value, local_reader=_read_local_image)
|
||||
|
||||
|
||||
def _media_item_image_bytes(tab: Any, item: dict[str, Any]) -> bytes:
|
||||
return _common_extract_media_item_bytes(tab, item, local_reader=_read_local_image)
|
||||
|
||||
|
||||
def validate_generated_images(
|
||||
uploaded_images: Iterable[Any],
|
||||
generated_media_items: Iterable[dict[str, Any]],
|
||||
*,
|
||||
tab: Any = None,
|
||||
) -> None:
|
||||
"""Reject Arena image output that is the uploaded reference image itself."""
|
||||
_common_validate_generated_images(
|
||||
uploaded_images,
|
||||
generated_media_items,
|
||||
tab=tab,
|
||||
max_dhash_distance=None,
|
||||
local_reader=_read_local_image,
|
||||
error_builder=lambda msg: ArenaImageGenerationError(
|
||||
ARENA_IMAGE_UNCHANGED_CODE,
|
||||
f"Arena 图片生成失败:{msg}",
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
from app.services.arena_image_stream_observer import ArenaImageStreamObserver
|
||||
|
||||
|
||||
__all__ = [
|
||||
"ARENA_IMAGE_GENERATION_FAILED_CODE",
|
||||
"ARENA_IMAGE_UNCHANGED_CODE",
|
||||
"ARENA_NATIVE_STOP_SELECTOR",
|
||||
"ARENA_NON_RETRYABLE_CODES",
|
||||
"ARENA_PROMPT_REJECTED_CODE",
|
||||
"ARENA_RESULT_BASELINE_PROPERTY",
|
||||
"ArenaImageGenerationError",
|
||||
"ArenaImageGenerationGuard",
|
||||
"ArenaImageGenerationObservation",
|
||||
"ArenaImageStreamObserver",
|
||||
"auto_skip_arena_direct_comparison",
|
||||
"detect_arena_render_crash",
|
||||
"evaluate_arena_direct_generation_state",
|
||||
"current_page_url",
|
||||
"capture_arena_result_baseline",
|
||||
"get_arena_generation_status",
|
||||
"image_signatures",
|
||||
"is_arena_image_generation_request",
|
||||
"is_arena_page_url",
|
||||
"is_interrupted_stream_reason",
|
||||
"is_visible_arena_stop",
|
||||
"looks_like_image_generation_request",
|
||||
"read_image_bytes",
|
||||
"same_image",
|
||||
"validate_generated_images",
|
||||
]
|
||||
|
||||
@@ -0,0 +1,204 @@
|
||||
"""
|
||||
app/services/arena_image_stream_observer.py - Arena 生图守卫与流式监控观察者
|
||||
|
||||
将 Arena 专属的生图状态机、跳过比较逻辑与原生停止按钮守卫封装为 StreamObserver,
|
||||
供 StreamMonitor 在生图会话中挂载。
|
||||
"""
|
||||
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
|
||||
from app.core.config import logger
|
||||
from app.core.stream_observer import (
|
||||
StreamObserver,
|
||||
register_stream_observer_factory,
|
||||
)
|
||||
from app.services.arena_image_generation import (
|
||||
ARENA_NATIVE_STOP_SELECTOR as ARENA_IMAGE_NATIVE_STOP_SELECTOR,
|
||||
ArenaImageGenerationError,
|
||||
ArenaImageGenerationGuard,
|
||||
auto_skip_arena_direct_comparison,
|
||||
evaluate_arena_direct_generation_state,
|
||||
is_arena_image_generation_request,
|
||||
)
|
||||
|
||||
|
||||
class ArenaImageStreamObserver(StreamObserver):
|
||||
"""Arena 生图守卫与流式状态机观察者。"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
guard: Optional[ArenaImageGenerationGuard] = None,
|
||||
image_config: Optional[Dict[str, Any]] = None,
|
||||
):
|
||||
self.guard = guard
|
||||
self.image_config = image_config or {}
|
||||
self.last_observation = None
|
||||
self.last_direct_eval: Optional[Dict[str, Any]] = None
|
||||
|
||||
def on_stream_start(self, monitor: Any, ctx: Any) -> None:
|
||||
if self.guard is None and hasattr(monitor, "tab"):
|
||||
selector = getattr(monitor, "selector", "")
|
||||
token = self.image_config.get("arena_result_baseline_token", "")
|
||||
prop = self.image_config.get("arena_result_baseline_property", "")
|
||||
self.guard = ArenaImageGenerationGuard(
|
||||
monitor.tab,
|
||||
result_selector=selector,
|
||||
baseline_token=token,
|
||||
baseline_property=prop,
|
||||
)
|
||||
|
||||
def _evaluate_direct_state(
|
||||
self,
|
||||
monitor: Any,
|
||||
ctx: Any,
|
||||
current_has_new_rendered_image: bool,
|
||||
still_generating: bool,
|
||||
) -> Tuple[bool, bool, Optional[Dict[str, Any]]]:
|
||||
image_config = getattr(monitor, "_image_config", None) or self.image_config or {}
|
||||
parser_id = str(image_config.get("_parser_id", "") or getattr(monitor, "_workflow_parser_id", "") or "").lower()
|
||||
parser_side = str(image_config.get("_parser_target_side", "") or "").strip().lower()
|
||||
is_battle = "battle" in parser_id or parser_side in {"left", "right"}
|
||||
tab = getattr(monitor, "tab", None)
|
||||
if is_battle or not tab:
|
||||
return current_has_new_rendered_image, still_generating, None
|
||||
|
||||
try:
|
||||
auto_skip_arena_direct_comparison(tab)
|
||||
baseline_depth = int(ctx.baseline_snapshot.get("groups_count") or 0) if ctx.baseline_snapshot else 0
|
||||
prompt_text = str(getattr(monitor, "_prompt", "") or getattr(monitor, "prompt", "") or getattr(ctx, "prompt", "") or "")
|
||||
eval_data = evaluate_arena_direct_generation_state(
|
||||
tab,
|
||||
baseline_depth=baseline_depth,
|
||||
current_prompt=prompt_text,
|
||||
stop_selector=ARENA_IMAGE_NATIVE_STOP_SELECTOR,
|
||||
)
|
||||
if isinstance(eval_data, dict):
|
||||
if eval_data.get("error_code"):
|
||||
raise ArenaImageGenerationError(
|
||||
str(eval_data.get("error_code")),
|
||||
str(eval_data.get("error_msg") or "Arena image generation error"),
|
||||
)
|
||||
status = eval_data.get("status")
|
||||
if status == "SUCCESS" and eval_data.get("image_urls"):
|
||||
urls = [u for u in eval_data.get("image_urls") or [] if isinstance(u, str) and u.strip()]
|
||||
if urls:
|
||||
monitor._final_image_urls = list(urls)
|
||||
if hasattr(monitor, "_prefetch_snapshot_image_urls"):
|
||||
monitor._prefetch_snapshot_image_urls({"image_urls": urls, "image_references": urls})
|
||||
return True, False, eval_data
|
||||
if status in {"GENERATING", "WAITING_HYDRATION"}:
|
||||
return current_has_new_rendered_image, True, eval_data
|
||||
return current_has_new_rendered_image, still_generating, eval_data
|
||||
return current_has_new_rendered_image, still_generating, None
|
||||
except ArenaImageGenerationError:
|
||||
raise
|
||||
except Exception as exc:
|
||||
logger.debug(f"Direct generation evaluation non-fatal error: {exc}")
|
||||
return current_has_new_rendered_image, still_generating, None
|
||||
|
||||
def on_poll_tick(
|
||||
self,
|
||||
monitor: Any,
|
||||
ctx: Any,
|
||||
snapshot: Dict[str, Any],
|
||||
state: Dict[str, Any],
|
||||
) -> None:
|
||||
current_has_new_rendered_image = state.get("current_has_new_rendered_image", False)
|
||||
still_generating = state.get("still_generating", False)
|
||||
interrupted_stream_recovery = state.get("interrupted_stream_recovery", False)
|
||||
stream_recovery_refresh_done = state.get("stream_recovery_refresh_done", False)
|
||||
post_refresh_settled = state.get("post_refresh_settled", True)
|
||||
|
||||
try:
|
||||
(
|
||||
current_has_new_rendered_image,
|
||||
still_generating,
|
||||
direct_eval,
|
||||
) = self._evaluate_direct_state(
|
||||
monitor,
|
||||
ctx,
|
||||
current_has_new_rendered_image,
|
||||
still_generating,
|
||||
)
|
||||
except ArenaImageGenerationError as exc:
|
||||
state["terminal_error"] = exc
|
||||
state["terminal_break"] = True
|
||||
return
|
||||
self.last_direct_eval = direct_eval
|
||||
state["current_has_new_rendered_image"] = current_has_new_rendered_image
|
||||
|
||||
if self.guard is not None:
|
||||
arena_observation = self.guard.observe(current_has_new_rendered_image)
|
||||
self.last_observation = arena_observation
|
||||
|
||||
if direct_eval is not None:
|
||||
still_generating = bool(direct_eval.get("still_generating", arena_observation.still_generating))
|
||||
else:
|
||||
still_generating = arena_observation.still_generating
|
||||
state["still_generating"] = still_generating
|
||||
|
||||
recovery_has_output = bool(arena_observation.has_new_image or arena_observation.terminal_error)
|
||||
state["recovery_has_output"] = recovery_has_output
|
||||
|
||||
if direct_eval is not None:
|
||||
recovery_confirmed = bool(
|
||||
direct_eval.get("is_complete")
|
||||
and (not stream_recovery_refresh_done or post_refresh_settled)
|
||||
)
|
||||
else:
|
||||
recovery_confirmed = bool(
|
||||
arena_observation.is_complete
|
||||
and (
|
||||
(
|
||||
interrupted_stream_recovery
|
||||
and stream_recovery_refresh_done
|
||||
and (post_refresh_settled or arena_observation.has_new_image)
|
||||
)
|
||||
or not interrupted_stream_recovery
|
||||
)
|
||||
)
|
||||
state["recovery_confirmed"] = recovery_confirmed
|
||||
|
||||
if arena_observation.terminal_error is not None:
|
||||
state["terminal_error"] = arena_observation.terminal_error
|
||||
|
||||
if recovery_confirmed:
|
||||
state["terminal_break"] = True
|
||||
logger.info("[Arena Image] 原生停止按钮已消失且已确认新图片,生成完成")
|
||||
|
||||
def get_active_stop_recovery_grace_seconds(self, default: float) -> Optional[float]:
|
||||
cfg = getattr(self.guard, "image_config", None) or self.image_config
|
||||
val = cfg.get("arena_active_stop_recovery_grace_seconds", 60.0)
|
||||
try:
|
||||
return float(val)
|
||||
except (TypeError, ValueError, OverflowError):
|
||||
return 60.0
|
||||
|
||||
def get_stream_recovery_max_refreshes(self, default: int) -> Optional[int]:
|
||||
cfg = getattr(self.guard, "image_config", None) or self.image_config
|
||||
hard_timeout = default * 20.0
|
||||
return ArenaImageGenerationGuard.max_refreshes(cfg, hard_timeout)
|
||||
|
||||
def get_interrupted_refresh_interval(self, default: float) -> Optional[float]:
|
||||
cfg = getattr(self.guard, "image_config", None) or self.image_config
|
||||
return ArenaImageGenerationGuard.refresh_interval_seconds(cfg)
|
||||
|
||||
|
||||
def _arena_image_observer_factory(
|
||||
tab: Any,
|
||||
user_input: Any,
|
||||
image_config: Dict[str, Any],
|
||||
) -> Optional[ArenaImageStreamObserver]:
|
||||
try:
|
||||
url = str(getattr(tab, "url", "") or "")
|
||||
uploaded = image_config.get("uploaded_image_paths") or []
|
||||
if is_arena_image_generation_request(url, user_input, image_config, uploaded):
|
||||
return ArenaImageStreamObserver(image_config=image_config)
|
||||
except Exception as exc:
|
||||
logger.debug(f"[StreamObserver] Arena 生图观察者工厂检查失败(忽略): {exc}")
|
||||
return None
|
||||
|
||||
|
||||
register_stream_observer_factory(_arena_image_observer_factory)
|
||||
|
||||
__all__ = ["ArenaImageStreamObserver"]
|
||||
@@ -0,0 +1,751 @@
|
||||
"""Arena direct model catalog persistence, migration, and dark pool filtering."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import copy
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import threading
|
||||
from datetime import datetime, timezone, timedelta
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional, Set, Tuple
|
||||
|
||||
from app.core.config import logger
|
||||
from app.utils.site_url import extract_remote_site_domain, route_domain_matches
|
||||
|
||||
|
||||
ARENA_MODEL_CATALOG_CONFIG_PATH = Path(
|
||||
os.getenv(
|
||||
"ARENA_MODEL_CATALOG_CONFIG_PATH",
|
||||
str(Path(__file__).resolve().parents[2] / "config" / "arena_model_catalog.local.json"),
|
||||
)
|
||||
)
|
||||
|
||||
ARENA_MODELS_CACHE_PATH = Path(
|
||||
os.getenv(
|
||||
"ARENA_MODELS_CACHE_PATH",
|
||||
str(Path(__file__).resolve().parents[2] / "scripts" / "arena_models_cache.json"),
|
||||
)
|
||||
)
|
||||
|
||||
MODEL_CATALOG_SOURCE = "arena_direct"
|
||||
|
||||
ONLINE_PUBLIC_VISIBLE_MODELS: Set[str] = {
|
||||
"gemini-3-flash", "glm-5.1", "qwen3.5-397b-a17b", "claude-sonnet-4-5-20250929",
|
||||
"gemini-3.1-pro-preview", "qwen3.7-plus", "minimax-m3", "claude-haiku-4-5-20251001",
|
||||
"gemini-2.5-pro", "glm-5v-turbo", "grok-4.20-beta-0309-reasoning", "gpt-5.2-high",
|
||||
"gpt-5.5-instant", "gpt-5.1", "gpt-5.2", "gemini-3.6-flash", "claude-sonnet-4-6",
|
||||
"grok-4.20-multi-agent-beta-0309", "qwen3.5-max-preview", "gemini-3.5-flash-lite",
|
||||
"glm-5", "claude-sonnet-4-5-20250929-thinking-32k", "gpt-5.1-high", "gpt-5.4-mini-high",
|
||||
"glm-4.7", "qwen3-max-preview", "gpt-5-high", "kimi-k2.5-instant", "o3-2025-04-16",
|
||||
"kimi-k2-thinking-turbo", "gpt-5-chat", "qwen3-max-2025-09-23", "qwen3-235b-a22b-instruct-2507",
|
||||
"kimi-k2-0711-preview", "kimi-k2-0905-preview", "qwen3.5-122b-a10b", "minimax-m2.7",
|
||||
"qwen3-vl-235b-a22b-instruct", "mistral-large-3", "gpt-4.1-2025-04-14", "gemini-2.5-flash",
|
||||
"mistral-medium-2508", "qwen3.5-27b", "inkling-small", "qwen3-235b-a22b-no-thinking",
|
||||
"gpt-5.4-nano-high", "longcat-flash-chat", "qwen3-next-80b-a3b-instruct",
|
||||
"claude-sonnet-4-20250514-thinking-32k", "qwen3-235b-a22b-thinking-2507", "qwen3.5-flash",
|
||||
"qwen3.5-35b-a3b", "hunyuan-vision-1.5-thinking", "qwen3-vl-235b-a22b-thinking",
|
||||
"step-3.5-flash", "minimax-m2.5", "o4-mini-2025-04-16", "gpt-5-mini-high",
|
||||
"claude-sonnet-4-20250514", "qwen3-coder-480b-a35b-instruct", "minimax-m2.1-preview",
|
||||
"qwen3-30b-a3b-instruct-2507", "gpt-4.1-mini-2025-04-14", "trinity-large-preview",
|
||||
"qwen3-235b-a22b", "trinity-large-thinking", "qwen3-next-80b-a3b-thinking",
|
||||
"gemma-3-27b-it", "minimax-m1", "gemini-2.0-flash-001", "intellect-3",
|
||||
"gemma-3-12b-it", "o3-mini-high", "gemma-3-4b-it", "mistral-small-3.2",
|
||||
"qwen3-vl-30b-a3b-instruct", "gemini-2.5-flash-lite", "qwen3-30b-a3b-thinking-2507",
|
||||
"qwen3-30b-a3b", "deepseek-v3.2", "gemma-3-1b-it", "deepseek-v4-flash",
|
||||
"o3-mini-2025-01-31", "glm-4-flash", "gemini-2.0-flash-thinking-exp-01-21",
|
||||
"qwen2.5-max-preview", "claude-3.5-haiku-20241022", "claude-3.7-sonnet",
|
||||
"step-2-16k-exp", "claude-3.7-sonnet-thinking", "deepseek-r1-distill-qwen-32b",
|
||||
"deepseek-v3.1", "deepseek-v4", "claude-3.5-sonnet-20241022", "gpt-4o-2024-11-20",
|
||||
"deepseek-v4-pro-max", "claude-sonnet-5-high", "gpt-5.4-mini", "gemini-3.7-pro",
|
||||
"gemini-3.7-pro-high", "glm-5.2 (max)", "deepseek-v4-flash-20260731",
|
||||
"gpt-5.4-high", "claude-sonnet-5-search", "gemini-3.5-pro-high",
|
||||
"gemini-3.5-flash-high", "deepseek-v4-flash-high", "gemini-3.5-flash",
|
||||
"gpt-5.4-turbo", "gpt-5.4", "gemini-3.5-pro", "gemini-2.5-pro-high",
|
||||
"gemini-3-flash-high", "gemini-3-pro-high", "gemini-3-flash-search",
|
||||
"gemini-2.5-ultra", "gemini-3-pro", "gemini-2.5-flash-thinking",
|
||||
"gemini-2.5-flash-high", "gemini-3.7-flash-high", "qwen3.5-122b-a10b-thinking",
|
||||
"qwen3.5-35b-a3b-thinking", "claude-opus-4.5", "claude-opus-4.5-thinking",
|
||||
"claude-opus-4.6", "claude-opus-4.6-thinking", "claude-opus-4.1",
|
||||
"claude-opus-4.1-thinking", "claude-opus-4", "claude-opus-4-thinking",
|
||||
"o4-mini-high", "o4-mini", "o4-max", "o4-high", "o3-pro-max", "o3-pro",
|
||||
"gpt-5.3-high", "gpt-5.3-mini", "gpt-5.3", "gpt-5.2-mini", "gpt-5.2-turbo",
|
||||
"gpt-5.1-turbo", "gpt-5.1-mini", "gpt-5.1-codex-max", "gpt-5-turbo", "gpt-5-mini",
|
||||
"inkling-low", "qwen3-omni-flash", "inkling-small-low", "inkling-medium",
|
||||
"gemini-3.7-flash", "mistral-medium-3.5", "gemini-3-flash (thinking-minimal)",
|
||||
"qwen3.7-plus-preview", "grok-4.6-medium", "grok-4.5"
|
||||
}
|
||||
|
||||
_catalog_io_lock = threading.RLock()
|
||||
|
||||
|
||||
def _canonical_modality(value: Any) -> str:
|
||||
"""Normalize Arena's page/API terminology to the catalog terminology."""
|
||||
modality = str(value or "").strip().lower()
|
||||
return {
|
||||
"webdev": "code",
|
||||
"web": "code",
|
||||
"chat": "text",
|
||||
}.get(modality, modality)
|
||||
|
||||
|
||||
def _keywords_list(source: Any) -> List[str]:
|
||||
if isinstance(source, str):
|
||||
source = re.split(r"[\n,]+", source)
|
||||
if not isinstance(source, (list, tuple, set)):
|
||||
return []
|
||||
result: List[str] = []
|
||||
seen = set()
|
||||
for item in source:
|
||||
keyword = str(item or "").strip()
|
||||
folded = keyword.casefold()
|
||||
if not keyword or folded in seen:
|
||||
continue
|
||||
seen.add(folded)
|
||||
result.append(keyword)
|
||||
return result
|
||||
|
||||
|
||||
def normalize_arena_model_catalog_config(value: Any) -> Dict[str, Any]:
|
||||
raw = value if isinstance(value, dict) else {}
|
||||
|
||||
result = {
|
||||
"enabled": bool(raw.get("enabled", False)),
|
||||
"source": str(raw.get("source") or MODEL_CATALOG_SOURCE).strip() or MODEL_CATALOG_SOURCE,
|
||||
"include_keywords": _keywords_list(raw.get("include_keywords")),
|
||||
"exclude_keywords": _keywords_list(raw.get("exclude_keywords")),
|
||||
"enable_dark_pool": bool(raw.get("enable_dark_pool", False)),
|
||||
"dark_pool_since": str(raw.get("dark_pool_since") or "").strip(),
|
||||
"dark_pool_whitelist_keywords": _keywords_list(raw.get("dark_pool_whitelist_keywords")),
|
||||
"dark_pool_blacklist_keywords": _keywords_list(raw.get("dark_pool_blacklist_keywords")),
|
||||
}
|
||||
if "modality" in raw and str(raw.get("modality") or "").strip():
|
||||
result["modality"] = _canonical_modality(raw.get("modality"))
|
||||
return result
|
||||
|
||||
|
||||
def _default_arena_model_catalog_config(domain: str, preset_name: str) -> Optional[Dict[str, Any]]:
|
||||
"""Return built-in catalog defaults for presets that should start enabled."""
|
||||
normalized_domain = str(domain or "").strip().lower().strip(".")
|
||||
normalized_preset = str(preset_name or "").strip()
|
||||
if route_domain_matches("arena.ai", normalized_domain) and normalized_preset == "主预设-直连模式":
|
||||
return normalize_arena_model_catalog_config(
|
||||
{
|
||||
"enabled": True,
|
||||
"source": MODEL_CATALOG_SOURCE,
|
||||
"modality": "text",
|
||||
}
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
def load_arena_model_catalog_data() -> Dict[str, Dict[str, Any]]:
|
||||
"""加载独立配置文件,返回 { domain: { preset_name: catalog_config } }"""
|
||||
with _catalog_io_lock:
|
||||
if not ARENA_MODEL_CATALOG_CONFIG_PATH.exists():
|
||||
return {}
|
||||
try:
|
||||
content = ARENA_MODEL_CATALOG_CONFIG_PATH.read_text(encoding="utf-8").strip()
|
||||
if not content:
|
||||
return {}
|
||||
data = json.loads(content)
|
||||
if not isinstance(data, dict):
|
||||
return {}
|
||||
normalized: Dict[str, Dict[str, Any]] = {}
|
||||
for domain, presets in data.items():
|
||||
if not isinstance(presets, dict):
|
||||
continue
|
||||
d_key = str(domain).strip().lower()
|
||||
normalized[d_key] = {}
|
||||
for preset_name, cat in presets.items():
|
||||
p_key = str(preset_name).strip()
|
||||
if isinstance(cat, dict):
|
||||
normalized[d_key][p_key] = normalize_arena_model_catalog_config(cat)
|
||||
return normalized
|
||||
except Exception as e:
|
||||
logger.warning(f"读取 Arena 独立模型目录配置失败: {e}")
|
||||
return {}
|
||||
|
||||
|
||||
def save_arena_model_catalog_data(data: Dict[str, Any]) -> bool:
|
||||
"""原子保存独立配置文件"""
|
||||
with _catalog_io_lock:
|
||||
try:
|
||||
ARENA_MODEL_CATALOG_CONFIG_PATH.parent.mkdir(parents=True, exist_ok=True)
|
||||
tmp_file = ARENA_MODEL_CATALOG_CONFIG_PATH.with_suffix(".tmp")
|
||||
with open(tmp_file, "w", encoding="utf-8", newline="\n") as f:
|
||||
json.dump(data, f, indent=2, ensure_ascii=False)
|
||||
f.flush()
|
||||
os.fsync(f.fileno())
|
||||
os.replace(tmp_file, ARENA_MODEL_CATALOG_CONFIG_PATH)
|
||||
logger.info(f"Arena 独立模型目录配置已保存: {ARENA_MODEL_CATALOG_CONFIG_PATH}")
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.error(f"保存 Arena 独立模型目录配置失败: {e}")
|
||||
return False
|
||||
|
||||
|
||||
def get_arena_model_catalog(
|
||||
domain: str = "arena.ai",
|
||||
preset_name: Optional[str] = None,
|
||||
config_engine: Any = None,
|
||||
) -> Dict[str, Any]:
|
||||
"""获取指定预设的 Arena 模型目录配置。支持旧配置兼容回退。"""
|
||||
normalized_domain = str(domain or "arena.ai").strip().lower().strip(".")
|
||||
catalog_data = load_arena_model_catalog_data()
|
||||
domain_catalogs = catalog_data.get(normalized_domain, {})
|
||||
|
||||
target_preset = str(preset_name or "").strip()
|
||||
if not target_preset and config_engine:
|
||||
try:
|
||||
target_preset = str(config_engine.get_default_preset(normalized_domain) or "").strip()
|
||||
except Exception:
|
||||
pass
|
||||
if not target_preset:
|
||||
target_preset = "主预设"
|
||||
|
||||
if target_preset in domain_catalogs:
|
||||
return copy.deepcopy(domain_catalogs[target_preset])
|
||||
|
||||
# 兼容期回退:若独立配置中无该预设,检查 config_engine 站点预设中是否有旧 model_catalog
|
||||
if config_engine:
|
||||
try:
|
||||
presets = None
|
||||
if hasattr(config_engine, "sites") and isinstance(config_engine.sites, dict):
|
||||
site = config_engine.sites.get(normalized_domain, {})
|
||||
if isinstance(site, dict):
|
||||
presets = site.get("presets")
|
||||
if presets is None and hasattr(config_engine, "presets") and isinstance(config_engine.presets, dict):
|
||||
presets = config_engine.presets
|
||||
|
||||
if isinstance(presets, dict):
|
||||
resolved_key = config_engine._resolve_preset_alias_key(target_preset, presets) if hasattr(config_engine, "_resolve_preset_alias_key") else target_preset
|
||||
preset_dict = presets.get(resolved_key)
|
||||
if isinstance(preset_dict, dict) and "model_catalog" in preset_dict:
|
||||
return normalize_arena_model_catalog_config(preset_dict.get("model_catalog"))
|
||||
|
||||
if hasattr(config_engine, "_get_site_data_readonly"):
|
||||
preset_dict = config_engine._get_site_data_readonly(normalized_domain, target_preset)
|
||||
if isinstance(preset_dict, dict) and "model_catalog" in preset_dict:
|
||||
return normalize_arena_model_catalog_config(preset_dict.get("model_catalog"))
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
default_catalog = _default_arena_model_catalog_config(normalized_domain, target_preset)
|
||||
if default_catalog is not None:
|
||||
return default_catalog
|
||||
|
||||
return normalize_arena_model_catalog_config({})
|
||||
|
||||
|
||||
def set_arena_model_catalog(
|
||||
domain: str = "arena.ai",
|
||||
preset_name: Optional[str] = None,
|
||||
catalog_config: Any = None,
|
||||
) -> Dict[str, Any]:
|
||||
"""设置并原子保存指定预设的 Arena 模型目录配置"""
|
||||
normalized_domain = str(domain or "arena.ai").strip().lower().strip(".")
|
||||
target_preset = str(preset_name or "主预设").strip()
|
||||
normalized_cat = normalize_arena_model_catalog_config(catalog_config)
|
||||
|
||||
with _catalog_io_lock:
|
||||
catalog_data = load_arena_model_catalog_data()
|
||||
if normalized_domain not in catalog_data:
|
||||
catalog_data[normalized_domain] = {}
|
||||
catalog_data[normalized_domain][target_preset] = normalized_cat
|
||||
if not save_arena_model_catalog_data(catalog_data):
|
||||
raise IOError(f"保存 Arena 独立模型目录配置失败: {normalized_domain} [{target_preset}]")
|
||||
|
||||
return normalized_cat
|
||||
|
||||
|
||||
def migrate_and_cleanup_sites_model_catalog(config_engine: Any) -> bool:
|
||||
"""扫描 config_engine.sites,将各预设下的 model_catalog 迁移到独立文件,并从 sites.json 中清理旧字段"""
|
||||
if not config_engine or not hasattr(config_engine, "sites") or not isinstance(config_engine.sites, dict):
|
||||
return False
|
||||
|
||||
with _catalog_io_lock:
|
||||
catalog_data = load_arena_model_catalog_data()
|
||||
catalog_data_changed = False
|
||||
to_cleanup = []
|
||||
|
||||
for domain, site_config in config_engine.sites.items():
|
||||
if domain.startswith("_") or not isinstance(site_config, dict):
|
||||
continue
|
||||
d_key = str(domain).strip().lower().strip(".")
|
||||
if not route_domain_matches("arena.ai", d_key):
|
||||
continue
|
||||
|
||||
presets = site_config.get("presets")
|
||||
if not isinstance(presets, dict):
|
||||
continue
|
||||
|
||||
if d_key not in catalog_data:
|
||||
catalog_data[d_key] = {}
|
||||
|
||||
for p_name, p_data in presets.items():
|
||||
if not isinstance(p_data, dict):
|
||||
continue
|
||||
p_key = str(p_name).strip()
|
||||
if "model_catalog" in p_data:
|
||||
old_cat = p_data.get("model_catalog")
|
||||
# 如果独立文件没有该预设配置,则补全迁移
|
||||
if p_key not in catalog_data[d_key]:
|
||||
catalog_data[d_key][p_key] = normalize_arena_model_catalog_config(old_cat)
|
||||
catalog_data_changed = True
|
||||
logger.info(f"已迁移站点 {domain} 预设 [{p_name}] 的 model_catalog 到独立配置文件")
|
||||
to_cleanup.append((p_data, d_key, p_key))
|
||||
|
||||
if catalog_data_changed:
|
||||
if not save_arena_model_catalog_data(catalog_data):
|
||||
logger.error("保存独立模型目录配置文件失败,取消清理 sites.json 中的 model_catalog")
|
||||
return False
|
||||
|
||||
# 只有在独立配置文件确认已包含该配置时,才安全清理内存与 sites.json
|
||||
sites_changed = False
|
||||
for p_data, d_key, p_key in to_cleanup:
|
||||
if d_key in catalog_data and p_key in catalog_data[d_key]:
|
||||
if "model_catalog" in p_data:
|
||||
p_data.pop("model_catalog", None)
|
||||
sites_changed = True
|
||||
|
||||
if sites_changed:
|
||||
try:
|
||||
config_engine.save_config()
|
||||
logger.info("已从 sites.json 中清理已迁移的 model_catalog 残留字段")
|
||||
except Exception as e:
|
||||
logger.warning(f"清理 sites.json 中的 model_catalog 后保存失败: {e}")
|
||||
|
||||
return catalog_data_changed or sites_changed
|
||||
|
||||
|
||||
def get_model_catalog_preset(config_engine: Any, domain: Any) -> Optional[Dict[str, Any]]:
|
||||
"""读取指定域名下已启用 model_catalog 的预设信息"""
|
||||
normalized_domain = str(domain or "").strip().lower().strip(".")
|
||||
if not normalized_domain or not route_domain_matches("arena.ai", normalized_domain):
|
||||
return None
|
||||
|
||||
try:
|
||||
if hasattr(config_engine, "refresh_if_changed"):
|
||||
config_engine.refresh_if_changed()
|
||||
site = config_engine.sites.get(normalized_domain) if hasattr(config_engine, "sites") else None
|
||||
except Exception:
|
||||
site = None
|
||||
|
||||
presets = site.get("presets") if isinstance(site, dict) else (
|
||||
getattr(config_engine, "presets", {}) if hasattr(config_engine, "presets") else {}
|
||||
)
|
||||
catalog_data = load_arena_model_catalog_data()
|
||||
domain_catalogs = catalog_data.get(normalized_domain, {})
|
||||
|
||||
# 优先检查默认预设
|
||||
default_preset = ""
|
||||
if hasattr(config_engine, "get_default_preset"):
|
||||
try:
|
||||
default_preset = str(config_engine.get_default_preset(normalized_domain) or "").strip()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
all_preset_names = []
|
||||
if default_preset:
|
||||
all_preset_names.append(default_preset)
|
||||
|
||||
if isinstance(presets, dict):
|
||||
for p_name in presets.keys():
|
||||
if str(p_name) not in all_preset_names:
|
||||
all_preset_names.append(str(p_name))
|
||||
for p_name in domain_catalogs.keys():
|
||||
if str(p_name) not in all_preset_names:
|
||||
all_preset_names.append(str(p_name))
|
||||
|
||||
for preset_name in all_preset_names:
|
||||
cat = domain_catalogs.get(preset_name)
|
||||
if not cat:
|
||||
# 兼容回退
|
||||
cat = get_arena_model_catalog(normalized_domain, preset_name, config_engine=config_engine)
|
||||
if cat and cat.get("enabled") and cat.get("source") == MODEL_CATALOG_SOURCE:
|
||||
preset = presets.get(preset_name, {}) if isinstance(presets, dict) else {}
|
||||
return {
|
||||
"preset_name": str(preset_name),
|
||||
"preset": preset,
|
||||
"catalog": cat,
|
||||
}
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def get_arena_direct_catalog_for_tab(
|
||||
config_engine: Any,
|
||||
tab: Any,
|
||||
*,
|
||||
preset_name: Any = None,
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
"""获取当前标签页对应的 Arena 直连目录配置"""
|
||||
if not isinstance(tab, dict):
|
||||
return None
|
||||
status = str(tab.get("status") or "").strip().lower()
|
||||
if status not in {"idle", "busy"} or bool(tab.get("terminating")):
|
||||
return None
|
||||
|
||||
current_url = str(tab.get("url") or tab.get("current_url") or "").strip()
|
||||
effective_preset_name = str(preset_name or tab.get("preset_name") or "").strip()
|
||||
try:
|
||||
config_engine.refresh_if_changed()
|
||||
if not effective_preset_name:
|
||||
effective_preset_name = str(
|
||||
config_engine.get_default_preset("arena.ai") or ""
|
||||
).strip()
|
||||
preset = config_engine._get_site_data_readonly(
|
||||
"arena.ai",
|
||||
effective_preset_name or None,
|
||||
)
|
||||
except Exception:
|
||||
return None
|
||||
if not isinstance(preset, dict):
|
||||
return None
|
||||
|
||||
catalog = get_arena_model_catalog("arena.ai", effective_preset_name, config_engine=config_engine)
|
||||
if not catalog.get("enabled") or catalog.get("source") != MODEL_CATALOG_SOURCE:
|
||||
return None
|
||||
|
||||
from app.services.arena_direct_models import _is_arena_direct_url
|
||||
if not _is_arena_direct_url(current_url, catalog_preset=preset, catalog_config=catalog):
|
||||
return None
|
||||
|
||||
return {
|
||||
"preset_name": effective_preset_name,
|
||||
"preset": preset,
|
||||
"catalog": catalog,
|
||||
}
|
||||
|
||||
|
||||
# ==============================================================================
|
||||
# 暗池模型判定、时间戳解析与全量缓存加载
|
||||
# ==============================================================================
|
||||
|
||||
def parse_uuidv7_timestamp(uuid_str: str) -> Optional[float]:
|
||||
"""从 UUIDv7 中解析毫秒级时间戳"""
|
||||
if not uuid_str or not isinstance(uuid_str, str):
|
||||
return None
|
||||
clean_hex = uuid_str.replace("-", "").strip().lower()
|
||||
if len(clean_hex) != 32 or clean_hex[12] != "7":
|
||||
return None
|
||||
try:
|
||||
ts_ms = int(clean_hex[:12], 16)
|
||||
if 1577836800000 <= ts_ms <= 1893456000000:
|
||||
return float(ts_ms)
|
||||
except Exception:
|
||||
pass
|
||||
return None
|
||||
|
||||
|
||||
def parse_date_to_timestamp(date_str: str) -> Optional[float]:
|
||||
"""将 YYYY-MM-DD 字符串转为 UTC 00:00:00 毫秒时间戳"""
|
||||
if not date_str or not isinstance(date_str, str):
|
||||
return None
|
||||
clean_str = date_str.strip()
|
||||
m = re.match(r"^(\d{4})[-_/](\d{1,2})[-_/](\d{1,2})$", clean_str)
|
||||
if m:
|
||||
try:
|
||||
year, month, day = int(m.group(1)), int(m.group(2)), int(m.group(3))
|
||||
dt = datetime(year, month, day, 0, 0, 0, tzinfo=timezone.utc)
|
||||
return dt.timestamp() * 1000.0
|
||||
except Exception:
|
||||
return None
|
||||
return None
|
||||
|
||||
|
||||
def get_model_timestamp(model: Dict[str, Any]) -> Optional[float]:
|
||||
"""获取模型入库时间戳(ms)"""
|
||||
mid = str(model.get("arena_model_id") or model.get("id") or "")
|
||||
ts = parse_uuidv7_timestamp(mid)
|
||||
if ts is not None:
|
||||
return ts
|
||||
|
||||
name_str = " ".join(
|
||||
str(model.get(k) or "") for k in ("name", "displayName", "display_name", "publicName", "public_name")
|
||||
)
|
||||
m = re.search(r"202[4-7][_-]?(0[1-9]|1[0-2])[_-]?([0-3][0-9])", name_str)
|
||||
if m:
|
||||
try:
|
||||
match_full = m.group(0).replace("_", "-")
|
||||
digits = re.sub(r"\D", "", match_full)
|
||||
if len(digits) == 8:
|
||||
year, month, day = int(digits[:4]), int(digits[4:6]), int(digits[6:])
|
||||
dt = datetime(year, month, day, 0, 0, 0, tzinfo=timezone.utc)
|
||||
return dt.timestamp() * 1000.0
|
||||
except Exception:
|
||||
pass
|
||||
return None
|
||||
|
||||
|
||||
def is_dark_pool_model(model: Dict[str, Any]) -> bool:
|
||||
"""判定是否属于暗池(在前端模型选择器中隐藏或来自离线暗池缓存)"""
|
||||
if "_is_dark_pool" in model:
|
||||
return bool(model["_is_dark_pool"])
|
||||
if model.get("is_dark_pool") is not None:
|
||||
return bool(model.get("is_dark_pool"))
|
||||
|
||||
disp = str(model.get("display_name") or model.get("displayName") or "").strip().lower()
|
||||
name = str(model.get("name") or "").strip().lower()
|
||||
pub = str(model.get("public_name") or model.get("publicName") or "").strip().lower()
|
||||
search_str = f"{disp} {name} {pub}"
|
||||
|
||||
# 包含明确内部/测试特征的直接归为暗池
|
||||
if any(k in search_str for k in ["internal", "test", "dlp", "fireworks", "node"]):
|
||||
return True
|
||||
|
||||
# 命中公开白名单的必定为明池
|
||||
for pub_name in ONLINE_PUBLIC_VISIBLE_MODELS:
|
||||
pub_lower = pub_name.lower()
|
||||
if disp == pub_lower or pub == pub_lower or name == pub_lower:
|
||||
return False
|
||||
|
||||
# 来自离线暗池全量缓存且不在公开白名单的归为暗池
|
||||
if model.get("_is_offline_cache"):
|
||||
return True
|
||||
|
||||
# 实时从页面抓取(或单测传入)的普通模型默认视为明池
|
||||
return False
|
||||
|
||||
|
||||
_offline_cache_lock = threading.Lock()
|
||||
_offline_cached_models: Optional[List[Dict[str, Any]]] = None
|
||||
|
||||
|
||||
def load_arena_offline_cache_models() -> List[Dict[str, Any]]:
|
||||
"""加载本地 scripts/arena_models_cache.json 中的模型并规范化"""
|
||||
global _offline_cached_models
|
||||
with _offline_cache_lock:
|
||||
if _offline_cached_models is not None:
|
||||
return _offline_cached_models
|
||||
|
||||
if not ARENA_MODELS_CACHE_PATH.exists():
|
||||
_offline_cached_models = []
|
||||
return []
|
||||
|
||||
try:
|
||||
with open(ARENA_MODELS_CACHE_PATH, "r", encoding="utf-8") as f:
|
||||
raw_list = json.load(f)
|
||||
if not isinstance(raw_list, list):
|
||||
_offline_cached_models = []
|
||||
return []
|
||||
|
||||
results = []
|
||||
for item in raw_list:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
mid = str(item.get("id") or "").strip()
|
||||
if not mid:
|
||||
continue
|
||||
|
||||
caps = item.get("capabilities") or {}
|
||||
out_caps = caps.get("outputCapabilities") or {}
|
||||
ranks = item.get("rankByModality") or {}
|
||||
|
||||
def rank_is_finite(key):
|
||||
val = ranks.get(key)
|
||||
return isinstance(val, (int, float)) and val < 9007199254740991
|
||||
|
||||
has_text = bool(out_caps.get("text"))
|
||||
has_image = bool(out_caps.get("image")) or rank_is_finite("image")
|
||||
has_video = bool(out_caps.get("video")) or rank_is_finite("video")
|
||||
# `outputCapabilities.web` is browsing/search support, not
|
||||
# the WebDev/code modality. Use only the webdev rank here.
|
||||
has_web = rank_is_finite("webdev") or bool(out_caps.get("web"))
|
||||
has_search = bool(out_caps.get("search")) or rank_is_finite("search")
|
||||
|
||||
disp_name = str(item.get("displayName") or item.get("publicName") or item.get("name") or mid).strip()
|
||||
internal_name = str(item.get("name") or item.get("id") or disp_name).strip()
|
||||
name_search = f"{internal_name} {disp_name} {mid}".lower()
|
||||
|
||||
is_known_image = any(k in name_search for k in [
|
||||
"gpt-image", "mona-lisa", "luna-lisa", "lina-alpha", "lina-f-alpha", "silver_halide",
|
||||
"flux", "seedream", "seededit", "imagine", "imagen", "z-image", "midjourney", "dall-e", "recraft", "krea"
|
||||
])
|
||||
|
||||
modality = "text"
|
||||
if has_video and not has_text and not is_known_image:
|
||||
modality = "video"
|
||||
elif is_known_image or (has_image and not has_text):
|
||||
modality = "image"
|
||||
elif has_web and not has_text:
|
||||
modality = "code"
|
||||
elif has_text:
|
||||
modality = "text"
|
||||
elif has_web:
|
||||
modality = "code"
|
||||
elif has_search:
|
||||
modality = "search"
|
||||
elif has_image:
|
||||
modality = "image"
|
||||
elif has_video:
|
||||
modality = "video"
|
||||
|
||||
results.append({
|
||||
"arena_model_id": mid,
|
||||
"name": internal_name,
|
||||
"public_name": disp_name,
|
||||
"display_name": disp_name,
|
||||
"provider": str(item.get("provider") or ""),
|
||||
"organization": str(item.get("organization") or item.get("provider") or "arena.ai"),
|
||||
"modality": modality,
|
||||
"_is_offline_cache": True,
|
||||
})
|
||||
|
||||
_offline_cached_models = results
|
||||
return results
|
||||
except Exception as e:
|
||||
logger.warning(f"读取 Arena 离线模型缓存失败: {e}")
|
||||
_offline_cached_models = []
|
||||
return []
|
||||
|
||||
|
||||
# ==============================================================================
|
||||
# 模型过滤引擎(明池 + 暗池)
|
||||
# ==============================================================================
|
||||
|
||||
def filter_arena_catalog_models(
|
||||
models: List[Dict[str, Any]],
|
||||
catalog_config: Any,
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""对模型列表执行明池与暗池的分别过滤与合并去重排序"""
|
||||
from app.services.arena_direct_models import (
|
||||
_canonical_modality,
|
||||
_natural_sort_key,
|
||||
get_arena_direct_model_public_id,
|
||||
)
|
||||
|
||||
catalog = normalize_arena_model_catalog_config(catalog_config)
|
||||
target_modality = _canonical_modality(catalog.get("modality") or "text")
|
||||
enable_dark_pool = bool(catalog.get("enable_dark_pool", False))
|
||||
|
||||
dark_since_ts = parse_date_to_timestamp(catalog.get("dark_pool_since", ""))
|
||||
dark_whitelist = [item.casefold() for item in catalog.get("dark_pool_whitelist_keywords", [])]
|
||||
dark_blacklist = [item.casefold() for item in catalog.get("dark_pool_blacklist_keywords", [])]
|
||||
|
||||
pub_includes = [item.casefold() for item in catalog.get("include_keywords", [])]
|
||||
pub_excludes = [item.casefold() for item in catalog.get("exclude_keywords", [])]
|
||||
|
||||
# 合并来自页面实时抓取与离线缓存的模型(按 arena_model_id 去重)
|
||||
all_candidates: List[Dict[str, Any]] = []
|
||||
seen_ids = set()
|
||||
for m in (models or []) + (load_arena_offline_cache_models() if enable_dark_pool else []):
|
||||
if not isinstance(m, dict):
|
||||
continue
|
||||
mid = str(m.get("arena_model_id") or "").strip().lower()
|
||||
if not mid or mid in seen_ids:
|
||||
continue
|
||||
seen_ids.add(mid)
|
||||
all_candidates.append(m)
|
||||
|
||||
video_keywords = (
|
||||
"video", "seedance", "dreamina", "sora", "veo", "gemini-omni", "kling",
|
||||
"hailuo", "runway", "pika", "mochi", "ltx", "vidu", "luma", "wan2",
|
||||
"wan-2", "wan_2", "wanx", "pixverse", "kandinsky-video", "kandinsky_video", "happyhorse"
|
||||
)
|
||||
|
||||
image_keywords = (
|
||||
"image", "mona-lisa", "luna-lisa", "lina-alpha", "lina-f-alpha",
|
||||
"silver_halide", "flux", "imagine", "seedream", "seededit", "imagen", "z-image",
|
||||
"midjourney", "dall-e", "recraft", "krea"
|
||||
)
|
||||
|
||||
result = []
|
||||
result_seen_ids = set()
|
||||
|
||||
for model in all_candidates:
|
||||
search_str = " ".join(
|
||||
str(model.get(k) or "") for k in ("name", "public_name", "display_name", "search_name")
|
||||
).lower()
|
||||
if target_modality != "video":
|
||||
if any((k in search_str and (k != "kling" or "inkling" not in search_str)) for k in video_keywords):
|
||||
continue
|
||||
|
||||
model_modality = _canonical_modality(model.get("modality"))
|
||||
if target_modality:
|
||||
if target_modality in {"code", "search", "video"}:
|
||||
if model_modality != target_modality:
|
||||
continue
|
||||
elif target_modality == "image":
|
||||
model_is_image = (model_modality == "image") or (
|
||||
not model_modality and any(k in search_str for k in image_keywords)
|
||||
)
|
||||
if not model_is_image:
|
||||
continue
|
||||
elif target_modality == "text":
|
||||
if model_modality and model_modality != "text":
|
||||
continue
|
||||
if not model_modality and any(k in search_str for k in image_keywords):
|
||||
continue
|
||||
else:
|
||||
if model_modality and model_modality != target_modality:
|
||||
continue
|
||||
|
||||
searchable = " ".join(
|
||||
str(model.get(key) or "")
|
||||
for key in (
|
||||
"name", "public_name", "display_name", "search_name", "provider", "organization"
|
||||
)
|
||||
).casefold()
|
||||
searchable += " " + " ".join(
|
||||
str(alias or "") for alias in (model.get("aliases") or [])
|
||||
).casefold()
|
||||
|
||||
is_dark = is_dark_pool_model(model)
|
||||
|
||||
if is_dark:
|
||||
# 暗池处理规则
|
||||
if not enable_dark_pool:
|
||||
continue
|
||||
|
||||
# 1. 黑名单最终否决
|
||||
if dark_blacklist and any(kw in searchable for kw in dark_blacklist):
|
||||
continue
|
||||
|
||||
# 2. 白名单与日期条件检查(若设置了白名单或日期,必须满足其中之一)
|
||||
if dark_whitelist or dark_since_ts is not None:
|
||||
whitelisted = bool(dark_whitelist and any(kw in searchable for kw in dark_whitelist))
|
||||
if not whitelisted:
|
||||
if dark_since_ts is None:
|
||||
continue
|
||||
model_ts = get_model_timestamp(model)
|
||||
if model_ts is None or model_ts < dark_since_ts:
|
||||
continue
|
||||
else:
|
||||
# 明池处理规则(不受暗池规则影响)
|
||||
if pub_includes and not any(keyword in searchable for keyword in pub_includes):
|
||||
continue
|
||||
if pub_excludes and any(keyword in searchable for keyword in pub_excludes):
|
||||
continue
|
||||
|
||||
mid_lower = str(model.get("arena_model_id") or "").lower()
|
||||
if mid_lower in result_seen_ids:
|
||||
continue
|
||||
result_seen_ids.add(mid_lower)
|
||||
result.append(model)
|
||||
|
||||
result.sort(key=lambda m: _natural_sort_key(get_arena_direct_model_public_id(m)))
|
||||
return result
|
||||
|
||||
|
||||
__all__ = [
|
||||
"ARENA_MODEL_CATALOG_CONFIG_PATH",
|
||||
"MODEL_CATALOG_SOURCE",
|
||||
"ONLINE_PUBLIC_VISIBLE_MODELS",
|
||||
"filter_arena_catalog_models",
|
||||
"get_arena_direct_catalog_for_tab",
|
||||
"get_arena_model_catalog",
|
||||
"get_model_catalog_preset",
|
||||
"get_model_timestamp",
|
||||
"is_dark_pool_model",
|
||||
"load_arena_model_catalog_data",
|
||||
"load_arena_offline_cache_models",
|
||||
"migrate_and_cleanup_sites_model_catalog",
|
||||
"normalize_arena_model_catalog_config",
|
||||
"parse_date_to_timestamp",
|
||||
"parse_uuidv7_timestamp",
|
||||
"save_arena_model_catalog_data",
|
||||
"set_arena_model_catalog",
|
||||
]
|
||||
@@ -0,0 +1,155 @@
|
||||
"""Shared Clash proxy rotation for Arena and other browser automation commands."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import time
|
||||
import urllib.parse
|
||||
from typing import Any, Callable, Dict, Iterable, Optional, Tuple
|
||||
|
||||
import requests
|
||||
|
||||
|
||||
ARENA_CLASH_API = os.getenv("CLASH_API", os.getenv("ARENA_CLASH_API", "http://127.0.0.1:9097")).rstrip("/")
|
||||
ARENA_CLASH_SECRET = os.getenv("CLASH_SECRET", os.getenv("ARENA_CLASH_SECRET", "1"))
|
||||
ARENA_CLASH_SELECTOR = os.getenv("CLASH_SELECTOR", os.getenv("ARENA_CLASH_SELECTOR", "主代理"))
|
||||
|
||||
DEFAULT_ARENA_PROXY_POOL = (
|
||||
"HK自动选择",
|
||||
"JP自动选择",
|
||||
"KR自动选择",
|
||||
"SG自动选择",
|
||||
"TW自动选择",
|
||||
"US自动选择",
|
||||
)
|
||||
|
||||
|
||||
def _get_proxy_pool_from_env() -> Tuple[str, ...]:
|
||||
raw = os.getenv("CLASH_PROXY_POOL", os.getenv("ARENA_PROXY_POOL", ""))
|
||||
if not raw:
|
||||
return DEFAULT_ARENA_PROXY_POOL
|
||||
try:
|
||||
if raw.startswith("["):
|
||||
parsed = json.loads(raw)
|
||||
if isinstance(parsed, list):
|
||||
return tuple(str(x).strip() for x in parsed if str(x).strip())
|
||||
except Exception:
|
||||
pass
|
||||
parts = [p.strip() for p in raw.split(",") if p.strip()]
|
||||
return tuple(parts) if parts else DEFAULT_ARENA_PROXY_POOL
|
||||
|
||||
|
||||
ARENA_PROXY_POOL = _get_proxy_pool_from_env()
|
||||
|
||||
|
||||
def rotate_arena_proxy(
|
||||
*,
|
||||
tab: Any,
|
||||
logger: Any,
|
||||
raise_if_cancelled: Optional[Callable[[], None]] = None,
|
||||
requests_module: Any = requests,
|
||||
sleep: Callable[[float], None] = time.sleep,
|
||||
proxy_pool: Optional[Iterable[str]] = None,
|
||||
clash_api: Optional[str] = None,
|
||||
clash_secret: Optional[str] = None,
|
||||
clash_selector: Optional[str] = None,
|
||||
) -> Dict[str, Any]:
|
||||
"""Move the Clash selector to the next node in the pool."""
|
||||
cancel = raise_if_cancelled or (lambda: None)
|
||||
effective_pool = tuple(str(item) for item in (proxy_pool if proxy_pool is not None else ARENA_PROXY_POOL) if str(item).strip())
|
||||
api_base = str(clash_api or os.getenv("CLASH_API", os.getenv("ARENA_CLASH_API", ARENA_CLASH_API))).rstrip("/")
|
||||
secret = str(clash_secret if clash_secret is not None else os.getenv("CLASH_SECRET", os.getenv("ARENA_CLASH_SECRET", ARENA_CLASH_SECRET)))
|
||||
selector_name = str(clash_selector or os.getenv("CLASH_SELECTOR", os.getenv("ARENA_CLASH_SELECTOR", ARENA_CLASH_SELECTOR)))
|
||||
|
||||
headers = {"Content-Type": "application/json"}
|
||||
if secret:
|
||||
headers["Authorization"] = f"Bearer {secret}"
|
||||
|
||||
selector_url = (
|
||||
f"{api_base}/proxies/"
|
||||
f"{urllib.parse.quote(selector_name, safe='')}"
|
||||
)
|
||||
try:
|
||||
cancel()
|
||||
response = requests_module.get(selector_url, headers=headers, timeout=5)
|
||||
response.raise_for_status()
|
||||
selector = response.json()
|
||||
available = [name for name in effective_pool if name in (selector.get("all") or [])]
|
||||
if len(available) != len(effective_pool):
|
||||
return {
|
||||
"ok": False,
|
||||
"error": "arena_proxy_pool_incomplete",
|
||||
"missing": [name for name in effective_pool if name not in available],
|
||||
}
|
||||
|
||||
current = str(selector.get("now") or "")
|
||||
next_proxy = (
|
||||
available[(available.index(current) + 1) % len(available)]
|
||||
if current in available
|
||||
else available[0]
|
||||
)
|
||||
if next_proxy == current:
|
||||
return {
|
||||
"ok": True,
|
||||
"switched": False,
|
||||
"selector": selector_name,
|
||||
"node": current,
|
||||
"pool_size": len(available),
|
||||
}
|
||||
|
||||
cancel()
|
||||
switch_response = requests_module.put(
|
||||
selector_url,
|
||||
json={"name": next_proxy},
|
||||
headers=headers,
|
||||
timeout=5,
|
||||
)
|
||||
switch_response.raise_for_status()
|
||||
try:
|
||||
requests_module.delete(
|
||||
f"{api_base}/connections",
|
||||
headers=headers,
|
||||
timeout=5,
|
||||
).raise_for_status()
|
||||
except Exception as close_error:
|
||||
logger.warning(
|
||||
f"[ARENA][IP-ROTATE] closing Clash connections failed: {close_error}"
|
||||
)
|
||||
|
||||
logger.info(
|
||||
f"[ARENA][IP-ROTATE] {selector_name}: "
|
||||
f"{current or '-'} -> {next_proxy}"
|
||||
)
|
||||
sleep(1.0)
|
||||
try:
|
||||
tab.refresh()
|
||||
sleep(2.0)
|
||||
except Exception as refresh_error:
|
||||
logger.warning(f"[ARENA][IP-ROTATE] page refresh failed: {refresh_error}")
|
||||
return {
|
||||
"ok": True,
|
||||
"switched": True,
|
||||
"selector": selector_name,
|
||||
"from": current,
|
||||
"to": next_proxy,
|
||||
"pool_size": len(available),
|
||||
}
|
||||
except Exception as error:
|
||||
logger.error(f"[ARENA][IP-ROTATE] Clash switch failed: {error}")
|
||||
return {"ok": False, "error": str(error)}
|
||||
|
||||
|
||||
# 通用别名
|
||||
rotate_clash_proxy = rotate_arena_proxy
|
||||
|
||||
|
||||
__all__ = [
|
||||
"ARENA_CLASH_API",
|
||||
"ARENA_CLASH_SECRET",
|
||||
"ARENA_CLASH_SELECTOR",
|
||||
"ARENA_PROXY_POOL",
|
||||
"DEFAULT_ARENA_PROXY_POOL",
|
||||
"rotate_arena_proxy",
|
||||
"rotate_clash_proxy",
|
||||
]
|
||||
@@ -0,0 +1,359 @@
|
||||
"""
|
||||
app/services/arena_rule_service.py - Arena 规则匹配与候选记录业务服务
|
||||
|
||||
职责:
|
||||
- 评估多规则匹配逻辑(必含/排除词/检测器评估);
|
||||
- 整合 Link Drawer 本地书签抽屉写入;
|
||||
- 生成标准化候选命中记录并调用 command_result_store 进行持久化。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import tempfile
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
from typing import Any, Callable, Dict, Iterable, List, Optional
|
||||
|
||||
def _get_store() -> Any:
|
||||
from app.services import command_result_store
|
||||
return command_result_store
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
try:
|
||||
import requests
|
||||
except ImportError: # pragma: no cover
|
||||
requests = None
|
||||
|
||||
_DRAWER_LOCK = threading.RLock()
|
||||
|
||||
|
||||
def _split_terms(value: Any) -> List[str]:
|
||||
if isinstance(value, list):
|
||||
parts = value
|
||||
else:
|
||||
parts = re.split(r"[,,\n\r]+", str(value or ""))
|
||||
return [str(part).strip() for part in parts if str(part).strip()]
|
||||
|
||||
|
||||
def _matches_rule(text: str, rule: Dict[str, Any]) -> tuple[bool, str]:
|
||||
folded = str(text or "").casefold()
|
||||
excluded = _split_terms(rule.get("excluded"))
|
||||
for token in excluded:
|
||||
if token.casefold() in folded:
|
||||
return False, f"excluded:{token}"
|
||||
|
||||
required_all = _split_terms(rule.get("required_all"))
|
||||
missing = [token for token in required_all if token.casefold() not in folded]
|
||||
if missing:
|
||||
return False, f"missing:{','.join(missing)}"
|
||||
|
||||
required_any = _split_terms(rule.get("required_any"))
|
||||
if required_any and not any(token.casefold() in folded for token in required_any):
|
||||
return False, "missing_any"
|
||||
return True, "matched"
|
||||
|
||||
|
||||
def _detector_accepts(
|
||||
prompt: str,
|
||||
response_text: str,
|
||||
rule: Dict[str, Any],
|
||||
values: Dict[str, Any],
|
||||
) -> tuple[bool, Dict[str, Any]]:
|
||||
req_module = getattr(_get_store(), "requests", requests)
|
||||
keyword = str(rule.get("detector_keyword") or "").strip()
|
||||
if not keyword:
|
||||
return True, {"skipped": True, "models": []}
|
||||
detector_url = str(values.get("detector_url") or "http://127.0.0.1:8765/api/judge").strip()
|
||||
if req_module is None:
|
||||
return True, {"unavailable": True, "error": "requests unavailable", "models": []}
|
||||
try:
|
||||
response = req_module.post(
|
||||
detector_url,
|
||||
json={"prompt": str(prompt or ""), "response": str(response_text or "")},
|
||||
timeout=20,
|
||||
)
|
||||
response.raise_for_status()
|
||||
payload = response.json()
|
||||
predictions = list(payload.get("predictions") or [])[:5] if isinstance(payload, dict) else []
|
||||
models = [str(item.get("model") or "").strip() for item in predictions if isinstance(item, dict)]
|
||||
accepted = any(keyword.casefold() in model.casefold() for model in models)
|
||||
return accepted, {"models": models, "best_model": payload.get("best_model") if isinstance(payload, dict) else ""}
|
||||
except Exception as error:
|
||||
# Preserve a text-filter hit if the optional local detector is offline.
|
||||
return True, {"unavailable": True, "error": str(error), "models": []}
|
||||
|
||||
|
||||
def _resolve_drawer_file(raw_path: Any) -> Path | None:
|
||||
text = os.path.expandvars(os.path.expanduser(str(raw_path or "").strip()))
|
||||
if not text:
|
||||
return None
|
||||
path = Path(text)
|
||||
return path / "drawer_data.json" if path.is_dir() or not path.suffix else path
|
||||
|
||||
|
||||
def _write_link_drawer(
|
||||
path: Path | None,
|
||||
url: str,
|
||||
title: str,
|
||||
category: str,
|
||||
controlled_browser: Optional[Dict[str, Any]] = None,
|
||||
) -> Dict[str, Any]:
|
||||
if path is None:
|
||||
return {"status": "disabled"}
|
||||
with _DRAWER_LOCK:
|
||||
try:
|
||||
if path.exists():
|
||||
with path.open("r", encoding="utf-8") as handle:
|
||||
payload = json.load(handle)
|
||||
else:
|
||||
payload = {"categories": ["默认分类"], "links": [], "settings": {}}
|
||||
categories = payload.get("categories") if isinstance(payload.get("categories"), list) else []
|
||||
links = payload.get("links") if isinstance(payload.get("links"), list) else []
|
||||
category = str(category or "默认分类").strip() or "默认分类"
|
||||
if category not in categories:
|
||||
categories.append(category)
|
||||
normalized_url = str(url or "").strip().rstrip("/")
|
||||
for item in links:
|
||||
if str(item.get("url") or "").strip().rstrip("/") == normalized_url:
|
||||
return {"status": "duplicate", "category": item.get("category", "")}
|
||||
link = {
|
||||
"id": uuid.uuid4().hex[:8],
|
||||
"url": str(url or "").strip(),
|
||||
"title": title,
|
||||
"category": category,
|
||||
"dateAdded": int(time.time() * 1000),
|
||||
}
|
||||
if isinstance(controlled_browser, dict) and controlled_browser:
|
||||
link["controlledBrowser"] = controlled_browser
|
||||
links.append(link)
|
||||
payload["categories"] = categories
|
||||
payload["links"] = links
|
||||
_get_store()._atomic_write_json(path, payload)
|
||||
return {"status": "added", "category": category}
|
||||
except Exception as error:
|
||||
return {"status": "error", "error": str(error)}
|
||||
|
||||
|
||||
def _clean_title_component(value: Any, fallback: str) -> str:
|
||||
text = re.sub(r"[\x00-\x1f\x7f]+", " ", str(value or "")).strip()
|
||||
return re.sub(r"\s+", " ", text) or fallback
|
||||
|
||||
|
||||
def _next_title(
|
||||
records: Iterable[Dict[str, Any]],
|
||||
profile: str,
|
||||
model: str,
|
||||
title_template: str = "",
|
||||
) -> str:
|
||||
raw_profile = str(profile or "").strip()
|
||||
raw_model = str(model or "").strip()
|
||||
profile = _clean_title_component(profile, "profile").replace("《", "〈").replace("》", "〉")
|
||||
model = _clean_title_component(model, "model")
|
||||
template = str(title_template or "").strip() or "《{profile}》-{model}-{index:03d}"
|
||||
matching_count = 0
|
||||
for item in records:
|
||||
if (
|
||||
str(item.get("browser_profile_name") or "").strip() != raw_profile
|
||||
or str(item.get("model_name") or "").strip() != raw_model
|
||||
):
|
||||
continue
|
||||
matching_count += 1
|
||||
index = matching_count + 1
|
||||
try:
|
||||
title = template.format(profile=profile, model=model, index=index)
|
||||
except (KeyError, ValueError, IndexError):
|
||||
title = f"《{profile}》-{model}-{index:03d}"
|
||||
return _clean_title_component(title, f"《{profile}》-{model}-{index:03d}")
|
||||
|
||||
|
||||
def _controlled_browser_metadata(values: Dict[str, Any], identity: Dict[str, Any]) -> Dict[str, Any]:
|
||||
profile = {
|
||||
key: str(identity.get(key) or "").strip()
|
||||
for key in (
|
||||
"name",
|
||||
"profile_directory",
|
||||
"profile_path",
|
||||
"user_data_dir",
|
||||
"browser_context_id",
|
||||
"source_tab_id",
|
||||
)
|
||||
if str(identity.get(key) or "").strip()
|
||||
}
|
||||
return {
|
||||
"version": 1,
|
||||
"apiUrl": str(
|
||||
values.get("controlled_browser_api_url")
|
||||
or "http://127.0.0.1:8199/api/browser/open-profile-url"
|
||||
).strip(),
|
||||
"profile": profile,
|
||||
}
|
||||
|
||||
|
||||
def record_arena_rule_candidates(
|
||||
command_id: str,
|
||||
values: Dict[str, Any],
|
||||
info: Dict[str, Any],
|
||||
prompt: str = "",
|
||||
source: str = "",
|
||||
profile_resolver: Optional[Callable[[], Any]] = None,
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
匹配 Arena 规则并保存候选结果到通用持久化存储与 Link Drawer。
|
||||
"""
|
||||
rules = values.get("rules") if isinstance(values.get("rules"), list) else []
|
||||
response_sides = info.get("response_sides") if isinstance(info, dict) else []
|
||||
if not isinstance(response_sides, (list, tuple)) or not response_sides:
|
||||
response_sides = [str((info or {}).get("visible_text") or "")]
|
||||
url = str((info or {}).get("url") or "").strip()
|
||||
if not url:
|
||||
return {"recorded": [], "matched": 0}
|
||||
|
||||
candidates: List[tuple[str, Dict[str, Any], int, int, str, Dict[str, Any]]] = []
|
||||
profile_identity: Optional[Dict[str, Any]] = None
|
||||
identity_unresolved = False
|
||||
|
||||
def _resolve_profile() -> Dict[str, Any]:
|
||||
nonlocal profile_identity
|
||||
if profile_identity is not None:
|
||||
return profile_identity
|
||||
resolved: Any = {}
|
||||
if callable(profile_resolver):
|
||||
try:
|
||||
resolved = profile_resolver() or {}
|
||||
except Exception:
|
||||
resolved = {}
|
||||
if isinstance(resolved, str):
|
||||
resolved = {"name": resolved}
|
||||
profile_identity = resolved if isinstance(resolved, dict) else {}
|
||||
return profile_identity
|
||||
|
||||
store = _get_store()
|
||||
existing_results = store.list_command_results(command_id, limit=2000)
|
||||
existing_hits = {
|
||||
str(item.get("rule_id") or "")
|
||||
for item in existing_results
|
||||
if isinstance(item, dict) and item.get("url") == url
|
||||
}
|
||||
|
||||
# Dispatch detector using command_result_store._detector_accepts so test patches take effect
|
||||
detector_fn = getattr(store, "_detector_accepts", _detector_accepts)
|
||||
matches_rule_fn = getattr(store, "_matches_rule", _matches_rule)
|
||||
|
||||
for rule_index, rule in enumerate(rules):
|
||||
if not isinstance(rule, dict) or rule.get("enabled", True) is False:
|
||||
continue
|
||||
rule_id = str(rule.get("id") or f"rule-{rule_index + 1}")
|
||||
if rule_id in existing_hits:
|
||||
continue
|
||||
for side_index, response_text in enumerate(response_sides):
|
||||
text = str(response_text or "").strip()
|
||||
if not text:
|
||||
continue
|
||||
|
||||
is_reasoning_side = False
|
||||
has_reasoning_sides = info.get("has_reasoning_sides") if isinstance(info, dict) else None
|
||||
if isinstance(has_reasoning_sides, list) and side_index < len(has_reasoning_sides):
|
||||
is_reasoning_side = bool(has_reasoning_sides[side_index])
|
||||
|
||||
if rule.get("exclude_reasoning") and is_reasoning_side:
|
||||
continue
|
||||
|
||||
matched, _ = matches_rule_fn(text, rule)
|
||||
if not matched:
|
||||
continue
|
||||
accepted, detector = detector_fn(prompt, text, rule, values)
|
||||
if accepted:
|
||||
candidates.append((rule_id, rule, rule_index, side_index, text, detector))
|
||||
break
|
||||
|
||||
if candidates:
|
||||
identity = _resolve_profile()
|
||||
if not str(identity.get("name") or "").strip():
|
||||
identity_unresolved = True
|
||||
logger.warning(
|
||||
"[CMD_RESULT] browser profile name 为空,%d 条候选命中将被跳过。"
|
||||
"请确认 profile_resolver 已正确返回 identity。",
|
||||
len(candidates),
|
||||
)
|
||||
|
||||
new_records: List[Dict[str, Any]] = []
|
||||
if not identity_unresolved and candidates:
|
||||
identity = _resolve_profile()
|
||||
current_records = store.get_all_records()
|
||||
for rule_id, rule, rule_index, side_index, text, detector in candidates:
|
||||
if any(
|
||||
item.get("command_id") == command_id
|
||||
and item.get("rule_id") == rule_id
|
||||
and item.get("url") == url
|
||||
for item in current_records
|
||||
if isinstance(item, dict)
|
||||
):
|
||||
continue
|
||||
model = str(rule.get("model_name") or rule.get("name") or f"model-{rule_index + 1}").strip()
|
||||
profile = str(identity.get("name") or "").strip()
|
||||
title = _next_title(current_records, profile, model, str(rule.get("title_template") or ""))
|
||||
drawer = _write_link_drawer(
|
||||
_resolve_drawer_file(values.get("link_drawer_path")),
|
||||
url,
|
||||
title,
|
||||
str(rule.get("drawer_group") or model),
|
||||
_controlled_browser_metadata(values, identity),
|
||||
)
|
||||
record = {
|
||||
"id": uuid.uuid4().hex,
|
||||
"command_id": command_id,
|
||||
"rule_id": rule_id,
|
||||
"rule_name": str(rule.get("name") or model),
|
||||
"model_name": model,
|
||||
"browser_profile_name": profile,
|
||||
"browser_profile": identity,
|
||||
"title": title,
|
||||
"url": url,
|
||||
"side": "A" if side_index == 0 else "B",
|
||||
"source": str(source or ""),
|
||||
"response_preview": text[:500],
|
||||
"detector": detector,
|
||||
"drawer": drawer,
|
||||
"created_at": time.time(),
|
||||
}
|
||||
current_records.append(record)
|
||||
new_records.append(record)
|
||||
|
||||
if new_records:
|
||||
# The final duplicate check and write must share one storage lock;
|
||||
# otherwise two concurrent bridge callbacks can persist the same hit.
|
||||
add_unique = getattr(store, "add_records_unique", None)
|
||||
if callable(add_unique):
|
||||
persisted_records = add_unique(
|
||||
new_records,
|
||||
("command_id", "rule_id", "url"),
|
||||
)
|
||||
else: # pragma: no cover - compatibility for lightweight test doubles
|
||||
persisted_records = store.add_records(new_records)
|
||||
new_records = persisted_records
|
||||
|
||||
return {
|
||||
"recorded": new_records,
|
||||
"matched": len(new_records),
|
||||
"identity_unresolved": identity_unresolved,
|
||||
}
|
||||
|
||||
|
||||
__all__ = [
|
||||
"_clean_title_component",
|
||||
"_controlled_browser_metadata",
|
||||
"_detector_accepts",
|
||||
"_matches_rule",
|
||||
"_next_title",
|
||||
"_resolve_drawer_file",
|
||||
"_split_terms",
|
||||
"_write_link_drawer",
|
||||
"record_arena_rule_candidates",
|
||||
]
|
||||
@@ -0,0 +1,358 @@
|
||||
"""
|
||||
app/services/arena_tab_listener.py - Arena 翻牌探测与结果桥接扩展监听器
|
||||
|
||||
从通用全局网络拦截层解耦的独立扩展模块。
|
||||
通过 register_response_listener 挂载到 _GlobalNetworkInterceptionManager,
|
||||
实现对 Arena 候选响应的事件桥接与前端 React Fiber store 的翻牌轮询。
|
||||
"""
|
||||
|
||||
import threading
|
||||
import time
|
||||
from typing import Any, Callable, Dict, Optional, Set
|
||||
|
||||
from app.core.config import logger
|
||||
from app.core.page_lifecycle import is_page_refresh_error
|
||||
from app.core.tab_pool_parts.session import TabSession
|
||||
from app.core.driver import driver_for_tab
|
||||
|
||||
|
||||
def is_explicit_arena_direct_url(url: Any) -> bool:
|
||||
"""Check if URL explicitly points to Arena direct chat page."""
|
||||
try:
|
||||
raw = str(url or "").strip()
|
||||
if not raw:
|
||||
return False
|
||||
from urllib.parse import parse_qs, urlparse
|
||||
parsed = urlparse(raw)
|
||||
path = (parsed.path or "").rstrip("/").lower()
|
||||
if path in {"/direct", "/text/direct", "/image/direct", "/code/direct", "/search/direct"}:
|
||||
return True
|
||||
if path.startswith(("/direct/", "/text/direct/", "/image/direct/", "/code/direct/", "/search/direct/")):
|
||||
return True
|
||||
if path.endswith("/direct"):
|
||||
return True
|
||||
query = parse_qs(parsed.query)
|
||||
if query.get("mode") == ["direct"]:
|
||||
return True
|
||||
return False
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
from app.core.tab_pool_parts._arena_snapshot import (
|
||||
_ARENA_STORE_SNAPSHOT_JS,
|
||||
_ARENA_STORE_SNAPSHOT_JSON_JS,
|
||||
)
|
||||
from app.core.cdp_hygiene import decode_js_json
|
||||
from app.core.tab_pool_parts.idle_maintenance import note_network_activity
|
||||
|
||||
|
||||
class ArenaTabListener:
|
||||
"""Arena 翻牌探测与结果桥接监听器。"""
|
||||
|
||||
ARENA_REVEAL_POLL_INTERVAL_SEC = 3.0
|
||||
ARENA_REVEAL_POLL_TIMEOUT_SEC = 120.0
|
||||
RESULT_BRIDGE_MAX_ACTIVE_PER_SESSION = 2
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
get_session_fn: Optional[Callable[[str], Optional[TabSession]]] = None,
|
||||
is_shutdown_fn: Optional[Callable[[], bool]] = None,
|
||||
):
|
||||
self._get_session = get_session_fn
|
||||
self._is_shutdown = is_shutdown_fn or (lambda: False)
|
||||
self._result_event_handler = self._create_result_event_handler()
|
||||
self._result_bridge_lock = threading.RLock()
|
||||
self._result_bridge_active_by_session: Dict[str, int] = {}
|
||||
self._arena_reveal_pollers: Dict[str, threading.Thread] = {}
|
||||
self._arena_reveal_lock = threading.RLock()
|
||||
self._arena_reveal_logged_signatures: Set[str] = set()
|
||||
|
||||
@staticmethod
|
||||
def _create_result_event_handler():
|
||||
try:
|
||||
from app.services.result_event_bridge import create_result_event_handler
|
||||
handler = create_result_event_handler()
|
||||
if handler:
|
||||
logger.info("[GlobalNet] Arena 结果事件桥接已启用(支持手动网页测试)")
|
||||
return handler
|
||||
except Exception as e:
|
||||
logger.debug(f"[GlobalNet] Arena 结果事件桥接初始化失败(忽略): {e}")
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def is_arena_candidate(event: Dict[str, Any]) -> bool:
|
||||
"""快速判断事件是否可能属于 Arena。"""
|
||||
url = str((event or {}).get("url") or "").strip().lower()
|
||||
if not url:
|
||||
return False
|
||||
return any(host in url for host in ("lmarena.ai", "arena.ai", "lmsys.org"))
|
||||
|
||||
@classmethod
|
||||
def _is_result_bridge_candidate(cls, event: Dict[str, Any]) -> bool:
|
||||
if not cls.is_arena_candidate(event):
|
||||
return False
|
||||
url = str((event or {}).get("url") or "").strip().lower()
|
||||
return any(
|
||||
token in url
|
||||
for token in (
|
||||
"/nextjs-api/stream/",
|
||||
"nextjs-api/stream",
|
||||
"create-evaluation",
|
||||
"post-to-evaluation",
|
||||
"stream/create",
|
||||
"stream/post",
|
||||
)
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def _is_reveal_snapshot_candidate(cls, event: Dict[str, Any]) -> bool:
|
||||
if not cls.is_arena_candidate(event):
|
||||
return False
|
||||
url = str((event or {}).get("url") or "").strip().lower()
|
||||
return any(
|
||||
token in url
|
||||
for token in (
|
||||
"/nextjs-api/stream/",
|
||||
"/rpc/i/",
|
||||
"/api/history/",
|
||||
"/c/",
|
||||
"_rsc=",
|
||||
)
|
||||
)
|
||||
|
||||
def _claim_result_bridge_slot(self, session_id: str) -> bool:
|
||||
key = str(session_id or "unknown")
|
||||
with self._result_bridge_lock:
|
||||
active = int(self._result_bridge_active_by_session.get(key, 0) or 0)
|
||||
if active >= self.RESULT_BRIDGE_MAX_ACTIVE_PER_SESSION:
|
||||
logger.debug_throttled(
|
||||
f"global_net.result_bridge_busy.{key}",
|
||||
f"[GlobalNet] Arena 结果桥接忙,跳过候选响应: {key}, active={active}",
|
||||
interval_sec=10.0,
|
||||
)
|
||||
return False
|
||||
self._result_bridge_active_by_session[key] = active + 1
|
||||
return True
|
||||
|
||||
def _release_result_bridge_slot(self, session_id: str) -> None:
|
||||
key = str(session_id or "unknown")
|
||||
with self._result_bridge_lock:
|
||||
active = int(self._result_bridge_active_by_session.get(key, 0) or 0)
|
||||
if active <= 1:
|
||||
self._result_bridge_active_by_session.pop(key, None)
|
||||
else:
|
||||
self._result_bridge_active_by_session[key] = active - 1
|
||||
|
||||
def _dispatch_result_bridge_async(
|
||||
self,
|
||||
session: TabSession,
|
||||
response: Any,
|
||||
event: Dict[str, Any],
|
||||
stop_event: threading.Event,
|
||||
) -> None:
|
||||
if not self._result_event_handler:
|
||||
return
|
||||
if not self._is_result_bridge_candidate(event):
|
||||
return
|
||||
|
||||
session_id = str(getattr(session, "id", "") or "unknown")
|
||||
if not self._claim_result_bridge_slot(session_id):
|
||||
return
|
||||
|
||||
try:
|
||||
thread = threading.Thread(
|
||||
target=self._dispatch_result_bridge,
|
||||
args=(session, response, dict(event or {}), stop_event),
|
||||
daemon=True,
|
||||
name=f"global-net-arena-{session_id}",
|
||||
)
|
||||
thread.start()
|
||||
except Exception as e:
|
||||
self._release_result_bridge_slot(session_id)
|
||||
logger.debug(f"[GlobalNet] Arena 结果桥接线程启动失败(忽略): {e}")
|
||||
|
||||
def _dispatch_result_bridge(
|
||||
self,
|
||||
session: TabSession,
|
||||
response: Any,
|
||||
event: Dict[str, Any],
|
||||
stop_event: threading.Event,
|
||||
) -> None:
|
||||
session_id = str(getattr(session, "id", "") or "unknown")
|
||||
try:
|
||||
from app.core.tab_pool_parts.network import _GlobalNetworkInterceptionManager
|
||||
raw_body, raw_body_source = _GlobalNetworkInterceptionManager.read_response_body(response, stop_event)
|
||||
if not raw_body:
|
||||
return
|
||||
|
||||
post_data = _GlobalNetworkInterceptionManager.extract_request_post_data(response)
|
||||
self._result_event_handler(
|
||||
{
|
||||
"event": event,
|
||||
"raw_body": raw_body,
|
||||
"raw_body_source": raw_body_source,
|
||||
"request_post_data": post_data,
|
||||
"parse_result": {"done": True},
|
||||
"parser_id": "lmarena_global",
|
||||
"session_id": getattr(session, "id", ""),
|
||||
"session": session,
|
||||
}
|
||||
)
|
||||
except Exception as e:
|
||||
logger.debug(f"[GlobalNet] Arena 结果事件桥接失败(忽略): {e}")
|
||||
finally:
|
||||
self._release_result_bridge_slot(session_id)
|
||||
|
||||
def _start_arena_reveal_poll(
|
||||
self,
|
||||
session: TabSession,
|
||||
event: Dict[str, Any],
|
||||
stop_event: threading.Event,
|
||||
reason: str,
|
||||
) -> None:
|
||||
if not session or not self._is_reveal_snapshot_candidate(event):
|
||||
return
|
||||
session_id = str(getattr(session, "id", "") or "")
|
||||
if not session_id:
|
||||
return
|
||||
|
||||
try:
|
||||
from app.core.workflow.arena_direct_guard import is_arena_direct_preset
|
||||
tab = getattr(session, "tab", None)
|
||||
current_url = getattr(tab, "url", "") if tab else ""
|
||||
if current_url and is_explicit_arena_direct_url(current_url):
|
||||
return
|
||||
preset_name = str(getattr(session, "preset_name", "") or getattr(session, "preset", "") or "")
|
||||
if preset_name and is_arena_direct_preset("arena.ai", preset_name):
|
||||
return
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
with self._arena_reveal_lock:
|
||||
current = self._arena_reveal_pollers.get(session_id)
|
||||
if current and current.is_alive():
|
||||
return
|
||||
thread = threading.Thread(
|
||||
target=self._arena_reveal_poll_loop,
|
||||
args=(session_id, stop_event, reason),
|
||||
daemon=True,
|
||||
name=f"global-net-reveal-{session_id}",
|
||||
)
|
||||
self._arena_reveal_pollers[session_id] = thread
|
||||
thread.start()
|
||||
|
||||
def _arena_reveal_poll_loop(
|
||||
self,
|
||||
session_id: str,
|
||||
stop_event: threading.Event,
|
||||
reason: str,
|
||||
) -> None:
|
||||
try:
|
||||
from app.services.result_event_bridge import emit_arena_snapshot_event
|
||||
except Exception as e:
|
||||
logger.debug(f"[GlobalNet] Arena 翻牌快照桥接不可用(忽略): {e}")
|
||||
return
|
||||
|
||||
deadline = time.time() + self.ARENA_REVEAL_POLL_TIMEOUT_SEC
|
||||
last_signature = ""
|
||||
try:
|
||||
while time.time() < deadline and not stop_event.is_set() and not self._is_shutdown():
|
||||
session = self._get_session(session_id) if callable(self._get_session) else None
|
||||
tab = getattr(session, "tab", None) if session is not None else None
|
||||
if tab is None:
|
||||
return
|
||||
# P0-6:翻牌轮询期间视为活跃,避免被空闲冻结
|
||||
note_network_activity(session)
|
||||
try:
|
||||
snapshot = decode_js_json(driver_for_tab(tab).run_js(_ARENA_STORE_SNAPSHOT_JSON_JS))
|
||||
except Exception as e:
|
||||
if not is_page_refresh_error(e):
|
||||
logger.debug_throttled(
|
||||
f"global_net.arena_reveal_snapshot.{session_id}",
|
||||
f"[GlobalNet] 读取 Arena 翻牌快照失败(忽略): {e}",
|
||||
interval_sec=10.0,
|
||||
)
|
||||
time.sleep(self.ARENA_REVEAL_POLL_INTERVAL_SEC)
|
||||
continue
|
||||
|
||||
if not isinstance(snapshot, dict):
|
||||
time.sleep(self.ARENA_REVEAL_POLL_INTERVAL_SEC)
|
||||
continue
|
||||
|
||||
is_direct = bool(
|
||||
snapshot.get("is_direct")
|
||||
or str(snapshot.get("mode") or "").strip().lower() == "direct"
|
||||
or is_explicit_arena_direct_url(snapshot.get("url"))
|
||||
)
|
||||
if is_direct:
|
||||
return
|
||||
|
||||
snapshot["session_id"] = session_id
|
||||
model_a = str(snapshot.get("model_a") or "").strip()
|
||||
model_b = str(snapshot.get("model_b") or "").strip()
|
||||
response_a = str(snapshot.get("response_a") or "")
|
||||
response_b = str(snapshot.get("response_b") or "")
|
||||
message_id_a = str(snapshot.get("message_id_a") or "").strip()
|
||||
message_id_b = str(snapshot.get("message_id_b") or "").strip()
|
||||
|
||||
if not (model_a and model_b and message_id_a and message_id_b and message_id_a != message_id_b):
|
||||
time.sleep(self.ARENA_REVEAL_POLL_INTERVAL_SEC)
|
||||
continue
|
||||
|
||||
signature = (
|
||||
f"{snapshot.get('conversation_id')}|{message_id_a}|"
|
||||
f"{message_id_b}|{model_a}|{model_b}"
|
||||
)
|
||||
if signature != last_signature:
|
||||
last_signature = signature
|
||||
log_signature = f"{session_id}|{signature}"
|
||||
with self._arena_reveal_lock:
|
||||
should_log = log_signature not in self._arena_reveal_logged_signatures
|
||||
if should_log:
|
||||
self._arena_reveal_logged_signatures.add(log_signature)
|
||||
if len(self._arena_reveal_logged_signatures) > 500:
|
||||
self._arena_reveal_logged_signatures.clear()
|
||||
if should_log:
|
||||
logger.debug(
|
||||
"[GlobalNet] Arena 翻牌快照更新: "
|
||||
f"reason={reason}, model_a={model_a}, model_b={model_b}, "
|
||||
f"a={len(response_a)}, b={len(response_b)}"
|
||||
)
|
||||
|
||||
if response_a and response_b:
|
||||
emit_arena_snapshot_event(snapshot)
|
||||
return
|
||||
|
||||
time.sleep(self.ARENA_REVEAL_POLL_INTERVAL_SEC)
|
||||
finally:
|
||||
with self._arena_reveal_lock:
|
||||
current = self._arena_reveal_pollers.get(session_id)
|
||||
if current is threading.current_thread():
|
||||
self._arena_reveal_pollers.pop(session_id, None)
|
||||
|
||||
def __call__(
|
||||
self,
|
||||
session: TabSession,
|
||||
response: Any,
|
||||
event: Dict[str, Any],
|
||||
stop_event: threading.Event,
|
||||
) -> None:
|
||||
"""作为全局 response listener 的回调入口。"""
|
||||
self._dispatch_result_bridge_async(session, response, event, stop_event)
|
||||
self._start_arena_reveal_poll(session, event, stop_event, "network-event")
|
||||
|
||||
|
||||
def register_arena_tab_listener(manager: Any) -> Optional[Callable[[], None]]:
|
||||
"""向全局网络管理器挂载 Arena 翻牌与结果监听器。"""
|
||||
if manager is None or not hasattr(manager, "register_response_listener"):
|
||||
return None
|
||||
|
||||
get_session_fn = getattr(manager, "_get_session", None)
|
||||
is_shutdown_fn = getattr(manager, "_is_shutdown", None)
|
||||
listener = ArenaTabListener(get_session_fn=get_session_fn, is_shutdown_fn=is_shutdown_fn)
|
||||
|
||||
return manager.register_response_listener(
|
||||
ArenaTabListener.is_arena_candidate,
|
||||
listener,
|
||||
)
|
||||
@@ -0,0 +1,425 @@
|
||||
"""
|
||||
app/services/catalog_routing.py - 通用动态模型目录与路由派发器
|
||||
|
||||
职责:
|
||||
- 统一管理各站点的动态模型目录提供者(CatalogProvider)
|
||||
- 抽象统一的 CatalogRouter,支持动态目录模型匹配、Tab 挑选及模型列表发现
|
||||
- 彻底解耦 API 层(chat.py)与具体业务模型目录(如 Arena Direct)的实现细节
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import random
|
||||
import sys
|
||||
from typing import Any, Callable, Dict, List, Optional, Protocol, Sequence, Set, Tuple
|
||||
|
||||
from app.core.config import get_logger
|
||||
from app.utils.site_url import route_domain_matches
|
||||
|
||||
logger = get_logger("SERVICES.CATALOG_ROUTING")
|
||||
|
||||
|
||||
def _get_chat_or_service_attr(name: str, fallback_module: Optional[Any] = None) -> Any:
|
||||
chat_mod = sys.modules.get("app.api.chat")
|
||||
if chat_mod and hasattr(chat_mod, name):
|
||||
return getattr(chat_mod, name)
|
||||
if fallback_module and hasattr(fallback_module, name):
|
||||
return getattr(fallback_module, name)
|
||||
return None
|
||||
|
||||
|
||||
def select_catalog_tab(
|
||||
browser: Any,
|
||||
candidates: List[Dict[str, Any]],
|
||||
preset_name: Any = None,
|
||||
cursor_prefix: str = "arena_catalog",
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
"""Select a tab from candidates respecting the tab pool's allocation policy."""
|
||||
if not candidates:
|
||||
return None
|
||||
|
||||
idle_candidates = [
|
||||
item
|
||||
for item in candidates
|
||||
if str(item.get("status") or "").strip().lower() == "idle"
|
||||
]
|
||||
pool = idle_candidates or candidates
|
||||
|
||||
try:
|
||||
allocation_mode = str(
|
||||
getattr(browser.tab_pool, "allocation_mode", "") or ""
|
||||
).strip().lower()
|
||||
except Exception:
|
||||
allocation_mode = ""
|
||||
if allocation_mode not in {"first_idle", "round_robin", "random"}:
|
||||
allocation_mode = "first_idle"
|
||||
|
||||
if allocation_mode == "round_robin":
|
||||
from app.api import tab_routes as tab_routes_api
|
||||
|
||||
cursor_key = f"{cursor_prefix}::{str(preset_name or '').strip().casefold()}"
|
||||
return tab_routes_api._select_round_robin_tab(pool, cursor_key)
|
||||
if allocation_mode == "random":
|
||||
return random.choice(pool)
|
||||
return min(pool, key=lambda item: int(item.get("persistent_index") or 0))
|
||||
|
||||
|
||||
class CatalogProvider(Protocol):
|
||||
"""Protocol defining a dynamic catalog provider for model discovery and routing."""
|
||||
|
||||
@property
|
||||
def name(self) -> str: ...
|
||||
|
||||
@property
|
||||
def route_domain(self) -> str: ...
|
||||
|
||||
def match_tab_model(
|
||||
self,
|
||||
browser: Any,
|
||||
config_engine: Any,
|
||||
tab: Dict[str, Any],
|
||||
requested_model: Any,
|
||||
) -> Optional[Dict[str, Any]]: ...
|
||||
|
||||
def collect_catalog_configs(
|
||||
self,
|
||||
config_engine: Any,
|
||||
tabs: List[Dict[str, Any]],
|
||||
) -> List[Dict[str, Any]]: ...
|
||||
|
||||
def is_alias_suppressed(
|
||||
self,
|
||||
item: Dict[str, Any],
|
||||
catalog_configs: List[Dict[str, Any]],
|
||||
) -> bool: ...
|
||||
|
||||
def list_catalog_model_entries(
|
||||
self,
|
||||
browser: Any,
|
||||
catalog_configs: List[Dict[str, Any]],
|
||||
created: int,
|
||||
) -> List[Dict[str, Any]]: ...
|
||||
|
||||
|
||||
class ArenaCatalogProvider:
|
||||
"""Dynamic catalog provider for Arena Direct models."""
|
||||
|
||||
@property
|
||||
def name(self) -> str:
|
||||
return "arena_direct"
|
||||
|
||||
@property
|
||||
def route_domain(self) -> str:
|
||||
return "arena.ai"
|
||||
|
||||
def match_tab_model(
|
||||
self,
|
||||
browser: Any,
|
||||
config_engine: Any,
|
||||
tab: Dict[str, Any],
|
||||
requested_model: Any,
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
import app.services.arena_direct_models as adm
|
||||
|
||||
get_catalog_fn = _get_chat_or_service_attr("get_arena_direct_catalog_for_tab", adm)
|
||||
list_models_fn = _get_chat_or_service_attr("list_arena_direct_models", adm)
|
||||
match_model_fn = _get_chat_or_service_attr("match_arena_direct_model", adm)
|
||||
get_public_id_fn = _get_chat_or_service_attr("get_arena_direct_model_public_id", adm)
|
||||
|
||||
catalog_info = get_catalog_fn(config_engine, tab) if callable(get_catalog_fn) else None
|
||||
catalog = catalog_info.get("catalog") if catalog_info else None
|
||||
if not isinstance(catalog, dict) or not catalog.get("enabled"):
|
||||
return None
|
||||
|
||||
try:
|
||||
models = list_models_fn(browser, catalog_config=catalog, tab=tab)
|
||||
except TypeError:
|
||||
models = list_models_fn(browser, catalog_config=catalog)
|
||||
|
||||
model = match_model_fn(models, requested_model)
|
||||
if not model:
|
||||
return None
|
||||
|
||||
preset_name = str(catalog_info.get("preset_name") or "")
|
||||
public_model_id = get_public_id_fn(model) if callable(get_public_id_fn) else ""
|
||||
return {
|
||||
"tab": tab,
|
||||
"model": model,
|
||||
"preset_name": preset_name,
|
||||
"public_model_id": public_model_id,
|
||||
"route_domain": self.route_domain,
|
||||
}
|
||||
|
||||
def collect_catalog_configs(
|
||||
self,
|
||||
config_engine: Any,
|
||||
tabs: List[Dict[str, Any]],
|
||||
) -> List[Dict[str, Any]]:
|
||||
import app.services.arena_direct_models as adm
|
||||
|
||||
get_catalog_fn = _get_chat_or_service_attr("get_arena_direct_catalog_for_tab", adm)
|
||||
normalize_config_fn = _get_chat_or_service_attr("normalize_model_catalog_config", adm)
|
||||
|
||||
catalog_presets: List[Dict[str, Any]] = []
|
||||
seen_catalog_keys: Set[Tuple[Any, ...]] = set()
|
||||
|
||||
for tab in tabs or []:
|
||||
candidate = get_catalog_fn(config_engine, tab) if callable(get_catalog_fn) else None
|
||||
cat_obj = candidate.get("catalog") if isinstance(candidate, dict) else None
|
||||
if isinstance(cat_obj, dict) and cat_obj.get("enabled"):
|
||||
cat = cat_obj
|
||||
else:
|
||||
cat = None
|
||||
if route_domain_matches(self.route_domain, tab.get("route_domain") or tab.get("current_domain") or ""):
|
||||
effective_preset_name = str(
|
||||
tab.get("preset_name") or config_engine.get_default_preset(self.route_domain) or ""
|
||||
).strip()
|
||||
try:
|
||||
preset = config_engine._get_site_data_readonly(self.route_domain, effective_preset_name or None)
|
||||
if isinstance(preset, dict) and callable(normalize_config_fn):
|
||||
cat_candidate = normalize_config_fn(preset.get("model_catalog"))
|
||||
if cat_candidate.get("enabled") and cat_candidate.get("source") == "arena_direct":
|
||||
cat = cat_candidate
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if cat and cat.get("enabled"):
|
||||
cat_key = (
|
||||
cat.get("source", ""),
|
||||
cat.get("modality", ""),
|
||||
tuple(cat.get("include_keywords") or []),
|
||||
tuple(cat.get("exclude_keywords") or []),
|
||||
bool(cat.get("enable_dark_pool", False)),
|
||||
str(cat.get("dark_pool_since") or "").strip(),
|
||||
tuple(cat.get("dark_pool_whitelist_keywords") or []),
|
||||
tuple(cat.get("dark_pool_blacklist_keywords") or []),
|
||||
)
|
||||
if cat_key not in seen_catalog_keys:
|
||||
seen_catalog_keys.add(cat_key)
|
||||
catalog_presets.append(cat)
|
||||
|
||||
return catalog_presets
|
||||
|
||||
def is_alias_suppressed(
|
||||
self,
|
||||
item: Dict[str, Any],
|
||||
catalog_configs: List[Dict[str, Any]],
|
||||
) -> bool:
|
||||
if not catalog_configs or not item.get("is_route_alias"):
|
||||
return False
|
||||
item_route_domains = list(item.get("route_domains") or [])
|
||||
if item.get("route_domain"):
|
||||
item_route_domains.append(item.get("route_domain"))
|
||||
return any(route_domain_matches(self.route_domain, domain) for domain in item_route_domains)
|
||||
|
||||
def list_catalog_model_entries(
|
||||
self,
|
||||
browser: Any,
|
||||
catalog_configs: List[Dict[str, Any]],
|
||||
created: int,
|
||||
) -> List[Dict[str, Any]]:
|
||||
import app.services.arena_direct_models as adm
|
||||
|
||||
list_models_fn = _get_chat_or_service_attr("list_arena_direct_models", adm)
|
||||
build_entries_fn = _get_chat_or_service_attr("build_openai_model_entries", adm)
|
||||
|
||||
entries: List[Dict[str, Any]] = []
|
||||
for cat in catalog_configs:
|
||||
cat_models = list_models_fn(browser, catalog_config=cat) if callable(list_models_fn) else []
|
||||
model_entries = build_entries_fn(cat_models, created=created) if callable(build_entries_fn) else []
|
||||
for item in model_entries:
|
||||
entries.append({
|
||||
"id": item.get("id"),
|
||||
"owned_by": item.get("owned_by") or self.route_domain,
|
||||
"display_name": item.get("display_name") or item.get("id"),
|
||||
})
|
||||
return entries
|
||||
|
||||
|
||||
class CatalogRouter:
|
||||
"""Universal dispatcher for dynamic catalog matching and model enumeration."""
|
||||
|
||||
def __init__(self, providers: Optional[Sequence[CatalogProvider]] = None):
|
||||
self._providers: List[CatalogProvider] = list(providers or [ArenaCatalogProvider()])
|
||||
|
||||
def register_provider(self, provider: CatalogProvider) -> None:
|
||||
self._providers.append(provider)
|
||||
|
||||
def match_route(
|
||||
self,
|
||||
browser: Any,
|
||||
config_engine: Any,
|
||||
tabs: List[Dict[str, Any]],
|
||||
requested_model: Any,
|
||||
preset_name: Any = None,
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
"""Match a requested model against all registered catalog providers across open tabs."""
|
||||
if not browser or not tabs:
|
||||
return None
|
||||
|
||||
# Check if chat API has monkeypatched match/select function
|
||||
chat_match_fn = _get_chat_or_service_attr("_match_arena_catalog_tab")
|
||||
if callable(chat_match_fn) and not getattr(chat_match_fn, "_is_default_wrapper", False):
|
||||
matched_tab_info = chat_match_fn(browser, config_engine, tabs, requested_model)
|
||||
if matched_tab_info:
|
||||
matched_tab = matched_tab_info.get("tab") or {}
|
||||
matched_model = matched_tab_info.get("model") or {}
|
||||
matched_preset = matched_tab_info.get("preset_name") or preset_name
|
||||
import app.services.arena_direct_models as adm
|
||||
get_public_id_fn = _get_chat_or_service_attr("get_arena_direct_model_public_id", adm)
|
||||
public_model_id = get_public_id_fn(matched_model) if callable(get_public_id_fn) else str(requested_model or "")
|
||||
catalog_tab_index = int(matched_tab.get("persistent_index") or 0) or None
|
||||
return {
|
||||
"route_domain": "arena.ai",
|
||||
"route_type": "model_catalog",
|
||||
"model_name": public_model_id or str(requested_model or ""),
|
||||
"matched_id": public_model_id or str(requested_model or ""),
|
||||
"match_type": "catalog",
|
||||
"preset_name": matched_preset,
|
||||
"available_model_ids": [public_model_id] if public_model_id else [],
|
||||
"catalog_tab": matched_tab,
|
||||
"catalog_tab_index": catalog_tab_index,
|
||||
"matched_model": matched_model,
|
||||
}
|
||||
return None
|
||||
|
||||
is_url_excluded = getattr(getattr(browser, "tab_pool", None), "is_url_excluded", None)
|
||||
|
||||
for provider in self._providers:
|
||||
matches: List[Dict[str, Any]] = []
|
||||
for tab in tabs:
|
||||
tab_url = str(tab.get("current_url") or tab.get("url") or "").strip()
|
||||
if callable(is_url_excluded) and is_url_excluded(tab_url):
|
||||
continue
|
||||
|
||||
matched = provider.match_tab_model(browser, config_engine, tab, requested_model)
|
||||
if matched:
|
||||
matches.append(matched)
|
||||
|
||||
if not matches:
|
||||
continue
|
||||
|
||||
select_tab_fn = _get_chat_or_service_attr("_select_arena_catalog_tab")
|
||||
if callable(select_tab_fn) and select_tab_fn is not select_catalog_tab:
|
||||
selected_tab = select_tab_fn(
|
||||
browser,
|
||||
[entry["tab"] for entry in matches],
|
||||
preset_name=preset_name,
|
||||
)
|
||||
else:
|
||||
selected_tab = select_catalog_tab(
|
||||
browser,
|
||||
[entry["tab"] for entry in matches],
|
||||
preset_name=preset_name,
|
||||
)
|
||||
|
||||
if not selected_tab:
|
||||
continue
|
||||
|
||||
selected_index = int(selected_tab.get("persistent_index") or 0)
|
||||
selected_tab_id = str(selected_tab.get("id") or selected_tab.get("tab_id") or "").strip()
|
||||
target_entry = next(
|
||||
(
|
||||
entry
|
||||
for entry in matches
|
||||
if entry["tab"] is selected_tab
|
||||
or (selected_tab_id and str(entry["tab"].get("id") or entry["tab"].get("tab_id") or "").strip() == selected_tab_id)
|
||||
or int(entry["tab"].get("persistent_index") or 0) == selected_index
|
||||
),
|
||||
None,
|
||||
)
|
||||
if not target_entry:
|
||||
continue
|
||||
|
||||
catalog_tab = target_entry["tab"]
|
||||
catalog_match = target_entry["model"]
|
||||
matched_preset_name = target_entry["preset_name"]
|
||||
public_model_id = target_entry.get("public_model_id") or ""
|
||||
catalog_tab_index = int(catalog_tab.get("persistent_index") or 0) or None
|
||||
|
||||
matched_uuid = str(
|
||||
catalog_match.get("id")
|
||||
or catalog_match.get("uuid")
|
||||
or catalog_match.get("arena_model_id")
|
||||
or ""
|
||||
)
|
||||
matched_public_name = str(
|
||||
catalog_match.get("public_name") or catalog_match.get("publicName") or ""
|
||||
)
|
||||
matched_display_name = str(
|
||||
catalog_match.get("display_name")
|
||||
or catalog_match.get("displayName")
|
||||
or catalog_match.get("name")
|
||||
or ""
|
||||
)
|
||||
|
||||
logger.info(
|
||||
f"[CATALOG_ROUTER:MATCH] 成功匹配动态目录模型: provider={provider.name}, "
|
||||
f"requested_model={requested_model!r}, preset={matched_preset_name!r}, "
|
||||
f"uuid={matched_uuid!r}, public_name={matched_public_name!r}, "
|
||||
f"display_name={matched_display_name!r}, public_model_id={public_model_id!r}, "
|
||||
f"target_tab_index={catalog_tab_index}, "
|
||||
f"target_tab_url={catalog_tab.get('current_url') or catalog_tab.get('url')}"
|
||||
)
|
||||
|
||||
return {
|
||||
"route_domain": provider.route_domain,
|
||||
"route_type": "model_catalog",
|
||||
"model_name": public_model_id or str(requested_model or ""),
|
||||
"matched_id": public_model_id or str(requested_model or ""),
|
||||
"match_type": "catalog",
|
||||
"preset_name": matched_preset_name,
|
||||
"available_model_ids": [public_model_id] if public_model_id else [],
|
||||
"catalog_tab": catalog_tab,
|
||||
"catalog_tab_index": catalog_tab_index,
|
||||
"matched_model": catalog_match,
|
||||
}
|
||||
|
||||
return None
|
||||
|
||||
def collect_models_for_catalog(
|
||||
self,
|
||||
browser: Any,
|
||||
config_engine: Any,
|
||||
tabs: List[Dict[str, Any]],
|
||||
append_entry: Callable[[str, Optional[str], Optional[str]], None],
|
||||
created: int,
|
||||
) -> None:
|
||||
"""Collect models across tabs and catalog providers for the /v1/models endpoint."""
|
||||
from app.utils.model_routing import collect_route_domain_models
|
||||
|
||||
# 1. 收集所有 Provider 的 Catalog 预设配置
|
||||
provider_configs: Dict[str, List[Dict[str, Any]]] = {}
|
||||
for provider in self._providers:
|
||||
configs = provider.collect_catalog_configs(config_engine, tabs)
|
||||
provider_configs[provider.name] = configs
|
||||
|
||||
# 2. 输出通用 Route Domain 模型,对被 Catalog 接管的 alias 模型进行抑制
|
||||
for item in collect_route_domain_models(tabs):
|
||||
suppressed = any(
|
||||
provider.is_alias_suppressed(item, provider_configs.get(provider.name, []))
|
||||
for provider in self._providers
|
||||
)
|
||||
if suppressed:
|
||||
continue
|
||||
|
||||
append_entry(
|
||||
item.get("id"),
|
||||
item.get("route_domain") or "universal-web-api",
|
||||
item.get("display_name") or item.get("id"),
|
||||
)
|
||||
|
||||
# 3. 输出各 Catalog Provider 发现的动态模型列表
|
||||
for provider in self._providers:
|
||||
configs = provider_configs.get(provider.name, [])
|
||||
if not configs:
|
||||
continue
|
||||
for model_entry in provider.list_catalog_model_entries(browser, configs, created):
|
||||
append_entry(
|
||||
model_entry.get("id"),
|
||||
model_entry.get("owned_by") or provider.route_domain,
|
||||
model_entry.get("display_name") or model_entry.get("id"),
|
||||
)
|
||||
|
||||
|
||||
catalog_router = CatalogRouter()
|
||||
@@ -0,0 +1,460 @@
|
||||
"""
|
||||
app/services/cf_turnstile_solver.py - Cloudflare Turnstile / 5s 盾通用自动过盾服务
|
||||
|
||||
职责:
|
||||
1. 探测并等待可点击人机验证按钮出现(支持动态选择器、自定义探测脚本或保底坐标);
|
||||
2. 复用项目的低熵模式移动轨迹(smooth_move_mouse + cdp_precise_click)平滑移动并精确点击;
|
||||
3. 确认验证通过(最多尝试指定次数,连续失败抛出异常);
|
||||
4. 通过后比对当前 URL 与本次请求实际占用的目标 URL,支持参数化校验与重定向。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
import random
|
||||
import time
|
||||
from typing import Any, Callable, Dict, List, Optional, Tuple
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
from app.core.config import logger as default_logger
|
||||
from app.utils.human_mouse import cdp_precise_click as default_cdp_precise_click, smooth_move_mouse as default_smooth_move_mouse
|
||||
from app.core.driver import driver_for_tab
|
||||
|
||||
_DEFAULT_MAX_ATTEMPTS = 3
|
||||
|
||||
DEFAULT_FIND_CLICK_POINT_JS = r"""
|
||||
return (() => {
|
||||
try {
|
||||
const visibleRect = (el) => {
|
||||
if (!el) return null;
|
||||
const style = window.getComputedStyle(el);
|
||||
const rect = el.getBoundingClientRect();
|
||||
if (
|
||||
style.display === 'none' ||
|
||||
style.visibility === 'hidden' ||
|
||||
Number(style.opacity || 1) <= 0.02 ||
|
||||
rect.width < 8 ||
|
||||
rect.height < 8
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
const vw = window.innerWidth || document.documentElement.clientWidth || 0;
|
||||
const vh = window.innerHeight || document.documentElement.clientHeight || 0;
|
||||
if (rect.right < 0 || rect.bottom < 0 || rect.left > vw || rect.top > vh) {
|
||||
return null;
|
||||
}
|
||||
const left = Math.max(0, rect.left);
|
||||
const top = Math.max(0, rect.top);
|
||||
const right = Math.min(vw, rect.right);
|
||||
const bottom = Math.min(vh, rect.bottom);
|
||||
return {
|
||||
left,
|
||||
top,
|
||||
right,
|
||||
bottom,
|
||||
width: right - left,
|
||||
height: bottom - top,
|
||||
rawWidth: rect.width,
|
||||
rawHeight: rect.height
|
||||
};
|
||||
};
|
||||
|
||||
const describe = (el) => {
|
||||
const attrs = [
|
||||
el.getAttribute('src'),
|
||||
el.getAttribute('title'),
|
||||
el.getAttribute('aria-label'),
|
||||
el.getAttribute('name'),
|
||||
el.id,
|
||||
el.className
|
||||
];
|
||||
return attrs.map((v) => String(v || '')).join(' ').toLowerCase();
|
||||
};
|
||||
|
||||
const candidates = [];
|
||||
const pushCandidate = (el, kind, score) => {
|
||||
const rect = visibleRect(el);
|
||||
if (!rect) return;
|
||||
const text = describe(el);
|
||||
const checkboxLike = /checkbox|anchor|recaptcha|hcaptcha|turnstile|challenge|cloudflare/.test(text);
|
||||
let clickX = rect.left + rect.width / 2;
|
||||
let clickY = rect.top + rect.height / 2;
|
||||
if (checkboxLike || rect.width >= 120) {
|
||||
clickX = rect.left + Math.min(Math.max(rect.width * 0.16, 24), 46);
|
||||
}
|
||||
const insetX = Math.min(4, Math.max(0, (rect.width - 1) / 2));
|
||||
const insetY = Math.min(4, Math.max(0, (rect.height - 1) / 2));
|
||||
clickX = Math.max(rect.left + insetX, Math.min(rect.left + rect.width - 1 - insetX, clickX));
|
||||
clickY = Math.max(rect.top + insetY, Math.min(rect.top + rect.height - 1 - insetY, clickY));
|
||||
candidates.push({
|
||||
kind,
|
||||
score: score + (checkboxLike ? 20 : 0) + Math.min(rect.width, 320) / 100,
|
||||
x: Math.round(clickX),
|
||||
y: Math.round(clickY),
|
||||
rect,
|
||||
hint: text.slice(0, 160)
|
||||
});
|
||||
};
|
||||
|
||||
for (const el of document.querySelectorAll('iframe')) {
|
||||
const text = describe(el);
|
||||
const rect = visibleRect(el);
|
||||
const isExpandedImage = !!rect && rect.width >= 360 && rect.height >= 300;
|
||||
if (isExpandedImage) continue;
|
||||
if (/recaptcha|google\.com\/recaptcha/.test(text)) pushCandidate(el, 'recaptcha_iframe', 100);
|
||||
else if (/hcaptcha|hcaptcha\.com/.test(text)) pushCandidate(el, 'hcaptcha_iframe', 95);
|
||||
else if (/turnstile|challenges\.cloudflare\.com|cloudflare|challenge/.test(text)) pushCandidate(el, 'cloudflare_iframe', 90);
|
||||
}
|
||||
|
||||
for (const selector of [
|
||||
'.cf-turnstile',
|
||||
'#challenge-stage',
|
||||
'#turnstile-wrapper',
|
||||
'.g-recaptcha',
|
||||
'.h-captcha',
|
||||
'[data-sitekey]',
|
||||
'[role="checkbox"]',
|
||||
'input[type="checkbox"]'
|
||||
]) {
|
||||
for (const el of document.querySelectorAll(selector)) {
|
||||
pushCandidate(el, selector, 50);
|
||||
}
|
||||
}
|
||||
|
||||
candidates.sort((a, b) => b.score - a.score);
|
||||
const vw = window.innerWidth || document.documentElement.clientWidth || 800;
|
||||
const vh = window.innerHeight || document.documentElement.clientHeight || 600;
|
||||
|
||||
if (candidates.length > 0) {
|
||||
return { ok: true, ...candidates[0], viewport: { width: vw, height: vh } };
|
||||
}
|
||||
return {
|
||||
ok: false,
|
||||
reason: 'no_candidate_element',
|
||||
viewport: { width: vw, height: vh },
|
||||
fallback: {
|
||||
x: Math.round(vw / 2 - 90),
|
||||
y: Math.round(vh / 2 + 10)
|
||||
}
|
||||
};
|
||||
} catch (e) {
|
||||
return { ok: false, error: String(e) };
|
||||
}
|
||||
})();
|
||||
"""
|
||||
|
||||
DEFAULT_PAGE_READY_PROBE_JS = r"""
|
||||
return (() => {
|
||||
try {
|
||||
const isVisible = (el) => {
|
||||
if (!el || !el.isConnected) return false;
|
||||
const r = el.getBoundingClientRect();
|
||||
const s = window.getComputedStyle(el);
|
||||
return r.width >= 8 && r.height >= 8 && r.bottom > 0 && r.right > 0 &&
|
||||
r.top < window.innerHeight && r.left < window.innerWidth &&
|
||||
s.display !== 'none' && s.visibility !== 'hidden' &&
|
||||
Number(s.opacity || 1) > 0.02;
|
||||
};
|
||||
const text = String(document.body && document.body.innerText ? document.body.innerText : '').toLowerCase();
|
||||
const hasVerifyText = text.includes('security verification') ||
|
||||
text.includes('protected by cloudflare') ||
|
||||
text.includes('verify you are human') ||
|
||||
text.includes('checking your browser') ||
|
||||
text.includes('confirm you are human') ||
|
||||
text.includes('确认您是真人') ||
|
||||
text.includes('人机身份验证') ||
|
||||
text.includes('请验证您是真人') ||
|
||||
text.includes('正在进行安全验证');
|
||||
const challengeIframes = Array.from(document.querySelectorAll(
|
||||
'iframe[src*="challenges.cloudflare.com"], iframe[src*="challenge-platform"], iframe[title*="challenge" i], iframe[title*="cloudflare" i], iframe[title*="turnstile" i]'
|
||||
));
|
||||
const hasVisibleChallengeIframe = challengeIframes.some(isVisible);
|
||||
if (hasVerifyText || hasVisibleChallengeIframe) {
|
||||
return { resolved: false, reason: 'challenge_still_present' };
|
||||
}
|
||||
const hasInput = !!document.querySelector('textarea, [contenteditable="true"], input[placeholder*="ask" i], input[placeholder*="Type" i]');
|
||||
const hasArenaUI = text.includes('new chat') || text.includes('leaderboard') || text.includes('battle mode') || hasInput;
|
||||
return { resolved: true, ready: hasArenaUI || hasInput };
|
||||
} catch (e) {
|
||||
return { resolved: false, error: String(e) };
|
||||
}
|
||||
})();
|
||||
"""
|
||||
|
||||
|
||||
def _interruptible_sleep(seconds: float, raise_if_cancelled=None, step: float = 0.1) -> None:
|
||||
deadline = time.time() + max(0.0, float(seconds))
|
||||
while time.time() < deadline:
|
||||
if raise_if_cancelled:
|
||||
raise_if_cancelled()
|
||||
remaining = deadline - time.time()
|
||||
time.sleep(min(step, max(0.01, remaining)))
|
||||
|
||||
|
||||
def is_valid_http_url(url: Optional[str]) -> bool:
|
||||
"""通用 HTTP/HTTPS URL 有效性校验(排除 Cloudflare Challenge 自身页面)。"""
|
||||
if not url or not isinstance(url, str):
|
||||
return False
|
||||
u = url.lower().strip()
|
||||
if not (u.startswith("http://") or u.startswith("https://")):
|
||||
return False
|
||||
if "challenges.cloudflare.com" in u or "challenge-platform" in u:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def _urls_match(url1: Optional[str], url2: Optional[str]) -> bool:
|
||||
u1 = str(url1 or "").strip().rstrip("/")
|
||||
u2 = str(url2 or "").strip().rstrip("/")
|
||||
if not u1 and not u2:
|
||||
return True
|
||||
if not u1 or not u2:
|
||||
return False
|
||||
if u1 == u2:
|
||||
return True
|
||||
try:
|
||||
p1 = urlsplit(u1)
|
||||
p2 = urlsplit(u2)
|
||||
if p1.netloc.lower() == p2.netloc.lower() and p1.path.rstrip("/") == p2.path.rstrip("/"):
|
||||
if p1.query == p2.query:
|
||||
return True
|
||||
except Exception:
|
||||
pass
|
||||
return False
|
||||
|
||||
|
||||
def _get_occupied_url(
|
||||
tab: Any,
|
||||
session: Any = None,
|
||||
default_url: str = "https://arena.ai/code",
|
||||
validator_fn: Optional[Callable[[Optional[str]], bool]] = None,
|
||||
) -> str:
|
||||
"""提取本次请求在过盾前绑定的实际占用 URL。"""
|
||||
validator = validator_fn or is_valid_http_url
|
||||
if session is not None:
|
||||
occupied = getattr(session, "_request_occupied_url", None)
|
||||
if occupied and validator(occupied):
|
||||
return str(occupied).strip()
|
||||
last_known = getattr(session, "last_known_url", None)
|
||||
if last_known and validator(last_known):
|
||||
return str(last_known).strip()
|
||||
try:
|
||||
tab_url = str(getattr(tab, "url", "") or "").strip()
|
||||
if tab_url and validator(tab_url):
|
||||
return tab_url
|
||||
except Exception:
|
||||
pass
|
||||
return default_url
|
||||
|
||||
|
||||
def _wait_for_clickable_point(
|
||||
tab: Any,
|
||||
timeout_sec: float = 6.0,
|
||||
raise_if_cancelled: Optional[Callable[[], None]] = None,
|
||||
fallback_coords: Optional[Tuple[int, int]] = None,
|
||||
find_click_point_js: Optional[str] = None,
|
||||
log: Any = default_logger,
|
||||
) -> Dict[str, Any]:
|
||||
"""等待并获取可点击按钮坐标,若无法准确定位元素则使用固定/自适应保底坐标。"""
|
||||
deadline = time.time() + max(1.0, float(timeout_sec))
|
||||
last_fallback = None
|
||||
js_code = find_click_point_js or DEFAULT_FIND_CLICK_POINT_JS
|
||||
|
||||
while time.time() < deadline:
|
||||
if raise_if_cancelled:
|
||||
raise_if_cancelled()
|
||||
try:
|
||||
probe = driver_for_tab(tab).run_js(js_code)
|
||||
if isinstance(probe, dict):
|
||||
if probe.get("ok"):
|
||||
x = max(0, int(probe["x"]) + random.randint(-2, 2))
|
||||
y = max(0, int(probe["y"]) + random.randint(-2, 2))
|
||||
return {
|
||||
"ok": True,
|
||||
"x": x,
|
||||
"y": y,
|
||||
"kind": probe.get("kind", "element"),
|
||||
"rect": probe.get("rect"),
|
||||
}
|
||||
elif probe.get("fallback"):
|
||||
last_fallback = probe["fallback"]
|
||||
except Exception as e:
|
||||
log.debug(f"[CF_SOLVER] 寻找点击坐标异常: {e}")
|
||||
|
||||
_interruptible_sleep(0.3, raise_if_cancelled=raise_if_cancelled)
|
||||
|
||||
# 无法准确定位元素时启用保底坐标
|
||||
if fallback_coords is not None:
|
||||
fx, fy = max(0, int(fallback_coords[0])), max(0, int(fallback_coords[1]))
|
||||
elif last_fallback and isinstance(last_fallback, dict):
|
||||
fx = max(0, int(last_fallback.get("x", 310)))
|
||||
fy = max(0, int(last_fallback.get("y", 310)))
|
||||
else:
|
||||
try:
|
||||
vw = int(driver_for_tab(tab).run_js("return window.innerWidth || document.documentElement.clientWidth || 800") or 800)
|
||||
vh = int(driver_for_tab(tab).run_js("return window.innerHeight || document.documentElement.clientHeight || 600") or 600)
|
||||
fx = max(0, int(vw / 2 - 90))
|
||||
fy = max(0, int(vh / 2 + 10))
|
||||
except Exception:
|
||||
fx, fy = 310, 310
|
||||
|
||||
fx = max(0, fx + random.randint(-4, 4))
|
||||
fy = max(0, fy + random.randint(-4, 4))
|
||||
log.info(f"[CF_SOLVER] 未定位到具体元素,启用固定/保底坐标点击: ({fx}, {fy})")
|
||||
return {"ok": True, "x": fx, "y": fy, "kind": "fixed_fallback"}
|
||||
|
||||
|
||||
def _wait_for_challenge_resolved(
|
||||
tab: Any,
|
||||
timeout_sec: float = 8.0,
|
||||
raise_if_cancelled: Optional[Callable[[], None]] = None,
|
||||
page_ready_probe_js: Optional[str] = None,
|
||||
log: Any = default_logger,
|
||||
) -> bool:
|
||||
"""轮询等待验证通过(特征文字与 iframe 消失且页面就绪)。"""
|
||||
deadline = time.time() + max(1.0, float(timeout_sec))
|
||||
js_code = page_ready_probe_js or DEFAULT_PAGE_READY_PROBE_JS
|
||||
while time.time() < deadline:
|
||||
if raise_if_cancelled:
|
||||
raise_if_cancelled()
|
||||
try:
|
||||
probe = driver_for_tab(tab).run_js(js_code)
|
||||
if isinstance(probe, dict) and probe.get("resolved"):
|
||||
return True
|
||||
except Exception as e:
|
||||
log.debug(f"[CF_SOLVER] 探测验证状态异常: {e}")
|
||||
_interruptible_sleep(0.5, raise_if_cancelled=raise_if_cancelled)
|
||||
return False
|
||||
|
||||
|
||||
def solve_turnstile_challenge(
|
||||
tab: Any,
|
||||
session: Any = None,
|
||||
target_url: Optional[str] = None,
|
||||
is_valid_url_fn: Optional[Callable[[Optional[str]], bool]] = None,
|
||||
ready_probe_js: Optional[str] = None,
|
||||
click_point_js: Optional[str] = None,
|
||||
fallback_coords: Optional[Tuple[int, int]] = None,
|
||||
max_attempts: int = _DEFAULT_MAX_ATTEMPTS,
|
||||
click_timeout_sec: float = 6.0,
|
||||
resolve_timeout_sec: float = 8.0,
|
||||
raise_if_cancelled: Optional[Callable[[], None]] = None,
|
||||
smooth_move_fn: Optional[Callable[..., Any]] = None,
|
||||
cdp_click_fn: Optional[Callable[..., Any]] = None,
|
||||
logger: Any = None,
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
通用 Cloudflare Turnstile / 5s 盾自动过盾流程。
|
||||
"""
|
||||
log = logger or default_logger
|
||||
smooth_move = smooth_move_fn or default_smooth_move_mouse
|
||||
cdp_click = cdp_click_fn or default_cdp_precise_click
|
||||
|
||||
if raise_if_cancelled:
|
||||
raise_if_cancelled()
|
||||
|
||||
occupied_url = target_url or _get_occupied_url(
|
||||
tab,
|
||||
session,
|
||||
validator_fn=is_valid_url_fn or is_valid_http_url,
|
||||
)
|
||||
log.info(f"[CF_SOLVER] 启动 Cloudflare 5s 盾自动处理: 实际占用目标 URL={occupied_url}")
|
||||
|
||||
attempts = 0
|
||||
passed = False
|
||||
|
||||
while attempts < max_attempts:
|
||||
attempts += 1
|
||||
if raise_if_cancelled:
|
||||
raise_if_cancelled()
|
||||
|
||||
log.info(f"[CF_SOLVER] 第 {attempts}/{max_attempts} 次尝试通过 5s 盾...")
|
||||
|
||||
click_target = _wait_for_clickable_point(
|
||||
tab,
|
||||
timeout_sec=click_timeout_sec,
|
||||
raise_if_cancelled=raise_if_cancelled,
|
||||
fallback_coords=fallback_coords,
|
||||
find_click_point_js=click_point_js,
|
||||
log=log,
|
||||
)
|
||||
|
||||
click_x = max(0, int(click_target["x"]))
|
||||
click_y = max(0, int(click_target["y"]))
|
||||
log.info(
|
||||
f"[CF_SOLVER] 准备低熵点击验证目标 (第 {attempts} 次): "
|
||||
f"({click_x}, {click_y}), kind={click_target.get('kind', 'fallback')}"
|
||||
)
|
||||
|
||||
start_x = max(0, click_x - random.randint(40, 80))
|
||||
start_y = max(0, click_y - random.randint(30, 60))
|
||||
smooth_move(tab, (start_x, start_y), (click_x, click_y))
|
||||
_interruptible_sleep(random.uniform(0.06, 0.14), raise_if_cancelled=raise_if_cancelled)
|
||||
click_ok = cdp_click(tab, click_x, click_y)
|
||||
if not click_ok:
|
||||
log.warning(f"[CF_SOLVER] CDP 精确点击未确认: ({click_x}, {click_y})")
|
||||
|
||||
passed = _wait_for_challenge_resolved(
|
||||
tab,
|
||||
timeout_sec=resolve_timeout_sec,
|
||||
raise_if_cancelled=raise_if_cancelled,
|
||||
page_ready_probe_js=ready_probe_js,
|
||||
log=log,
|
||||
)
|
||||
if passed:
|
||||
log.info(f"[CF_SOLVER] ✅ 第 {attempts} 次尝试后 5s 盾验证通过!")
|
||||
break
|
||||
else:
|
||||
log.warning(f"[CF_SOLVER] 第 {attempts} 次点击后 5s 盾仍未通过,准备重试...")
|
||||
_interruptible_sleep(random.uniform(1.5, 2.5), raise_if_cancelled=raise_if_cancelled)
|
||||
|
||||
if not passed:
|
||||
err_msg = f"连续尝试 {max_attempts} 次仍未通过 Cloudflare 5s 盾人机验证"
|
||||
log.error(f"[CF_SOLVER] ❌ {err_msg}")
|
||||
raise RuntimeError(f"cf_turnstile_verify_failed_after_{max_attempts}_attempts")
|
||||
|
||||
current_url = str(getattr(tab, "url", "") or "").strip()
|
||||
url_matches = _urls_match(current_url, occupied_url)
|
||||
|
||||
validator = is_valid_url_fn or is_valid_http_url
|
||||
if not url_matches and occupied_url and validator(occupied_url):
|
||||
log.warning(
|
||||
f"[CF_SOLVER] 过盾后 URL 不一致: 当前 URL ({current_url}) != 实际占用 URL ({occupied_url}),"
|
||||
f"正在跳转回实际占用 URL..."
|
||||
)
|
||||
try:
|
||||
tab.get(occupied_url)
|
||||
_interruptible_sleep(random.uniform(1.0, 2.0), raise_if_cancelled=raise_if_cancelled)
|
||||
current_url = str(getattr(tab, "url", "") or "").strip()
|
||||
log.info(f"[CF_SOLVER] 已跳转回实际占用 URL: {current_url}")
|
||||
except Exception as e:
|
||||
log.error(f"[CF_SOLVER] 跳转回实际占用 URL 失败: {e}")
|
||||
raise
|
||||
else:
|
||||
log.info(f"[CF_SOLVER] 过盾后 URL 一致 ({current_url}),无需重定向。")
|
||||
|
||||
return {
|
||||
"ok": True,
|
||||
"attempts": attempts,
|
||||
"current_url": current_url,
|
||||
"occupied_url": occupied_url,
|
||||
"url_redirected": not url_matches,
|
||||
}
|
||||
|
||||
|
||||
# 兼容别名
|
||||
solve_arena_turnstile_challenge = solve_turnstile_challenge
|
||||
|
||||
|
||||
__all__ = [
|
||||
"DEFAULT_FIND_CLICK_POINT_JS",
|
||||
"DEFAULT_PAGE_READY_PROBE_JS",
|
||||
"_get_occupied_url",
|
||||
"_interruptible_sleep",
|
||||
"_urls_match",
|
||||
"_wait_for_challenge_resolved",
|
||||
"_wait_for_clickable_point",
|
||||
"is_valid_http_url",
|
||||
"solve_arena_turnstile_challenge",
|
||||
"solve_turnstile_challenge",
|
||||
]
|
||||
@@ -0,0 +1,110 @@
|
||||
"""
|
||||
app/services/command_defs.py - Shared command definitions
|
||||
|
||||
Responsibilities:
|
||||
- Default command payload
|
||||
- Trigger/action type metadata
|
||||
- Internal command-flow control exception
|
||||
"""
|
||||
|
||||
import uuid
|
||||
from typing import Any, Dict
|
||||
|
||||
TRIGGER_TYPES = {
|
||||
"request_count": "请求计数",
|
||||
"error_count": "错误计数",
|
||||
"idle_timeout": "空闲超时",
|
||||
"page_check": "页面检查,适合 Cloudflare 场景",
|
||||
"command_check": "命令检查",
|
||||
"command_triggered": "命令已触发",
|
||||
"command_result_match": "命令返回结果",
|
||||
"command_result_event": "命令结果事件",
|
||||
"network_request_error": "网络异常",
|
||||
}
|
||||
|
||||
ACTION_TYPES = {
|
||||
"clear_cookies": "清除 Cookie",
|
||||
"refresh_page": "刷新页面",
|
||||
"new_chat": "新建对话",
|
||||
"run_js": "执行 JavaScript",
|
||||
"run_js_file": "执行 JS 文件",
|
||||
"wait": "等待",
|
||||
"execute_preset": "切换预设",
|
||||
"execute_workflow": "执行工作流",
|
||||
"switch_preset": "切换预设",
|
||||
"navigate": "跳转 URL",
|
||||
"switch_proxy": "切换 Clash 代理",
|
||||
"send_webhook": "发送 Webhook / 通知",
|
||||
"send_napcat": "发送 NapCat QQ 消息",
|
||||
"execute_command_group": "执行命令组",
|
||||
"abort_task": "中断任务",
|
||||
"release_tab_lock": "解除标签页占用",
|
||||
"click_element": "点击元素",
|
||||
"click_coordinates": "点击坐标",
|
||||
"click_captcha_challenge": "点击人机验证",
|
||||
"write_element": "写入元素",
|
||||
"read_element": "读取元素",
|
||||
"http_request": "页面内请求",
|
||||
"append_file": "追加到文件",
|
||||
}
|
||||
|
||||
|
||||
class CommandFlowAbort(Exception):
|
||||
"""用于在动作链中提前中断后续步骤。"""
|
||||
pass
|
||||
|
||||
|
||||
def _new_command_id() -> str:
|
||||
return f"cmd_{uuid.uuid4().hex[:8]}"
|
||||
|
||||
|
||||
def get_default_command() -> Dict[str, Any]:
|
||||
"""返回默认命令结构。"""
|
||||
return {
|
||||
"id": _new_command_id(),
|
||||
"name": "新命令",
|
||||
"enabled": True,
|
||||
"log_enabled": True,
|
||||
"log_level": "GLOBAL",
|
||||
"mode": "simple",
|
||||
"trigger": {
|
||||
"type": "request_count",
|
||||
"value": 10,
|
||||
"command_id": "",
|
||||
"action_ref": "",
|
||||
"match_rule": "equals",
|
||||
"expected_value": "",
|
||||
"match_mode": "keyword",
|
||||
"status_codes": "403,429,500,502,503,504",
|
||||
"abort_on_match": True,
|
||||
"command_ids": [],
|
||||
"listen_all_commands": False,
|
||||
"informative_only": True,
|
||||
"scope": "all",
|
||||
"domain": "",
|
||||
"tab_index": None,
|
||||
"priority": 2,
|
||||
"allow_during_workflow": True,
|
||||
"interrupt_policy": "auto",
|
||||
"interrupt_message": "",
|
||||
},
|
||||
"actions": [
|
||||
{"type": "clear_cookies"},
|
||||
{"type": "refresh_page"},
|
||||
],
|
||||
"group_name": "",
|
||||
"script": "",
|
||||
"script_lang": "javascript",
|
||||
"advanced_ui": {},
|
||||
"last_triggered": None,
|
||||
"trigger_count": 0,
|
||||
}
|
||||
|
||||
|
||||
__all__ = [
|
||||
'ACTION_TYPES',
|
||||
'CommandFlowAbort',
|
||||
'TRIGGER_TYPES',
|
||||
'_new_command_id',
|
||||
'get_default_command',
|
||||
]
|
||||
@@ -0,0 +1,649 @@
|
||||
"""
|
||||
app/services/command_engine.py - 命令引擎
|
||||
|
||||
职责:
|
||||
- 命令的 CRUD 管理
|
||||
- 触发条件检查(在标签页释放后调用)
|
||||
- 动作执行调度
|
||||
- 高级模式脚本执行(JavaScript / Python)
|
||||
|
||||
存储位置:config/commands.json
|
||||
"""
|
||||
|
||||
import copy
|
||||
import os
|
||||
import threading
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from typing import Dict, List, Optional, Any, TYPE_CHECKING
|
||||
|
||||
from app.core.page_lifecycle import (
|
||||
BACKGROUND_WAKE_JS_TIMEOUT,
|
||||
)
|
||||
from app.services.command_defs import ACTION_TYPES, TRIGGER_TYPES, get_default_command
|
||||
from app.services.command_engine_actions import CommandEngineActionsMixin
|
||||
from app.services.command_engine_results import CommandEngineResultsMixin
|
||||
from app.services.command_engine_runtime import CommandEngineRuntimeMixin
|
||||
from app.services.command_engine_pages import CommandEnginePagesMixin
|
||||
from app.services.command_engine_scheduler import CommandEngineSchedulerMixin
|
||||
from app.services.command_engine_store import CommandEngineStoreMixin
|
||||
from app.services.command_engine_crud import CommandEngineCrudMixin
|
||||
from app.services.command_engine_triggers import CommandEngineTriggersMixin
|
||||
from app.services.command_engine_page_check import CommandEnginePageCheckMixin
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from app.core.tab_pool import TabSession
|
||||
|
||||
# R2-3:模块级常量、日志器与可选依赖迁到 command_engine_common,这里重新导出以保持兼容
|
||||
from app.services.command_engine_common import ( # noqa: E402,F401
|
||||
FOLLOW_DEFAULT_PRESET,
|
||||
HAS_REQUESTS,
|
||||
_NON_ASCII_RE,
|
||||
_WHITESPACE_RE,
|
||||
_WORD_LIKE_KEYWORD_RE,
|
||||
logger,
|
||||
)
|
||||
|
||||
|
||||
# ================= 常量 =================
|
||||
|
||||
class CommandEngine(
|
||||
CommandEnginePagesMixin,
|
||||
CommandEngineSchedulerMixin,
|
||||
CommandEngineStoreMixin,
|
||||
CommandEngineCrudMixin,
|
||||
CommandEngineTriggersMixin,
|
||||
CommandEnginePageCheckMixin,
|
||||
CommandEngineRuntimeMixin,
|
||||
CommandEngineResultsMixin,
|
||||
CommandEngineActionsMixin,
|
||||
):
|
||||
"""命令引擎"""
|
||||
def __init__(self):
|
||||
self._config_engine = None
|
||||
self._browser = None
|
||||
self._commands_file = None
|
||||
self._commands_local_file = None
|
||||
self._commands_mtime = 0.0
|
||||
self._commands_local_mtime = 0.0
|
||||
# B8:最近一次「读失败」的命令文件 mtime;同一 mtime 不重复读取/刷错误日志
|
||||
self._commands_failed_mtime: Optional[float] = None
|
||||
# (cache_key, compact_text):page_check 去空白页面文本的单槽缓存
|
||||
self._compact_haystack_cache: Optional[tuple] = None
|
||||
self._commands_loaded = False
|
||||
self._commands_cache: List[Dict[str, Any]] = []
|
||||
self._command_runtime_stats: Dict[str, Dict[str, Any]] = {}
|
||||
self._lock = threading.RLock()
|
||||
self._commands_lock = threading.RLock()
|
||||
self._scheduler_lifecycle_lock = threading.RLock()
|
||||
self._shutdown_requested = False
|
||||
|
||||
# 触发状态:{(command_id, tab_id): {"req": int, "err": int, ...}}
|
||||
self._trigger_states: Dict[tuple, Dict[str, Any]] = {}
|
||||
self._pending_async_trigger_meta: Dict[tuple, Dict[str, Any]] = {}
|
||||
# 最近命令执行结果:{(source_command_id, tab_id): {...}}
|
||||
self._command_results: Dict[tuple, Dict[str, Any]] = {}
|
||||
# 命令结果事件:{tab_id: [event, ...]}
|
||||
self._command_result_events: Dict[str, List[Dict[str, Any]]] = {}
|
||||
# 最近网络事件:{tab_id: [event, ...]}
|
||||
self._network_events: Dict[str, List[Dict[str, Any]]] = {}
|
||||
# 正在执行的命令(防止重复触发)
|
||||
self._executing: set = set()
|
||||
self._periodic_next_run: Dict[tuple, float] = {}
|
||||
self._periodic_stop_event = threading.Event()
|
||||
self._periodic_thread: Optional[threading.Thread] = None
|
||||
self._pending_high_by_session: Dict[str, int] = {}
|
||||
self._running_high_by_session: Dict[str, int] = {}
|
||||
self._pending_high_by_domain: Dict[str, int] = {}
|
||||
self._running_high_by_domain: Dict[str, int] = {}
|
||||
try:
|
||||
_max_async_workers = int(os.getenv("CMD_ASYNC_MAX_WORKERS", "20"))
|
||||
except Exception:
|
||||
_max_async_workers = 20
|
||||
self._command_executor = ThreadPoolExecutor(
|
||||
max_workers=max(1, _max_async_workers),
|
||||
thread_name_prefix="cmd-exec",
|
||||
)
|
||||
try:
|
||||
_baseline = int(os.getenv("CMD_REQUEST_PRIORITY_BASELINE", "2"))
|
||||
except Exception:
|
||||
_baseline = 2
|
||||
self._request_priority_baseline = _baseline
|
||||
self._activate_tab_on_command = str(
|
||||
os.getenv("CMD_ACTIVATE_TAB_ON_COMMAND", "false")
|
||||
).strip().lower() in {"1", "true", "yes", "y", "on"}
|
||||
self._use_focus_emulation_on_command = str(
|
||||
os.getenv("CMD_USE_FOCUS_EMULATION_ON_COMMAND", "true")
|
||||
).strip().lower() in {"1", "true", "yes", "y", "on"}
|
||||
self._wake_tab_before_page_check = str(
|
||||
os.getenv("CMD_WAKE_TAB_BEFORE_PAGE_CHECK", "true")
|
||||
).strip().lower() in {"1", "true", "yes", "y", "on"}
|
||||
# 唤醒标签页的 CDP 状态(focus / visibility / Page.setWebLifecycleState=active)
|
||||
# 是粘性的,不需要每条 page_check 命令都重发一遍。
|
||||
# 每次唤醒是 4~6 次 CDP 往返 + 一次整页可见性补丁 JS,
|
||||
# 而一个会话上往往同时挂着多条 page_check 命令,实际调用频率是每秒数次。
|
||||
# 默认 2s 限频:既能把唤醒风暴压下去,最坏也只比原来晚 2s 唤醒
|
||||
# (page_check 自身的最小轮询间隔就是 1.5s,量级相当)。
|
||||
# 设为 0 可恢复"每次都唤醒"的旧行为;若观察到后台页仍被冻结可调小。
|
||||
try:
|
||||
_wake_min_interval = float(os.getenv("CMD_WAKE_TAB_MIN_INTERVAL_SEC", "2"))
|
||||
except Exception:
|
||||
_wake_min_interval = 2.0
|
||||
self._wake_tab_min_interval_sec = max(0.0, _wake_min_interval)
|
||||
self._tab_pool_auto_refresh = str(
|
||||
os.getenv("CMD_TAB_POOL_AUTO_REFRESH", "true")
|
||||
).strip().lower() in {"1", "true", "yes", "y", "on"}
|
||||
try:
|
||||
_refresh_interval = float(os.getenv("CMD_TAB_POOL_REFRESH_INTERVAL_SEC", "5"))
|
||||
except Exception:
|
||||
_refresh_interval = 5.0
|
||||
self._tab_pool_refresh_interval_sec = max(1.0, _refresh_interval)
|
||||
self._last_tab_pool_refresh_at = 0.0
|
||||
# In memory saver mode, do not wake every idle tab on a fixed cadence.
|
||||
# A page_check or real workflow still wakes its target tab on demand.
|
||||
memory_saver_enabled = str(
|
||||
os.getenv("BROWSER_MEMORY_SAVER", "false")
|
||||
).strip().lower() in {"1", "true", "yes", "y", "on"}
|
||||
# P0-6:开启空闲冻结时,周期 keepalive 会不断唤醒标签页,默认关闭
|
||||
# (显式设置 CMD_PERIODIC_KEEPALIVE_ENABLED 仍以用户为准)。
|
||||
try:
|
||||
from app.core.tab_pool_parts.idle_maintenance import is_idle_freeze_enabled
|
||||
|
||||
idle_freeze_enabled = is_idle_freeze_enabled()
|
||||
except Exception:
|
||||
idle_freeze_enabled = False
|
||||
keepalive_default = "false" if (memory_saver_enabled or idle_freeze_enabled) else "true"
|
||||
self._periodic_keepalive_enabled = str(
|
||||
os.getenv("CMD_PERIODIC_KEEPALIVE_ENABLED", keepalive_default)
|
||||
).strip().lower() in {"1", "true", "yes", "y", "on"}
|
||||
try:
|
||||
_keepalive_interval = float(os.getenv("CMD_PERIODIC_KEEPALIVE_INTERVAL_SEC", "20"))
|
||||
except Exception:
|
||||
_keepalive_interval = 20.0
|
||||
self._periodic_keepalive_interval_sec = max(5.0, _keepalive_interval)
|
||||
try:
|
||||
_page_check_js_timeout = float(os.getenv("CMD_PAGE_CHECK_JS_TIMEOUT_SEC", str(BACKGROUND_WAKE_JS_TIMEOUT)))
|
||||
except Exception:
|
||||
_page_check_js_timeout = BACKGROUND_WAKE_JS_TIMEOUT
|
||||
self._page_check_js_timeout_sec = max(0.1, min(2.0, _page_check_js_timeout))
|
||||
try:
|
||||
_page_check_failure_backoff = float(os.getenv("CMD_PAGE_CHECK_FAILURE_BACKOFF_SEC", "30"))
|
||||
except Exception:
|
||||
_page_check_failure_backoff = 30.0
|
||||
self._page_check_failure_backoff_sec = max(1.0, _page_check_failure_backoff)
|
||||
try:
|
||||
_page_check_refresh_grace = float(
|
||||
os.getenv("CMD_PAGE_CHECK_REFRESH_GRACE_SEC", "2")
|
||||
)
|
||||
except Exception:
|
||||
_page_check_refresh_grace = 2.0
|
||||
self._page_check_refresh_grace_sec = max(0.2, min(10.0, _page_check_refresh_grace))
|
||||
self._last_keepalive_by_session: Dict[str, float] = {}
|
||||
self._last_tab_pool_wait_log_at = 0.0
|
||||
self._last_periodic_summary_log_at = 0.0
|
||||
self._periodic_active_log_interval_sec = 8.0
|
||||
self._periodic_idle_log_interval_sec = 30.0
|
||||
self._periodic_summary_log_enabled = str(
|
||||
os.getenv("CMD_PERIODIC_SUMMARY_LOG_ENABLED", "false")
|
||||
).strip().lower() in {"1", "true", "yes", "y", "on"}
|
||||
# page_check observer: {session_id: set_of_keywords_installed}
|
||||
self._observer_keywords_by_session: Dict[str, set] = {}
|
||||
|
||||
logger.debug("命令引擎已初始化")
|
||||
if self._should_auto_start_scheduler():
|
||||
self._start_periodic_scheduler()
|
||||
def evict_session(self, session_id: str):
|
||||
"""Evict runtime caches and pending state associated with a removed tab session."""
|
||||
session_key = str(session_id or "").strip()
|
||||
if not session_key:
|
||||
return
|
||||
|
||||
logger.info(f"逐出会话缓存: {session_key}")
|
||||
with self._lock:
|
||||
keys_to_remove = [
|
||||
key
|
||||
for key in self._trigger_states
|
||||
if isinstance(key, tuple) and len(key) > 1 and key[1] == session_key
|
||||
]
|
||||
for key in keys_to_remove:
|
||||
self._trigger_states.pop(key, None)
|
||||
|
||||
meta_keys_to_remove = [
|
||||
key
|
||||
for key in self._pending_async_trigger_meta
|
||||
if isinstance(key, tuple) and len(key) > 1 and key[1] == session_key
|
||||
]
|
||||
for key in meta_keys_to_remove:
|
||||
self._pending_async_trigger_meta.pop(key, None)
|
||||
|
||||
result_keys_to_remove = [
|
||||
key
|
||||
for key in self._command_results
|
||||
if isinstance(key, tuple) and len(key) > 1 and key[1] == session_key
|
||||
]
|
||||
for key in result_keys_to_remove:
|
||||
self._command_results.pop(key, None)
|
||||
|
||||
exec_keys_to_remove = [
|
||||
key
|
||||
for key in self._executing
|
||||
if isinstance(key, tuple) and len(key) > 1 and key[1] == session_key
|
||||
]
|
||||
for key in exec_keys_to_remove:
|
||||
self._executing.discard(key)
|
||||
|
||||
run_keys_to_remove = [
|
||||
key
|
||||
for key in self._periodic_next_run
|
||||
if isinstance(key, tuple) and len(key) > 1 and key[1] == session_key
|
||||
]
|
||||
for key in run_keys_to_remove:
|
||||
self._periodic_next_run.pop(key, None)
|
||||
|
||||
self._command_result_events.pop(session_key, None)
|
||||
self._network_events.pop(session_key, None)
|
||||
self._pending_high_by_session.pop(session_key, None)
|
||||
self._running_high_by_session.pop(session_key, None)
|
||||
self._last_keepalive_by_session.pop(session_key, None)
|
||||
self._observer_keywords_by_session.pop(session_key, None)
|
||||
def _get_config_engine(self):
|
||||
if self._config_engine is None:
|
||||
from app.services.config_engine import config_engine
|
||||
self._config_engine = config_engine
|
||||
return self._config_engine
|
||||
def _get_browser(self):
|
||||
if self._browser is None:
|
||||
from app.core.browser import get_browser
|
||||
self._browser = get_browser(auto_connect=False)
|
||||
return self._browser
|
||||
@staticmethod
|
||||
def _format_scope_label(scope: str) -> str:
|
||||
mapping = {
|
||||
"all": "全部标签页",
|
||||
"domain": "同域",
|
||||
"tab": "当前标签页",
|
||||
}
|
||||
return mapping.get(str(scope or "").strip().lower(), str(scope or "未指定"))
|
||||
def _set_pending_async_trigger_meta(
|
||||
self,
|
||||
command: Dict[str, Any],
|
||||
session: 'TabSession',
|
||||
meta: Optional[Dict[str, Any]],
|
||||
):
|
||||
key = (str(command.get("id", "")).strip(), str(getattr(session, "id", "") or ""))
|
||||
if not all(key):
|
||||
return
|
||||
with self._lock:
|
||||
if meta:
|
||||
self._pending_async_trigger_meta[key] = copy.deepcopy(meta)
|
||||
else:
|
||||
self._pending_async_trigger_meta.pop(key, None)
|
||||
def _take_pending_async_trigger_meta(
|
||||
self,
|
||||
command: Dict[str, Any],
|
||||
session: 'TabSession',
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
key = (str(command.get("id", "")).strip(), str(getattr(session, "id", "") or ""))
|
||||
if not all(key):
|
||||
return None
|
||||
with self._lock:
|
||||
meta = self._pending_async_trigger_meta.pop(key, None)
|
||||
return copy.deepcopy(meta) if meta else None
|
||||
_PAGE_CHECK_OBSERVER_JS = r"""
|
||||
(function() {
|
||||
var kws = %KEYWORDS%;
|
||||
function newSeen() {
|
||||
// Set 判重是 O(1);旧实现用数组 + indexOf 是 O(n),整页遍历退化成
|
||||
// O(n^2),长对话页面每次快照要跑上千万次比较,是浏览器端 CPU 的主要来源。
|
||||
try {
|
||||
if (typeof Set === 'function') {
|
||||
var s = new Set();
|
||||
return {
|
||||
has: function(n) { return s.has(n); },
|
||||
add: function(n) { s.add(n); }
|
||||
};
|
||||
}
|
||||
} catch (e) {}
|
||||
var arr = [];
|
||||
return {
|
||||
has: function(n) { return arr.indexOf(n) !== -1; },
|
||||
add: function(n) { arr.push(n); }
|
||||
};
|
||||
}
|
||||
function appendText(parts, value) {
|
||||
if (typeof value === 'string' && value) parts.push(value);
|
||||
}
|
||||
function collectText(node, parts, seen) {
|
||||
if (!node || seen.has(node)) return;
|
||||
seen.add(node);
|
||||
|
||||
if (node.nodeType === 1) {
|
||||
var tag = String(node.tagName || '').toUpperCase();
|
||||
if (
|
||||
tag === 'SCRIPT' ||
|
||||
tag === 'STYLE' ||
|
||||
tag === 'NOSCRIPT' ||
|
||||
tag === 'TEMPLATE' ||
|
||||
tag === 'META' ||
|
||||
tag === 'HEAD'
|
||||
) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
if (node.shadowRoot) collectText(node.shadowRoot, parts, seen);
|
||||
} catch (e) {}
|
||||
if (tag === 'IFRAME') {
|
||||
try {
|
||||
var doc = node.contentDocument || (node.contentWindow && node.contentWindow.document);
|
||||
if (doc) collectText(doc.documentElement || doc.body, parts, seen);
|
||||
} catch (e) {}
|
||||
}
|
||||
}
|
||||
|
||||
if (node.nodeType === 3) {
|
||||
appendText(parts, String(node.nodeValue || '').trim());
|
||||
}
|
||||
|
||||
var child = null;
|
||||
try {
|
||||
child = node.firstChild;
|
||||
} catch (e) {}
|
||||
while (child) {
|
||||
collectText(child, parts, seen);
|
||||
try {
|
||||
child = child.nextSibling;
|
||||
} catch (e) {
|
||||
child = null;
|
||||
}
|
||||
}
|
||||
}
|
||||
function collectRootText(root, parts, seen) {
|
||||
collectText(root, parts, seen);
|
||||
}
|
||||
function isElementVisible(el) {
|
||||
if (!el) return false;
|
||||
try {
|
||||
var style = window.getComputedStyle ? window.getComputedStyle(el) : null;
|
||||
var rect = el.getBoundingClientRect ? el.getBoundingClientRect() : null;
|
||||
if (style) {
|
||||
if (style.display === 'none' || style.visibility === 'hidden') return false;
|
||||
if (Number(style.opacity || 1) <= 0.02) return false;
|
||||
}
|
||||
return !!rect && rect.width > 5 && rect.height > 5;
|
||||
} catch (e) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
function hasVisibleSelector(selector) {
|
||||
try {
|
||||
var els = document.querySelectorAll(selector);
|
||||
for (var i = 0; i < els.length; i++) {
|
||||
if (isElementVisible(els[i])) return true;
|
||||
}
|
||||
return false;
|
||||
} catch (e) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
function buildSnapshot() {
|
||||
var parts = [];
|
||||
var seen = newSeen();
|
||||
appendText(parts, document.title || '');
|
||||
collectRootText(document.documentElement || document.body, parts, seen);
|
||||
var text = parts.join('\n').toLowerCase();
|
||||
var cfIndicators = [];
|
||||
if (text.indexOf('security verification') !== -1) cfIndicators.push('security verification');
|
||||
if (text.indexOf('protected by cloudflare') !== -1) cfIndicators.push('protected by cloudflare');
|
||||
if (text.indexOf('verify you are human') !== -1) cfIndicators.push('verify you are human');
|
||||
if (text.indexOf('checking your browser') !== -1) cfIndicators.push('checking your browser');
|
||||
if (text.indexOf('确认您是真人') !== -1) cfIndicators.push('确认您是真人');
|
||||
if (
|
||||
hasVisibleSelector('iframe[src*="challenges.cloudflare.com"]') ||
|
||||
hasVisibleSelector('iframe[src*="turnstile"]') ||
|
||||
hasVisibleSelector('.cf-turnstile') ||
|
||||
hasVisibleSelector('[name="cf-turnstile-response"]') ||
|
||||
hasVisibleSelector('[data-testid*="cf" i]')
|
||||
) {
|
||||
cfIndicators.push('cloudflare');
|
||||
} else if (text.indexOf('cloudflare') !== -1) {
|
||||
cfIndicators.push('cloudflare');
|
||||
}
|
||||
if (cfIndicators.length) {
|
||||
text += '\n' + cfIndicators.join('\n');
|
||||
}
|
||||
var recaptchaIndicators = [];
|
||||
if (text.indexOf('protected by recaptcha') !== -1) recaptchaIndicators.push('protected by recaptcha');
|
||||
if (
|
||||
hasVisibleSelector('iframe[src*="google.com/recaptcha"]') ||
|
||||
hasVisibleSelector('iframe[src*="recaptcha"]') ||
|
||||
hasVisibleSelector('.g-recaptcha') ||
|
||||
hasVisibleSelector('[name="g-recaptcha-response"]') ||
|
||||
hasVisibleSelector('[title*="recaptcha" i]') ||
|
||||
hasVisibleSelector('[aria-label*="recaptcha" i]')
|
||||
) {
|
||||
recaptchaIndicators.push('recaptcha');
|
||||
recaptchaIndicators.push('\u4eba\u673a\u8eab\u4efd\u9a8c\u8bc1');
|
||||
} else if (text.indexOf('recaptcha') !== -1) {
|
||||
recaptchaIndicators.push('recaptcha');
|
||||
}
|
||||
if (recaptchaIndicators.length) {
|
||||
text += '\n' + recaptchaIndicators.join('\n');
|
||||
}
|
||||
return text;
|
||||
}
|
||||
if (window.__pcObserver && window.__pcKeywords) {
|
||||
var same = kws.length === window.__pcKeywords.length &&
|
||||
kws.every(function(k){ return window.__pcKeywords.indexOf(k) !== -1; });
|
||||
if (same) return 'already_installed';
|
||||
window.__pcObserver.disconnect();
|
||||
window.__pcObserver = null;
|
||||
}
|
||||
window.__pcKeywords = kws;
|
||||
window.__pcHits = {};
|
||||
var pending = null;
|
||||
var MIN_DELAY_MS = 200;
|
||||
var MAX_DELAY_MS = 800;
|
||||
// 自适应节流:下一次扫描的间隔取"上次扫描耗时 × 4",钳在 200~800ms。
|
||||
// 换成 Set 判重后,即使 6 万节点的页面单次扫描也只要 30ms 左右,
|
||||
// 算出来仍是下限 200ms —— 正常页面行为完全不变。
|
||||
// 只有扫描耗时超过 50ms 的病态页面才会退避,且最多退到 800ms,
|
||||
// 不会拖慢 page_check 对 Cloudflare 盾 / 报错文案的识别。
|
||||
var nextDelayMs = MIN_DELAY_MS;
|
||||
function doCheck() {
|
||||
pending = null;
|
||||
var startedAt = 0;
|
||||
try {
|
||||
startedAt = (window.performance && performance.now) ? performance.now() : Date.now();
|
||||
} catch (e) {}
|
||||
try {
|
||||
var text = buildSnapshot();
|
||||
window.__pcSnapshot = text;
|
||||
for (var i = 0; i < window.__pcKeywords.length; i++) {
|
||||
var k = window.__pcKeywords[i];
|
||||
window.__pcHits[k] = text.indexOf(k) !== -1;
|
||||
}
|
||||
} catch(e) {}
|
||||
try {
|
||||
var endedAt = (window.performance && performance.now) ? performance.now() : Date.now();
|
||||
var cost = Math.max(0, endedAt - startedAt);
|
||||
nextDelayMs = Math.min(MAX_DELAY_MS, Math.max(MIN_DELAY_MS, Math.round(cost * 4)));
|
||||
} catch (e) {
|
||||
nextDelayMs = MIN_DELAY_MS;
|
||||
}
|
||||
}
|
||||
doCheck();
|
||||
window.__pcObserver = new MutationObserver(function() {
|
||||
if (!pending) pending = setTimeout(doCheck, nextDelayMs);
|
||||
});
|
||||
var target = document.body || document.documentElement;
|
||||
if (target) {
|
||||
window.__pcObserver.observe(target, {
|
||||
childList: true, subtree: true, characterData: true
|
||||
});
|
||||
}
|
||||
return 'installed';
|
||||
})();
|
||||
"""
|
||||
_PAGE_CHECK_SNAPSHOT_JS = r"""
|
||||
return (function() {
|
||||
function newSeen() {
|
||||
// 同 _PAGE_CHECK_OBSERVER_JS:数组 indexOf 判重会让整页遍历退化成 O(n^2)。
|
||||
try {
|
||||
if (typeof Set === 'function') {
|
||||
var s = new Set();
|
||||
return {
|
||||
has: function(n) { return s.has(n); },
|
||||
add: function(n) { s.add(n); }
|
||||
};
|
||||
}
|
||||
} catch (e) {}
|
||||
var arr = [];
|
||||
return {
|
||||
has: function(n) { return arr.indexOf(n) !== -1; },
|
||||
add: function(n) { arr.push(n); }
|
||||
};
|
||||
}
|
||||
function appendText(parts, value) {
|
||||
if (typeof value === 'string' && value) parts.push(value);
|
||||
}
|
||||
function collectText(node, parts, seen) {
|
||||
if (!node || seen.has(node)) return;
|
||||
seen.add(node);
|
||||
|
||||
if (node.nodeType === 1) {
|
||||
var tag = String(node.tagName || '').toUpperCase();
|
||||
if (
|
||||
tag === 'SCRIPT' ||
|
||||
tag === 'STYLE' ||
|
||||
tag === 'NOSCRIPT' ||
|
||||
tag === 'TEMPLATE' ||
|
||||
tag === 'META' ||
|
||||
tag === 'HEAD'
|
||||
) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
if (node.shadowRoot) collectText(node.shadowRoot, parts, seen);
|
||||
} catch (e) {}
|
||||
if (tag === 'IFRAME') {
|
||||
try {
|
||||
var doc = node.contentDocument || (node.contentWindow && node.contentWindow.document);
|
||||
if (doc) collectText(doc.documentElement || doc.body, parts, seen);
|
||||
} catch (e) {}
|
||||
}
|
||||
}
|
||||
|
||||
if (node.nodeType === 3) {
|
||||
appendText(parts, String(node.nodeValue || '').trim());
|
||||
}
|
||||
|
||||
var child = null;
|
||||
try {
|
||||
child = node.firstChild;
|
||||
} catch (e) {}
|
||||
while (child) {
|
||||
collectText(child, parts, seen);
|
||||
try {
|
||||
child = child.nextSibling;
|
||||
} catch (e) {
|
||||
child = null;
|
||||
}
|
||||
}
|
||||
}
|
||||
function collectRootText(root, parts, seen) {
|
||||
collectText(root, parts, seen);
|
||||
}
|
||||
function isElementVisible(el) {
|
||||
if (!el) return false;
|
||||
try {
|
||||
var style = window.getComputedStyle ? window.getComputedStyle(el) : null;
|
||||
var rect = el.getBoundingClientRect ? el.getBoundingClientRect() : null;
|
||||
if (style) {
|
||||
if (style.display === 'none' || style.visibility === 'hidden') return false;
|
||||
if (Number(style.opacity || 1) <= 0.02) return false;
|
||||
}
|
||||
return !!rect && rect.width > 5 && rect.height > 5;
|
||||
} catch (e) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
function hasVisibleSelector(selector) {
|
||||
try {
|
||||
var els = document.querySelectorAll(selector);
|
||||
for (var i = 0; i < els.length; i++) {
|
||||
if (isElementVisible(els[i])) return true;
|
||||
}
|
||||
return false;
|
||||
} catch (e) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
var parts = [];
|
||||
appendText(parts, document.title || '');
|
||||
collectRootText(document.documentElement || document.body, parts, newSeen());
|
||||
var text = parts.join('\n').toLowerCase();
|
||||
var cfIndicators = [];
|
||||
if (text.indexOf('security verification') !== -1) cfIndicators.push('security verification');
|
||||
if (text.indexOf('protected by cloudflare') !== -1) cfIndicators.push('protected by cloudflare');
|
||||
if (text.indexOf('verify you are human') !== -1) cfIndicators.push('verify you are human');
|
||||
if (text.indexOf('checking your browser') !== -1) cfIndicators.push('checking your browser');
|
||||
if (text.indexOf('确认您是真人') !== -1) cfIndicators.push('确认您是真人');
|
||||
if (
|
||||
hasVisibleSelector('iframe[src*="challenges.cloudflare.com"]') ||
|
||||
hasVisibleSelector('iframe[src*="turnstile"]') ||
|
||||
hasVisibleSelector('.cf-turnstile') ||
|
||||
hasVisibleSelector('[name="cf-turnstile-response"]') ||
|
||||
hasVisibleSelector('[data-testid*="cf" i]')
|
||||
) {
|
||||
cfIndicators.push('cloudflare');
|
||||
} else if (text.indexOf('cloudflare') !== -1) {
|
||||
cfIndicators.push('cloudflare');
|
||||
}
|
||||
if (cfIndicators.length) {
|
||||
text += '\n' + cfIndicators.join('\n');
|
||||
}
|
||||
var recaptchaIndicators = [];
|
||||
if (text.indexOf('protected by recaptcha') !== -1) recaptchaIndicators.push('protected by recaptcha');
|
||||
if (
|
||||
hasVisibleSelector('iframe[src*="google.com/recaptcha"]') ||
|
||||
hasVisibleSelector('iframe[src*="recaptcha"]') ||
|
||||
hasVisibleSelector('.g-recaptcha') ||
|
||||
hasVisibleSelector('[name="g-recaptcha-response"]') ||
|
||||
hasVisibleSelector('[title*="recaptcha" i]') ||
|
||||
hasVisibleSelector('[aria-label*="recaptcha" i]')
|
||||
) {
|
||||
recaptchaIndicators.push('recaptcha');
|
||||
recaptchaIndicators.push('\u4eba\u673a\u8eab\u4efd\u9a8c\u8bc1');
|
||||
} else if (text.indexOf('recaptcha') !== -1) {
|
||||
recaptchaIndicators.push('recaptcha');
|
||||
}
|
||||
if (recaptchaIndicators.length) {
|
||||
text += '\n' + recaptchaIndicators.join('\n');
|
||||
}
|
||||
return text;
|
||||
})();
|
||||
"""
|
||||
def shutdown(self):
|
||||
with self._scheduler_lifecycle_lock:
|
||||
self._shutdown_requested = True
|
||||
self._periodic_stop_event.set()
|
||||
thread = self._periodic_thread
|
||||
if thread and thread.is_alive():
|
||||
thread.join(timeout=1.0)
|
||||
try:
|
||||
self._command_executor.shutdown(wait=False, cancel_futures=True)
|
||||
except TypeError:
|
||||
self._command_executor.shutdown(wait=False)
|
||||
except Exception as e:
|
||||
logger.debug(f"[CMD] 命令线程池关闭失败(忽略): {e}")
|
||||
|
||||
|
||||
# ================= 单例 =================
|
||||
command_engine = CommandEngine()
|
||||
|
||||
__all__ = [
|
||||
'CommandEngine',
|
||||
'command_engine',
|
||||
'TRIGGER_TYPES',
|
||||
'ACTION_TYPES',
|
||||
'get_default_command',
|
||||
]
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,21 @@
|
||||
"""CommandEngine 及其 mixin 共用的模块级对象(R2-3 拆分时从 command_engine.py 迁出,原处仍然重新导出)。"""
|
||||
|
||||
import re
|
||||
|
||||
from app.core.config import get_logger
|
||||
|
||||
try:
|
||||
import requests # noqa: F401
|
||||
HAS_REQUESTS = True
|
||||
except ImportError: # pragma: no cover - requests 为必装依赖
|
||||
requests = None # type: ignore[assignment]
|
||||
HAS_REQUESTS = False
|
||||
|
||||
logger = get_logger("CMD_ENG")
|
||||
FOLLOW_DEFAULT_PRESET = "__DEFAULT__"
|
||||
|
||||
# 页面文本匹配相关的正则预编译:这些会在 page_check 高频路径上
|
||||
# 对整页快照(可能几十上百 KB)反复求值,逐次 re.* 的模式缓存查找是不必要的开销。
|
||||
_WHITESPACE_RE = re.compile(r"\s+")
|
||||
_NON_ASCII_RE = re.compile(r"[^\x00-\x7F]")
|
||||
_WORD_LIKE_KEYWORD_RE = re.compile(r"[a-z0-9 _-]+")
|
||||
@@ -0,0 +1,553 @@
|
||||
"""CommandEngine 的一部分(R2-3 拆分):命令与分组的增删改查、分组执行与预览。
|
||||
|
||||
方法原样从 command_engine.py 迁出,行为不变;通过 mixin 组合回 CommandEngine,
|
||||
类属性与实例状态都在 CommandEngine(command_engine.py)里定义。
|
||||
"""
|
||||
|
||||
import copy
|
||||
import uuid
|
||||
from typing import Dict, List, Optional, Any, TYPE_CHECKING
|
||||
from app.services.command_defs import _new_command_id, get_default_command
|
||||
from app.services.command_engine_common import logger
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from app.core.tab_pool import TabSession # noqa: F401
|
||||
|
||||
|
||||
class CommandEngineCrudMixin:
|
||||
"""命令与分组的增删改查、分组执行与预览"""
|
||||
|
||||
def list_commands(self) -> List[Dict]:
|
||||
"""获取所有命令"""
|
||||
return self._merge_runtime_stats_into_commands(self._load_commands())
|
||||
def get_command(self, command_id: str) -> Optional[Dict]:
|
||||
command_key = str(command_id or "").strip()
|
||||
if not command_key:
|
||||
return None
|
||||
command = None
|
||||
with self._commands_lock:
|
||||
self._get_config_engine()
|
||||
self._refresh_commands_if_changed()
|
||||
for cmd in self._commands_cache:
|
||||
if not isinstance(cmd, dict):
|
||||
continue
|
||||
if str(cmd.get("id", "")).strip() == command_key:
|
||||
command = copy.deepcopy(cmd)
|
||||
break
|
||||
if command is None:
|
||||
return None
|
||||
return self._merge_runtime_stats_into_command(command)
|
||||
def get_command_config(self, command_id: str) -> Optional[Dict]:
|
||||
"""获取单条命令原始配置,不合并展示用运行态统计。"""
|
||||
command_key = str(command_id or "").strip()
|
||||
if not command_key:
|
||||
return None
|
||||
with self._commands_lock:
|
||||
self._get_config_engine()
|
||||
self._refresh_commands_if_changed()
|
||||
for cmd in self._commands_cache:
|
||||
if not isinstance(cmd, dict):
|
||||
continue
|
||||
if str(cmd.get("id", "")).strip() == command_key:
|
||||
return copy.deepcopy(cmd)
|
||||
return None
|
||||
def add_command(self, command: Dict = None) -> Optional[Dict]:
|
||||
if command is None:
|
||||
command = get_default_command()
|
||||
else:
|
||||
if not command.get("id"):
|
||||
command["id"] = _new_command_id()
|
||||
|
||||
with self._commands_lock:
|
||||
commands = self._load_commands()
|
||||
command["name"] = self._ensure_unique_command_name(command.get("name"), commands)
|
||||
command["group_name"] = self._normalize_group_name(command.get("group_name"))
|
||||
command["advanced_ui"] = self._normalize_advanced_ui(command.get("advanced_ui"))
|
||||
self._normalize_command_logging(command)
|
||||
commands.append(command)
|
||||
if not self._save_commands(commands):
|
||||
return None
|
||||
|
||||
logger.info(f"[OK] 命令已添加: {command.get('name')} ({command['id']})")
|
||||
return copy.deepcopy(command)
|
||||
def duplicate_command(self, command_id: str) -> Optional[Dict]:
|
||||
"""复制单条命令,并将副本紧跟在原命令之后。"""
|
||||
command_key = str(command_id or "").strip()
|
||||
if not command_key:
|
||||
return None
|
||||
|
||||
with self._commands_lock:
|
||||
commands = self._load_commands()
|
||||
source_index = next(
|
||||
(index for index, item in enumerate(commands) if item.get("id") == command_key),
|
||||
-1,
|
||||
)
|
||||
if source_index < 0:
|
||||
return None
|
||||
|
||||
duplicated = copy.deepcopy(commands[source_index])
|
||||
duplicated["id"] = _new_command_id()
|
||||
duplicated["name"] = self._ensure_unique_copy_name(
|
||||
duplicated.get("name"),
|
||||
{str(item.get("name") or "").strip() for item in commands},
|
||||
)
|
||||
self._normalize_command_logging(duplicated)
|
||||
commands.insert(source_index + 1, duplicated)
|
||||
if not self._save_commands(commands):
|
||||
return None
|
||||
|
||||
logger.info(f"[OK] 命令已复制: {command_key} -> {duplicated['id']}")
|
||||
return copy.deepcopy(duplicated)
|
||||
def duplicate_group(self, group_name: str) -> Optional[Dict[str, Any]]:
|
||||
"""复制整个命令组,并重映射副本组内的命令引用。"""
|
||||
source_group = self._normalize_group_name(group_name)
|
||||
if not source_group:
|
||||
return None
|
||||
|
||||
with self._commands_lock:
|
||||
commands = self._load_commands()
|
||||
source_commands = [
|
||||
item
|
||||
for item in commands
|
||||
if self._normalize_group_name(item.get("group_name")) == source_group
|
||||
]
|
||||
if not source_commands:
|
||||
return None
|
||||
|
||||
existing_group_names = {
|
||||
self._normalize_group_name(item.get("group_name"))
|
||||
for item in commands
|
||||
if self._normalize_group_name(item.get("group_name"))
|
||||
}
|
||||
target_group = self._ensure_unique_copy_name(source_group, existing_group_names)
|
||||
command_id_map = {
|
||||
str(item.get("id") or "").strip(): _new_command_id()
|
||||
for item in source_commands
|
||||
if str(item.get("id") or "").strip()
|
||||
}
|
||||
existing_command_names = {
|
||||
str(item.get("name") or "").strip()
|
||||
for item in commands
|
||||
if str(item.get("name") or "").strip()
|
||||
}
|
||||
duplicated_commands: List[Dict[str, Any]] = []
|
||||
|
||||
for source in source_commands:
|
||||
duplicated = copy.deepcopy(source)
|
||||
source_id = str(source.get("id") or "").strip()
|
||||
duplicated["id"] = command_id_map.get(source_id) or _new_command_id()
|
||||
duplicated["name"] = self._ensure_unique_copy_name(
|
||||
source.get("name"),
|
||||
existing_command_names,
|
||||
)
|
||||
existing_command_names.add(duplicated["name"])
|
||||
duplicated["group_name"] = target_group
|
||||
self._remap_duplicated_command_references(
|
||||
duplicated,
|
||||
command_id_map,
|
||||
source_group_name=source_group,
|
||||
target_group_name=target_group,
|
||||
)
|
||||
self._normalize_command_logging(duplicated)
|
||||
duplicated_commands.append(duplicated)
|
||||
|
||||
commands.extend(duplicated_commands)
|
||||
if not self._save_commands(commands):
|
||||
return None
|
||||
|
||||
logger.info(
|
||||
f"[OK] 命令组已复制: {source_group} -> {target_group} "
|
||||
f"({len(duplicated_commands)} 条命令)"
|
||||
)
|
||||
return {
|
||||
"group_name": target_group,
|
||||
"commands": copy.deepcopy(duplicated_commands),
|
||||
"count": len(duplicated_commands),
|
||||
}
|
||||
def update_command(self, command_id: str, updates: Dict) -> Optional[Dict]:
|
||||
updates = dict(updates or {})
|
||||
with self._commands_lock:
|
||||
commands = self._load_commands()
|
||||
|
||||
for i, cmd in enumerate(commands):
|
||||
if cmd.get("id") == command_id:
|
||||
updates.pop("id", None)
|
||||
if "name" in updates:
|
||||
updates["name"] = self._ensure_unique_command_name(
|
||||
updates.get("name"),
|
||||
commands,
|
||||
exclude_id=command_id,
|
||||
)
|
||||
if "group_name" in updates:
|
||||
updates["group_name"] = self._normalize_group_name(updates.get("group_name"))
|
||||
if "advanced_ui" in updates:
|
||||
updates["advanced_ui"] = self._normalize_advanced_ui(updates.get("advanced_ui"))
|
||||
cmd.update(updates)
|
||||
self._normalize_command_logging(cmd)
|
||||
commands[i] = cmd
|
||||
if not self._save_commands(commands):
|
||||
return None
|
||||
logger.debug(f"[OK] 命令已更新: {cmd.get('name')} ({command_id})")
|
||||
return copy.deepcopy(cmd)
|
||||
|
||||
return None
|
||||
def delete_command(self, command_id: str) -> bool:
|
||||
with self._commands_lock:
|
||||
commands = self._load_commands()
|
||||
new_commands = [c for c in commands if c.get("id") != command_id]
|
||||
|
||||
if len(new_commands) == len(commands):
|
||||
return False
|
||||
|
||||
if not self._save_commands(new_commands):
|
||||
return False
|
||||
|
||||
# 清理触发状态
|
||||
with self._lock:
|
||||
keys_to_remove = [k for k in self._trigger_states if k[0] == command_id]
|
||||
for k in keys_to_remove:
|
||||
del self._trigger_states[k]
|
||||
result_keys = [k for k in self._command_results if k[0] == command_id]
|
||||
for k in result_keys:
|
||||
del self._command_results[k]
|
||||
self._command_runtime_stats.pop(command_id, None)
|
||||
|
||||
logger.info(f"[OK] 命令已删除: {command_id}")
|
||||
return True
|
||||
def reorder_commands(self, command_ids: List[str]) -> bool:
|
||||
with self._commands_lock:
|
||||
commands = self._load_commands()
|
||||
cmd_map = {c["id"]: c for c in commands}
|
||||
new_commands = []
|
||||
|
||||
for cid in command_ids:
|
||||
if cid in cmd_map:
|
||||
new_commands.append(cmd_map.pop(cid))
|
||||
|
||||
for remaining in cmd_map.values():
|
||||
new_commands.append(remaining)
|
||||
|
||||
if len(new_commands) == len(commands) and all(
|
||||
current is updated for current, updated in zip(commands, new_commands)
|
||||
):
|
||||
return True
|
||||
|
||||
return self._save_commands(new_commands)
|
||||
return True
|
||||
def set_commands_group(self, command_ids: List[str], group_name: str) -> int:
|
||||
"""批量设置命令分组。group_name 为空时表示解散选中的命令。"""
|
||||
target_ids = {str(cid).strip() for cid in (command_ids or []) if str(cid).strip()}
|
||||
if not target_ids:
|
||||
return 0
|
||||
|
||||
normalized_group = self._normalize_group_name(group_name)
|
||||
updated = 0
|
||||
|
||||
with self._commands_lock:
|
||||
commands = self._load_commands()
|
||||
for cmd in commands:
|
||||
if cmd.get("id") not in target_ids:
|
||||
continue
|
||||
if self._normalize_group_name(cmd.get("group_name")) == normalized_group:
|
||||
continue
|
||||
cmd["group_name"] = normalized_group
|
||||
updated += 1
|
||||
if updated > 0 and not self._save_commands(commands):
|
||||
return -1
|
||||
|
||||
return updated
|
||||
def rename_group(self, old_group_name: str, new_group_name: str) -> int:
|
||||
"""重命名命令组。"""
|
||||
source_name = self._normalize_group_name(old_group_name)
|
||||
target_name = self._normalize_group_name(new_group_name)
|
||||
if not source_name or not target_name or source_name == target_name:
|
||||
return 0
|
||||
|
||||
updated = 0
|
||||
with self._commands_lock:
|
||||
commands = self._load_commands()
|
||||
for cmd in commands:
|
||||
if self._normalize_group_name(cmd.get("group_name")) != source_name:
|
||||
continue
|
||||
cmd["group_name"] = target_name
|
||||
updated += 1
|
||||
if updated > 0 and not self._save_commands(commands):
|
||||
return -1
|
||||
return updated
|
||||
def set_commands_enabled(self, command_ids: List[str], enabled: bool) -> int:
|
||||
"""批量更新命令启用状态。"""
|
||||
target_ids = {str(cid).strip() for cid in (command_ids or []) if str(cid).strip()}
|
||||
if not target_ids:
|
||||
return 0
|
||||
|
||||
desired_enabled = bool(enabled)
|
||||
updated = 0
|
||||
|
||||
with self._commands_lock:
|
||||
commands = self._load_commands()
|
||||
for cmd in commands:
|
||||
if cmd.get("id") not in target_ids:
|
||||
continue
|
||||
current_enabled = bool(cmd.get("enabled", True))
|
||||
if current_enabled == desired_enabled:
|
||||
continue
|
||||
cmd["enabled"] = desired_enabled
|
||||
updated += 1
|
||||
if updated > 0 and not self._save_commands(commands):
|
||||
return -1
|
||||
|
||||
return updated
|
||||
def disband_group(self, group_name: str) -> int:
|
||||
"""解散整个命令组。"""
|
||||
normalized_group = self._normalize_group_name(group_name)
|
||||
if not normalized_group:
|
||||
return 0
|
||||
|
||||
updated = 0
|
||||
with self._commands_lock:
|
||||
commands = self._load_commands()
|
||||
for cmd in commands:
|
||||
if self._normalize_group_name(cmd.get("group_name")) != normalized_group:
|
||||
continue
|
||||
cmd["group_name"] = ""
|
||||
updated += 1
|
||||
if updated > 0 and not self._save_commands(commands):
|
||||
return -1
|
||||
return updated
|
||||
def set_group_enabled(self, group_name: str, enabled: bool) -> int:
|
||||
"""直接更新整个命令组的启用状态。"""
|
||||
normalized_group = self._normalize_group_name(group_name)
|
||||
if not normalized_group:
|
||||
return 0
|
||||
|
||||
desired_enabled = bool(enabled)
|
||||
updated = 0
|
||||
with self._commands_lock:
|
||||
commands = self._load_commands()
|
||||
for cmd in commands:
|
||||
if self._normalize_group_name(cmd.get("group_name")) != normalized_group:
|
||||
continue
|
||||
current_enabled = bool(cmd.get("enabled", True))
|
||||
if current_enabled == desired_enabled:
|
||||
continue
|
||||
cmd["enabled"] = desired_enabled
|
||||
updated += 1
|
||||
if updated > 0 and not self._save_commands(commands):
|
||||
return -1
|
||||
return updated
|
||||
def list_command_groups(self) -> List[Dict[str, Any]]:
|
||||
groups: Dict[str, Dict[str, Any]] = {}
|
||||
for cmd in self._load_commands_for_checks():
|
||||
group_name = self._normalize_group_name(cmd.get("group_name"))
|
||||
if not group_name:
|
||||
continue
|
||||
bucket = groups.setdefault(group_name, {
|
||||
"name": group_name,
|
||||
"count": 0,
|
||||
"enabled_count": 0,
|
||||
"command_ids": [],
|
||||
})
|
||||
bucket["count"] += 1
|
||||
bucket["enabled_count"] += 1 if cmd.get("enabled", True) else 0
|
||||
bucket["command_ids"].append(cmd.get("id"))
|
||||
|
||||
return [groups[name] for name in sorted(groups.keys())]
|
||||
def execute_command_group(
|
||||
self,
|
||||
group_name: str,
|
||||
session: 'TabSession',
|
||||
include_disabled: bool = False,
|
||||
source_command_id: Optional[str] = None,
|
||||
ancestry_chain: Optional[List[str]] = None,
|
||||
acquire_policy: Optional[str] = None,
|
||||
prepared_plan: Optional[Dict[str, Any]] = None,
|
||||
) -> Dict[str, Any]:
|
||||
"""在当前会话中顺序执行命令组内的命令。"""
|
||||
normalized_group = self._normalize_group_name(group_name)
|
||||
if not normalized_group:
|
||||
return {"ok": False, "error": "empty_group_name"}
|
||||
effective_policy = self._normalize_group_acquire_policy(acquire_policy)
|
||||
|
||||
plan = dict(prepared_plan or {}) if isinstance(prepared_plan, dict) else {}
|
||||
if not plan:
|
||||
plan = self.preview_command_group(
|
||||
group_name=normalized_group,
|
||||
session=session,
|
||||
include_disabled=include_disabled,
|
||||
source_command_id=source_command_id,
|
||||
ancestry_chain=ancestry_chain,
|
||||
)
|
||||
candidates: List[Dict[str, Any]] = list(plan.pop("_candidate_commands", []))
|
||||
results: List[Dict[str, Any]] = []
|
||||
initial_scope_skipped = int(plan.get("scope_skipped", 0) or 0)
|
||||
|
||||
if not candidates:
|
||||
return {
|
||||
"ok": False,
|
||||
"error": "group_empty_or_no_runnable_commands",
|
||||
"group_name": normalized_group,
|
||||
"scope_skipped": initial_scope_skipped,
|
||||
"runnable_count": plan.get("runnable_count", 0),
|
||||
}
|
||||
|
||||
chain_seed = list(ancestry_chain or [])
|
||||
if not chain_seed and source_command_id:
|
||||
chain_seed = [source_command_id]
|
||||
group_acquired = False
|
||||
group_task_id = f"group_{normalized_group}_{uuid.uuid4().hex[:12]}"
|
||||
can_acquire = hasattr(session, "acquire_for_command")
|
||||
if effective_policy != "inherit_session":
|
||||
if not can_acquire:
|
||||
if effective_policy == "require_acquire":
|
||||
return {
|
||||
"ok": False,
|
||||
"error": "group_acquire_unavailable",
|
||||
"group_name": normalized_group,
|
||||
"acquire_policy": effective_policy,
|
||||
}
|
||||
else:
|
||||
group_acquired = bool(session.acquire_for_command(group_task_id))
|
||||
if not group_acquired and effective_policy == "require_acquire":
|
||||
return {
|
||||
"ok": False,
|
||||
"error": "group_acquire_failed",
|
||||
"group_name": normalized_group,
|
||||
"acquire_policy": effective_policy,
|
||||
}
|
||||
|
||||
try:
|
||||
for cmd in candidates:
|
||||
command_id = cmd.get("id")
|
||||
if not command_id:
|
||||
continue
|
||||
if not self._matches_scope(cmd, session):
|
||||
results.append({
|
||||
"id": command_id,
|
||||
"name": cmd.get("name", command_id),
|
||||
"ok": False,
|
||||
"error": "scope_mismatch",
|
||||
})
|
||||
continue
|
||||
exec_key = (command_id, session.id)
|
||||
with self._lock:
|
||||
if exec_key in self._executing:
|
||||
results.append({
|
||||
"id": command_id,
|
||||
"name": cmd.get("name", command_id),
|
||||
"ok": False,
|
||||
"error": "already_executing",
|
||||
})
|
||||
continue
|
||||
self._executing.add(exec_key)
|
||||
with self._command_logging_context(cmd):
|
||||
try:
|
||||
execution_result = self._execute_command(cmd, session, chain=chain_seed)
|
||||
command_ok = not self._execution_needs_page_check_retry(execution_result)
|
||||
results.append({
|
||||
"id": command_id,
|
||||
"name": cmd.get("name", command_id),
|
||||
"ok": command_ok,
|
||||
"result": execution_result,
|
||||
**({"error": "execution_not_ok"} if not command_ok else {}),
|
||||
})
|
||||
except Exception as e:
|
||||
logger.error(f"trigger check failed [{cmd.get('name')}]: {e}")
|
||||
results.append({
|
||||
"id": command_id,
|
||||
"name": cmd.get("name", command_id),
|
||||
"ok": False,
|
||||
"error": str(e),
|
||||
})
|
||||
finally:
|
||||
with self._lock:
|
||||
self._executing.discard(exec_key)
|
||||
finally:
|
||||
if group_acquired:
|
||||
try:
|
||||
browser = self._get_browser()
|
||||
pool = getattr(browser, "_tab_pool", None)
|
||||
if pool is not None and hasattr(pool, "release"):
|
||||
pool.release(
|
||||
session.id,
|
||||
check_triggers=False,
|
||||
expected_task_id=group_task_id,
|
||||
)
|
||||
else:
|
||||
session.release(
|
||||
clear_page=False,
|
||||
check_triggers=False,
|
||||
expected_task_id=group_task_id,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.debug(f"[CMD] 命令组释放标签页失败(忽略): {e}")
|
||||
|
||||
success_count = sum(1 for item in results if item.get("ok"))
|
||||
failure_count = sum(1 for item in results if not item.get("ok"))
|
||||
return {
|
||||
"ok": len(results) > 0 and failure_count == 0,
|
||||
"partial_ok": success_count > 0,
|
||||
"group_name": normalized_group,
|
||||
"executed": success_count,
|
||||
"total": len(results),
|
||||
"failures": failure_count,
|
||||
"results": results,
|
||||
"acquire_policy": effective_policy,
|
||||
"acquired": group_acquired,
|
||||
"scope_skipped": sum(1 for item in results if item.get("error") == "scope_mismatch"),
|
||||
"runnable_count": plan.get("runnable_count", 0),
|
||||
}
|
||||
def preview_command_group(
|
||||
self,
|
||||
group_name: str,
|
||||
session: 'TabSession',
|
||||
include_disabled: bool = False,
|
||||
source_command_id: Optional[str] = None,
|
||||
ancestry_chain: Optional[List[str]] = None,
|
||||
) -> Dict[str, Any]:
|
||||
normalized_group = self._normalize_group_name(group_name)
|
||||
if not normalized_group:
|
||||
return {
|
||||
"ok": False,
|
||||
"error": "empty_group_name",
|
||||
"group_name": normalized_group,
|
||||
"_candidate_commands": [],
|
||||
"_scope_skipped_results": [],
|
||||
}
|
||||
|
||||
with self._commands_lock:
|
||||
commands = self._load_commands()
|
||||
|
||||
candidates: List[Dict[str, Any]] = []
|
||||
scope_skipped_results: List[Dict[str, Any]] = []
|
||||
total_candidates = 0
|
||||
ancestors = {str(item or "").strip() for item in (ancestry_chain or []) if str(item or "").strip()}
|
||||
if source_command_id:
|
||||
ancestors.add(str(source_command_id).strip())
|
||||
for cmd in commands:
|
||||
if self._normalize_group_name(cmd.get("group_name")) != normalized_group:
|
||||
continue
|
||||
if not include_disabled and not cmd.get("enabled", True):
|
||||
continue
|
||||
if str(cmd.get("id", "")).strip() in ancestors:
|
||||
continue
|
||||
total_candidates += 1
|
||||
candidates.append(cmd)
|
||||
if not self._matches_scope(cmd, session):
|
||||
scope_skipped_results.append({
|
||||
"id": cmd.get("id"),
|
||||
"name": cmd.get("name", cmd.get("id", "")),
|
||||
"ok": False,
|
||||
"error": "scope_mismatch",
|
||||
})
|
||||
continue
|
||||
|
||||
return {
|
||||
"ok": bool(candidates) and not scope_skipped_results,
|
||||
"fully_runnable": bool(candidates) and not scope_skipped_results,
|
||||
"group_name": normalized_group,
|
||||
"total_candidates": total_candidates,
|
||||
"runnable_count": total_candidates - len(scope_skipped_results),
|
||||
"scope_skipped": len(scope_skipped_results),
|
||||
"_candidate_commands": candidates,
|
||||
"_scope_skipped_results": scope_skipped_results,
|
||||
}
|
||||
@@ -0,0 +1,346 @@
|
||||
"""CommandEngine 的一部分(R2-3 拆分):页面内容检查:表达式解析、文本匹配、快照求值与锁存。
|
||||
|
||||
方法原样从 command_engine.py 迁出,行为不变;通过 mixin 组合回 CommandEngine,
|
||||
类属性与实例状态都在 CommandEngine(command_engine.py)里定义。
|
||||
"""
|
||||
|
||||
import re
|
||||
import time
|
||||
from typing import Dict, List, Optional, Any, TYPE_CHECKING
|
||||
from app.services.command_engine_common import _NON_ASCII_RE, _WHITESPACE_RE, _WORD_LIKE_KEYWORD_RE, logger
|
||||
from app.core.driver import driver_for_tab
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from app.core.tab_pool import TabSession # noqa: F401
|
||||
|
||||
|
||||
class CommandEnginePageCheckMixin:
|
||||
"""页面内容检查:表达式解析、文本匹配、快照求值与锁存"""
|
||||
|
||||
@staticmethod
|
||||
def _parse_page_check_expression(value: str):
|
||||
"""解析 page_check 的 value 字段,支持 || (OR) 和 && (AND) 语法。
|
||||
|
||||
返回 (operator, keywords_list):
|
||||
- ("or", [kw1, kw2, ...]) — 任意一个关键词命中即触发
|
||||
- ("and", [kw1, kw2, ...]) — 所有关键词都命中才触发
|
||||
- ("single", [value]) — 单关键词(向后兼容)
|
||||
|
||||
当 || 和 && 同时出现时,|| 优先拆分。
|
||||
"""
|
||||
raw = str(value or "").strip()
|
||||
if not raw:
|
||||
return ("single", [])
|
||||
if "||" in raw:
|
||||
parts = [p.strip() for p in raw.split("||") if p.strip()]
|
||||
return ("or", parts) if parts else ("single", [])
|
||||
if "&&" in raw:
|
||||
parts = [p.strip() for p in raw.split("&&") if p.strip()]
|
||||
return ("and", parts) if parts else ("single", [])
|
||||
return ("single", [raw])
|
||||
@staticmethod
|
||||
def _normalize_match_text(value: Any) -> str:
|
||||
text = str(value or "").strip().lower()
|
||||
if not text:
|
||||
return ""
|
||||
return _WHITESPACE_RE.sub(" ", text)
|
||||
def _get_compact_haystack(self, hay: str) -> str:
|
||||
"""去空白版页面文本(带缓存)。
|
||||
|
||||
page_check 每个非 ASCII 关键词都要一份去空白文本,
|
||||
而页面快照往往有几十上百 KB;原实现按关键词重复做全文 re.sub,
|
||||
一次检查里同一段文本会被重复扫描 N 次。这里按内容缓存一份即可。
|
||||
"""
|
||||
if not hay:
|
||||
return ""
|
||||
cached = self._compact_haystack_cache
|
||||
# 用对象身份比较:同一轮检查里各个关键词拿到的 hay 是同一个字符串对象,
|
||||
# 比 (len, hash) 更简单,也不存在哈希碰撞返回错误结果的可能。
|
||||
if cached is not None and cached[0] is hay:
|
||||
return cached[1]
|
||||
compact = _WHITESPACE_RE.sub("", hay)
|
||||
self._compact_haystack_cache = (hay, compact)
|
||||
return compact
|
||||
def _text_contains_needle(
|
||||
self,
|
||||
haystack: str,
|
||||
needle: str,
|
||||
pre_normalized_haystack: Optional[str] = None,
|
||||
) -> bool:
|
||||
hay = pre_normalized_haystack if pre_normalized_haystack is not None else self._normalize_match_text(haystack)
|
||||
ned = self._normalize_match_text(needle)
|
||||
if not hay or not ned:
|
||||
return False
|
||||
|
||||
if _NON_ASCII_RE.search(ned):
|
||||
compact_hay = self._get_compact_haystack(hay)
|
||||
compact_ned = _WHITESPACE_RE.sub("", ned)
|
||||
if compact_hay and compact_ned and compact_ned in compact_hay:
|
||||
return True
|
||||
|
||||
# For plain word-like keywords (for example "battle"), prefer whole-word
|
||||
# matching to reduce accidental substring hits.
|
||||
if _WORD_LIKE_KEYWORD_RE.fullmatch(ned):
|
||||
pattern = rf"(?<![a-z0-9]){re.escape(ned)}(?![a-z0-9])"
|
||||
return re.search(pattern, hay) is not None
|
||||
|
||||
return ned in hay
|
||||
def _get_page_check_snapshot_text(self, session: 'TabSession') -> str:
|
||||
if self._is_session_closed(session):
|
||||
return ""
|
||||
if self._is_page_check_refreshing(session):
|
||||
return ""
|
||||
now = time.time()
|
||||
cached = getattr(session, "_pc_snapshot_cached", None)
|
||||
if isinstance(cached, tuple) and len(cached) == 2:
|
||||
cached_ts, cached_text = cached
|
||||
try:
|
||||
cached_ts = float(cached_ts or 0.0)
|
||||
except Exception:
|
||||
cached_ts = 0.0
|
||||
if cached_ts > 0.0 and now - cached_ts < 0.5:
|
||||
return str(cached_text or "")
|
||||
|
||||
if self._is_page_check_backing_off(session):
|
||||
return ""
|
||||
|
||||
self._try_wake_tab(session, reason="page_check")
|
||||
|
||||
try:
|
||||
snapshot = self._run_page_check_js(
|
||||
session,
|
||||
"return window.__pcObserver ? String(window.__pcSnapshot || '') : null"
|
||||
)
|
||||
self._record_page_check_js_success(session)
|
||||
if snapshot is not None:
|
||||
snapshot_text = str(snapshot or "")
|
||||
if snapshot_text.strip():
|
||||
setattr(session, "_pc_snapshot_cached", (now, snapshot_text))
|
||||
return snapshot_text
|
||||
except Exception as e:
|
||||
if self._is_session_closed(session):
|
||||
return ""
|
||||
self._record_page_check_js_failure(session, e, "snapshot_observer")
|
||||
if self._mark_session_closed_if_disconnected(session, e, "page_check_snapshot_observer"):
|
||||
return ""
|
||||
if self._is_page_check_refreshing(session):
|
||||
return ""
|
||||
pass
|
||||
|
||||
try:
|
||||
page_text = str(self._run_page_check_js(session, self._PAGE_CHECK_SNAPSHOT_JS) or "")
|
||||
self._record_page_check_js_success(session)
|
||||
if page_text.strip():
|
||||
setattr(session, "_pc_snapshot_cached", (now, page_text))
|
||||
return page_text
|
||||
except Exception as e:
|
||||
if self._is_session_closed(session):
|
||||
return ""
|
||||
self._record_page_check_js_failure(session, e, "snapshot_body")
|
||||
if self._mark_session_closed_if_disconnected(session, e, "page_check_snapshot_body"):
|
||||
return ""
|
||||
if self._is_page_check_refreshing(session):
|
||||
return ""
|
||||
pass
|
||||
|
||||
try:
|
||||
title_text = str(self._run_page_check_js(session, "return document.title || '';") or "")
|
||||
self._record_page_check_js_success(session)
|
||||
setattr(session, "_pc_snapshot_cached", (now, title_text))
|
||||
return title_text
|
||||
except Exception as e:
|
||||
if self._is_session_closed(session):
|
||||
return ""
|
||||
self._record_page_check_js_failure(session, e, "snapshot_title")
|
||||
self._mark_session_closed_if_disconnected(session, e, "page_check_snapshot_title")
|
||||
return ""
|
||||
def _build_page_check_snapshot_preview(
|
||||
self,
|
||||
snapshot: str,
|
||||
matched_keywords: Optional[List[str]] = None,
|
||||
limit: int = 180,
|
||||
) -> str:
|
||||
normalized_snapshot = self._normalize_match_text(snapshot)
|
||||
if not normalized_snapshot:
|
||||
return ""
|
||||
|
||||
start = 0
|
||||
for keyword in matched_keywords or []:
|
||||
normalized_keyword = self._normalize_match_text(keyword)
|
||||
if not normalized_keyword:
|
||||
continue
|
||||
index = normalized_snapshot.find(normalized_keyword)
|
||||
if index != -1:
|
||||
start = max(0, index - 60)
|
||||
break
|
||||
|
||||
end = min(len(normalized_snapshot), start + max(40, limit))
|
||||
preview = normalized_snapshot[start:end]
|
||||
if start > 0:
|
||||
preview = "..." + preview
|
||||
if end < len(normalized_snapshot):
|
||||
preview = preview + "..."
|
||||
return preview.replace("'", '"')
|
||||
def _evaluate_page_check_snapshot(
|
||||
self,
|
||||
snapshot: str,
|
||||
op: str,
|
||||
keywords: List[str],
|
||||
) -> Dict[str, Any]:
|
||||
if not keywords:
|
||||
return {
|
||||
"hit": False,
|
||||
"matched_keywords": [],
|
||||
"snapshot_preview": "",
|
||||
}
|
||||
|
||||
normalized_snapshot = self._normalize_match_text(snapshot)
|
||||
matched_keywords = [
|
||||
keyword for keyword in keywords
|
||||
if self._text_contains_needle(
|
||||
snapshot,
|
||||
keyword,
|
||||
pre_normalized_haystack=normalized_snapshot,
|
||||
)
|
||||
]
|
||||
if op == "or":
|
||||
hit = bool(matched_keywords)
|
||||
else:
|
||||
hit = len(matched_keywords) == len(keywords)
|
||||
|
||||
return {
|
||||
"hit": hit,
|
||||
"matched_keywords": matched_keywords,
|
||||
"snapshot_preview": self._build_page_check_snapshot_preview(snapshot, matched_keywords),
|
||||
}
|
||||
def _evaluate_page_check_expr(
|
||||
self,
|
||||
session: 'TabSession',
|
||||
op: str,
|
||||
keywords: List[str],
|
||||
) -> Dict[str, Any]:
|
||||
snapshot = self._get_page_check_snapshot_text(session)
|
||||
result = self._evaluate_page_check_snapshot(snapshot, op, keywords)
|
||||
result["snapshot_text"] = snapshot
|
||||
return result
|
||||
def _evaluate_page_check_probe(
|
||||
self,
|
||||
session: 'TabSession',
|
||||
code: str,
|
||||
) -> Dict[str, Any]:
|
||||
self._try_wake_tab(session, reason="page_check_probe")
|
||||
context = {
|
||||
"parser_id": str(getattr(session, "_workflow_parser_id", "") or "").strip().lower(),
|
||||
"target_side": str(getattr(session, "_workflow_target_side", "") or "").strip().lower(),
|
||||
"runtime_id": str(getattr(session, "_workflow_runtime_id", "") or "").strip(),
|
||||
}
|
||||
try:
|
||||
try:
|
||||
driver_for_tab(session.tab).run_js(
|
||||
"return (() => { window.__codexWorkflowContext = arguments[0] || {}; return true; })()",
|
||||
context,
|
||||
)
|
||||
except Exception as context_error:
|
||||
logger.debug(f"[CMD] 页面检查上下文注入失败(继续探测): {context_error}")
|
||||
result = self._run_command_js(session.tab, code)
|
||||
except Exception as e:
|
||||
message = f"probe_js_failed: {e}"
|
||||
return {
|
||||
"hit": False,
|
||||
"result": message,
|
||||
"summary": message,
|
||||
}
|
||||
|
||||
if isinstance(result, dict):
|
||||
hit = bool(result.get("hit"))
|
||||
summary = str(
|
||||
result.get("summary", result.get("result", result))
|
||||
or ""
|
||||
).strip()
|
||||
else:
|
||||
hit = bool(result)
|
||||
summary = "" if result in (None, False, "", 0) else str(result).strip()
|
||||
|
||||
return {
|
||||
"hit": hit,
|
||||
"result": result,
|
||||
"summary": summary[:180],
|
||||
}
|
||||
def _log_page_check_hit_details(
|
||||
self,
|
||||
command: Dict,
|
||||
session: 'TabSession',
|
||||
match_info: Dict[str, Any],
|
||||
):
|
||||
matched_keywords = match_info.get("matched_keywords") or []
|
||||
preview = str(match_info.get("snapshot_preview") or "").strip()
|
||||
probe_summary = str(match_info.get("probe_summary") or "").strip()
|
||||
probe_suffix = f", JS探测='{probe_summary}'" if probe_summary else ""
|
||||
logger.info(
|
||||
f"[CMD] 页面检查命中详情: {command.get('name')} "
|
||||
f"(标签页={session.id}, 命中关键词={matched_keywords}, 快照预览='{preview}'{probe_suffix})"
|
||||
)
|
||||
def _check_page_content_expr(
|
||||
self, session: 'TabSession', op: str, keywords: list
|
||||
) -> bool:
|
||||
"""根据解析后的表达式 (op, keywords) 检查页面内容。
|
||||
|
||||
- op="or" 任意一个关键词命中即返回 True
|
||||
- op="and" 所有关键词都命中才返回 True
|
||||
- op="single" 等同于 and(只有一个关键词)
|
||||
"""
|
||||
return bool(self._evaluate_page_check_expr(session, op, keywords).get("hit"))
|
||||
def _check_page_content(self, session: 'TabSession', text: str) -> bool:
|
||||
needle = str(text or "").strip()
|
||||
if not needle:
|
||||
return False
|
||||
snapshot = self._get_page_check_snapshot_text(session)
|
||||
return self._text_contains_needle(snapshot, needle)
|
||||
def _reset_page_check_latch(self, command: Dict, session: 'TabSession', reason: str = ""):
|
||||
"""Allow page_check commands to retrigger when previous execution did not complete successfully."""
|
||||
trigger = command.get("trigger", {}) or {}
|
||||
if str(trigger.get("type", "")).strip().lower() != "page_check":
|
||||
return
|
||||
if not trigger.get("reset_latch_on_failure", True):
|
||||
# Only block latch reset if the command actually started executing (not blocked by session acquire timeout, scheduling failures, or page navigation)
|
||||
if reason not in {"acquire_timeout", "workflow_schedule_failed"} and not (
|
||||
reason.startswith("navigated") or reason.startswith("tab_navigated")
|
||||
):
|
||||
return
|
||||
|
||||
key = (command.get("id"), getattr(session, "id", ""))
|
||||
if not key[0] or not key[1]:
|
||||
return
|
||||
|
||||
normalized_text = str(trigger.get("value", "") or "").strip().lower()
|
||||
with self._lock:
|
||||
state = self._trigger_states.get(key)
|
||||
if not state:
|
||||
return
|
||||
state["page_key"] = normalized_text
|
||||
state["page_hit"] = False
|
||||
state["page_stable"] = False
|
||||
state["page_hit_since"] = 0.0
|
||||
|
||||
if bool(trigger.get("once_per_request", False)):
|
||||
current_request_id = ""
|
||||
try:
|
||||
for attr_name in ("_command_request_id", "_bound_request_id"):
|
||||
req_id = str(getattr(session, attr_name, "") or "").strip()
|
||||
if req_id:
|
||||
current_request_id = req_id
|
||||
break
|
||||
if not current_request_id:
|
||||
task_id = str(getattr(session, "current_task_id", "") or "").strip()
|
||||
if task_id.startswith("req-"):
|
||||
current_request_id = task_id
|
||||
except Exception:
|
||||
pass
|
||||
if current_request_id and "triggered_requests" in state:
|
||||
state["triggered_requests"].discard(current_request_id)
|
||||
|
||||
if reason:
|
||||
logger.debug(
|
||||
f"[CMD] 页面检查锁存已重置: {command.get('name')} "
|
||||
f"(标签页={session.id}, 原因={reason})"
|
||||
)
|
||||
@@ -0,0 +1,439 @@
|
||||
"""CommandEngine 的一部分(R2-3 拆分):标签页交互:网络监听挂起/恢复、会话状态判断、焦点模拟、唤醒节流、页面检查脚本与观察者。
|
||||
|
||||
方法原样从 command_engine.py 迁出,行为不变;通过 mixin 组合回 CommandEngine,
|
||||
类属性与实例状态都在 CommandEngine(command_engine.py)里定义。
|
||||
"""
|
||||
|
||||
import copy
|
||||
import json
|
||||
import os
|
||||
import time
|
||||
from typing import Dict, List, Any, TYPE_CHECKING
|
||||
from app.core.page_lifecycle import (
|
||||
BACKGROUND_WAKE_CDP_TIMEOUT,
|
||||
BACKGROUND_WAKE_JS_TIMEOUT,
|
||||
install_visibility_emulation,
|
||||
is_page_refresh_error,
|
||||
restore_visibility_emulation,
|
||||
)
|
||||
from app.services.command_engine_common import logger
|
||||
from app.core.driver import driver_for_tab
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from app.core.tab_pool import TabSession # noqa: F401
|
||||
|
||||
|
||||
class CommandEnginePagesMixin:
|
||||
"""标签页交互:网络监听挂起/恢复、会话状态判断、焦点模拟、唤醒节流、页面检查脚本与观察者"""
|
||||
|
||||
def _suspend_tab_global_network(self, session: 'TabSession', reason: str = "command"):
|
||||
"""命令执行期间暂停标签页全局网络监听,避免和工作流监听冲突。"""
|
||||
try:
|
||||
browser = self._get_browser()
|
||||
pool = getattr(browser, "_tab_pool", None)
|
||||
if pool is not None and hasattr(pool, "suspend_global_network_monitor"):
|
||||
pool.suspend_global_network_monitor(session.id, reason=reason)
|
||||
except Exception as e:
|
||||
logger.debug(f"[CMD] 暂停全局网络监听失败(忽略): {e}")
|
||||
def _resume_tab_global_network(self, session: 'TabSession', reason: str = "command"):
|
||||
"""命令执行结束后恢复标签页全局网络监听。"""
|
||||
try:
|
||||
browser = self._get_browser()
|
||||
pool = getattr(browser, "_tab_pool", None)
|
||||
if pool is not None and hasattr(pool, "resume_global_network_monitor"):
|
||||
pool.resume_global_network_monitor(session.id, reason=reason)
|
||||
except Exception as e:
|
||||
logger.debug(f"[CMD] 恢复全局网络监听失败(忽略): {e}")
|
||||
@staticmethod
|
||||
def _session_status_value(session: 'TabSession') -> str:
|
||||
return str(getattr(getattr(session, "status", None), "value", "")).lower()
|
||||
def _is_session_closed(self, session: 'TabSession') -> bool:
|
||||
return self._session_status_value(session) == "closed"
|
||||
@staticmethod
|
||||
def _looks_like_page_disconnected_error(error: Any) -> bool:
|
||||
text = str(error or "").lower()
|
||||
return (
|
||||
"连接已断开" in text
|
||||
or "connection disconnected" in text
|
||||
or "target closed" in text
|
||||
or "session closed" in text
|
||||
or "websocket" in text and "closed" in text
|
||||
)
|
||||
@staticmethod
|
||||
def _looks_like_page_refresh_error(error: Any) -> bool:
|
||||
return is_page_refresh_error(error)
|
||||
def _mark_session_closed_if_disconnected(self, session: 'TabSession', error: Any, reason: str) -> bool:
|
||||
if not self._looks_like_page_disconnected_error(error):
|
||||
return False
|
||||
if getattr(session, "_cdp_recycle_in_progress", False):
|
||||
# P0-3:空闲 CDP 会话回收期间的短暂断开不是标签页关闭
|
||||
return True
|
||||
try:
|
||||
if hasattr(session, "mark_closed"):
|
||||
session.mark_closed(reason)
|
||||
except Exception:
|
||||
pass
|
||||
return True
|
||||
def _set_focus_emulation(self, session: 'TabSession', enabled: bool):
|
||||
"""Best-effort focus emulation without stealing OS/browser foreground focus."""
|
||||
try:
|
||||
driver_for_tab(session.tab).run_cdp(
|
||||
"Emulation.setFocusEmulationEnabled",
|
||||
enabled=bool(enabled),
|
||||
_timeout=BACKGROUND_WAKE_CDP_TIMEOUT,
|
||||
)
|
||||
if enabled:
|
||||
install_visibility_emulation(
|
||||
session.tab,
|
||||
owner=session,
|
||||
reason="command_focus_emulation",
|
||||
)
|
||||
else:
|
||||
restore_visibility_emulation(
|
||||
session.tab,
|
||||
owner=session,
|
||||
reason="command_focus_emulation_end",
|
||||
)
|
||||
except Exception as e:
|
||||
logger.debug(f"[CMD] 焦点模拟设置失败(忽略): enabled={enabled}, 错误={e}")
|
||||
def _should_wake_tab_now(self, session: 'TabSession') -> bool:
|
||||
"""按会话限频唤醒动作:唤醒后的 CDP 状态是粘性的,不必每轮都重发。"""
|
||||
interval = float(self._wake_tab_min_interval_sec or 0.0)
|
||||
if interval <= 0.0:
|
||||
return True
|
||||
try:
|
||||
# 上一轮 page_check JS 已经出错,说明页面可能真的被冻结/丢弃了。
|
||||
# 这种情况下不限频,保持原来"每轮都唤醒"的行为,
|
||||
# 避免连续两轮失败把会话推进 30s 退避。
|
||||
if int(getattr(session, "_pc_js_failures", 0) or 0) > 0:
|
||||
setattr(session, "_pc_last_wake_at", time.time())
|
||||
return True
|
||||
except Exception:
|
||||
return True
|
||||
try:
|
||||
now = time.time()
|
||||
last_at = float(getattr(session, "_pc_last_wake_at", 0.0) or 0.0)
|
||||
if last_at > 0.0 and (now - last_at) < interval:
|
||||
return False
|
||||
setattr(session, "_pc_last_wake_at", now)
|
||||
except Exception:
|
||||
return True
|
||||
return True
|
||||
def _invalidate_wake_throttle(self, session: 'TabSession') -> None:
|
||||
"""让下一次 page_check 立即重新唤醒(JS 失败/页面可能已被冻结时调用)。"""
|
||||
try:
|
||||
setattr(session, "_pc_last_wake_at", 0.0)
|
||||
except Exception:
|
||||
pass
|
||||
def _try_wake_tab(self, session: 'TabSession', reason: str = ""):
|
||||
"""
|
||||
Best-effort wake-up for background/discard-prone tabs.
|
||||
Uses lifecycle and lightweight JS ping, without forcing browser focus.
|
||||
"""
|
||||
if self._is_session_closed(session):
|
||||
return
|
||||
# P0-6:page_check / keepalive 之前先解除空闲冻结(与唤醒节流无关,必须执行)
|
||||
try:
|
||||
from app.core.tab_pool_parts.idle_maintenance import resume_if_frozen
|
||||
|
||||
resume_if_frozen(session, reason=reason or "wake_tab")
|
||||
except Exception:
|
||||
pass
|
||||
if not self._wake_tab_before_page_check:
|
||||
return
|
||||
if self._is_page_check_backing_off(session):
|
||||
return
|
||||
if not self._should_wake_tab_now(session):
|
||||
return
|
||||
focus_emulation_set = False
|
||||
try:
|
||||
driver_for_tab(session.tab).run_cdp(
|
||||
"Emulation.setFocusEmulationEnabled",
|
||||
enabled=True,
|
||||
_timeout=BACKGROUND_WAKE_CDP_TIMEOUT,
|
||||
)
|
||||
focus_emulation_set = True
|
||||
except Exception as e:
|
||||
if self._mark_session_closed_if_disconnected(session, e, "wake_focus_emulation"):
|
||||
return
|
||||
try:
|
||||
install_visibility_emulation(session.tab, owner=session, reason=reason or "wake_tab")
|
||||
except Exception as e:
|
||||
if self._mark_session_closed_if_disconnected(session, e, "wake_visibility"):
|
||||
return
|
||||
try:
|
||||
driver_for_tab(session.tab).run_cdp(
|
||||
"Page.setWebLifecycleState",
|
||||
state="active",
|
||||
_timeout=BACKGROUND_WAKE_CDP_TIMEOUT,
|
||||
)
|
||||
except Exception as e:
|
||||
if self._mark_session_closed_if_disconnected(session, e, "wake_lifecycle"):
|
||||
return
|
||||
try:
|
||||
driver_for_tab(session.tab).run_js("return document.readyState || '';", timeout=BACKGROUND_WAKE_JS_TIMEOUT)
|
||||
except Exception as e:
|
||||
if self._mark_session_closed_if_disconnected(session, e, "wake_ready_state"):
|
||||
return
|
||||
finally:
|
||||
if focus_emulation_set:
|
||||
try:
|
||||
driver_for_tab(session.tab).run_cdp(
|
||||
"Emulation.setFocusEmulationEnabled",
|
||||
enabled=False,
|
||||
_timeout=BACKGROUND_WAKE_CDP_TIMEOUT,
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
def _run_page_check_js(self, session: 'TabSession', script: str) -> Any:
|
||||
if self._is_session_closed(session):
|
||||
raise RuntimeError("page_check session is closed")
|
||||
return driver_for_tab(session.tab).run_js(script, timeout=self._page_check_js_timeout_sec)
|
||||
def _is_page_check_backing_off(self, session: 'TabSession') -> bool:
|
||||
until = float(getattr(session, "_pc_js_backoff_until", 0.0) or 0.0)
|
||||
return until > time.time()
|
||||
def _is_page_check_refreshing(self, session: 'TabSession') -> bool:
|
||||
until = float(getattr(session, "_pc_refresh_grace_until", 0.0) or 0.0)
|
||||
return until > time.time()
|
||||
def _mark_page_check_refreshing(self, session: 'TabSession') -> None:
|
||||
"""Pause page checks briefly while navigation replaces the document."""
|
||||
self._invalidate_wake_throttle(session)
|
||||
sid = str(getattr(session, "id", "") or "")
|
||||
if sid:
|
||||
with self._lock:
|
||||
self._observer_keywords_by_session.pop(sid, None)
|
||||
try:
|
||||
setattr(
|
||||
session,
|
||||
"_pc_refresh_grace_until",
|
||||
time.time() + self._page_check_refresh_grace_sec,
|
||||
)
|
||||
setattr(session, "_pc_js_failures", 0)
|
||||
setattr(session, "_pc_js_backoff_until", 0.0)
|
||||
setattr(session, "_pc_snapshot_cached", None)
|
||||
except Exception:
|
||||
pass
|
||||
def _record_page_check_js_success(self, session: 'TabSession') -> None:
|
||||
try:
|
||||
setattr(session, "_pc_js_failures", 0)
|
||||
setattr(session, "_pc_js_backoff_until", 0.0)
|
||||
setattr(session, "_pc_refresh_grace_until", 0.0)
|
||||
except Exception:
|
||||
pass
|
||||
def _record_page_check_js_failure(self, session: 'TabSession', error: Any, context: str) -> None:
|
||||
if self._looks_like_page_refresh_error(error):
|
||||
self._mark_page_check_refreshing(session)
|
||||
return
|
||||
# JS 执行失败通常意味着页面被冻结/丢弃或已导航,
|
||||
# 清掉唤醒限频,让下一轮立刻重新唤醒,不受最小间隔限制。
|
||||
self._invalidate_wake_throttle(session)
|
||||
try:
|
||||
failures = int(getattr(session, "_pc_js_failures", 0) or 0) + 1
|
||||
setattr(session, "_pc_js_failures", failures)
|
||||
if failures >= 2:
|
||||
until = time.time() + self._page_check_failure_backoff_sec
|
||||
setattr(session, "_pc_js_backoff_until", until)
|
||||
logger.debug(
|
||||
f"[CMD] 页面检查 JS 连续失败,暂停后台检查: "
|
||||
f"标签页={getattr(session, 'id', '-')}, failures={failures}, "
|
||||
f"backoff={self._page_check_failure_backoff_sec:.0f}s, context={context}, error={error}"
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
def _collect_page_check_keywords(
|
||||
self,
|
||||
commands: List[Dict],
|
||||
session: 'TabSession',
|
||||
) -> set:
|
||||
"""Collect all page_check keywords that apply to this session."""
|
||||
keywords: set = set()
|
||||
for cmd in commands:
|
||||
if not cmd.get("enabled", True):
|
||||
continue
|
||||
trigger = cmd.get("trigger", {}) or {}
|
||||
if str(trigger.get("type", "")).strip().lower() != "page_check":
|
||||
continue
|
||||
if not bool(trigger.get("periodic_enabled", True)):
|
||||
continue
|
||||
if not self._matches_scope(cmd, session):
|
||||
continue
|
||||
value = str(trigger.get("value", "") or "").strip()
|
||||
if value:
|
||||
_, parts = self._parse_page_check_expression(value)
|
||||
for part in parts:
|
||||
kw = part.lower().strip()
|
||||
if kw:
|
||||
keywords.add(kw)
|
||||
return keywords
|
||||
def _sync_session_bootstrap_js_files(
|
||||
self,
|
||||
commands: List[Dict],
|
||||
session: 'TabSession',
|
||||
) -> None:
|
||||
"""Install explicitly opted-in JS files when a managed tab becomes idle.
|
||||
|
||||
`run_js_file` actions normally run only after their trigger fires. Some
|
||||
page-resilience scripts must also be registered before a later reload,
|
||||
so they opt in with ``bootstrap_on_session_ready``. The per-session
|
||||
mtime cache keeps this out of the hot periodic path while still
|
||||
replacing a registered script when its source changes.
|
||||
"""
|
||||
state = getattr(session, "_command_bootstrap_js_files", None)
|
||||
if not isinstance(state, dict):
|
||||
state = {}
|
||||
setattr(session, "_command_bootstrap_js_files", state)
|
||||
|
||||
for command in commands:
|
||||
if not isinstance(command, dict) or not command.get("enabled", True):
|
||||
continue
|
||||
if not self._matches_scope(command, session):
|
||||
continue
|
||||
|
||||
command_id = str(command.get("id", "") or "").strip()
|
||||
for action in command.get("actions", []) or []:
|
||||
if not isinstance(action, dict):
|
||||
continue
|
||||
if str(action.get("type", "")).strip().lower() != "run_js_file":
|
||||
continue
|
||||
if not self._coerce_action_bool(action.get("bootstrap_on_session_ready"), False):
|
||||
continue
|
||||
|
||||
resolved_path = self._resolve_action_file_path(action.get("file_path", ""))
|
||||
if not resolved_path or not os.path.isfile(resolved_path):
|
||||
logger.warning(
|
||||
f"[CMD] 启动 JS 文件不存在: command={command_id or '-'}, path={resolved_path or action.get('file_path', '')!r}"
|
||||
)
|
||||
continue
|
||||
|
||||
try:
|
||||
source_mtime = os.path.getmtime(resolved_path)
|
||||
except OSError as error:
|
||||
logger.warning(
|
||||
f"[CMD] 读取启动 JS 文件时间失败: command={command_id or '-'}, path={resolved_path!r}, error={error}"
|
||||
)
|
||||
continue
|
||||
|
||||
action_id = str(action.get("action_id", "") or "").strip()
|
||||
cache_key = (command_id, action_id, resolved_path.lower())
|
||||
if state.get(cache_key) == source_mtime:
|
||||
continue
|
||||
|
||||
try:
|
||||
result = self._execute_action(copy.deepcopy(action), session)
|
||||
except Exception as error:
|
||||
logger.warning(
|
||||
f"[CMD] 启动 JS 文件同步失败: command={command_id or '-'}, path={resolved_path!r}, error={error}"
|
||||
)
|
||||
continue
|
||||
|
||||
if self._is_action_soft_failure(result):
|
||||
logger.warning(
|
||||
f"[CMD] 启动 JS 文件同步未完成: command={command_id or '-'}, path={resolved_path!r}, result={result!r}"
|
||||
)
|
||||
continue
|
||||
|
||||
state[cache_key] = source_mtime
|
||||
logger.debug(
|
||||
f"[CMD] 已同步启动 JS 文件: command={command_id or '-'}, path={resolved_path!r}"
|
||||
)
|
||||
def _ensure_page_check_observer(
|
||||
self,
|
||||
session: 'TabSession',
|
||||
keywords: set,
|
||||
):
|
||||
"""Inject or update MutationObserver for real-time page_check text detection."""
|
||||
if not keywords:
|
||||
return
|
||||
if self._is_session_closed(session):
|
||||
return
|
||||
sid = str(getattr(session, "id", "") or "")
|
||||
if not sid:
|
||||
return
|
||||
if self._is_page_check_backing_off(session):
|
||||
return
|
||||
if self._is_page_check_refreshing(session):
|
||||
return
|
||||
setattr(session, "_pc_observer_empty_cleanup_done", False)
|
||||
# Skip if already installed with the same keywords AND observer is still alive
|
||||
with self._lock:
|
||||
installed = self._observer_keywords_by_session.get(sid)
|
||||
if installed == keywords:
|
||||
# Quick liveness check with throttling (outside lock — run_js is I/O).
|
||||
now = time.time()
|
||||
last_check = float(getattr(session, "_last_pc_observer_check_at", 0.0) or 0.0)
|
||||
if now - last_check < 5.0:
|
||||
return
|
||||
setattr(session, "_last_pc_observer_check_at", now)
|
||||
try:
|
||||
alive = self._run_page_check_js(session, "return !!window.__pcObserver")
|
||||
if alive:
|
||||
self._record_page_check_js_success(session)
|
||||
return
|
||||
except Exception as e:
|
||||
if self._is_session_closed(session):
|
||||
return
|
||||
self._record_page_check_js_failure(session, e, "observer_alive")
|
||||
if self._mark_session_closed_if_disconnected(session, e, "page_check_observer_alive"):
|
||||
return
|
||||
if self._is_page_check_refreshing(session):
|
||||
return
|
||||
pass
|
||||
# Observer lost — clear cache, re-inject below
|
||||
with self._lock:
|
||||
if self._observer_keywords_by_session.get(sid) == installed:
|
||||
self._observer_keywords_by_session.pop(sid, None)
|
||||
# Inject observer
|
||||
sorted_kws = sorted(keywords)
|
||||
js = self._PAGE_CHECK_OBSERVER_JS.replace(
|
||||
"%KEYWORDS%", json.dumps(sorted_kws)
|
||||
)
|
||||
try:
|
||||
result = self._run_page_check_js(session, js)
|
||||
self._record_page_check_js_success(session)
|
||||
with self._lock:
|
||||
self._observer_keywords_by_session[sid] = set(keywords)
|
||||
if result != "already_installed":
|
||||
logger.debug(
|
||||
f"[CMD] 页面检查观察器已注入: "
|
||||
f"标签页={sid}, 关键词={sorted_kws}"
|
||||
)
|
||||
except Exception as e:
|
||||
if self._is_session_closed(session):
|
||||
return
|
||||
self._record_page_check_js_failure(session, e, "observer_install")
|
||||
self._mark_session_closed_if_disconnected(session, e, "page_check_observer_install")
|
||||
def _clear_page_check_observer(self, session: 'TabSession') -> None:
|
||||
if self._is_session_closed(session):
|
||||
return
|
||||
sid = str(getattr(session, "id", "") or "")
|
||||
if not sid:
|
||||
return
|
||||
with self._lock:
|
||||
installed = self._observer_keywords_by_session.pop(sid, None)
|
||||
if installed is None and bool(getattr(session, "_pc_observer_empty_cleanup_done", False)):
|
||||
return
|
||||
if self._is_page_check_backing_off(session):
|
||||
return
|
||||
|
||||
cleanup_js = """
|
||||
return (() => {
|
||||
if (window.__pcObserver && typeof window.__pcObserver.disconnect === 'function') {
|
||||
window.__pcObserver.disconnect();
|
||||
}
|
||||
window.__pcObserver = null;
|
||||
window.__pcKeywords = [];
|
||||
window.__pcHits = {};
|
||||
window.__pcSnapshot = '';
|
||||
return true;
|
||||
})();
|
||||
"""
|
||||
try:
|
||||
self._run_page_check_js(session, cleanup_js)
|
||||
self._record_page_check_js_success(session)
|
||||
setattr(session, "_pc_observer_empty_cleanup_done", True)
|
||||
except Exception as e:
|
||||
if self._is_session_closed(session):
|
||||
return
|
||||
self._record_page_check_js_failure(session, e, "observer_cleanup")
|
||||
self._mark_session_closed_if_disconnected(session, e, "page_check_observer_cleanup")
|
||||
logger.debug(f"[CMD] 页面检查观察器注入失败: {e}")
|
||||
@@ -0,0 +1,962 @@
|
||||
import copy
|
||||
import json
|
||||
import re
|
||||
import time
|
||||
import uuid
|
||||
from typing import Any, Dict, List, Optional, TYPE_CHECKING
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
from app.core.config import get_logger
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from app.core.tab_pool import TabSession
|
||||
|
||||
|
||||
logger = get_logger("CMD_ENG")
|
||||
|
||||
|
||||
class CommandEngineResultsMixin:
|
||||
MAX_EVENT_QUEUE_ITEMS = 50
|
||||
MAX_COMMAND_RESULT_EVENTS = 50
|
||||
|
||||
@staticmethod
|
||||
def _normalize_domain_host(value: Any) -> str:
|
||||
text = str(value or "").strip().lower()
|
||||
if not text:
|
||||
return ""
|
||||
try:
|
||||
if "://" in text:
|
||||
parsed = urlsplit(text)
|
||||
text = parsed.hostname or text
|
||||
else:
|
||||
text = text.split("/", 1)[0]
|
||||
except Exception:
|
||||
pass
|
||||
if text.count(":") == 1:
|
||||
host, port = text.rsplit(":", 1)
|
||||
if port.isdigit():
|
||||
text = host
|
||||
return text.strip().strip(".")
|
||||
|
||||
def _domain_matches(self, target_domain: Any, actual_domain: Any) -> bool:
|
||||
target = self._normalize_domain_host(target_domain)
|
||||
actual = self._normalize_domain_host(actual_domain)
|
||||
if not target or not actual:
|
||||
return False
|
||||
return actual == target or actual.endswith(f".{target}")
|
||||
|
||||
def _stringify_result(self, value: Any) -> str:
|
||||
if value is None:
|
||||
return ""
|
||||
if isinstance(value, str):
|
||||
return value
|
||||
if isinstance(value, (dict, list, tuple)):
|
||||
try:
|
||||
return json.dumps(value, ensure_ascii=False, sort_keys=True)
|
||||
except Exception:
|
||||
return str(value)
|
||||
return str(value)
|
||||
|
||||
def _match_value_rule(self, actual: str, expected: str, rule: str) -> bool:
|
||||
normalized_rule = self._normalize_match_rule(rule)
|
||||
if normalized_rule == "contains":
|
||||
if not expected:
|
||||
return False
|
||||
return expected in actual
|
||||
if normalized_rule == "not_equals":
|
||||
return actual != expected
|
||||
return actual == expected
|
||||
|
||||
def _get_command_result(self, command_id: str, tab_id: str) -> Optional[Dict[str, Any]]:
|
||||
with self._lock:
|
||||
return copy.deepcopy(self._command_results.get((command_id, tab_id)))
|
||||
|
||||
def _match_command_result_trigger(
|
||||
self,
|
||||
command: Dict[str, Any],
|
||||
session: 'TabSession',
|
||||
consume: bool = False,
|
||||
) -> bool:
|
||||
trigger = command.get("trigger", {})
|
||||
source_id = str(trigger.get("command_id", "")).strip()
|
||||
if not source_id:
|
||||
return False
|
||||
|
||||
result_entry = self._get_command_result(source_id, session.id)
|
||||
if not result_entry:
|
||||
return False
|
||||
|
||||
action_ref = str(trigger.get("action_ref", "")).strip()
|
||||
actual = result_entry.get("result", "")
|
||||
if action_ref:
|
||||
actual = result_entry.get("step_results", {}).get(action_ref, actual)
|
||||
|
||||
expected = self._stringify_result(trigger.get("expected_value", ""))
|
||||
rule = trigger.get("match_rule", "equals")
|
||||
if not self._match_value_rule(self._stringify_result(actual), expected, rule):
|
||||
return False
|
||||
|
||||
if consume:
|
||||
state, _ = self._ensure_trigger_state(command["id"], session)
|
||||
token = str(result_entry.get("token", ""))
|
||||
if state.get("result_token") == token:
|
||||
return False
|
||||
with self._lock:
|
||||
state["result_token"] = token
|
||||
|
||||
return True
|
||||
|
||||
def _prepare_command_result_trigger_dispatch(
|
||||
self,
|
||||
command: Dict[str, Any],
|
||||
session: 'TabSession',
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
trigger = command.get("trigger", {})
|
||||
source_id = str(trigger.get("command_id", "")).strip()
|
||||
if not source_id:
|
||||
return None
|
||||
|
||||
result_entry = self._get_command_result(source_id, session.id)
|
||||
if not result_entry:
|
||||
return None
|
||||
|
||||
action_ref = str(trigger.get("action_ref", "")).strip()
|
||||
actual = result_entry.get("result", "")
|
||||
if action_ref:
|
||||
actual = result_entry.get("step_results", {}).get(action_ref, actual)
|
||||
|
||||
expected = self._stringify_result(trigger.get("expected_value", ""))
|
||||
rule = trigger.get("match_rule", "equals")
|
||||
if not self._match_value_rule(self._stringify_result(actual), expected, rule):
|
||||
return None
|
||||
|
||||
token = str(result_entry.get("token", ""))
|
||||
state, _ = self._ensure_trigger_state(command["id"], session)
|
||||
with self._lock:
|
||||
if state.get("result_token") == token:
|
||||
return None
|
||||
state["result_token"] = token
|
||||
|
||||
return {
|
||||
"rollback": {"kind": "result_token", "token": token},
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _extract_command_check_actual(
|
||||
trigger: Dict[str, Any],
|
||||
execution_result: Dict[str, Any],
|
||||
) -> str:
|
||||
action_ref = str(trigger.get("action_ref", "")).strip()
|
||||
actual: Any = execution_result.get("result", "")
|
||||
if action_ref:
|
||||
for step in execution_result.get("steps", []) or []:
|
||||
if str(step.get("action_ref", "")).strip() == action_ref:
|
||||
actual = step.get("result", "")
|
||||
break
|
||||
return actual
|
||||
|
||||
def _run_command_check_source(
|
||||
self,
|
||||
command: Dict[str, Any],
|
||||
source_command: Dict[str, Any],
|
||||
session: 'TabSession',
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
source_id = str((source_command or {}).get("id", "")).strip()
|
||||
session_id = str(getattr(session, "id", "") or "").strip()
|
||||
if not source_id or not session_id:
|
||||
return None
|
||||
|
||||
exec_key = (source_id, session_id)
|
||||
with self._lock:
|
||||
if exec_key in self._executing:
|
||||
return None
|
||||
self._executing.add(exec_key)
|
||||
|
||||
with self._command_logging_context(source_command):
|
||||
try:
|
||||
return self._execute_command(
|
||||
source_command,
|
||||
session,
|
||||
chain=[str((command or {}).get("id", "")).strip()],
|
||||
record_result=False,
|
||||
emit_followups=False,
|
||||
update_trigger_stats=False,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
f"[CMD] 命令检查执行失败: {source_command.get('name', source_id)} "
|
||||
f"(标签页={session_id}, 错误={e})"
|
||||
)
|
||||
return {
|
||||
"mode": str(source_command.get("mode", "") or ""),
|
||||
"result": f"command_check_failed: {e}",
|
||||
"steps": [],
|
||||
"error": str(e),
|
||||
}
|
||||
finally:
|
||||
with self._lock:
|
||||
self._executing.discard(exec_key)
|
||||
|
||||
def _prepare_command_check_dispatch(
|
||||
self,
|
||||
command: Dict[str, Any],
|
||||
session: 'TabSession',
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
trigger = command.get("trigger", {}) or {}
|
||||
source_id = str(trigger.get("command_id", "")).strip()
|
||||
target_id = str(command.get("id", "")).strip()
|
||||
if not source_id or not target_id or source_id == target_id:
|
||||
return None
|
||||
|
||||
source_command = self.get_command(source_id)
|
||||
if not source_command:
|
||||
return None
|
||||
|
||||
execution_result = self._run_command_check_source(command, source_command, session)
|
||||
if not execution_result:
|
||||
return None
|
||||
|
||||
actual_raw = self._extract_command_check_actual(trigger, execution_result)
|
||||
actual = self._stringify_result(actual_raw)
|
||||
expected = self._stringify_result(trigger.get("expected_value", ""))
|
||||
rule = trigger.get("match_rule", "equals")
|
||||
matched = self._match_value_rule(actual, expected, rule)
|
||||
|
||||
state, _ = self._ensure_trigger_state(command["id"], session)
|
||||
signature = f"{source_id}|{str(trigger.get('action_ref', '')).strip()}|{actual}"
|
||||
with self._lock:
|
||||
if matched:
|
||||
if str(state.get("command_check_sig", "") or "") == signature:
|
||||
return None
|
||||
state["command_check_sig"] = signature
|
||||
state["command_check_actual"] = actual[:500]
|
||||
else:
|
||||
state["command_check_sig"] = ""
|
||||
state["command_check_actual"] = actual[:500]
|
||||
return None
|
||||
|
||||
return {
|
||||
"interrupt_context": {
|
||||
"command_check": {
|
||||
"source_command_id": source_id,
|
||||
"source_command_name": str(source_command.get("name", "") or source_id),
|
||||
"actual_result": actual[:500],
|
||||
"expected_value": expected[:200],
|
||||
"match_rule": str(rule or "equals"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def _record_command_result(
|
||||
self,
|
||||
command: Dict[str, Any],
|
||||
session: 'TabSession',
|
||||
execution_result: Dict[str, Any],
|
||||
):
|
||||
if not command.get("id"):
|
||||
return
|
||||
|
||||
token = f"{int(time.time() * 1000)}_{uuid.uuid4().hex[:6]}"
|
||||
step_results = {}
|
||||
for step in execution_result.get("steps", []) or []:
|
||||
ref = str(step.get("action_ref", "")).strip()
|
||||
if not ref:
|
||||
continue
|
||||
step_results[ref] = self._stringify_result(step.get("result", ""))
|
||||
|
||||
entry = {
|
||||
"token": token,
|
||||
"timestamp": time.time(),
|
||||
"result": self._stringify_result(execution_result.get("result", "")),
|
||||
"step_results": step_results,
|
||||
"mode": execution_result.get("mode", ""),
|
||||
}
|
||||
logger.debug(f"[CMD] 执行结果已记录: {command.get('name')} (标签页={session.id})")
|
||||
event = self._build_command_result_event(command, execution_result, entry, session)
|
||||
with self._lock:
|
||||
self._command_results[(command["id"], session.id)] = entry
|
||||
self._append_bounded_event(self._command_result_events, session.id, event)
|
||||
|
||||
def _build_command_result_event(
|
||||
self,
|
||||
command: Dict[str, Any],
|
||||
execution_result: Dict[str, Any],
|
||||
entry: Dict[str, Any],
|
||||
session: 'TabSession',
|
||||
) -> Dict[str, Any]:
|
||||
summary, informative = self._summarize_command_result(command, execution_result, entry)
|
||||
return {
|
||||
"token": entry.get("token", ""),
|
||||
"timestamp": float(entry.get("timestamp", time.time()) or time.time()),
|
||||
"source_command_id": str(command.get("id", "")).strip(),
|
||||
"source_command_name": str(command.get("name", "")).strip(),
|
||||
"source_group_name": self._normalize_group_name(command.get("group_name")),
|
||||
"result": str(entry.get("result", "") or ""),
|
||||
"summary": summary,
|
||||
"informative": bool(informative),
|
||||
"mode": str(entry.get("mode", "") or ""),
|
||||
"task_id": str(getattr(session, "current_task_id", "") or ""),
|
||||
}
|
||||
|
||||
def _summarize_command_result(
|
||||
self,
|
||||
command: Dict[str, Any],
|
||||
execution_result: Dict[str, Any],
|
||||
entry: Dict[str, Any],
|
||||
) -> tuple[str, bool]:
|
||||
error_text = str(execution_result.get("error", "") or "").strip()
|
||||
if error_text:
|
||||
return (error_text, True)
|
||||
|
||||
result_text = str(entry.get("result", "") or "").strip()
|
||||
if result_text:
|
||||
generic_prefixes = (
|
||||
"waited:",
|
||||
"page_refreshed",
|
||||
"cookies_cleared",
|
||||
"new_chat_clicked",
|
||||
"element_clicked:",
|
||||
"element_stealth_clicked:",
|
||||
"coordinates_clicked:",
|
||||
"navigated:",
|
||||
"preset_switched:",
|
||||
)
|
||||
informative_values = {"VERIFY", "BLACK", "READY", "UNKNOWN"}
|
||||
if result_text in informative_values:
|
||||
return (result_text, True)
|
||||
lowered = result_text.lower()
|
||||
if "failed" in lowered or "error" in lowered or "timeout" in lowered:
|
||||
return (result_text, True)
|
||||
if not lowered.startswith(generic_prefixes):
|
||||
return (result_text, True)
|
||||
|
||||
for step in execution_result.get("steps", []) or []:
|
||||
if not bool(step.get("ok", True)):
|
||||
text = self._stringify_result(step.get("result", "")).strip()
|
||||
return (text or "步骤执行失败", True)
|
||||
|
||||
cmd_name = str(command.get("name", "")).strip() or "命令"
|
||||
return (f"{cmd_name} 执行完成", False)
|
||||
|
||||
def _get_latest_command_result_event(self, tab_id: str) -> Optional[Dict[str, Any]]:
|
||||
with self._lock:
|
||||
items = self._command_result_events.get(tab_id) or []
|
||||
if not items:
|
||||
return None
|
||||
return copy.deepcopy(items[-1])
|
||||
|
||||
def _append_bounded_event(
|
||||
self,
|
||||
bucket: Dict[str, List[Dict[str, Any]]],
|
||||
tab_id: str,
|
||||
event: Dict[str, Any],
|
||||
*,
|
||||
max_events: Optional[int] = None,
|
||||
) -> None:
|
||||
queue = bucket.setdefault(tab_id, [])
|
||||
queue.append(event)
|
||||
limit = max(1, int(max_events or self.MAX_EVENT_QUEUE_ITEMS))
|
||||
overflow = len(queue) - limit
|
||||
if overflow > 0:
|
||||
del queue[:overflow]
|
||||
|
||||
def _get_command_result_events(self, tab_id: str) -> List[Dict[str, Any]]:
|
||||
with self._lock:
|
||||
return copy.deepcopy(list(self._command_result_events.get(tab_id) or []))
|
||||
|
||||
def emit_external_command_result_event(
|
||||
self,
|
||||
session: 'TabSession',
|
||||
*,
|
||||
source_command_id: str,
|
||||
source_command_name: str,
|
||||
summary: Any,
|
||||
result: Any = "",
|
||||
informative: bool = True,
|
||||
mode: str = "external",
|
||||
group_name: Any = "",
|
||||
trigger_commands: bool = True,
|
||||
extra_fields: Optional[Dict[str, Any]] = None,
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
if session is None:
|
||||
return None
|
||||
|
||||
normalized_command_id = str(source_command_id or "").strip()
|
||||
normalized_name = str(source_command_name or "").strip()
|
||||
normalized_summary = str(summary or "").strip()
|
||||
if not normalized_command_id or not normalized_name or not normalized_summary:
|
||||
return None
|
||||
|
||||
event = {
|
||||
"token": f"{int(time.time() * 1000)}_{uuid.uuid4().hex[:6]}",
|
||||
"timestamp": time.time(),
|
||||
"source_command_id": normalized_command_id,
|
||||
"source_command_name": normalized_name,
|
||||
"source_group_name": self._normalize_group_name(group_name),
|
||||
"result": self._stringify_result(result),
|
||||
"summary": normalized_summary,
|
||||
"informative": bool(informative),
|
||||
"mode": str(mode or "external").strip() or "external",
|
||||
"task_id": str(getattr(session, "current_task_id", "") or ""),
|
||||
}
|
||||
if isinstance(extra_fields, dict):
|
||||
for key, value in extra_fields.items():
|
||||
normalized_key = str(key or "").strip()
|
||||
if not normalized_key or normalized_key in event:
|
||||
continue
|
||||
event[normalized_key] = copy.deepcopy(value)
|
||||
|
||||
with self._lock:
|
||||
self._append_bounded_event(
|
||||
self._command_result_events,
|
||||
session.id,
|
||||
copy.deepcopy(event),
|
||||
)
|
||||
|
||||
logger.info(
|
||||
f"[CMD] 外部结果事件已记录: {normalized_name} "
|
||||
f"(标签页={session.id}, 摘要={normalized_summary})"
|
||||
)
|
||||
|
||||
if trigger_commands:
|
||||
pseudo_command = {
|
||||
"id": normalized_command_id,
|
||||
"name": normalized_name,
|
||||
"group_name": self._normalize_group_name(group_name),
|
||||
}
|
||||
self._trigger_result_event_commands(pseudo_command, session)
|
||||
|
||||
return copy.deepcopy(event)
|
||||
|
||||
def _parse_result_event_source_ids(self, trigger: Dict[str, Any]) -> tuple[bool, set[str]]:
|
||||
listen_all = bool(trigger.get("listen_all_commands", False))
|
||||
raw = trigger.get("command_ids", [])
|
||||
values = raw if isinstance(raw, list) else str(raw or "").split(",")
|
||||
normalized = {str(item or "").strip() for item in values if str(item or "").strip()}
|
||||
if "*" in normalized:
|
||||
listen_all = True
|
||||
normalized.discard("*")
|
||||
return listen_all, normalized
|
||||
|
||||
@staticmethod
|
||||
def _get_consumed_history(state: Dict[str, Any], field: str) -> List[str]:
|
||||
history_field = f"{field}s"
|
||||
raw_history = state.get(history_field, [])
|
||||
if isinstance(raw_history, list):
|
||||
history = [str(item or "").strip() for item in raw_history if str(item or "").strip()]
|
||||
else:
|
||||
history = []
|
||||
latest = str(state.get(field, "") or "").strip()
|
||||
if latest and latest not in history:
|
||||
history.append(latest)
|
||||
return history[-50:]
|
||||
|
||||
def _result_event_matches_trigger(self, trigger: Dict[str, Any], event: Dict[str, Any]) -> bool:
|
||||
listen_all = bool(trigger.get("listen_all_commands", False))
|
||||
raw = trigger.get("command_ids", [])
|
||||
values = raw if isinstance(raw, list) else str(raw or "").split(",")
|
||||
command_ids = {str(item or "").strip() for item in values if str(item or "").strip()}
|
||||
if "*" in command_ids:
|
||||
listen_all = True
|
||||
command_ids.discard("*")
|
||||
|
||||
source_id = str(event.get("source_command_id", "")).strip()
|
||||
if not listen_all and command_ids and source_id not in command_ids:
|
||||
return False
|
||||
if not listen_all and not command_ids:
|
||||
return False
|
||||
if bool(trigger.get("informative_only", True)) and not bool(event.get("informative", False)):
|
||||
return False
|
||||
|
||||
expected = self._stringify_result(
|
||||
trigger.get("expected_value")
|
||||
if trigger.get("expected_value") not in (None, "")
|
||||
else trigger.get("value", "")
|
||||
)
|
||||
if expected:
|
||||
actual_parts = []
|
||||
for field in (
|
||||
"summary",
|
||||
"result",
|
||||
"default_response",
|
||||
"response_a",
|
||||
"response_b",
|
||||
"source_command_name",
|
||||
"source_command_id",
|
||||
):
|
||||
value = self._stringify_result(event.get(field, ""))
|
||||
if value:
|
||||
actual_parts.append(value)
|
||||
actual = "\n".join(actual_parts)
|
||||
if not self._match_value_rule(actual, expected, trigger.get("match_rule", "contains")):
|
||||
return False
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
def _normalize_result_event_dedupe_fields(trigger: Dict[str, Any]) -> List[str]:
|
||||
raw = trigger.get("dedupe_key_fields", trigger.get("result_event_dedupe_fields", []))
|
||||
if isinstance(raw, str):
|
||||
values = str(raw or "").replace(",", ",").split(",")
|
||||
elif isinstance(raw, (list, tuple, set)):
|
||||
values = list(raw)
|
||||
else:
|
||||
values = []
|
||||
fields = [str(item or "").strip() for item in values if str(item or "").strip()]
|
||||
return fields
|
||||
|
||||
def _build_result_event_dedupe_signature(
|
||||
self,
|
||||
trigger: Dict[str, Any],
|
||||
event: Dict[str, Any],
|
||||
) -> str:
|
||||
fields = self._normalize_result_event_dedupe_fields(trigger)
|
||||
if not fields:
|
||||
return ""
|
||||
|
||||
parts: List[str] = []
|
||||
for field in fields:
|
||||
value = event.get(field, "")
|
||||
parts.append(f"{field}={str(value or '').strip()}")
|
||||
return "|".join(parts)
|
||||
|
||||
def _remember_consumed_token(
|
||||
self,
|
||||
state: Dict[str, Any],
|
||||
field: str,
|
||||
token: str,
|
||||
) -> None:
|
||||
token = str(token or "").strip()
|
||||
if not token:
|
||||
return
|
||||
history = self._get_consumed_history(state, field)
|
||||
if token in history:
|
||||
history.remove(token)
|
||||
history.append(token)
|
||||
state[field] = token
|
||||
overflow = len(history) - 50
|
||||
if overflow > 0:
|
||||
del history[:overflow]
|
||||
state[f"{field}s"] = history
|
||||
|
||||
def _get_next_command_result_event(
|
||||
self,
|
||||
command: Dict[str, Any],
|
||||
session: 'TabSession',
|
||||
consume: bool = False,
|
||||
copy_event: bool = True,
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
trigger = command.get("trigger", {}) or {}
|
||||
state, _ = self._ensure_trigger_state(command["id"], session)
|
||||
with self._lock:
|
||||
items = list(self._command_result_events.get(session.id) or [])
|
||||
consumed = set(self._get_consumed_history(state, "result_event_token"))
|
||||
latched_signature = str(state.get("result_event_latch_sig", "") or "").strip()
|
||||
for event in items:
|
||||
token = str(event.get("token", "")).strip()
|
||||
if not token or token in consumed:
|
||||
continue
|
||||
if not self._result_event_matches_trigger(trigger, event):
|
||||
continue
|
||||
signature = self._build_result_event_dedupe_signature(trigger, event)
|
||||
if signature and signature == latched_signature:
|
||||
if consume:
|
||||
self._remember_consumed_token(state, "result_event_token", token)
|
||||
continue
|
||||
if consume:
|
||||
self._remember_consumed_token(state, "result_event_token", token)
|
||||
if signature:
|
||||
state["result_event_latch_sig"] = signature
|
||||
return copy.deepcopy(event) if copy_event else event
|
||||
return None
|
||||
|
||||
def _match_command_result_event_trigger(
|
||||
self,
|
||||
command: Dict[str, Any],
|
||||
session: 'TabSession',
|
||||
consume: bool = False,
|
||||
) -> bool:
|
||||
return self._get_next_command_result_event(
|
||||
command,
|
||||
session,
|
||||
consume=consume,
|
||||
copy_event=False,
|
||||
) is not None
|
||||
|
||||
def _prepare_command_result_event_dispatch(
|
||||
self,
|
||||
command: Dict[str, Any],
|
||||
session: 'TabSession',
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
event = self._get_next_command_result_event(command, session, consume=True)
|
||||
if not event:
|
||||
return None
|
||||
token = str(event.get("token", "")).strip()
|
||||
|
||||
return {
|
||||
"rollback": {"kind": "result_event_token", "token": token},
|
||||
"interrupt_context": {"command_result_event": event},
|
||||
}
|
||||
|
||||
def _normalize_status_codes(self, raw_codes: Any) -> set[int]:
|
||||
if isinstance(raw_codes, (list, tuple, set)):
|
||||
values = raw_codes
|
||||
else:
|
||||
values = str(raw_codes or "").replace(",", ",").split(",")
|
||||
|
||||
result: set[int] = set()
|
||||
for item in values:
|
||||
text = str(item).strip()
|
||||
if not text:
|
||||
continue
|
||||
try:
|
||||
result.add(int(text))
|
||||
except Exception:
|
||||
continue
|
||||
return result
|
||||
|
||||
def _pattern_to_listen_hint(self, pattern: str, mode: str) -> str:
|
||||
raw = str(pattern or "").strip()
|
||||
if not raw:
|
||||
return ""
|
||||
|
||||
if str(mode or "").strip().lower() == "regex":
|
||||
if "|" in raw:
|
||||
return "http"
|
||||
tokens = re.split(r"[\^\$\.\*\+\?\(\)\[\]\{\}\|\\]+", raw)
|
||||
tokens = [t.strip() for t in tokens if t and len(t.strip()) >= 3]
|
||||
if not tokens:
|
||||
wildcard_fallback = raw.replace(".*", "").replace("\\/", "/")
|
||||
wildcard_fallback = wildcard_fallback.strip("* ").strip()
|
||||
return wildcard_fallback[:120]
|
||||
tokens.sort(key=len, reverse=True)
|
||||
return tokens[0][:120]
|
||||
|
||||
simplified = raw.strip("* ").strip()
|
||||
return simplified[:120]
|
||||
|
||||
# H12:网络事件 URL 正则的求值预算(与工作流 matches 运算符一致:25ms)
|
||||
URL_REGEX_MAX_PATTERN = 512
|
||||
URL_REGEX_MAX_INPUT = 8192
|
||||
URL_REGEX_TIMEOUT_SEC = 0.025
|
||||
|
||||
@classmethod
|
||||
def _bounded_regex_search(cls, pattern: str, url: str) -> Optional[bool]:
|
||||
"""带超时的正则匹配:返回 True/False;模式无效返回 None;超时视为不匹配。"""
|
||||
import regex as _regex
|
||||
|
||||
try:
|
||||
return bool(_regex.search(
|
||||
pattern,
|
||||
url[: cls.URL_REGEX_MAX_INPUT],
|
||||
flags=_regex.IGNORECASE,
|
||||
timeout=cls.URL_REGEX_TIMEOUT_SEC,
|
||||
))
|
||||
except TimeoutError:
|
||||
logger.warning(f"[CMD] URL 正则超过 25ms 求值预算,按不匹配处理: {pattern[:120]}")
|
||||
return False
|
||||
except _regex.error:
|
||||
return None
|
||||
|
||||
def _matches_url_rule(self, url: str, pattern: str, mode: str) -> bool:
|
||||
if not pattern:
|
||||
return True
|
||||
if mode == "regex":
|
||||
if len(pattern) > self.URL_REGEX_MAX_PATTERN:
|
||||
logger.warning(f"[CMD] URL 正则超过 {self.URL_REGEX_MAX_PATTERN} 字符,按关键词匹配")
|
||||
return pattern.lower() in url.lower()
|
||||
matched = self._bounded_regex_search(pattern, url)
|
||||
if matched is not None:
|
||||
return matched
|
||||
logger.warning(f"[CMD] 无效正则,回退通配/关键词匹配: {pattern}")
|
||||
wildcard = str(pattern).replace(".", r"\.").replace("*", ".*")
|
||||
matched = self._bounded_regex_search(wildcard, url)
|
||||
if matched is not None:
|
||||
return matched
|
||||
simplified = str(pattern).replace("*", "").strip()
|
||||
if simplified:
|
||||
return simplified.lower() in url.lower()
|
||||
return pattern.lower() in url.lower()
|
||||
|
||||
def _matches_network_trigger(self, trigger: Dict[str, Any], event: Dict[str, Any]) -> bool:
|
||||
url = str(event.get("url", "") or "")
|
||||
status = event.get("status")
|
||||
pattern = str(trigger.get("url_pattern") or trigger.get("value") or "").strip()
|
||||
match_mode = str(trigger.get("match_mode", "keyword")).strip().lower()
|
||||
codes = self._normalize_status_codes(trigger.get("status_codes", ""))
|
||||
|
||||
if not self._matches_url_rule(url, pattern, match_mode):
|
||||
return False
|
||||
if codes and int(status or 0) not in codes:
|
||||
return False
|
||||
return True
|
||||
|
||||
def _build_network_signature(self, event: Dict[str, Any]) -> str:
|
||||
event_id = str(event.get("event_id", "")).strip()
|
||||
if event_id:
|
||||
return event_id
|
||||
ts = str(event.get("timestamp", ""))
|
||||
return f"{ts}:{event.get('status')}:{event.get('url', '')}"
|
||||
|
||||
def _consume_network_signature(self, command_id: str, session: 'TabSession', signature: str) -> bool:
|
||||
if not command_id:
|
||||
return False
|
||||
state, _ = self._ensure_trigger_state(command_id, session)
|
||||
with self._lock:
|
||||
if signature in self._get_consumed_history(state, "net_sig"):
|
||||
return False
|
||||
self._remember_consumed_token(state, "net_sig", signature)
|
||||
return True
|
||||
|
||||
def _prepare_network_trigger_dispatch(
|
||||
self,
|
||||
command: Dict[str, Any],
|
||||
session: 'TabSession',
|
||||
event: Optional[Dict[str, Any]] = None,
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
trigger = command.get("trigger", {}) or {}
|
||||
if event is None:
|
||||
for candidate in self._snapshot_network_events(session.id):
|
||||
if not self._matches_network_trigger(trigger, candidate):
|
||||
continue
|
||||
signature = self._build_network_signature(candidate)
|
||||
if not self._consume_network_signature(command.get("id"), session, signature):
|
||||
continue
|
||||
event = candidate
|
||||
break
|
||||
else:
|
||||
if not self._matches_network_trigger(trigger, event):
|
||||
return None
|
||||
signature = self._build_network_signature(event)
|
||||
if not self._consume_network_signature(command.get("id"), session, signature):
|
||||
return None
|
||||
if event is None:
|
||||
return None
|
||||
signature = self._build_network_signature(event)
|
||||
|
||||
return {
|
||||
"rollback": {"kind": "net_sig", "token": signature},
|
||||
"interrupt_context": {"network_event": copy.deepcopy(event)},
|
||||
}
|
||||
|
||||
def _get_latest_network_event(self, tab_id: str) -> Optional[Dict[str, Any]]:
|
||||
with self._lock:
|
||||
items = self._network_events.get(tab_id) or []
|
||||
if not items:
|
||||
return None
|
||||
return copy.deepcopy(items[-1])
|
||||
|
||||
def _snapshot_network_events(self, tab_id: str) -> List[Dict[str, Any]]:
|
||||
with self._lock:
|
||||
return list(self._network_events.get(tab_id) or [])
|
||||
|
||||
def _get_network_events(self, tab_id: str) -> List[Dict[str, Any]]:
|
||||
with self._lock:
|
||||
return copy.deepcopy(list(self._network_events.get(tab_id) or []))
|
||||
|
||||
def _matches_scope(self, command: Dict, session: 'TabSession') -> bool:
|
||||
trigger = command.get("trigger", {})
|
||||
scope = trigger.get("scope", "all")
|
||||
|
||||
if scope == "domain":
|
||||
target_domain = str(trigger.get("domain", "") or "").strip().lower()
|
||||
session_domain = self._get_session_domain(session)
|
||||
if target_domain and session_domain:
|
||||
return self._domain_matches(target_domain, session_domain)
|
||||
return not target_domain
|
||||
|
||||
if scope == "tab":
|
||||
target_index = trigger.get("tab_index")
|
||||
return target_index is None or session.persistent_index == target_index
|
||||
|
||||
return True
|
||||
|
||||
def _rollback_trigger_consumption(
|
||||
self,
|
||||
command: Dict[str, Any],
|
||||
session: 'TabSession',
|
||||
rollback: Optional[Dict[str, Any]],
|
||||
):
|
||||
if not rollback:
|
||||
return
|
||||
state, _ = self._ensure_trigger_state(command["id"], session)
|
||||
field_map = {
|
||||
"result_token": "result_token",
|
||||
"result_event_token": "result_event_token",
|
||||
"net_sig": "net_sig",
|
||||
}
|
||||
field = field_map.get(str(rollback.get("kind", "")).strip())
|
||||
token = str(rollback.get("token", "") or "")
|
||||
if not field or not token:
|
||||
return
|
||||
with self._lock:
|
||||
history = [item for item in self._get_consumed_history(state, field) if item != token]
|
||||
state[f"{field}s"] = history
|
||||
state[field] = history[-1] if history else ""
|
||||
|
||||
def _trigger_chained_commands(
|
||||
self,
|
||||
source_command: Dict,
|
||||
session: 'TabSession',
|
||||
chain: Optional[List[str]] = None,
|
||||
interrupt_context: Optional[Dict[str, Any]] = None,
|
||||
):
|
||||
source_id = source_command.get("id")
|
||||
if not source_id:
|
||||
return
|
||||
|
||||
chain = list(chain or [])
|
||||
next_chain = chain + [source_id]
|
||||
|
||||
try:
|
||||
commands = self._load_commands()
|
||||
except Exception as e:
|
||||
logger.debug(f"链式命令加载失败,跳过: {e}")
|
||||
return
|
||||
|
||||
for cmd in commands:
|
||||
if not cmd.get("enabled", True):
|
||||
continue
|
||||
|
||||
target_id = cmd.get("id")
|
||||
trigger = cmd.get("trigger", {})
|
||||
if trigger.get("type") != "command_triggered":
|
||||
continue
|
||||
if trigger.get("command_id") != source_id:
|
||||
continue
|
||||
if not target_id or target_id in next_chain:
|
||||
continue
|
||||
if not self._matches_scope(cmd, session):
|
||||
continue
|
||||
|
||||
exec_key = (target_id, session.id)
|
||||
with self._lock:
|
||||
if exec_key in self._executing:
|
||||
continue
|
||||
|
||||
logger.info(
|
||||
f"[CMD] 链式触发: {source_command.get('name')} -> {cmd.get('name')} "
|
||||
f"(标签页={session.id})"
|
||||
)
|
||||
if interrupt_context is not None:
|
||||
self._dispatch_interrupt_followup_command(
|
||||
cmd, session, next_chain, interrupt_context
|
||||
)
|
||||
else:
|
||||
self._execute_command_async(cmd, session, chain=next_chain)
|
||||
|
||||
def _trigger_result_match_commands(
|
||||
self,
|
||||
source_command: Dict[str, Any],
|
||||
session: 'TabSession',
|
||||
chain: Optional[List[str]] = None,
|
||||
interrupt_context: Optional[Dict[str, Any]] = None,
|
||||
):
|
||||
source_id = source_command.get("id")
|
||||
if not source_id:
|
||||
return
|
||||
|
||||
chain = list(chain or [])
|
||||
next_chain = chain + [source_id]
|
||||
|
||||
try:
|
||||
commands = self._load_commands()
|
||||
except Exception as e:
|
||||
logger.debug(f"条件分支命令加载失败,跳过: {e}")
|
||||
return
|
||||
|
||||
for cmd in commands:
|
||||
if not cmd.get("enabled", True):
|
||||
continue
|
||||
|
||||
target_id = cmd.get("id")
|
||||
trigger = cmd.get("trigger", {})
|
||||
if trigger.get("type") != "command_result_match":
|
||||
continue
|
||||
if str(trigger.get("command_id", "")).strip() != source_id:
|
||||
continue
|
||||
if not target_id or target_id in next_chain:
|
||||
continue
|
||||
if not self._matches_scope(cmd, session):
|
||||
continue
|
||||
dispatch = self._prepare_command_result_trigger_dispatch(cmd, session)
|
||||
if not dispatch:
|
||||
continue
|
||||
|
||||
logger.info(
|
||||
f"[CMD] 条件分支触发: {source_command.get('name')} -> {cmd.get('name')} "
|
||||
f"(标签页={session.id})"
|
||||
)
|
||||
combined_context = copy.deepcopy(interrupt_context) if interrupt_context is not None else None
|
||||
extra_context = dispatch.get("interrupt_context")
|
||||
if extra_context:
|
||||
combined_context = combined_context or {}
|
||||
combined_context.update(copy.deepcopy(extra_context))
|
||||
if interrupt_context is not None:
|
||||
self._dispatch_interrupt_followup_command(
|
||||
cmd, session, next_chain, combined_context or interrupt_context
|
||||
)
|
||||
else:
|
||||
self._execute_command_async(
|
||||
cmd,
|
||||
session,
|
||||
chain=next_chain,
|
||||
interrupt_context=combined_context,
|
||||
trigger_rollback=dispatch.get("rollback"),
|
||||
)
|
||||
|
||||
def _trigger_result_event_commands(
|
||||
self,
|
||||
source_command: Dict[str, Any],
|
||||
session: 'TabSession',
|
||||
chain: Optional[List[str]] = None,
|
||||
interrupt_context: Optional[Dict[str, Any]] = None,
|
||||
):
|
||||
source_id = str(source_command.get("id", "")).strip()
|
||||
if not source_id:
|
||||
return
|
||||
|
||||
chain = list(chain or [])
|
||||
next_chain = chain + [source_id]
|
||||
|
||||
try:
|
||||
commands = self._load_commands()
|
||||
except Exception as e:
|
||||
logger.debug(f"结果事件命令加载失败,跳过: {e}")
|
||||
return
|
||||
|
||||
for cmd in commands:
|
||||
if not cmd.get("enabled", True):
|
||||
continue
|
||||
target_id = str(cmd.get("id", "")).strip()
|
||||
if not target_id or target_id in next_chain:
|
||||
continue
|
||||
trigger = cmd.get("trigger", {}) or {}
|
||||
if str(trigger.get("type", "")).strip() != "command_result_event":
|
||||
continue
|
||||
if not self._matches_scope(cmd, session):
|
||||
continue
|
||||
dispatch = self._prepare_command_result_event_dispatch(cmd, session)
|
||||
if not dispatch:
|
||||
continue
|
||||
|
||||
logger.info(
|
||||
f"[CMD] 结果事件触发: {source_command.get('name')} -> {cmd.get('name')} "
|
||||
f"(标签页={session.id})"
|
||||
)
|
||||
combined_context = copy.deepcopy(interrupt_context) if interrupt_context is not None else None
|
||||
extra_context = dispatch.get("interrupt_context")
|
||||
if extra_context:
|
||||
combined_context = combined_context or {}
|
||||
combined_context.update(copy.deepcopy(extra_context))
|
||||
if interrupt_context is not None:
|
||||
self._dispatch_interrupt_followup_command(
|
||||
cmd, session, next_chain, combined_context or interrupt_context
|
||||
)
|
||||
else:
|
||||
self._execute_command_async(
|
||||
cmd,
|
||||
session,
|
||||
chain=next_chain,
|
||||
interrupt_context=combined_context,
|
||||
trigger_rollback=dispatch.get("rollback"),
|
||||
)
|
||||
@@ -0,0 +1,581 @@
|
||||
import copy
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
from typing import Any, Dict, List, Optional, TYPE_CHECKING
|
||||
|
||||
from app.core.config import get_logger
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from app.core.tab_pool import TabSession
|
||||
|
||||
|
||||
logger = get_logger("CMD_ENG")
|
||||
|
||||
|
||||
class CommandEngineRuntimeMixin:
|
||||
def _workflow_queue_key(
|
||||
self,
|
||||
command: Dict[str, Any],
|
||||
trigger_rollback: Optional[Dict[str, Any]] = None,
|
||||
) -> str:
|
||||
cmd_id = str((command or {}).get("id", "")).strip()
|
||||
rollback = trigger_rollback or {}
|
||||
kind = str(rollback.get("kind", "") or "").strip()
|
||||
token = str(rollback.get("token", "") or "").strip()
|
||||
if cmd_id and kind and token:
|
||||
return f"{cmd_id}:{kind}:{token}"
|
||||
return cmd_id
|
||||
|
||||
def _dedupe_deferred_items(self, items: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
|
||||
deduped: List[Dict[str, Any]] = []
|
||||
seen: set[str] = set()
|
||||
for item in items or []:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
queue_key = str(item.get("queue_key", "") or "").strip()
|
||||
if not queue_key:
|
||||
queue_key = self._workflow_queue_key(
|
||||
item.get("command") or {},
|
||||
item.get("trigger_rollback") if isinstance(item.get("trigger_rollback"), dict) else None,
|
||||
)
|
||||
if queue_key:
|
||||
item = dict(item)
|
||||
item["queue_key"] = queue_key
|
||||
if queue_key and queue_key in seen:
|
||||
continue
|
||||
if queue_key:
|
||||
seen.add(queue_key)
|
||||
deduped.append(item)
|
||||
return deduped
|
||||
|
||||
def _has_active_workflow(self, session: 'TabSession') -> bool:
|
||||
runtime = self._get_active_workflow_runtime(session)
|
||||
return bool(runtime and runtime.get("active"))
|
||||
|
||||
def begin_workflow_runtime(
|
||||
self,
|
||||
session: 'TabSession',
|
||||
*,
|
||||
task_id: str = "",
|
||||
preset_name: str = "",
|
||||
priority: Optional[int] = None,
|
||||
) -> Dict[str, Any]:
|
||||
runtime = {
|
||||
"runtime_id": uuid.uuid4().hex,
|
||||
"task_id": str(task_id or "").strip(),
|
||||
"preset_name": str(preset_name or "").strip(),
|
||||
"parser_id": str(getattr(session, "_workflow_parser_id", "") or "").strip().lower(),
|
||||
"target_side": str(getattr(session, "_workflow_target_side", "") or "").strip().lower(),
|
||||
"priority": self._normalize_priority(priority, self._get_request_priority_baseline()),
|
||||
"current_step_index": -1,
|
||||
"current_step_action": "",
|
||||
"pending_interrupts": [],
|
||||
"pending_interrupt_ids": set(),
|
||||
"deferred_commands": [],
|
||||
"deferred_command_ids": set(),
|
||||
"active": True,
|
||||
"interrupting": False,
|
||||
}
|
||||
with self._lock:
|
||||
stack = list(getattr(session, "_workflow_runtime_stack", None) or [])
|
||||
stack.append(runtime)
|
||||
setattr(session, "_workflow_runtime_stack", stack)
|
||||
setattr(session, "_workflow_runtime_id", runtime["runtime_id"])
|
||||
setattr(session, "_workflow_target_side", runtime["target_side"])
|
||||
return runtime
|
||||
|
||||
def update_workflow_runtime_step(
|
||||
self,
|
||||
session: 'TabSession',
|
||||
step_index: int,
|
||||
step: Optional[Dict[str, Any]] = None,
|
||||
):
|
||||
runtime = self._get_active_workflow_runtime(session)
|
||||
if not runtime:
|
||||
return
|
||||
runtime["current_step_index"] = int(step_index)
|
||||
runtime["current_step_action"] = str((step or {}).get("action", "") or "")
|
||||
|
||||
def update_workflow_target_side(self, session: 'TabSession', side: str) -> bool:
|
||||
normalized = str(side or "").strip().lower()
|
||||
if normalized not in {"left", "right"}:
|
||||
return False
|
||||
runtime = self._get_active_workflow_runtime(session)
|
||||
if not runtime:
|
||||
return False
|
||||
runtime["target_side"] = normalized
|
||||
setattr(session, "_workflow_target_side", normalized)
|
||||
return True
|
||||
|
||||
def finish_workflow_runtime(
|
||||
self,
|
||||
session: 'TabSession',
|
||||
*,
|
||||
aborted: bool = False,
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
with self._lock:
|
||||
stack = list(getattr(session, "_workflow_runtime_stack", None) or [])
|
||||
if not stack:
|
||||
return None
|
||||
runtime = stack.pop()
|
||||
setattr(session, "_workflow_runtime_stack", stack)
|
||||
|
||||
pending_interrupts = list(runtime.get("pending_interrupts") or [])
|
||||
deferred_commands = list(runtime.get("deferred_commands") or [])
|
||||
stop_reason = str(getattr(session, "_workflow_stop_reason", "") or "").strip()
|
||||
preserve_pending_interrupts = bool(pending_interrupts) and (
|
||||
not aborted or stop_reason in {"command_interrupt", "command_interrupt_abort"}
|
||||
)
|
||||
if preserve_pending_interrupts:
|
||||
deferred_ids = {
|
||||
str(item.get("queue_key", "") or "").strip()
|
||||
or self._workflow_queue_key(
|
||||
item.get("command") or {},
|
||||
item.get("trigger_rollback") if isinstance(item.get("trigger_rollback"), dict) else None,
|
||||
)
|
||||
for item in deferred_commands
|
||||
if isinstance(item, dict)
|
||||
}
|
||||
for item in pending_interrupts:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
command = copy.deepcopy(item.get("command") or {})
|
||||
queue_key = str(item.get("queue_key", "") or "").strip() or self._workflow_queue_key(
|
||||
command,
|
||||
item.get("trigger_rollback") if isinstance(item.get("trigger_rollback"), dict) else None,
|
||||
)
|
||||
if queue_key and queue_key in deferred_ids:
|
||||
continue
|
||||
deferred_commands.append({
|
||||
"command": command,
|
||||
"chain": list(item.get("chain") or []),
|
||||
"interrupt_context": copy.deepcopy(item.get("interrupt_context") or {}),
|
||||
"trigger_rollback": copy.deepcopy(item.get("trigger_rollback") or {}),
|
||||
"queue_key": queue_key,
|
||||
"queued_at": float(item.get("requested_at", time.time()) or time.time()),
|
||||
})
|
||||
if queue_key:
|
||||
deferred_ids.add(queue_key)
|
||||
|
||||
runtime["active"] = False
|
||||
runtime["interrupting"] = False
|
||||
runtime["pending_interrupts"] = []
|
||||
runtime["pending_interrupt_ids"] = set()
|
||||
|
||||
preserve_deferred = bool(deferred_commands) and (
|
||||
not aborted or stop_reason == "command_interrupt_abort"
|
||||
)
|
||||
if stack:
|
||||
parent = stack[-1]
|
||||
if preserve_deferred:
|
||||
parent_deferred = parent.setdefault("deferred_commands", [])
|
||||
parent_ids = parent.setdefault("deferred_command_ids", set())
|
||||
for item in deferred_commands:
|
||||
queue_key = str(item.get("queue_key", "") or "").strip() or self._workflow_queue_key(
|
||||
item.get("command") or {},
|
||||
item.get("trigger_rollback") if isinstance(item.get("trigger_rollback"), dict) else None,
|
||||
)
|
||||
if queue_key and queue_key in parent_ids:
|
||||
continue
|
||||
parent_deferred.append(item)
|
||||
if queue_key:
|
||||
parent_ids.add(queue_key)
|
||||
elif preserve_deferred:
|
||||
existing = list(getattr(session, "_pending_post_workflow_commands", None) or [])
|
||||
existing_ids = {
|
||||
str((item.get("command") or {}).get("id", "")).strip()
|
||||
for item in existing
|
||||
if isinstance(item, dict)
|
||||
}
|
||||
merged = self._dedupe_deferred_items(existing + deferred_commands)
|
||||
setattr(session, "_pending_post_workflow_commands", merged)
|
||||
else:
|
||||
setattr(session, "_pending_post_workflow_commands", [])
|
||||
|
||||
parent = stack[-1] if stack else None
|
||||
if parent:
|
||||
setattr(session, "_workflow_runtime_id", str(parent.get("runtime_id") or ""))
|
||||
setattr(session, "_workflow_parser_id", str(parent.get("parser_id") or ""))
|
||||
setattr(session, "_workflow_target_side", str(parent.get("target_side") or ""))
|
||||
else:
|
||||
setattr(session, "_workflow_runtime_id", "")
|
||||
setattr(session, "_workflow_parser_id", "")
|
||||
setattr(session, "_workflow_target_side", "")
|
||||
|
||||
return runtime
|
||||
|
||||
def flush_deferred_workflow_commands(self, session: 'TabSession'):
|
||||
if bool(getattr(self, "_shutdown_requested", False)):
|
||||
return
|
||||
with self._lock:
|
||||
items = list(getattr(session, "_pending_post_workflow_commands", None) or [])
|
||||
setattr(session, "_pending_post_workflow_commands", [])
|
||||
if not items:
|
||||
return
|
||||
|
||||
requeued = 0
|
||||
for item in items:
|
||||
command = copy.deepcopy((item or {}).get("command") or {})
|
||||
if not command or not command.get("enabled", True):
|
||||
continue
|
||||
dispatched = self._execute_command_async(
|
||||
command,
|
||||
session,
|
||||
chain=list((item or {}).get("chain") or []),
|
||||
interrupt_context=copy.deepcopy((item or {}).get("interrupt_context") or {}) or None,
|
||||
trigger_rollback=copy.deepcopy((item or {}).get("trigger_rollback") or {}) or None,
|
||||
)
|
||||
if dispatched:
|
||||
requeued += 1
|
||||
|
||||
logger.info(
|
||||
f"[CMD] 已将延后命令交回命令调度器执行 "
|
||||
f"(标签页={getattr(session, 'id', '')}, count={requeued})"
|
||||
)
|
||||
|
||||
def schedule_deferred_workflow_commands(
|
||||
self,
|
||||
session: 'TabSession',
|
||||
*,
|
||||
delay_sec: float = 0.25,
|
||||
):
|
||||
if bool(getattr(self, "_shutdown_requested", False)):
|
||||
return False
|
||||
delay = max(0.0, float(delay_sec or 0.0))
|
||||
|
||||
thread = threading.Thread(
|
||||
target=self.flush_deferred_workflow_commands,
|
||||
args=(session,),
|
||||
daemon=True,
|
||||
name=f"cmd-resume-delay-{getattr(session, 'id', 'tab')}",
|
||||
)
|
||||
if delay > 0:
|
||||
# Timer waits before dispatching, while command execution itself stays in
|
||||
# the command scheduler path instead of touching the tab from this helper.
|
||||
thread = threading.Timer(delay, self.flush_deferred_workflow_commands, args=(session,))
|
||||
thread.daemon = True
|
||||
thread.name = f"cmd-resume-delay-{getattr(session, 'id', 'tab')}"
|
||||
thread.start()
|
||||
return True
|
||||
|
||||
def _get_workflow_interrupt_policy(self, command: Dict[str, Any]) -> str:
|
||||
trigger = command.get("trigger", {}) or {}
|
||||
policy = str(
|
||||
trigger.get("interrupt_policy", command.get("interrupt_policy", "auto"))
|
||||
or "auto"
|
||||
).strip().lower()
|
||||
return policy if policy in {"auto", "resume", "abort"} else "auto"
|
||||
|
||||
def _build_workflow_interrupt_message(self, command: Dict[str, Any]) -> str:
|
||||
trigger = command.get("trigger", {}) or {}
|
||||
custom = str(
|
||||
trigger.get("interrupt_message", command.get("interrupt_message", ""))
|
||||
or ""
|
||||
).strip()
|
||||
if custom:
|
||||
return custom
|
||||
return f"触发 {command.get('name', '命令')},后续工作流已打断,请重试"
|
||||
|
||||
def _allow_command_during_workflow(self, command: Dict[str, Any]) -> bool:
|
||||
trigger = command.get("trigger", {}) or {}
|
||||
raw = trigger.get("allow_during_workflow", None)
|
||||
if raw is None:
|
||||
trigger_type = str(trigger.get("type", "")).strip().lower()
|
||||
if trigger_type != "page_check":
|
||||
return False
|
||||
policy = self._get_workflow_interrupt_policy(command)
|
||||
if policy in {"resume", "abort"}:
|
||||
return True
|
||||
return self._get_command_priority(command) > self._get_request_priority_baseline()
|
||||
return bool(raw)
|
||||
|
||||
def _queue_workflow_interrupt(
|
||||
self,
|
||||
command: Dict[str, Any],
|
||||
session: 'TabSession',
|
||||
chain: Optional[List[str]] = None,
|
||||
interrupt_context: Optional[Dict[str, Any]] = None,
|
||||
trigger_rollback: Optional[Dict[str, Any]] = None,
|
||||
) -> bool:
|
||||
runtime = self._get_active_workflow_runtime(session)
|
||||
if not runtime or not runtime.get("active"):
|
||||
return False
|
||||
if not self._allow_command_during_workflow(command):
|
||||
return False
|
||||
|
||||
queue_key = self._workflow_queue_key(command, trigger_rollback)
|
||||
if not queue_key:
|
||||
return False
|
||||
|
||||
with self._lock:
|
||||
pending_ids = runtime.setdefault("pending_interrupt_ids", set())
|
||||
if queue_key in pending_ids:
|
||||
return False
|
||||
|
||||
runtime.setdefault("pending_interrupts", []).append({
|
||||
"command": copy.deepcopy(command),
|
||||
"chain": list(chain or []),
|
||||
"interrupt_context": copy.deepcopy(interrupt_context or {}),
|
||||
"trigger_rollback": copy.deepcopy(trigger_rollback or {}),
|
||||
"queue_key": queue_key,
|
||||
"requested_at": time.time(),
|
||||
"priority": self._get_command_priority(command),
|
||||
})
|
||||
pending_ids.add(queue_key)
|
||||
setattr(session, "_workflow_stop_reason", "command_interrupt")
|
||||
logger.info(
|
||||
f"[CMD] 请求暂停工作流: {command.get('name')} "
|
||||
f"(标签页={session.id}, 工作流优先级={runtime.get('priority')}, "
|
||||
f"命令优先级={self._get_command_priority(command)})"
|
||||
)
|
||||
return True
|
||||
|
||||
def _schedule_command_for_active_workflow(
|
||||
self,
|
||||
command: Dict[str, Any],
|
||||
session: 'TabSession',
|
||||
chain: Optional[List[str]] = None,
|
||||
interrupt_context: Optional[Dict[str, Any]] = None,
|
||||
trigger_rollback: Optional[Dict[str, Any]] = None,
|
||||
) -> bool:
|
||||
runtime = self._get_active_workflow_runtime(session)
|
||||
if not runtime or not runtime.get("active"):
|
||||
return False
|
||||
|
||||
if self._allow_command_during_workflow(command):
|
||||
return self._queue_workflow_interrupt(
|
||||
command,
|
||||
session,
|
||||
chain=chain,
|
||||
interrupt_context=interrupt_context,
|
||||
trigger_rollback=trigger_rollback,
|
||||
)
|
||||
|
||||
self._defer_command_until_workflow_resume(
|
||||
runtime,
|
||||
command,
|
||||
chain=chain,
|
||||
interrupt_context=interrupt_context,
|
||||
trigger_rollback=trigger_rollback,
|
||||
)
|
||||
logger.info(
|
||||
f"[CMD] 工作流忙碌,延后执行命令: {command.get('name')} "
|
||||
f"(标签页={session.id}, 工作流优先级={runtime.get('priority')}, "
|
||||
f"命令优先级={self._get_command_priority(command)})"
|
||||
)
|
||||
return True
|
||||
|
||||
def workflow_interrupt_requested(self, session: 'TabSession') -> bool:
|
||||
runtime = self._get_active_workflow_runtime(session)
|
||||
if not runtime:
|
||||
return False
|
||||
with self._lock:
|
||||
return bool(runtime.get("pending_interrupts"))
|
||||
|
||||
def _defer_command_until_workflow_resume(
|
||||
self,
|
||||
runtime: Dict[str, Any],
|
||||
command: Dict[str, Any],
|
||||
chain: Optional[List[str]] = None,
|
||||
interrupt_context: Optional[Dict[str, Any]] = None,
|
||||
trigger_rollback: Optional[Dict[str, Any]] = None,
|
||||
):
|
||||
queue_key = self._workflow_queue_key(command, trigger_rollback)
|
||||
if not queue_key:
|
||||
return
|
||||
with self._lock:
|
||||
deferred_ids = runtime.setdefault("deferred_command_ids", set())
|
||||
if queue_key in deferred_ids:
|
||||
return
|
||||
runtime.setdefault("deferred_commands", []).append({
|
||||
"command": copy.deepcopy(command),
|
||||
"chain": list(chain or []),
|
||||
"interrupt_context": copy.deepcopy(interrupt_context or {}),
|
||||
"trigger_rollback": copy.deepcopy(trigger_rollback or {}),
|
||||
"queue_key": queue_key,
|
||||
"queued_at": time.time(),
|
||||
})
|
||||
deferred_ids.add(queue_key)
|
||||
|
||||
def _mark_interrupt_abort(
|
||||
self,
|
||||
command: Dict[str, Any],
|
||||
interrupt_context: Dict[str, Any],
|
||||
):
|
||||
if interrupt_context.get("abort"):
|
||||
return
|
||||
policy = self._get_workflow_interrupt_policy(command)
|
||||
priority = self._get_command_priority(command)
|
||||
workflow_priority = self._normalize_priority(
|
||||
interrupt_context.get("workflow_priority", self._get_request_priority_baseline()),
|
||||
self._get_request_priority_baseline(),
|
||||
)
|
||||
if policy == "abort" or (policy == "auto" and priority > workflow_priority):
|
||||
interrupt_context["abort"] = True
|
||||
interrupt_context["message"] = self._build_workflow_interrupt_message(command)
|
||||
interrupt_context["abort_by"] = command.get("name", "")
|
||||
|
||||
def _dispatch_interrupt_followup_command(
|
||||
self,
|
||||
command: Dict[str, Any],
|
||||
session: 'TabSession',
|
||||
chain: Optional[List[str]],
|
||||
interrupt_context: Dict[str, Any],
|
||||
):
|
||||
runtime = interrupt_context.get("runtime") or {}
|
||||
workflow_priority = self._normalize_priority(
|
||||
interrupt_context.get("workflow_priority", self._get_request_priority_baseline()),
|
||||
self._get_request_priority_baseline(),
|
||||
)
|
||||
priority = self._get_command_priority(command)
|
||||
policy = self._get_workflow_interrupt_policy(command)
|
||||
|
||||
if priority > workflow_priority or policy == "abort":
|
||||
logger.info(
|
||||
f"[CMD] 工作流暂停期间继续执行高优先级命令: {command.get('name')} "
|
||||
f"(标签页={session.id}, 命令优先级={priority}, 工作流优先级={workflow_priority})"
|
||||
)
|
||||
self._execute_command(command, session, chain=chain, interrupt_context=interrupt_context)
|
||||
self._mark_interrupt_abort(command, interrupt_context)
|
||||
return
|
||||
|
||||
logger.info(
|
||||
f"[CMD] 延后命令至工作流恢复后执行: {command.get('name')} "
|
||||
f"(标签页={session.id}, 命令优先级={priority}, 工作流优先级={workflow_priority})"
|
||||
)
|
||||
self._defer_command_until_workflow_resume(
|
||||
runtime,
|
||||
command,
|
||||
chain=chain,
|
||||
interrupt_context=interrupt_context,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _interrupt_item_sort_key(item: Dict[str, Any]) -> tuple[int, float]:
|
||||
try:
|
||||
priority = int(item.get("priority", 0) or 0)
|
||||
except (TypeError, ValueError):
|
||||
priority = 0
|
||||
try:
|
||||
requested_at = float(item.get("requested_at", 0.0) or 0.0)
|
||||
except (TypeError, ValueError):
|
||||
requested_at = 0.0
|
||||
return -priority, requested_at
|
||||
|
||||
def _take_next_pending_interrupt(
|
||||
self,
|
||||
runtime: Dict[str, Any],
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
with self._lock:
|
||||
raw_pending = runtime.get("pending_interrupts") or []
|
||||
pending = raw_pending if isinstance(raw_pending, list) else list(raw_pending)
|
||||
pending = [item for item in pending if isinstance(item, dict)]
|
||||
if not pending:
|
||||
runtime["pending_interrupts"] = []
|
||||
runtime["pending_interrupt_ids"] = set()
|
||||
return None
|
||||
|
||||
best_index = 0
|
||||
best_key = self._interrupt_item_sort_key(pending[0])
|
||||
for index in range(1, len(pending)):
|
||||
key = self._interrupt_item_sort_key(pending[index])
|
||||
if key < best_key:
|
||||
best_index = index
|
||||
best_key = key
|
||||
|
||||
item = pending.pop(best_index)
|
||||
runtime["pending_interrupts"] = pending
|
||||
return item
|
||||
|
||||
def _drop_pending_interrupts_by_key(
|
||||
self,
|
||||
runtime: Dict[str, Any],
|
||||
queue_key: str,
|
||||
) -> None:
|
||||
if not queue_key:
|
||||
return
|
||||
with self._lock:
|
||||
runtime["pending_interrupts"] = [
|
||||
existing for existing in runtime.get("pending_interrupts", [])
|
||||
if isinstance(existing, dict)
|
||||
and (
|
||||
str(existing.get("queue_key", "") or "").strip()
|
||||
or self._workflow_queue_key(
|
||||
(existing.get("command") or {}),
|
||||
existing.get("trigger_rollback") if isinstance(existing.get("trigger_rollback"), dict) else None,
|
||||
)
|
||||
) != queue_key
|
||||
]
|
||||
|
||||
def handle_pending_workflow_interrupts(self, session: 'TabSession') -> Dict[str, Any]:
|
||||
runtime = self._get_active_workflow_runtime(session)
|
||||
if not runtime or runtime.get("interrupting"):
|
||||
return {"handled": False, "abort": False, "message": ""}
|
||||
|
||||
with self._lock:
|
||||
pending = list(runtime.get("pending_interrupts") or [])
|
||||
if not pending:
|
||||
return {"handled": False, "abort": False, "message": ""}
|
||||
|
||||
with self._lock:
|
||||
if runtime.get("interrupting"):
|
||||
return {"handled": False, "abort": False, "message": ""}
|
||||
runtime["interrupting"] = True
|
||||
interrupt_context = {
|
||||
"workflow_priority": runtime.get("priority", self._get_request_priority_baseline()),
|
||||
"runtime": runtime,
|
||||
"abort": False,
|
||||
"message": "",
|
||||
"abort_by": "",
|
||||
}
|
||||
|
||||
try:
|
||||
while True:
|
||||
item = self._take_next_pending_interrupt(runtime)
|
||||
if not item:
|
||||
break
|
||||
cmd = copy.deepcopy(item.get("command") or {})
|
||||
chain = list(item.get("chain") or [])
|
||||
queue_key = str(item.get("queue_key", "") or "").strip() or self._workflow_queue_key(
|
||||
cmd,
|
||||
item.get("trigger_rollback") if isinstance(item.get("trigger_rollback"), dict) else None,
|
||||
)
|
||||
interrupt_payload = copy.deepcopy(item.get("interrupt_context") or {})
|
||||
|
||||
self._drop_pending_interrupts_by_key(runtime, queue_key)
|
||||
with self._lock:
|
||||
runtime.setdefault("pending_interrupt_ids", set()).discard(queue_key)
|
||||
|
||||
if not cmd or not cmd.get("enabled", True):
|
||||
continue
|
||||
|
||||
logger.info(
|
||||
f"[CMD] 工作流已暂停,执行插队命令: {cmd.get('name')} "
|
||||
f"(标签页={session.id}, 步骤={runtime.get('current_step_index')})"
|
||||
)
|
||||
merged_context = dict(interrupt_context)
|
||||
if interrupt_payload:
|
||||
merged_context.update(interrupt_payload)
|
||||
self._execute_command(cmd, session, chain=chain, interrupt_context=merged_context)
|
||||
self._mark_interrupt_abort(cmd, interrupt_context)
|
||||
if interrupt_context.get("abort"):
|
||||
with self._lock:
|
||||
runtime["pending_interrupts"] = []
|
||||
runtime["pending_interrupt_ids"] = set()
|
||||
break
|
||||
finally:
|
||||
with self._lock:
|
||||
runtime["interrupting"] = False
|
||||
|
||||
if interrupt_context.get("abort"):
|
||||
setattr(session, "_workflow_stop_reason", "command_interrupt_abort")
|
||||
else:
|
||||
setattr(session, "_workflow_stop_reason", None)
|
||||
|
||||
return {
|
||||
"handled": True,
|
||||
"abort": bool(interrupt_context.get("abort")),
|
||||
"message": str(interrupt_context.get("message", "") or ""),
|
||||
"abort_by": str(interrupt_context.get("abort_by", "") or ""),
|
||||
}
|
||||
@@ -0,0 +1,266 @@
|
||||
"""CommandEngine 的一部分(R2-3 拆分):周期调度:保活、标签页池刷新、周期检查循环。
|
||||
|
||||
方法原样从 command_engine.py 迁出,行为不变;通过 mixin 组合回 CommandEngine,
|
||||
类属性与实例状态都在 CommandEngine(command_engine.py)里定义。
|
||||
"""
|
||||
|
||||
import os
|
||||
import random
|
||||
import threading
|
||||
import time
|
||||
from typing import Dict, List, Any, TYPE_CHECKING
|
||||
from app.services.command_engine_common import logger
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from app.core.tab_pool import TabSession # noqa: F401
|
||||
|
||||
|
||||
class CommandEngineSchedulerMixin:
|
||||
"""周期调度:保活、标签页池刷新、周期检查循环"""
|
||||
|
||||
@staticmethod
|
||||
def _should_auto_start_scheduler() -> bool:
|
||||
# The application lifespan explicitly owns scheduler startup/shutdown.
|
||||
# Starting here makes a plain module import (including pytest test
|
||||
# collection) leak a background thread with no matching shutdown.
|
||||
return str(os.getenv("CMD_ENGINE_AUTO_START", "false")).strip().lower() in {
|
||||
"1",
|
||||
"true",
|
||||
"yes",
|
||||
"y",
|
||||
"on",
|
||||
}
|
||||
def _should_log_periodic_summary(self, now_ts: float, due_total: int) -> bool:
|
||||
if not self._periodic_summary_log_enabled:
|
||||
return False
|
||||
interval = (
|
||||
self._periodic_active_log_interval_sec
|
||||
if due_total > 0
|
||||
else self._periodic_idle_log_interval_sec
|
||||
)
|
||||
if (now_ts - self._last_periodic_summary_log_at) < interval:
|
||||
return False
|
||||
self._last_periodic_summary_log_at = now_ts
|
||||
return True
|
||||
def _refresh_tab_pool_if_due(self, pool: Any):
|
||||
if not self._tab_pool_auto_refresh:
|
||||
return
|
||||
now = time.time()
|
||||
if (now - self._last_tab_pool_refresh_at) < self._tab_pool_refresh_interval_sec:
|
||||
return
|
||||
self._last_tab_pool_refresh_at = now
|
||||
try:
|
||||
if hasattr(pool, "refresh_tabs"):
|
||||
pool.refresh_tabs()
|
||||
except Exception as e:
|
||||
logger.debug(f"[CMD] 刷新标签页池失败(忽略): {e}")
|
||||
def _maybe_periodic_keepalive(self, session: 'TabSession', now_ts: float):
|
||||
if not self._periodic_keepalive_enabled:
|
||||
return
|
||||
sid = str(getattr(session, "id", "") or "")
|
||||
if not sid:
|
||||
return
|
||||
last_at = float(self._last_keepalive_by_session.get(sid, 0.0) or 0.0)
|
||||
if (now_ts - last_at) < self._periodic_keepalive_interval_sec:
|
||||
return
|
||||
self._last_keepalive_by_session[sid] = now_ts
|
||||
self._try_wake_tab(session, reason="periodic_keepalive")
|
||||
def _start_periodic_scheduler(self):
|
||||
with self._scheduler_lifecycle_lock:
|
||||
if self._shutdown_requested:
|
||||
return False
|
||||
if self._periodic_thread and self._periodic_thread.is_alive():
|
||||
return True
|
||||
self._periodic_stop_event.clear()
|
||||
self._periodic_thread = threading.Thread(
|
||||
target=self._periodic_loop,
|
||||
daemon=True,
|
||||
name="cmd-periodic-checker",
|
||||
)
|
||||
self._periodic_thread.start()
|
||||
logger.debug("[CMD] 周期调度器已启动")
|
||||
return True
|
||||
def is_scheduler_running(self) -> bool:
|
||||
with self._scheduler_lifecycle_lock:
|
||||
thread = self._periodic_thread
|
||||
return bool(
|
||||
not self._shutdown_requested
|
||||
and thread
|
||||
and thread.is_alive()
|
||||
)
|
||||
def ensure_scheduler_running(self):
|
||||
"""Best-effort watchdog: start periodic checker if it is not running."""
|
||||
return self._start_periodic_scheduler()
|
||||
def _periodic_loop(self):
|
||||
while not self._periodic_stop_event.wait(1.0):
|
||||
try:
|
||||
self._run_periodic_checks()
|
||||
except Exception as e:
|
||||
logger.debug(f"[CMD] 周期调度循环异常(忽略): {e}")
|
||||
def _get_periodic_trigger_timing(self, trigger: Dict, session_id: str) -> tuple[float, float]:
|
||||
interval = max(1.0, self._coerce_float(trigger.get("periodic_interval_sec", 8), 8.0))
|
||||
jitter = max(0.0, self._coerce_float(trigger.get("periodic_jitter_sec", 2), 2.0))
|
||||
|
||||
trigger_type = str(trigger.get("type", "")).strip().lower()
|
||||
has_keyword_expression = bool(str(trigger.get("value", "") or "").strip())
|
||||
if trigger_type == "page_check" and has_keyword_expression:
|
||||
with self._lock:
|
||||
has_observer = session_id in self._observer_keywords_by_session
|
||||
if has_observer:
|
||||
interval = min(interval, 1.5)
|
||||
jitter = 0.0
|
||||
|
||||
return interval, jitter
|
||||
def _run_periodic_checks(self):
|
||||
try:
|
||||
commands = self._load_commands_for_checks()
|
||||
except Exception:
|
||||
return
|
||||
if not commands:
|
||||
return
|
||||
|
||||
try:
|
||||
browser = self._get_browser()
|
||||
except Exception:
|
||||
return
|
||||
|
||||
pool = getattr(browser, "_tab_pool", None)
|
||||
if pool is None:
|
||||
try:
|
||||
pool = browser.tab_pool
|
||||
logger.debug("[CMD] 周期调度器已初始化标签页池")
|
||||
except Exception as e:
|
||||
now = time.time()
|
||||
if (now - self._last_tab_pool_wait_log_at) >= 10:
|
||||
self._last_tab_pool_wait_log_at = now
|
||||
logger.debug(f"[CMD] 周期调度器等待标签页池初始化: {e}")
|
||||
return
|
||||
|
||||
if not hasattr(pool, "get_idle_sessions_snapshot"):
|
||||
return
|
||||
self._refresh_tab_pool_if_due(pool)
|
||||
|
||||
if hasattr(pool, "get_sessions_snapshot"):
|
||||
sessions = pool.get_sessions_snapshot()
|
||||
else:
|
||||
sessions = pool.get_idle_sessions_snapshot()
|
||||
if not sessions:
|
||||
return
|
||||
current_session_ids = {str(getattr(s, "id", "") or "") for s in sessions}
|
||||
|
||||
now = time.time()
|
||||
active_keys = set()
|
||||
due_total = 0
|
||||
|
||||
enabled_commands = [
|
||||
(idx, cmd) for idx, cmd in enumerate(commands)
|
||||
if cmd.get("enabled", True)
|
||||
]
|
||||
|
||||
for session in sessions:
|
||||
session_id = str(getattr(session, "id", "") or "")
|
||||
if not session_id or session_id not in current_session_ids:
|
||||
continue
|
||||
session_status = self._session_status_value(session)
|
||||
if session_status not in {"idle", "busy"}:
|
||||
continue
|
||||
is_busy_workflow = session_status == "busy" and self._has_active_workflow(session)
|
||||
if session_status == "busy" and not is_busy_workflow:
|
||||
continue
|
||||
if session_status == "idle":
|
||||
self._maybe_periodic_keepalive(session, now)
|
||||
try:
|
||||
self._sync_session_bootstrap_js_files(commands, session)
|
||||
except Exception as error:
|
||||
logger.debug(f"[CMD] 启动 JS 文件同步异常(忽略): {error}")
|
||||
|
||||
# Inject/update MutationObserver for page_check keywords on this session
|
||||
try:
|
||||
pc_keywords = self._collect_page_check_keywords(commands, session)
|
||||
if pc_keywords:
|
||||
self._ensure_page_check_observer(session, pc_keywords)
|
||||
else:
|
||||
self._clear_page_check_observer(session)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
due_commands: List[tuple[int, int, Dict[str, Any]]] = []
|
||||
for idx, cmd in enabled_commands:
|
||||
cmd_id = str(cmd.get("id", "")).strip()
|
||||
if not cmd_id:
|
||||
continue
|
||||
|
||||
trigger = cmd.get("trigger", {}) or {}
|
||||
if not bool(trigger.get("periodic_enabled", True)):
|
||||
continue
|
||||
|
||||
trigger_type = str(trigger.get("type", "")).strip().lower()
|
||||
if (
|
||||
is_busy_workflow
|
||||
and trigger_type == "page_check"
|
||||
and not self._should_evaluate_page_check_while_busy_workflow(cmd)
|
||||
):
|
||||
# While a workflow is actively running, low-priority page_check
|
||||
# commands should not consume transient intermediate states.
|
||||
continue
|
||||
|
||||
key = (cmd_id, session.id)
|
||||
active_keys.add(key)
|
||||
|
||||
interval, jitter = self._get_periodic_trigger_timing(trigger, session_id)
|
||||
|
||||
with self._lock:
|
||||
next_at = float(self._periodic_next_run.get(key, 0.0))
|
||||
if now < next_at:
|
||||
continue
|
||||
|
||||
delay = interval + (random.uniform(0.0, jitter) if jitter > 0 else 0.0)
|
||||
with self._lock:
|
||||
self._periodic_next_run[key] = now + delay
|
||||
|
||||
due_commands.append((self._get_command_priority(cmd), idx, cmd))
|
||||
|
||||
due_total += len(due_commands)
|
||||
due_commands.sort(key=lambda item: (-item[0], item[1]))
|
||||
for _, _, cmd in due_commands:
|
||||
with self._command_logging_context(cmd):
|
||||
current_status = str(getattr(getattr(session, "status", None), "value", "")).lower()
|
||||
if current_status == "busy" and not self._has_active_workflow(session):
|
||||
break
|
||||
if current_status not in {"idle", "busy"}:
|
||||
break
|
||||
if self._should_trigger(cmd, session):
|
||||
meta = self._take_pending_async_trigger_meta(cmd, session) or {}
|
||||
if current_status == "busy":
|
||||
scheduled = self._schedule_command_for_active_workflow(
|
||||
cmd,
|
||||
session,
|
||||
interrupt_context=meta.get("interrupt_context"),
|
||||
trigger_rollback=meta.get("rollback"),
|
||||
)
|
||||
if not scheduled:
|
||||
if meta.get("rollback"):
|
||||
self._rollback_trigger_consumption(cmd, session, meta.get("rollback"))
|
||||
self._finalize_request_count_trigger_state(cmd, session, rollback=True)
|
||||
self._reset_page_check_latch(cmd, session, reason="workflow_schedule_failed")
|
||||
else:
|
||||
self._execute_command_async(
|
||||
cmd,
|
||||
session,
|
||||
interrupt_context=meta.get("interrupt_context"),
|
||||
trigger_rollback=meta.get("rollback"),
|
||||
)
|
||||
|
||||
if self._should_log_periodic_summary(now, due_total):
|
||||
logger.debug(
|
||||
f"[CMD] 周期检查: 会话数={len(sessions)}, "
|
||||
f"已启用周期命令={len(active_keys)}, 本轮到点={due_total}"
|
||||
)
|
||||
|
||||
with self._lock:
|
||||
stale_keys = [k for k in self._periodic_next_run if k not in active_keys]
|
||||
for key in stale_keys:
|
||||
self._periodic_next_run.pop(key, None)
|
||||
stale_keepalive_keys = [k for k in self._last_keepalive_by_session if k not in {s.id for s in sessions}]
|
||||
for key in stale_keepalive_keys:
|
||||
self._last_keepalive_by_session.pop(key, None)
|
||||
@@ -0,0 +1,680 @@
|
||||
"""CommandEngine 的一部分(R2-3 拆分):命令的读写与规范化:commands.json / 本地状态、运行统计合并、命名去重、run_js_file 清理。
|
||||
|
||||
方法原样从 command_engine.py 迁出,行为不变;通过 mixin 组合回 CommandEngine,
|
||||
类属性与实例状态都在 CommandEngine(command_engine.py)里定义。
|
||||
"""
|
||||
|
||||
import copy
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
from typing import Dict, List, Optional, Any, TYPE_CHECKING
|
||||
from app.core.config import command_log_context
|
||||
from app.services.command_engine_common import FOLLOW_DEFAULT_PRESET, logger
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from app.core.tab_pool import TabSession # noqa: F401
|
||||
|
||||
|
||||
class CommandEngineStoreMixin:
|
||||
"""命令的读写与规范化:commands.json / 本地状态、运行统计合并、命名去重、run_js_file 清理"""
|
||||
|
||||
def _get_commands_file(self) -> str:
|
||||
if self._commands_file is None:
|
||||
from app.services.config_engine import ConfigConstants
|
||||
self._commands_file = ConfigConstants.COMMANDS_FILE
|
||||
return self._commands_file
|
||||
def _get_commands_local_file(self) -> str:
|
||||
if self._commands_local_file is None:
|
||||
from app.services.config_engine import ConfigConstants
|
||||
self._commands_local_file = ConfigConstants.COMMANDS_LOCAL_FILE
|
||||
return self._commands_local_file
|
||||
def _read_commands_file(self) -> Optional[List[Dict]]:
|
||||
"""读取命令配置。
|
||||
|
||||
B8:返回 ``None`` 表示「读取失败」(JSON 损坏、结构不对、IO 错误),调用方必须保留
|
||||
last-known-good;只有文件不存在或合法的空列表才返回 ``[]``。旧实现把两者混为 ``[]``,
|
||||
编辑器半写入等短暂损坏会清空全部命令并触发 run_js_file 清理。
|
||||
"""
|
||||
commands_file = self._get_commands_file()
|
||||
commands = []
|
||||
|
||||
if os.path.exists(commands_file):
|
||||
try:
|
||||
with open(commands_file, "r", encoding="utf-8-sig") as f:
|
||||
data = json.load(f)
|
||||
|
||||
if isinstance(data, dict):
|
||||
data = data.get("commands", [])
|
||||
|
||||
if isinstance(data, list):
|
||||
commands = [entry for entry in data if isinstance(entry, dict)]
|
||||
for entry in commands:
|
||||
self._normalize_command_logging(entry)
|
||||
entry["advanced_ui"] = self._normalize_advanced_ui(entry.get("advanced_ui"))
|
||||
else:
|
||||
logger.warning(f"命令配置文件格式无效: {commands_file}")
|
||||
return None
|
||||
except json.JSONDecodeError as e:
|
||||
logger.error(f"命令配置文件格式错误: {e}")
|
||||
return None
|
||||
except Exception as e:
|
||||
logger.error(f"加载命令配置失败: {e}")
|
||||
return None
|
||||
|
||||
return self._apply_local_command_state(commands)
|
||||
def _load_command_state_entries(self) -> Optional[List[Dict[str, Any]]]:
|
||||
local_file = self._get_commands_local_file()
|
||||
if not os.path.exists(local_file):
|
||||
self._commands_local_mtime = 0.0
|
||||
return []
|
||||
|
||||
try:
|
||||
with open(local_file, "r", encoding="utf-8-sig") as f:
|
||||
data = json.load(f)
|
||||
|
||||
self._commands_local_mtime = os.path.getmtime(local_file)
|
||||
if not isinstance(data, dict):
|
||||
logger.error(f"本地命令状态文件格式无效: {local_file}")
|
||||
return None
|
||||
entries = data.get("commands", [])
|
||||
if not isinstance(entries, list):
|
||||
logger.error(f"本地命令状态 commands 字段格式无效: {local_file}")
|
||||
return None
|
||||
return [entry for entry in entries if isinstance(entry, dict)]
|
||||
except json.JSONDecodeError as e:
|
||||
logger.error(f"本地命令状态文件格式错误: {e}")
|
||||
return None
|
||||
except Exception as e:
|
||||
logger.error(f"加载本地命令状态失败: {e}")
|
||||
return None
|
||||
def _fallback_local_command_state_entries(self) -> List[Dict[str, Any]]:
|
||||
entries: List[Dict[str, Any]] = []
|
||||
for cmd in self._commands_cache or []:
|
||||
if not isinstance(cmd, dict):
|
||||
continue
|
||||
entries.append({
|
||||
"id": str(cmd.get("id", "")).strip(),
|
||||
"name": str(cmd.get("name", "")).strip(),
|
||||
"enabled": bool(cmd.get("enabled", True)),
|
||||
"group_name": self._normalize_group_name(cmd.get("group_name")),
|
||||
})
|
||||
if entries:
|
||||
logger.warning("本地命令状态读取失败,沿用内存中的上一轮状态覆盖")
|
||||
return entries
|
||||
def _apply_local_command_state(self, commands: List[Dict]) -> List[Dict]:
|
||||
entries = self._load_command_state_entries()
|
||||
if entries is None:
|
||||
entries = self._fallback_local_command_state_entries()
|
||||
if not entries or not commands:
|
||||
return commands
|
||||
|
||||
by_id = {}
|
||||
by_name = {}
|
||||
for entry in entries:
|
||||
command_id = str(entry.get("id", "")).strip()
|
||||
command_name = str(entry.get("name", "")).strip()
|
||||
if command_id:
|
||||
by_id[command_id] = entry
|
||||
if command_name:
|
||||
by_name[command_name] = entry
|
||||
|
||||
applied = 0
|
||||
for cmd in commands:
|
||||
if not isinstance(cmd, dict):
|
||||
continue
|
||||
command_id = str(cmd.get("id", "")).strip()
|
||||
command_name = str(cmd.get("name", "")).strip()
|
||||
entry = by_id.get(command_id) or by_name.get(command_name)
|
||||
if not entry:
|
||||
continue
|
||||
if "enabled" in entry:
|
||||
cmd["enabled"] = bool(entry.get("enabled"))
|
||||
if "group_name" in entry:
|
||||
cmd["group_name"] = self._normalize_group_name(entry.get("group_name"))
|
||||
applied += 1
|
||||
|
||||
if applied > 0:
|
||||
logger.debug(f"已应用 {applied} 条本地命令状态覆盖")
|
||||
return commands
|
||||
def _save_local_command_state(self, commands: List[Dict[str, Any]]) -> bool:
|
||||
local_file = self._get_commands_local_file()
|
||||
tmp_file = local_file + ".tmp"
|
||||
entries = []
|
||||
for cmd in commands:
|
||||
if not isinstance(cmd, dict):
|
||||
continue
|
||||
entries.append({
|
||||
"id": str(cmd.get("id", "")).strip(),
|
||||
"name": str(cmd.get("name", "")).strip(),
|
||||
"enabled": bool(cmd.get("enabled", True)),
|
||||
"group_name": self._normalize_group_name(cmd.get("group_name")),
|
||||
})
|
||||
|
||||
try:
|
||||
os.makedirs(os.path.dirname(local_file), exist_ok=True)
|
||||
with open(tmp_file, "w", encoding="utf-8") as f:
|
||||
json.dump({"commands": entries}, f, indent=2, ensure_ascii=False)
|
||||
f.flush()
|
||||
os.fsync(f.fileno())
|
||||
|
||||
os.replace(tmp_file, local_file)
|
||||
try:
|
||||
self._commands_local_mtime = os.path.getmtime(local_file) if os.path.exists(local_file) else 0.0
|
||||
except Exception as mtime_error:
|
||||
logger.warning(f"本地命令状态已保存但更新时间戳失败: {mtime_error}")
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.error(f"保存本地命令状态失败: {e}")
|
||||
try:
|
||||
if os.path.exists(tmp_file):
|
||||
os.remove(tmp_file)
|
||||
except Exception:
|
||||
pass
|
||||
return False
|
||||
def _snapshot_local_command_state(self) -> Optional[tuple[bool, bytes]]:
|
||||
local_file = self._get_commands_local_file()
|
||||
try:
|
||||
if not os.path.exists(local_file):
|
||||
return (False, b"")
|
||||
with open(local_file, "rb") as f:
|
||||
return (True, f.read())
|
||||
except Exception as e:
|
||||
logger.error(f"读取本地命令状态快照失败: {e}")
|
||||
return None
|
||||
def _restore_local_command_state(self, snapshot: Optional[tuple[bool, bytes]]) -> None:
|
||||
if snapshot is None:
|
||||
return
|
||||
|
||||
local_file = self._get_commands_local_file()
|
||||
tmp_file = local_file + ".restore.tmp"
|
||||
existed, payload = snapshot
|
||||
try:
|
||||
if existed:
|
||||
os.makedirs(os.path.dirname(local_file), exist_ok=True)
|
||||
with open(tmp_file, "wb") as f:
|
||||
f.write(payload)
|
||||
f.flush()
|
||||
os.fsync(f.fileno())
|
||||
os.replace(tmp_file, local_file)
|
||||
elif os.path.exists(local_file):
|
||||
os.remove(local_file)
|
||||
try:
|
||||
self._commands_local_mtime = os.path.getmtime(local_file) if os.path.exists(local_file) else 0.0
|
||||
except Exception as mtime_error:
|
||||
logger.warning(f"本地命令状态已恢复但更新时间戳失败: {mtime_error}")
|
||||
except Exception as e:
|
||||
logger.error(f"恢复本地命令状态失败: {e}")
|
||||
try:
|
||||
if os.path.exists(tmp_file):
|
||||
os.remove(tmp_file)
|
||||
except Exception:
|
||||
pass
|
||||
@staticmethod
|
||||
def _iter_run_js_file_actions(command: Optional[Dict[str, Any]]):
|
||||
for action in list((command or {}).get("actions") or []):
|
||||
if not isinstance(action, dict):
|
||||
continue
|
||||
if str(action.get("type", "")).strip().lower() != "run_js_file":
|
||||
continue
|
||||
yield action
|
||||
def _collect_enabled_run_js_file_keys(
|
||||
self,
|
||||
commands: List[Dict[str, Any]],
|
||||
*,
|
||||
preinject_only: bool = False,
|
||||
) -> set[str]:
|
||||
keys: set[str] = set()
|
||||
for command in commands or []:
|
||||
if not isinstance(command, dict) or not bool(command.get("enabled", True)):
|
||||
continue
|
||||
for action in self._iter_run_js_file_actions(command):
|
||||
if preinject_only and not self._coerce_action_bool(
|
||||
action.get("inject_on_new_document", True),
|
||||
True,
|
||||
):
|
||||
continue
|
||||
resolved_path = self._resolve_action_file_path(action.get("file_path", ""))
|
||||
if resolved_path:
|
||||
keys.add(resolved_path.lower())
|
||||
return keys
|
||||
def _collect_run_js_file_cleanup_actions(
|
||||
self,
|
||||
previous_commands: List[Dict[str, Any]],
|
||||
next_commands: List[Dict[str, Any]],
|
||||
) -> List[Dict[str, Any]]:
|
||||
active_keys = self._collect_enabled_run_js_file_keys(next_commands)
|
||||
active_preinject_keys = self._collect_enabled_run_js_file_keys(
|
||||
next_commands,
|
||||
preinject_only=True,
|
||||
)
|
||||
next_by_id: Dict[str, Dict[str, Any]] = {}
|
||||
next_by_name: Dict[str, Dict[str, Any]] = {}
|
||||
for command in next_commands or []:
|
||||
if not isinstance(command, dict):
|
||||
continue
|
||||
command_id = str(command.get("id", "")).strip()
|
||||
command_name = str(command.get("name", "")).strip()
|
||||
if command_id:
|
||||
next_by_id[command_id] = command
|
||||
if command_name:
|
||||
next_by_name[command_name] = command
|
||||
|
||||
cleanup_actions: List[Dict[str, Any]] = []
|
||||
seen_keys: set[str] = set()
|
||||
for command in previous_commands or []:
|
||||
if not isinstance(command, dict) or not bool(command.get("enabled", True)):
|
||||
continue
|
||||
command_id = str(command.get("id", "")).strip()
|
||||
command_name = str(command.get("name", "")).strip()
|
||||
next_command = next_by_id.get(command_id) or next_by_name.get(command_name)
|
||||
command_still_enabled = next_command is not None and bool(next_command.get("enabled", True))
|
||||
for action in self._iter_run_js_file_actions(command):
|
||||
resolved_path = self._resolve_action_file_path(action.get("file_path", ""))
|
||||
action_key = resolved_path.lower() if resolved_path else f"{command_id}:{action.get('action_id', '')}"
|
||||
if command_still_enabled:
|
||||
was_preinjected = self._coerce_action_bool(
|
||||
action.get("inject_on_new_document", True),
|
||||
True,
|
||||
)
|
||||
if not was_preinjected:
|
||||
continue
|
||||
if resolved_path and resolved_path.lower() in active_preinject_keys:
|
||||
continue
|
||||
elif resolved_path and resolved_path.lower() in active_keys:
|
||||
continue
|
||||
if action_key in seen_keys:
|
||||
continue
|
||||
seen_keys.add(action_key)
|
||||
cleanup_actions.append(copy.deepcopy(action))
|
||||
return cleanup_actions
|
||||
def _cleanup_disabled_run_js_file_actions(self, actions: List[Dict[str, Any]]) -> None:
|
||||
if not actions:
|
||||
return
|
||||
|
||||
try:
|
||||
browser = self._get_browser()
|
||||
pool = getattr(browser, "_tab_pool", None)
|
||||
if pool is None or not hasattr(pool, "get_sessions_snapshot"):
|
||||
return
|
||||
sessions = list(pool.get_sessions_snapshot() or [])
|
||||
except Exception as e:
|
||||
logger.debug(f"[CMD] run_js_file 清理前获取会话失败(忽略): {e}")
|
||||
return
|
||||
|
||||
cleaned = 0
|
||||
for session in sessions:
|
||||
for action in actions:
|
||||
try:
|
||||
result = self._cleanup_run_js_file_action(
|
||||
session,
|
||||
action,
|
||||
reason="command_disabled",
|
||||
)
|
||||
if result.get("removed_init_script") or result.get("ran_teardown"):
|
||||
cleaned += 1
|
||||
if not result.get("ok", True):
|
||||
logger.debug(
|
||||
f"[CMD] run_js_file 清理存在告警: "
|
||||
f"session={getattr(session, 'id', '')}, path={result.get('path', '')}, "
|
||||
f"errors={result.get('errors', [])}"
|
||||
)
|
||||
except Exception as e:
|
||||
logger.debug(
|
||||
f"[CMD] run_js_file 清理失败(忽略): "
|
||||
f"session={getattr(session, 'id', '')}, error={e}"
|
||||
)
|
||||
|
||||
if cleaned > 0:
|
||||
logger.info(f"[CMD] 已清理 {cleaned} 个 run_js_file 注入实例")
|
||||
def _refresh_commands_if_changed(self, force: bool = False):
|
||||
with self._commands_lock:
|
||||
commands_file = self._get_commands_file()
|
||||
current_mtime = os.path.getmtime(commands_file) if os.path.exists(commands_file) else 0.0
|
||||
commands_local_file = self._get_commands_local_file()
|
||||
current_local_mtime = os.path.getmtime(commands_local_file) if os.path.exists(commands_local_file) else 0.0
|
||||
|
||||
if (
|
||||
force
|
||||
or not self._commands_loaded
|
||||
or current_mtime != self._commands_mtime
|
||||
or current_local_mtime != self._commands_local_mtime
|
||||
):
|
||||
if (
|
||||
not force
|
||||
and self._commands_loaded
|
||||
and self._commands_failed_mtime is not None
|
||||
and current_mtime == self._commands_failed_mtime
|
||||
and current_local_mtime == self._commands_local_mtime
|
||||
):
|
||||
# 同一份损坏文件已判定过失败:继续使用 last-known-good,等待文件被修好
|
||||
return
|
||||
previous_snapshot = copy.deepcopy(self._commands_cache) if self._commands_loaded else []
|
||||
next_snapshot = self._read_commands_file()
|
||||
if next_snapshot is None:
|
||||
# B8:读失败 ≠ 合法空配置。保留上一版命令与预注入脚本,不推进 mtime、不做清理;
|
||||
# 记录失败 mtime 以免每次轮询重复读取/刷日志,文件修复(mtime 变化)后自动重载。
|
||||
self._commands_failed_mtime = current_mtime
|
||||
if self._commands_loaded:
|
||||
logger.warning(
|
||||
f"[CMD] 命令配置读取失败,继续使用上一版 {len(self._commands_cache)} 条命令"
|
||||
"(修复 commands.json 后将自动重新加载)"
|
||||
)
|
||||
else:
|
||||
# 首次加载就失败:没有 last-known-good,只能以空配置启动,但不标记为已加载成功的 mtime
|
||||
self._commands_cache = []
|
||||
self._commands_loaded = True
|
||||
return
|
||||
self._commands_failed_mtime = None
|
||||
cleanup_actions = self._collect_run_js_file_cleanup_actions(previous_snapshot, next_snapshot)
|
||||
self._commands_cache = next_snapshot
|
||||
self._commands_mtime = current_mtime
|
||||
self._commands_loaded = True
|
||||
self._cleanup_disabled_run_js_file_actions(cleanup_actions)
|
||||
def _save_commands(self, commands: List[Dict]) -> bool:
|
||||
commands_file = self._get_commands_file()
|
||||
tmp_file = commands_file + ".tmp"
|
||||
local_snapshot: Optional[tuple[bool, bytes]] = None
|
||||
local_state_written = False
|
||||
|
||||
try:
|
||||
with self._commands_lock:
|
||||
commands_snapshot = copy.deepcopy(commands)
|
||||
previous_snapshot = copy.deepcopy(self._commands_cache) if self._commands_loaded else []
|
||||
os.makedirs(os.path.dirname(commands_file), exist_ok=True)
|
||||
local_snapshot = self._snapshot_local_command_state()
|
||||
if local_snapshot is None:
|
||||
return False
|
||||
if not self._save_local_command_state(commands_snapshot):
|
||||
self._restore_local_command_state(local_snapshot)
|
||||
return False
|
||||
local_state_written = True
|
||||
with open(tmp_file, "w", encoding="utf-8") as f:
|
||||
json.dump({"commands": commands_snapshot}, f, indent=2, ensure_ascii=False)
|
||||
f.flush()
|
||||
os.fsync(f.fileno())
|
||||
|
||||
os.replace(tmp_file, commands_file)
|
||||
try:
|
||||
self._commands_mtime = os.path.getmtime(commands_file) if os.path.exists(commands_file) else 0.0
|
||||
except Exception as mtime_error:
|
||||
logger.warning(f"命令配置已保存但更新时间戳失败: {mtime_error}")
|
||||
self._commands_loaded = True
|
||||
self._commands_failed_mtime = None
|
||||
self._commands_cache = commands_snapshot
|
||||
cleanup_actions = self._collect_run_js_file_cleanup_actions(previous_snapshot, commands_snapshot)
|
||||
self._cleanup_disabled_run_js_file_actions(cleanup_actions)
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.error(f"保存命令配置失败: {e}")
|
||||
try:
|
||||
if os.path.exists(tmp_file):
|
||||
os.remove(tmp_file)
|
||||
except Exception:
|
||||
pass
|
||||
if local_state_written:
|
||||
self._restore_local_command_state(local_snapshot)
|
||||
return False
|
||||
def _runtime_stats_for_command_ids(self, command_ids: set[str]) -> Dict[str, Dict[str, Any]]:
|
||||
if not command_ids:
|
||||
return {}
|
||||
with self._lock:
|
||||
return {
|
||||
command_id: copy.deepcopy(stats)
|
||||
for command_id, stats in self._command_runtime_stats.items()
|
||||
if command_id in command_ids and isinstance(stats, dict)
|
||||
}
|
||||
def _merge_runtime_stats_into_commands(self, commands: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
|
||||
if not commands:
|
||||
return commands
|
||||
command_ids = {
|
||||
str(cmd.get("id", "")).strip()
|
||||
for cmd in commands
|
||||
if isinstance(cmd, dict) and str(cmd.get("id", "")).strip()
|
||||
}
|
||||
runtime_stats = self._runtime_stats_for_command_ids(command_ids)
|
||||
if not runtime_stats:
|
||||
return commands
|
||||
for cmd in commands:
|
||||
cmd_id = str(cmd.get("id", "")).strip()
|
||||
if not cmd_id:
|
||||
continue
|
||||
stats = runtime_stats.get(cmd_id)
|
||||
if not stats:
|
||||
continue
|
||||
cmd.update(stats)
|
||||
return commands
|
||||
def _merge_runtime_stats_into_command(self, command: Dict[str, Any]) -> Dict[str, Any]:
|
||||
command_id = str((command or {}).get("id", "")).strip()
|
||||
if not command_id:
|
||||
return command
|
||||
runtime_stats = self._runtime_stats_for_command_ids({command_id})
|
||||
stats = runtime_stats.get(command_id)
|
||||
if stats:
|
||||
command.update(stats)
|
||||
return command
|
||||
def _normalize_group_name(self, group_name: Any) -> str:
|
||||
return str(group_name or "").strip()
|
||||
@staticmethod
|
||||
def _normalize_advanced_ui(ui: Any) -> Dict[str, Any]:
|
||||
if not isinstance(ui, dict):
|
||||
return {}
|
||||
|
||||
kind = str(ui.get("kind") or "none").strip().lower()
|
||||
if kind not in {"none", "form"}:
|
||||
kind = "none"
|
||||
|
||||
title = str(ui.get("title") or "").strip()
|
||||
description = str(ui.get("description") or "").strip()
|
||||
|
||||
fields = ui.get("fields")
|
||||
if not isinstance(fields, list):
|
||||
fields = []
|
||||
normalized_fields: List[Dict[str, Any]] = []
|
||||
for field in fields:
|
||||
if not isinstance(field, dict):
|
||||
continue
|
||||
field_type = str(field.get("type") or "text").strip().lower()
|
||||
if field_type not in {"text", "textarea", "number", "boolean", "select", "password", "rule_list", "image"}:
|
||||
continue
|
||||
options = field.get("options")
|
||||
if not isinstance(options, list):
|
||||
options = []
|
||||
rows = field.get("rows")
|
||||
try:
|
||||
rows = int(rows)
|
||||
except (TypeError, ValueError, OverflowError):
|
||||
rows = rows
|
||||
normalized_fields.append({
|
||||
"key": str(field.get("key") or "").strip(),
|
||||
"label": str(field.get("label") or "").strip(),
|
||||
"type": field_type,
|
||||
"default": field.get("default"),
|
||||
"help": str(field.get("help") or "").strip(),
|
||||
"placeholder": str(field.get("placeholder") or "").strip(),
|
||||
"required": bool(field.get("required", False)),
|
||||
"options": options,
|
||||
"rows": rows,
|
||||
"item_defaults": field.get("item_defaults") if isinstance(field.get("item_defaults"), dict) else {},
|
||||
})
|
||||
|
||||
values = ui.get("values")
|
||||
if not isinstance(values, dict):
|
||||
values = {}
|
||||
|
||||
return {
|
||||
"kind": kind,
|
||||
"title": title,
|
||||
"description": description,
|
||||
"results_enabled": bool(ui.get("results_enabled", False)),
|
||||
"fields": normalized_fields,
|
||||
"values": values,
|
||||
}
|
||||
@staticmethod
|
||||
def _coerce_bool_flag(value: Any, default: bool = True) -> bool:
|
||||
if value is None:
|
||||
return default
|
||||
if isinstance(value, bool):
|
||||
return value
|
||||
if isinstance(value, (int, float)):
|
||||
return bool(value)
|
||||
text = str(value).strip().lower()
|
||||
if text in {"1", "true", "yes", "y", "on"}:
|
||||
return True
|
||||
if text in {"0", "false", "no", "n", "off"}:
|
||||
return False
|
||||
return default
|
||||
@staticmethod
|
||||
def _normalize_command_log_level(value: Any) -> str:
|
||||
level = str(value or "GLOBAL").strip().upper()
|
||||
return level if level in {"GLOBAL", "DEBUG", "INFO", "WARNING", "ERROR"} else "GLOBAL"
|
||||
def _normalize_command_logging(self, command: Optional[Dict[str, Any]]) -> Optional[Dict[str, Any]]:
|
||||
if not isinstance(command, dict):
|
||||
return command
|
||||
command["log_enabled"] = self._coerce_bool_flag(command.get("log_enabled", True), True)
|
||||
command["log_level"] = self._normalize_command_log_level(command.get("log_level"))
|
||||
return command
|
||||
def _get_command_log_context(self, command: Optional[Dict[str, Any]]) -> Dict[str, Any]:
|
||||
normalized = self._normalize_command_logging(command if isinstance(command, dict) else {}) or {}
|
||||
return {
|
||||
"enabled": bool(normalized.get("log_enabled", True)),
|
||||
"level": self._normalize_command_log_level(normalized.get("log_level")),
|
||||
}
|
||||
def _command_logging_context(self, command: Optional[Dict[str, Any]]):
|
||||
return command_log_context(self._get_command_log_context(command))
|
||||
@staticmethod
|
||||
def _normalize_group_acquire_policy(value: Any) -> str:
|
||||
policy = str(value or "inherit_session").strip().lower()
|
||||
return policy if policy in {"inherit_session", "try_acquire", "require_acquire"} else "inherit_session"
|
||||
def _repair_mojibake_text(self, text: Any) -> str:
|
||||
value = str(text or "").strip()
|
||||
if not value:
|
||||
return ""
|
||||
|
||||
candidates = [value]
|
||||
for source_encoding in ("latin-1", "cp1252"):
|
||||
try:
|
||||
repaired = value.encode(source_encoding).decode("utf-8")
|
||||
except (UnicodeEncodeError, UnicodeDecodeError):
|
||||
continue
|
||||
if repaired and repaired not in candidates:
|
||||
candidates.append(repaired)
|
||||
return candidates[-1]
|
||||
def _should_follow_default_preset(self, preset_name: Any) -> bool:
|
||||
value = str(preset_name or "").strip()
|
||||
return value in {"", FOLLOW_DEFAULT_PRESET}
|
||||
def _resolve_preset_name(self, preset_name: Any, session: Optional['TabSession'] = None) -> str:
|
||||
if self._should_follow_default_preset(preset_name):
|
||||
return ""
|
||||
|
||||
raw_name = str(preset_name or "").strip()
|
||||
if not raw_name:
|
||||
return ""
|
||||
|
||||
repaired_name = self._repair_mojibake_text(raw_name)
|
||||
if repaired_name == raw_name:
|
||||
return raw_name
|
||||
|
||||
domain = str(getattr(session, "current_domain", "") or "").strip() if session else ""
|
||||
if not domain:
|
||||
logger.warning(f"[CMD] 检测到乱码预设名,已自动修正: {raw_name} -> {repaired_name}")
|
||||
return repaired_name
|
||||
|
||||
try:
|
||||
config_engine = self._get_config_engine()
|
||||
site = getattr(config_engine, "sites", {}).get(domain, {}) or {}
|
||||
presets = site.get("presets", {}) or {}
|
||||
if not presets or repaired_name in presets:
|
||||
logger.warning(f"[CMD] 检测到乱码预设名,已自动修正: {raw_name} -> {repaired_name}")
|
||||
return repaired_name
|
||||
except Exception as e:
|
||||
logger.debug(f"[CMD] 预设名乱码修正校验失败(忽略): {e}")
|
||||
|
||||
return raw_name
|
||||
def _ensure_unique_command_name(
|
||||
self,
|
||||
raw_name: Any,
|
||||
commands: List[Dict[str, Any]],
|
||||
exclude_id: Optional[str] = None,
|
||||
) -> str:
|
||||
existing = {
|
||||
str(cmd.get("name", "")).strip()
|
||||
for cmd in commands
|
||||
if cmd.get("id") != exclude_id and str(cmd.get("name", "")).strip()
|
||||
}
|
||||
|
||||
base_name = str(raw_name or "").strip() or "新命令"
|
||||
if base_name != "新命令" and base_name not in existing:
|
||||
return base_name
|
||||
|
||||
root = re.sub(r"\d+$", "", base_name).rstrip() or "新命令"
|
||||
pattern = re.compile(rf"^{re.escape(root)}(\d+)$")
|
||||
next_num = 1
|
||||
for name in existing:
|
||||
match = pattern.match(name)
|
||||
if match:
|
||||
next_num = max(next_num, int(match.group(1)) + 1)
|
||||
|
||||
candidate = f"{root}{next_num}"
|
||||
while candidate in existing:
|
||||
next_num += 1
|
||||
candidate = f"{root}{next_num}"
|
||||
return candidate
|
||||
@staticmethod
|
||||
def _ensure_unique_copy_name(raw_name: Any, existing_names: set[str], max_length: int = 100) -> str:
|
||||
source_name = str(raw_name or "").strip() or "未命名"
|
||||
root = re.sub(r"\s*-\s*副本(?:\s+\d+)?$", "", source_name).strip() or source_name
|
||||
|
||||
def build_candidate(copy_number: int) -> str:
|
||||
suffix = " - 副本" if copy_number == 1 else f" - 副本 {copy_number}"
|
||||
available = max(1, max_length - len(suffix))
|
||||
return f"{root[:available].rstrip()}{suffix}"
|
||||
|
||||
copy_number = 1
|
||||
candidate = build_candidate(copy_number)
|
||||
while candidate in existing_names:
|
||||
copy_number += 1
|
||||
candidate = build_candidate(copy_number)
|
||||
return candidate
|
||||
@staticmethod
|
||||
def _remap_duplicated_command_references(
|
||||
command: Dict[str, Any],
|
||||
command_id_map: Dict[str, str],
|
||||
source_group_name: str = "",
|
||||
target_group_name: str = "",
|
||||
) -> None:
|
||||
trigger = command.get("trigger")
|
||||
if isinstance(trigger, dict):
|
||||
source_id = str(trigger.get("command_id") or "").strip()
|
||||
if source_id in command_id_map:
|
||||
trigger["command_id"] = command_id_map[source_id]
|
||||
|
||||
source_ids = trigger.get("command_ids")
|
||||
if isinstance(source_ids, list):
|
||||
trigger["command_ids"] = [
|
||||
command_id_map.get(str(command_id or "").strip(), command_id)
|
||||
for command_id in source_ids
|
||||
]
|
||||
|
||||
if not source_group_name or not target_group_name:
|
||||
return
|
||||
for action in command.get("actions") or []:
|
||||
if not isinstance(action, dict) or action.get("type") != "execute_command_group":
|
||||
continue
|
||||
if str(action.get("group_name") or "").strip() == source_group_name:
|
||||
action["group_name"] = target_group_name
|
||||
def _load_commands(self) -> List[Dict]:
|
||||
"""从配置引擎加载命令列表原始快照,避免共享可变引用。"""
|
||||
with self._commands_lock:
|
||||
self._get_config_engine()
|
||||
self._refresh_commands_if_changed()
|
||||
snapshot = self._commands_cache
|
||||
return copy.deepcopy(snapshot)
|
||||
def _load_commands_for_checks(self) -> List[Dict]:
|
||||
"""Load a lightweight command snapshot for read-mostly trigger checks."""
|
||||
with self._commands_lock:
|
||||
self._get_config_engine()
|
||||
self._refresh_commands_if_changed()
|
||||
snapshot = self._commands_cache
|
||||
commands = [dict(cmd) for cmd in snapshot if isinstance(cmd, dict)]
|
||||
return commands
|
||||
@@ -0,0 +1,918 @@
|
||||
"""CommandEngine 的一部分(R2-3 拆分):触发判定:请求计数、网络事件、优先级与忙碌判断、导航事件分发。
|
||||
|
||||
方法原样从 command_engine.py 迁出,行为不变;通过 mixin 组合回 CommandEngine,
|
||||
类属性与实例状态都在 CommandEngine(command_engine.py)里定义。
|
||||
"""
|
||||
|
||||
import time
|
||||
import uuid
|
||||
from typing import Dict, List, Optional, Any, TYPE_CHECKING
|
||||
from app.utils.site_url import extract_remote_site_domain
|
||||
from app.services.command_engine_common import logger
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from app.core.tab_pool import TabSession # noqa: F401
|
||||
|
||||
|
||||
class CommandEngineTriggersMixin:
|
||||
"""触发判定:请求计数、网络事件、优先级与忙碌判断、导航事件分发"""
|
||||
|
||||
def check_triggers(self, session: 'TabSession'):
|
||||
"""
|
||||
检查所有命令的触发条件
|
||||
|
||||
在 TabSession.release() 后调用(锁外、后台,不阻塞主流程)
|
||||
"""
|
||||
self.ensure_scheduler_running()
|
||||
try:
|
||||
commands = self._load_commands_for_checks()
|
||||
except Exception as e:
|
||||
logger.debug(f"命令加载失败,跳过触发检查: {e}")
|
||||
return
|
||||
|
||||
if not commands:
|
||||
return
|
||||
|
||||
|
||||
ordered_commands = [
|
||||
(idx, cmd) for idx, cmd in enumerate(commands)
|
||||
if cmd.get("enabled", True)
|
||||
]
|
||||
ordered_commands.sort(key=lambda item: (-self._get_command_priority(item[1]), item[0]))
|
||||
|
||||
for _, cmd in ordered_commands:
|
||||
with self._command_logging_context(cmd):
|
||||
try:
|
||||
if self._should_trigger(cmd, session):
|
||||
meta = self._take_pending_async_trigger_meta(cmd, session) or {}
|
||||
self._execute_command_async(
|
||||
cmd,
|
||||
session,
|
||||
interrupt_context=meta.get("interrupt_context"),
|
||||
trigger_rollback=meta.get("rollback"),
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"trigger check failed [{cmd.get('name')}]: {e}")
|
||||
def check_workflow_triggers_now(self, session: 'TabSession') -> bool:
|
||||
"""Check commands for an active workflow and enqueue matching interrupts immediately."""
|
||||
if not self._has_active_workflow(session):
|
||||
return False
|
||||
|
||||
self.ensure_scheduler_running()
|
||||
try:
|
||||
commands = self._load_commands_for_checks()
|
||||
except Exception as e:
|
||||
logger.debug(f"命令加载失败,跳过工作流触发检查: {e}")
|
||||
return False
|
||||
|
||||
ordered_commands = [
|
||||
(idx, cmd) for idx, cmd in enumerate(commands)
|
||||
if cmd.get("enabled", True)
|
||||
]
|
||||
ordered_commands.sort(key=lambda item: (-self._get_command_priority(item[1]), item[0]))
|
||||
|
||||
scheduled_any = False
|
||||
for _, cmd in ordered_commands:
|
||||
with self._command_logging_context(cmd):
|
||||
try:
|
||||
trigger = cmd.get("trigger", {}) or {}
|
||||
trigger_type = str(trigger.get("type", "")).strip().lower()
|
||||
if (
|
||||
trigger_type == "page_check"
|
||||
and not self._should_evaluate_page_check_while_busy_workflow(cmd)
|
||||
):
|
||||
continue
|
||||
if not self._should_trigger(cmd, session):
|
||||
continue
|
||||
meta = self._take_pending_async_trigger_meta(cmd, session) or {}
|
||||
scheduled = self._schedule_command_for_active_workflow(
|
||||
cmd,
|
||||
session,
|
||||
interrupt_context=meta.get("interrupt_context"),
|
||||
trigger_rollback=meta.get("rollback"),
|
||||
)
|
||||
if scheduled:
|
||||
scheduled_any = True
|
||||
elif meta.get("rollback"):
|
||||
self._rollback_trigger_consumption(cmd, session, meta.get("rollback"))
|
||||
except Exception as e:
|
||||
logger.error(f"workflow trigger check failed [{cmd.get('name')}]: {e}")
|
||||
|
||||
return scheduled_any
|
||||
def submit_background_task(self, fn, *args, **kwargs):
|
||||
"""Submit non-trigger command work to the bounded command executor."""
|
||||
return self._command_executor.submit(fn, *args, **kwargs)
|
||||
def handle_network_event(self, session: 'TabSession', event: Dict[str, Any]) -> bool:
|
||||
"""
|
||||
处理实时网络事件。
|
||||
|
||||
返回值:
|
||||
- True: 命中了网络拦截,当前监听应暂停并交回工作流处理
|
||||
- False: 不需要暂停当前监听
|
||||
"""
|
||||
self.ensure_scheduler_running()
|
||||
if not event:
|
||||
return False
|
||||
|
||||
event_copy = dict(event)
|
||||
if not event_copy.get("event_id"):
|
||||
event_copy["event_id"] = f"net_{uuid.uuid4().hex[:10]}"
|
||||
event_copy.setdefault("timestamp", time.time())
|
||||
|
||||
with self._lock:
|
||||
self._append_bounded_event(self._network_events, session.id, event_copy)
|
||||
|
||||
try:
|
||||
commands = self._load_commands_for_checks()
|
||||
except Exception as e:
|
||||
logger.debug(f"命令加载失败,跳过网络事件触发: {e}")
|
||||
return False
|
||||
|
||||
should_interrupt_listener = False
|
||||
|
||||
for cmd in commands:
|
||||
if not cmd.get("enabled", True):
|
||||
continue
|
||||
trigger = cmd.get("trigger", {})
|
||||
if trigger.get("type") != "network_request_error":
|
||||
continue
|
||||
if not self._matches_scope(cmd, session):
|
||||
continue
|
||||
with self._command_logging_context(cmd):
|
||||
dispatch = self._prepare_network_trigger_dispatch(cmd, session, event_copy)
|
||||
if not dispatch:
|
||||
continue
|
||||
if self._has_active_workflow(session):
|
||||
scheduled = self._schedule_command_for_active_workflow(
|
||||
cmd,
|
||||
session,
|
||||
interrupt_context=dispatch.get("interrupt_context"),
|
||||
trigger_rollback=dispatch.get("rollback"),
|
||||
)
|
||||
if scheduled:
|
||||
should_interrupt_listener = (
|
||||
should_interrupt_listener
|
||||
or self.workflow_interrupt_requested(session)
|
||||
)
|
||||
else:
|
||||
scheduled = self._execute_command_async(
|
||||
cmd,
|
||||
session,
|
||||
interrupt_context=dispatch.get("interrupt_context"),
|
||||
trigger_rollback=dispatch.get("rollback"),
|
||||
)
|
||||
if scheduled:
|
||||
should_interrupt_listener = (
|
||||
should_interrupt_listener
|
||||
or bool(trigger.get("abort_on_match", True))
|
||||
)
|
||||
if scheduled:
|
||||
continue
|
||||
if dispatch.get("rollback"):
|
||||
self._rollback_trigger_consumption(cmd, session, dispatch.get("rollback"))
|
||||
|
||||
return should_interrupt_listener
|
||||
def has_network_interception_for_session(self, session: 'TabSession') -> bool:
|
||||
"""当前会话是否存在可生效的网络异常拦截触发器。"""
|
||||
try:
|
||||
commands = self._load_commands_for_checks()
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
for cmd in commands:
|
||||
if not cmd.get("enabled", True):
|
||||
continue
|
||||
trigger = cmd.get("trigger", {})
|
||||
if trigger.get("type") != "network_request_error":
|
||||
continue
|
||||
if self._matches_scope(cmd, session):
|
||||
return True
|
||||
return False
|
||||
def get_network_listen_pattern(self, session: 'TabSession') -> str:
|
||||
"""
|
||||
依据网络异常拦截命令推断一个 listen_pattern。
|
||||
仅用于事件监听,不要求完全精准。
|
||||
"""
|
||||
try:
|
||||
commands = self._load_commands_for_checks()
|
||||
except Exception:
|
||||
return "http"
|
||||
|
||||
hints: List[str] = []
|
||||
for cmd in commands:
|
||||
if not cmd.get("enabled", True):
|
||||
continue
|
||||
trigger = cmd.get("trigger", {})
|
||||
if trigger.get("type") != "network_request_error":
|
||||
continue
|
||||
if not self._matches_scope(cmd, session):
|
||||
continue
|
||||
|
||||
pattern = str(trigger.get("url_pattern") or trigger.get("value") or "").strip()
|
||||
if not pattern:
|
||||
continue
|
||||
hint = self._pattern_to_listen_hint(pattern, str(trigger.get("match_mode", "keyword")))
|
||||
if hint:
|
||||
hints.append(hint)
|
||||
|
||||
if not hints:
|
||||
return "http"
|
||||
hints.sort(key=len, reverse=True)
|
||||
return hints[0]
|
||||
def _ensure_trigger_state(
|
||||
self,
|
||||
command_id: str,
|
||||
session: 'TabSession',
|
||||
state_key: Optional[tuple] = None,
|
||||
initial_req: Optional[int] = None,
|
||||
initial_err: Optional[int] = None,
|
||||
) -> tuple[Dict[str, Any], bool]:
|
||||
state_key = state_key or (command_id, session.id)
|
||||
req_baseline = session.request_count if initial_req is None else int(initial_req)
|
||||
err_baseline = session.error_count if initial_err is None else int(initial_err)
|
||||
with self._lock:
|
||||
state = self._trigger_states.get(state_key)
|
||||
if state is None:
|
||||
state = {
|
||||
"req": req_baseline,
|
||||
"err": err_baseline,
|
||||
"result_token": "",
|
||||
"net_sig": "",
|
||||
"page_key": "",
|
||||
"page_hit": False,
|
||||
"page_last_fire_at": 0.0,
|
||||
}
|
||||
self._trigger_states[state_key] = state
|
||||
return state, True
|
||||
return state, False
|
||||
def _coerce_int(self, value: Any, default: int) -> int:
|
||||
try:
|
||||
return int(value)
|
||||
except (TypeError, ValueError, OverflowError):
|
||||
return default
|
||||
def _coerce_float(self, value: Any, default: float) -> float:
|
||||
try:
|
||||
return float(value)
|
||||
except (TypeError, ValueError, OverflowError):
|
||||
return default
|
||||
@staticmethod
|
||||
def _counter_inc(counter: Dict[str, int], key: str):
|
||||
if not key:
|
||||
return
|
||||
counter[key] = int(counter.get(key, 0)) + 1
|
||||
@staticmethod
|
||||
def _counter_dec(counter: Dict[str, int], key: str):
|
||||
if not key:
|
||||
return
|
||||
next_value = int(counter.get(key, 0)) - 1
|
||||
if next_value > 0:
|
||||
counter[key] = next_value
|
||||
else:
|
||||
counter.pop(key, None)
|
||||
def _normalize_priority(self, value: Any, default: int = 2) -> int:
|
||||
try:
|
||||
p = int(value)
|
||||
except (TypeError, ValueError, OverflowError):
|
||||
p = int(default)
|
||||
return p
|
||||
def _get_request_priority_baseline(self) -> int:
|
||||
return self._normalize_priority(getattr(self, "_request_priority_baseline", 2), 2)
|
||||
def _get_command_priority(self, command: Dict) -> int:
|
||||
trigger = command.get("trigger", {}) or {}
|
||||
raw = trigger.get("priority", trigger.get("command_priority", 2))
|
||||
return self._normalize_priority(raw, 2)
|
||||
def _should_evaluate_page_check_while_busy_workflow(self, command: Dict) -> bool:
|
||||
trigger = command.get("trigger", {}) or {}
|
||||
raw = trigger.get("check_while_busy_workflow", None)
|
||||
if raw is None:
|
||||
return True
|
||||
return bool(raw)
|
||||
def _command_affects_domain(self, command: Dict) -> bool:
|
||||
actions = command.get("actions", [])
|
||||
for action in actions or []:
|
||||
if str((action or {}).get("type", "")).strip() == "clear_cookies":
|
||||
return True
|
||||
return False
|
||||
def _has_busy_peer_on_domain(self, domain: str, exclude_session_id: str = "") -> bool:
|
||||
normalized = str(domain or "").strip().lower()
|
||||
if not normalized:
|
||||
return False
|
||||
try:
|
||||
browser = self._get_browser()
|
||||
pool = getattr(browser, "_tab_pool", None)
|
||||
if pool is None:
|
||||
return False
|
||||
status = pool.get_status() if hasattr(pool, "get_status") else {}
|
||||
for tab in status.get("tabs", []) or []:
|
||||
sid = str(tab.get("id", "") or "")
|
||||
if exclude_session_id and sid == exclude_session_id:
|
||||
continue
|
||||
if str(tab.get("status", "")).lower() != "busy":
|
||||
continue
|
||||
tab_domain = str(tab.get("current_domain", "") or "").strip().lower()
|
||||
if self._domain_matches(normalized, tab_domain):
|
||||
return True
|
||||
except Exception:
|
||||
return False
|
||||
return False
|
||||
def should_block_request_for_session(self, session: "TabSession", task_id: str = "") -> bool:
|
||||
if session is None:
|
||||
return False
|
||||
task = str(task_id or "").strip().lower()
|
||||
if task.startswith("cmd_") or task.startswith("group_") or task.startswith("cmd_test_") or task.startswith("group_test_"):
|
||||
return False
|
||||
|
||||
session_id = str(getattr(session, "id", "") or "")
|
||||
domain = self._get_session_domain(session)
|
||||
with self._lock:
|
||||
if int(self._pending_high_by_session.get(session_id, 0)) > 0:
|
||||
return True
|
||||
if int(self._running_high_by_session.get(session_id, 0)) > 0:
|
||||
return True
|
||||
if domain and int(self._pending_high_by_domain.get(domain, 0)) > 0:
|
||||
return True
|
||||
if domain and int(self._running_high_by_domain.get(domain, 0)) > 0:
|
||||
return True
|
||||
return False
|
||||
def _get_session_domain(self, session: 'TabSession') -> str:
|
||||
try:
|
||||
url = str(getattr(session.tab, "url", "") or "")
|
||||
domain = str(extract_remote_site_domain(url) or "").strip().lower()
|
||||
if domain:
|
||||
session.current_domain = domain
|
||||
return domain
|
||||
except Exception:
|
||||
pass
|
||||
domain = str(getattr(session, "current_domain", "") or "").strip().lower()
|
||||
if domain:
|
||||
return domain
|
||||
return ""
|
||||
def _get_active_workflow_runtime(self, session: 'TabSession') -> Optional[Dict[str, Any]]:
|
||||
stack = getattr(session, "_workflow_runtime_stack", None) or []
|
||||
if not stack:
|
||||
return None
|
||||
runtime = stack[-1]
|
||||
return runtime if isinstance(runtime, dict) else None
|
||||
def _build_request_count_state_key(self, command: Dict, session: 'TabSession') -> tuple:
|
||||
trigger = command.get("trigger", {}) or {}
|
||||
scope = str(trigger.get("scope", "all") or "all").strip().lower()
|
||||
|
||||
if scope == "all":
|
||||
return (command["id"], "__scope:all")
|
||||
|
||||
if scope == "domain":
|
||||
domain_key = str(trigger.get("domain", "") or "").strip().lower()
|
||||
if not domain_key:
|
||||
domain_key = self._get_session_domain(session)
|
||||
return (command["id"], f"__scope:domain:{domain_key or '_'}")
|
||||
|
||||
return (command["id"], session.id)
|
||||
def _get_scope_request_count(self, command: Dict, session: 'TabSession') -> int:
|
||||
trigger = command.get("trigger", {}) or {}
|
||||
scope = str(trigger.get("scope", "all") or "all").strip().lower()
|
||||
|
||||
if scope == "tab":
|
||||
try:
|
||||
return int(getattr(session, "request_count", 0) or 0)
|
||||
except Exception:
|
||||
return 0
|
||||
|
||||
try:
|
||||
browser = self._get_browser()
|
||||
pool = getattr(browser, "_tab_pool", None)
|
||||
if pool is None:
|
||||
return int(getattr(session, "request_count", 0) or 0)
|
||||
status = pool.get_status() if hasattr(pool, "get_status") else {}
|
||||
tabs = status.get("tabs", []) or []
|
||||
except Exception:
|
||||
return int(getattr(session, "request_count", 0) or 0)
|
||||
|
||||
if scope == "all":
|
||||
total = 0
|
||||
for tab in tabs:
|
||||
try:
|
||||
total += int(tab.get("request_count", 0) or 0)
|
||||
except Exception:
|
||||
continue
|
||||
return total
|
||||
|
||||
if scope == "domain":
|
||||
target_domain = str(trigger.get("domain", "") or "").strip().lower()
|
||||
if not target_domain:
|
||||
target_domain = self._get_session_domain(session)
|
||||
if not target_domain:
|
||||
return int(getattr(session, "request_count", 0) or 0)
|
||||
|
||||
total = 0
|
||||
for tab in tabs:
|
||||
try:
|
||||
tab_domain = str(tab.get("current_domain", "") or "").strip().lower()
|
||||
if not tab_domain:
|
||||
url = str(tab.get("url", "") or "")
|
||||
if "://" in url:
|
||||
tab_domain = url.split("//", 1)[1].split("/", 1)[0].strip().lower()
|
||||
if self._domain_matches(target_domain, tab_domain):
|
||||
total += int(tab.get("request_count", 0) or 0)
|
||||
except Exception:
|
||||
continue
|
||||
return total
|
||||
|
||||
try:
|
||||
return int(getattr(session, "request_count", 0) or 0)
|
||||
except Exception:
|
||||
return 0
|
||||
def _should_trigger(self, command: Dict, session: 'TabSession') -> bool:
|
||||
trigger = command.get("trigger", {})
|
||||
trigger_type = trigger.get("type", "")
|
||||
scope = trigger.get("scope", "all")
|
||||
|
||||
# Scope pre-check
|
||||
if scope == "domain":
|
||||
target_domain = str(trigger.get("domain", "") or "").strip().lower()
|
||||
session_domain = self._get_session_domain(session)
|
||||
if target_domain and session_domain:
|
||||
if not self._domain_matches(target_domain, session_domain):
|
||||
return False
|
||||
elif target_domain:
|
||||
return False
|
||||
elif scope == "tab":
|
||||
target_index = trigger.get("tab_index")
|
||||
if target_index is not None and session.persistent_index != target_index:
|
||||
return False
|
||||
|
||||
# Skip if same command already executing on this tab
|
||||
exec_key = (command["id"], session.id)
|
||||
with self._lock:
|
||||
if exec_key in self._executing:
|
||||
return False
|
||||
|
||||
request_state_key = None
|
||||
scope_request_count = None
|
||||
initial_req = None
|
||||
if trigger_type == "request_count":
|
||||
request_state_key = self._build_request_count_state_key(command, session)
|
||||
scope_request_count = self._get_scope_request_count(command, session)
|
||||
try:
|
||||
# Count the current completed request as the first hit so
|
||||
# a threshold of N fires on the Nth completed request rather
|
||||
# than the (N+1)th after state initialization.
|
||||
initial_req = max(0, int(scope_request_count) - 1)
|
||||
except Exception:
|
||||
initial_req = scope_request_count
|
||||
|
||||
# Initialize or load trigger state
|
||||
state, is_new = self._ensure_trigger_state(
|
||||
command["id"],
|
||||
session,
|
||||
state_key=request_state_key,
|
||||
initial_req=initial_req if trigger_type == "request_count" else scope_request_count,
|
||||
)
|
||||
if is_new and trigger_type not in {"page_check", "command_check"}:
|
||||
return False # Newly initialized: wait for next check cycle
|
||||
|
||||
# Evaluate trigger condition by trigger type
|
||||
if trigger_type == "request_count":
|
||||
threshold = max(1, self._coerce_int(trigger.get("value", 10), 10))
|
||||
current_count = (
|
||||
int(scope_request_count)
|
||||
if scope_request_count is not None
|
||||
else self._get_scope_request_count(command, session)
|
||||
)
|
||||
with self._lock:
|
||||
baseline = int(state.get("req", 0))
|
||||
delta = current_count - baseline
|
||||
should_fire = delta >= threshold
|
||||
if should_fire:
|
||||
# 先记录旧基线;若后续因标签页忙碌/超时未实际执行,可回滚以便重试。
|
||||
state["req_prev"] = baseline
|
||||
state["req_pending"] = True
|
||||
state["req"] = current_count
|
||||
logger.debug_throttled(
|
||||
f"cmd.request_count.{command.get('id')}:{session.id}:{self._format_scope_label(scope)}",
|
||||
f"[CMD] 请求计数检查: {command.get('name')} "
|
||||
f"(当前={current_count}, 基线={baseline}, 增量={delta}, 阈值={threshold}, "
|
||||
f"标签页={session.id}, 范围={self._format_scope_label(scope)})",
|
||||
interval_sec=10.0,
|
||||
)
|
||||
if should_fire:
|
||||
logger.info(
|
||||
f"[CMD] 触发命令: {command.get('name')} "
|
||||
f"(请求增量={delta}, 阈值={threshold}, 标签页={session.id}, "
|
||||
f"范围={self._format_scope_label(scope)})"
|
||||
)
|
||||
return True
|
||||
|
||||
elif trigger_type == "error_count":
|
||||
threshold = max(1, self._coerce_int(trigger.get("value", 3), 3))
|
||||
delta = session.error_count - state["err"]
|
||||
if delta >= threshold:
|
||||
logger.info(
|
||||
f"[CMD] 触发命令: {command.get('name')} "
|
||||
f"(错误增量={delta}, 阈值={threshold})"
|
||||
)
|
||||
with self._lock:
|
||||
state["err"] = session.error_count
|
||||
return True
|
||||
|
||||
elif trigger_type == "idle_timeout":
|
||||
threshold_sec = max(1.0, self._coerce_float(trigger.get("value", 300), 300.0))
|
||||
idle = time.time() - session.last_used_at
|
||||
if idle >= threshold_sec:
|
||||
logger.info(
|
||||
f"[CMD] 触发命令: {command.get('name')} "
|
||||
f"(空闲时长={idle:.0f}秒, 阈值={threshold_sec}秒)"
|
||||
)
|
||||
return True
|
||||
|
||||
elif trigger_type == "page_check":
|
||||
check_text = str(trigger.get("value", ""))
|
||||
normalized_text = check_text.lower().strip()
|
||||
op, keywords = self._parse_page_check_expression(check_text)
|
||||
probe_js = str(trigger.get("probe_js", "") or "").strip()
|
||||
match_info = (
|
||||
self._evaluate_page_check_expr(session, op, keywords)
|
||||
if keywords else {
|
||||
"hit": True if probe_js else False,
|
||||
"matched_keywords": [],
|
||||
"snapshot_preview": "",
|
||||
}
|
||||
)
|
||||
current_hit = bool(match_info.get("hit"))
|
||||
probe_always = bool(trigger.get("probe_always", False))
|
||||
if probe_js and (current_hit or probe_always):
|
||||
probe_info = self._evaluate_page_check_probe(session, probe_js)
|
||||
match_info["probe_hit"] = bool(probe_info.get("hit"))
|
||||
match_info["probe_result"] = probe_info.get("result")
|
||||
match_info["probe_summary"] = str(probe_info.get("summary") or "").strip()
|
||||
current_hit = bool(probe_info.get("hit"))
|
||||
match_info["hit"] = current_hit
|
||||
fire_mode = str(trigger.get("fire_mode", "edge") or "edge").strip().lower()
|
||||
cooldown_sec = max(0.0, self._coerce_float(trigger.get("cooldown_sec", 0), 0.0))
|
||||
stable_for_sec = max(0.0, self._coerce_float(trigger.get("stable_for_sec", 0), 0.0))
|
||||
now_ts = time.time()
|
||||
|
||||
once_per_request = bool(trigger.get("once_per_request", False))
|
||||
current_request_id = ""
|
||||
if once_per_request:
|
||||
try:
|
||||
for attr_name in ("_command_request_id", "_bound_request_id"):
|
||||
req_id = str(getattr(session, attr_name, "") or "").strip()
|
||||
if req_id:
|
||||
current_request_id = req_id
|
||||
break
|
||||
if not current_request_id:
|
||||
task_id = str(getattr(session, "current_task_id", "") or "").strip()
|
||||
if task_id.startswith("req-"):
|
||||
current_request_id = task_id
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
with self._lock:
|
||||
if once_per_request and current_request_id:
|
||||
triggered_reqs = state.setdefault("triggered_requests", set())
|
||||
if current_request_id in triggered_reqs:
|
||||
return False
|
||||
|
||||
prev_key = str(state.get("page_key", ""))
|
||||
prev_hit = bool(state.get("page_hit", False)) if prev_key == normalized_text else False
|
||||
prev_stable = bool(state.get("page_stable", False)) if prev_key == normalized_text else False
|
||||
hit_since = float(state.get("page_hit_since", 0.0) or 0.0) if prev_key == normalized_text else 0.0
|
||||
if current_hit:
|
||||
if not prev_hit or hit_since <= 0:
|
||||
hit_since = now_ts
|
||||
else:
|
||||
hit_since = 0.0
|
||||
state["page_key"] = normalized_text
|
||||
state["page_hit"] = current_hit
|
||||
state["page_hit_since"] = hit_since
|
||||
last_fire_at = float(state.get("page_last_fire_at", 0.0) or 0.0)
|
||||
stable_hit = bool(current_hit and ((now_ts - hit_since) >= stable_for_sec))
|
||||
state["page_stable"] = stable_hit
|
||||
|
||||
if fire_mode == "level":
|
||||
if stable_hit and (cooldown_sec <= 0 or (now_ts - last_fire_at) >= cooldown_sec):
|
||||
if once_per_request and current_request_id:
|
||||
triggered_reqs = state.setdefault("triggered_requests", set())
|
||||
if current_request_id in triggered_reqs:
|
||||
return False
|
||||
triggered_reqs.add(current_request_id)
|
||||
state["page_last_fire_at"] = now_ts
|
||||
logger.info(
|
||||
f"[CMD] 触发命令: {command.get('name')} "
|
||||
f"(页面检查命中, 模式=持续触发, 文本='{check_text[:30]}', "
|
||||
f"冷却={cooldown_sec}秒)"
|
||||
)
|
||||
self._log_page_check_hit_details(command, session, match_info)
|
||||
return True
|
||||
else:
|
||||
if stable_hit and not prev_stable:
|
||||
if once_per_request and current_request_id:
|
||||
triggered_reqs = state.setdefault("triggered_requests", set())
|
||||
if current_request_id in triggered_reqs:
|
||||
return False
|
||||
triggered_reqs.add(current_request_id)
|
||||
state["page_last_fire_at"] = now_ts
|
||||
logger.info(
|
||||
f"[CMD] 触发命令: {command.get('name')} "
|
||||
f"(页面检查命中, 模式=边沿触发, 文本='{check_text[:30]}')"
|
||||
)
|
||||
self._log_page_check_hit_details(command, session, match_info)
|
||||
return True
|
||||
|
||||
elif trigger_type == "command_result_match":
|
||||
dispatch = self._prepare_command_result_trigger_dispatch(command, session)
|
||||
if dispatch:
|
||||
self._set_pending_async_trigger_meta(command, session, dispatch)
|
||||
logger.info(
|
||||
f"[CMD] 触发命令: {command.get('name')} "
|
||||
f"(命中命令结果匹配条件)"
|
||||
)
|
||||
return True
|
||||
|
||||
elif trigger_type == "command_check":
|
||||
dispatch = self._prepare_command_check_dispatch(command, session)
|
||||
if dispatch:
|
||||
self._set_pending_async_trigger_meta(command, session, dispatch)
|
||||
check_info = (dispatch.get("interrupt_context") or {}).get("command_check", {})
|
||||
logger.info(
|
||||
f"[CMD] 触发命令: {command.get('name')} "
|
||||
f"(命令检查命中: 来源={check_info.get('source_command_name', '')}, "
|
||||
f"结果={str(check_info.get('actual_result', '') or '')[:80]})"
|
||||
)
|
||||
return True
|
||||
|
||||
elif trigger_type == "command_result_event":
|
||||
dispatch = self._prepare_command_result_event_dispatch(command, session)
|
||||
if dispatch:
|
||||
self._set_pending_async_trigger_meta(command, session, dispatch)
|
||||
logger.info(
|
||||
f"[CMD] 触发命令: {command.get('name')} "
|
||||
f"(命中命令结果事件条件)"
|
||||
)
|
||||
return True
|
||||
|
||||
elif trigger_type == "network_request_error":
|
||||
dispatch = self._prepare_network_trigger_dispatch(command, session)
|
||||
if dispatch:
|
||||
event = (dispatch.get("interrupt_context") or {}).get("network_event", {})
|
||||
self._set_pending_async_trigger_meta(command, session, dispatch)
|
||||
if (dispatch.get("rollback") or {}).get("token"):
|
||||
logger.info(
|
||||
f"[CMD] 触发命令: {command.get('name')} "
|
||||
f"(网络异常状态码={event.get('status')}, 地址={event.get('url', '')[:80]})"
|
||||
)
|
||||
return True
|
||||
|
||||
return False
|
||||
def _is_command_applicable_on_navigation(self, cmd: Dict[str, Any]) -> bool:
|
||||
"""判断命令是否适用于页面导航/刷新事件触发。"""
|
||||
if not isinstance(cmd, dict) or not bool(cmd.get("enabled", True)):
|
||||
return False
|
||||
|
||||
trigger = cmd.get("trigger", {}) or {}
|
||||
# 1. 显式配置 check_on_navigation / check_on_page_load
|
||||
if bool(trigger.get("check_on_navigation", False)) or bool(cmd.get("check_on_navigation", False)):
|
||||
return True
|
||||
if bool(trigger.get("check_on_page_load", False)) or bool(cmd.get("check_on_page_load", False)):
|
||||
return True
|
||||
|
||||
# 2. page_check 触发器天然依赖页面 DOM/内容,页面跳转刷新后必须立即重新检查
|
||||
trigger_type = str(trigger.get("type", "")).strip().lower()
|
||||
if trigger_type == "page_check":
|
||||
return True
|
||||
|
||||
# 3. 包含 bootstrap_on_session_ready 的 run_js_file 命令需要在页面就绪后注入/执行
|
||||
for action in list(cmd.get("actions") or []):
|
||||
if isinstance(action, dict) and bool(action.get("bootstrap_on_session_ready", False)):
|
||||
return True
|
||||
|
||||
return False
|
||||
def notify_tab_navigated(
|
||||
self,
|
||||
session: 'TabSession',
|
||||
reason: str = "page_reload",
|
||||
async_dispatch: bool = True,
|
||||
) -> None:
|
||||
"""
|
||||
通用的页面导航/刷新通知接口。
|
||||
|
||||
当标签页发生刷新(F5、location.reload、断流恢复刷新等)或导航(URL 跳转、新标签页打开等)时调用。
|
||||
|
||||
执行操作:
|
||||
1. 即时重置 JS 执行失败退避锁(_pc_js_backoff_until = 0.0)和失败计数(_pc_js_failures = 0)
|
||||
2. 清空页面快照缓存与刷新宽限期,重置 observer 探测节流与缓存
|
||||
3. 清除 observer 缓存与 haystack 缓存,重置适用命令的周期倒计时与 edge 锁存
|
||||
4. 异步或同步派发页面导航后评估(_dispatch_tab_navigated_evaluation):
|
||||
- 同步 bootstrap 启动 JS 文件注入(Page.addScriptToEvaluateOnNewDocument + 立即执行)
|
||||
- 重新注入/同步 MutationObserver
|
||||
- 毫秒级触发并评估适用命令
|
||||
"""
|
||||
if session is None:
|
||||
return
|
||||
|
||||
session_id = getattr(session, "id", "")
|
||||
if not session_id:
|
||||
return
|
||||
|
||||
# 1. 同步清理 session 上的快照与退避状态
|
||||
try:
|
||||
setattr(session, "_pc_js_failures", 0)
|
||||
setattr(session, "_pc_js_backoff_until", 0.0)
|
||||
setattr(session, "_pc_refresh_grace_until", 0.0)
|
||||
setattr(session, "_pc_snapshot_cached", None)
|
||||
setattr(session, "_last_pc_observer_check_at", 0.0)
|
||||
setattr(session, "_pc_observer_empty_cleanup_done", False)
|
||||
if hasattr(self, "_invalidate_wake_throttle"):
|
||||
self._invalidate_wake_throttle(session)
|
||||
except Exception as e:
|
||||
logger.debug(f"[CMD] 重置会话退避状态失败(忽略): {e}")
|
||||
|
||||
# 2. 清理引擎层全局缓存与适用命令的状态
|
||||
try:
|
||||
commands = self._load_commands_for_checks()
|
||||
except Exception as e:
|
||||
logger.debug(f"[CMD] 导航通知加载命令失败: {e}")
|
||||
commands = []
|
||||
|
||||
applicable_cmds = []
|
||||
with self._lock:
|
||||
self._observer_keywords_by_session.pop(session_id, None)
|
||||
self._compact_haystack_cache = None
|
||||
for cmd in commands:
|
||||
cmd_id = cmd.get("id")
|
||||
if not cmd_id:
|
||||
continue
|
||||
if self._is_command_applicable_on_navigation(cmd):
|
||||
self._periodic_next_run[(cmd_id, session_id)] = 0.0
|
||||
applicable_cmds.append(cmd)
|
||||
|
||||
for cmd in applicable_cmds:
|
||||
self._reset_page_check_latch(cmd, session, reason=f"navigated:{reason}")
|
||||
|
||||
logger.debug(
|
||||
f"[CMD] 标签页导航/刷新通知: session={session_id}, reason={reason}, async={async_dispatch}"
|
||||
)
|
||||
|
||||
# 3. 派发评估
|
||||
if async_dispatch:
|
||||
self.submit_background_task(self._dispatch_tab_navigated_evaluation, session, reason)
|
||||
else:
|
||||
self._dispatch_tab_navigated_evaluation(session, reason)
|
||||
def _dispatch_tab_navigated_evaluation(
|
||||
self,
|
||||
session: 'TabSession',
|
||||
reason: str = "page_reload",
|
||||
) -> None:
|
||||
"""在页面导航/刷新后执行 bootstrap 脚本注入、observer 同步及命令评估调度。"""
|
||||
if self._is_session_closed(session):
|
||||
return
|
||||
|
||||
self.ensure_scheduler_running()
|
||||
try:
|
||||
commands = self._load_commands_for_checks()
|
||||
except Exception as e:
|
||||
logger.debug(f"[CMD] 导航后评估加载命令失败: {e}")
|
||||
return
|
||||
|
||||
if not commands:
|
||||
return
|
||||
|
||||
# 1. 优先执行/注入 bootstrap 启动脚本(例如 Page.addScriptToEvaluateOnNewDocument 及即时注入)
|
||||
try:
|
||||
self._sync_session_bootstrap_js_files(commands, session)
|
||||
except Exception as e:
|
||||
logger.warning(f"[CMD] 导航后同步 bootstrap 脚本失败: {e}")
|
||||
|
||||
# 2. 同步 Page Check Observer(如果有 page_check 关键词)
|
||||
try:
|
||||
pc_keywords = self._collect_page_check_keywords(commands, session)
|
||||
if pc_keywords:
|
||||
self._ensure_page_check_observer(session, pc_keywords)
|
||||
else:
|
||||
self._clear_page_check_observer(session)
|
||||
except Exception as e:
|
||||
logger.debug(f"[CMD] 导航后同步 observer 失败: {e}")
|
||||
|
||||
# 3. 评估并调度适用的命令
|
||||
applicable_commands = [
|
||||
(idx, cmd) for idx, cmd in enumerate(commands)
|
||||
if self._is_command_applicable_on_navigation(cmd) and self._matches_scope(cmd, session)
|
||||
]
|
||||
applicable_commands.sort(key=lambda item: (-self._get_command_priority(item[1]), item[0]))
|
||||
|
||||
for _, cmd in applicable_commands:
|
||||
with self._command_logging_context(cmd):
|
||||
try:
|
||||
current_status = str(getattr(getattr(session, "status", None), "value", "")).lower()
|
||||
if current_status == "busy" and not self._has_active_workflow(session):
|
||||
continue
|
||||
if current_status not in {"idle", "busy"}:
|
||||
continue
|
||||
if self._should_trigger(cmd, session):
|
||||
meta = self._take_pending_async_trigger_meta(cmd, session) or {}
|
||||
if current_status == "busy":
|
||||
scheduled = self._schedule_command_for_active_workflow(
|
||||
cmd,
|
||||
session,
|
||||
interrupt_context=meta.get("interrupt_context"),
|
||||
trigger_rollback=meta.get("rollback"),
|
||||
)
|
||||
if not scheduled:
|
||||
if meta.get("rollback"):
|
||||
self._rollback_trigger_consumption(cmd, session, meta.get("rollback"))
|
||||
self._finalize_request_count_trigger_state(cmd, session, rollback=True)
|
||||
self._reset_page_check_latch(cmd, session, reason="workflow_schedule_failed")
|
||||
else:
|
||||
self._execute_command_async(
|
||||
cmd,
|
||||
session,
|
||||
interrupt_context=meta.get("interrupt_context"),
|
||||
trigger_rollback=meta.get("rollback"),
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"[CMD] 导航后触发检查失败 [{cmd.get('name')}]: {e}")
|
||||
def _finalize_request_count_trigger_state(
|
||||
self,
|
||||
command: Dict,
|
||||
session: 'TabSession',
|
||||
*,
|
||||
rollback: bool = False
|
||||
):
|
||||
"""
|
||||
收尾 request_count 触发状态:
|
||||
- rollback=True: 未实际执行时回滚 req 到触发前基线,避免触发被“吃掉”
|
||||
- rollback=False: 实际开始执行后清理 pending 标记
|
||||
"""
|
||||
trigger = command.get("trigger", {}) or {}
|
||||
if str(trigger.get("type", "")).strip().lower() != "request_count":
|
||||
return
|
||||
|
||||
key = self._build_request_count_state_key(command, session)
|
||||
with self._lock:
|
||||
state = self._trigger_states.get(key)
|
||||
if not state:
|
||||
return
|
||||
|
||||
pending = bool(state.pop("req_pending", False))
|
||||
prev_req = state.pop("req_prev", None)
|
||||
if rollback and pending and prev_req is not None:
|
||||
try:
|
||||
state["req"] = int(prev_req)
|
||||
except Exception:
|
||||
pass
|
||||
@staticmethod
|
||||
def _execution_needs_page_check_retry(execution_result: Any) -> bool:
|
||||
"""
|
||||
Determine whether a page_check-triggered command should be retried.
|
||||
Retry signal is inferred from:
|
||||
- Step-level ok flag being False (exception during execution)
|
||||
- Action results shaped like {"ok": False, ...}
|
||||
- String results containing known failure prefixes (e.g. "js_failed:")
|
||||
"""
|
||||
if not isinstance(execution_result, dict):
|
||||
return False
|
||||
|
||||
_FAIL_PREFIXES = (
|
||||
"js_failed:",
|
||||
"python_failed:",
|
||||
"unsupported_lang:",
|
||||
"ERROR:",
|
||||
"refresh_failed:",
|
||||
"navigate_failed:",
|
||||
)
|
||||
|
||||
direct_result = execution_result.get("result")
|
||||
if isinstance(direct_result, dict) and direct_result.get("ok") is False:
|
||||
return True
|
||||
if isinstance(direct_result, str) and direct_result.startswith(_FAIL_PREFIXES):
|
||||
return True
|
||||
|
||||
steps = execution_result.get("steps")
|
||||
if isinstance(steps, list):
|
||||
for step in steps:
|
||||
if not isinstance(step, dict):
|
||||
continue
|
||||
# Check step-level ok flag (set by _execute_simple on exceptions)
|
||||
if step.get("ok") is False:
|
||||
return True
|
||||
step_result = step.get("result")
|
||||
if isinstance(step_result, dict) and step_result.get("ok") is False:
|
||||
return True
|
||||
if isinstance(step_result, str) and step_result.startswith(_FAIL_PREFIXES):
|
||||
return True
|
||||
return False
|
||||
def _normalize_match_rule(self, rule: Any) -> str:
|
||||
rule_value = str(rule or "").strip().lower()
|
||||
mapping = {
|
||||
"eq": "equals",
|
||||
"equal": "equals",
|
||||
"equals": "equals",
|
||||
"is": "equals",
|
||||
"contains": "contains",
|
||||
"include": "contains",
|
||||
"includes": "contains",
|
||||
"ne": "not_equals",
|
||||
"not_equal": "not_equals",
|
||||
"not_equals": "not_equals",
|
||||
}
|
||||
return mapping.get(rule_value, "equals")
|
||||
@@ -0,0 +1,225 @@
|
||||
"""
|
||||
app/services/command_result_store.py - 通用命令执行结果持久化存储层 (CRUD)
|
||||
|
||||
职责:
|
||||
- 提供命令结果记录的底层 JSON 文件读写与增删改查;
|
||||
- 维护原子写入与最大记录数上限控制;
|
||||
- 存储层保持纯粹 CRUD,不耦合特定厂商或业务规则。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import tempfile
|
||||
import threading
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
from typing import Any, Callable, Dict, Iterable, List, Optional
|
||||
|
||||
try:
|
||||
import requests
|
||||
except ImportError: # pragma: no cover
|
||||
requests = None
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_PROJECT_ROOT = Path(__file__).resolve().parents[2]
|
||||
RESULTS_FILE = _PROJECT_ROOT / "config" / "command_results.local.json"
|
||||
_LOCK = threading.RLock()
|
||||
_MAX_RECORDS = 5000
|
||||
|
||||
|
||||
def _atomic_write_json(path: Path, payload: Dict[str, Any]) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
fd, temp_name = tempfile.mkstemp(prefix=f"{path.stem}_", suffix=".tmp", dir=str(path.parent))
|
||||
try:
|
||||
with os.fdopen(fd, "w", encoding="utf-8", newline="\n") as handle:
|
||||
json.dump(payload, handle, ensure_ascii=False, indent=2)
|
||||
handle.write("\n")
|
||||
handle.flush()
|
||||
os.fsync(handle.fileno())
|
||||
os.replace(temp_name, path)
|
||||
finally:
|
||||
if os.path.exists(temp_name):
|
||||
os.remove(temp_name)
|
||||
|
||||
|
||||
def _load_results() -> Dict[str, Any]:
|
||||
if not RESULTS_FILE.exists():
|
||||
return {"version": 1, "records": []}
|
||||
try:
|
||||
with RESULTS_FILE.open("r", encoding="utf-8") as handle:
|
||||
payload = json.load(handle)
|
||||
except (OSError, ValueError):
|
||||
return {"version": 1, "records": []}
|
||||
records = payload.get("records") if isinstance(payload, dict) else []
|
||||
return {"version": 1, "records": records if isinstance(records, list) else []}
|
||||
|
||||
|
||||
def get_all_records() -> List[Dict[str, Any]]:
|
||||
"""获取所有记录副本。"""
|
||||
with _LOCK:
|
||||
return list(_load_results()["records"])
|
||||
|
||||
|
||||
def get_record(record_id: str) -> Optional[Dict[str, Any]]:
|
||||
"""根据 ID 获取单条记录。"""
|
||||
rid = str(record_id or "").strip()
|
||||
if not rid:
|
||||
return None
|
||||
with _LOCK:
|
||||
for item in _load_results()["records"]:
|
||||
if isinstance(item, dict) and str(item.get("id") or "") == rid:
|
||||
return dict(item)
|
||||
return None
|
||||
|
||||
|
||||
def list_command_results(command_id: str, limit: int = 500) -> List[Dict[str, Any]]:
|
||||
"""根据 command_id 检索结果列表,按创建时间倒序。"""
|
||||
command_id = str(command_id or "").strip()
|
||||
with _LOCK:
|
||||
records = _load_results()["records"]
|
||||
filtered = [item for item in records if isinstance(item, dict) and item.get("command_id") == command_id]
|
||||
filtered.sort(key=lambda item: float(item.get("created_at") or 0), reverse=True)
|
||||
return filtered[: max(1, min(int(limit or 500), 2000))]
|
||||
|
||||
|
||||
def add_record(record: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"""新增单条命令执行结果。"""
|
||||
if not isinstance(record, dict):
|
||||
raise ValueError("record must be a dict")
|
||||
rec = dict(record)
|
||||
if not rec.get("id"):
|
||||
rec["id"] = uuid.uuid4().hex
|
||||
with _LOCK:
|
||||
payload = _load_results()
|
||||
records = payload["records"]
|
||||
records.append(rec)
|
||||
payload["records"] = records[-_MAX_RECORDS:]
|
||||
_atomic_write_json(RESULTS_FILE, payload)
|
||||
return rec
|
||||
|
||||
|
||||
def add_records(records: Iterable[Dict[str, Any]]) -> List[Dict[str, Any]]:
|
||||
"""批量新增命令执行结果。"""
|
||||
items: List[Dict[str, Any]] = []
|
||||
for raw in records:
|
||||
if isinstance(raw, dict):
|
||||
item = dict(raw)
|
||||
if not item.get("id"):
|
||||
item["id"] = uuid.uuid4().hex
|
||||
items.append(item)
|
||||
if not items:
|
||||
return []
|
||||
with _LOCK:
|
||||
payload = _load_results()
|
||||
existing = payload["records"]
|
||||
existing.extend(items)
|
||||
payload["records"] = existing[-_MAX_RECORDS:]
|
||||
_atomic_write_json(RESULTS_FILE, payload)
|
||||
return items
|
||||
|
||||
|
||||
def add_records_unique(
|
||||
records: Iterable[Dict[str, Any]],
|
||||
unique_fields: Iterable[str],
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""Add records while atomically suppressing duplicates by business key."""
|
||||
fields = tuple(str(field) for field in unique_fields if str(field))
|
||||
if not fields:
|
||||
return add_records(records)
|
||||
|
||||
items: List[Dict[str, Any]] = []
|
||||
for raw in records:
|
||||
if isinstance(raw, dict):
|
||||
item = dict(raw)
|
||||
if not item.get("id"):
|
||||
item["id"] = uuid.uuid4().hex
|
||||
items.append(item)
|
||||
if not items:
|
||||
return []
|
||||
|
||||
def key(item: Dict[str, Any]) -> tuple[Any, ...]:
|
||||
return tuple(str(item.get(field) or "") for field in fields)
|
||||
|
||||
with _LOCK:
|
||||
payload = _load_results()
|
||||
existing = payload["records"]
|
||||
seen = {key(item) for item in existing if isinstance(item, dict)}
|
||||
accepted = []
|
||||
for item in items:
|
||||
item_key = key(item)
|
||||
if item_key in seen:
|
||||
continue
|
||||
seen.add(item_key)
|
||||
accepted.append(item)
|
||||
if accepted:
|
||||
existing.extend(accepted)
|
||||
payload["records"] = existing[-_MAX_RECORDS:]
|
||||
_atomic_write_json(RESULTS_FILE, payload)
|
||||
return accepted
|
||||
|
||||
|
||||
def delete_record(record_id: str) -> bool:
|
||||
"""根据 ID 删除单条记录。"""
|
||||
rid = str(record_id or "").strip()
|
||||
if not rid:
|
||||
return False
|
||||
with _LOCK:
|
||||
payload = _load_results()
|
||||
before = len(payload["records"])
|
||||
payload["records"] = [
|
||||
item for item in payload["records"]
|
||||
if not (isinstance(item, dict) and str(item.get("id") or "") == rid)
|
||||
]
|
||||
removed = before != len(payload["records"])
|
||||
if removed:
|
||||
_atomic_write_json(RESULTS_FILE, payload)
|
||||
return removed
|
||||
|
||||
|
||||
def clear_command_results(command_id: str, rule_id: str = "") -> int:
|
||||
"""清空指定 command_id(及可选 rule_id)的结果。"""
|
||||
command_id = str(command_id or "").strip()
|
||||
rule_id = str(rule_id or "").strip()
|
||||
with _LOCK:
|
||||
payload = _load_results()
|
||||
before = len(payload["records"])
|
||||
payload["records"] = [
|
||||
item
|
||||
for item in payload["records"]
|
||||
if item.get("command_id") != command_id
|
||||
or (rule_id and str(item.get("rule_id") or "") != rule_id)
|
||||
]
|
||||
removed = before - len(payload["records"])
|
||||
if removed:
|
||||
_atomic_write_json(RESULTS_FILE, payload)
|
||||
return removed
|
||||
|
||||
|
||||
# 业务转发兼容层(保持既有测试与导入不中断)
|
||||
from app.services.arena_rule_service import ( # noqa: E402
|
||||
_detector_accepts,
|
||||
_matches_rule,
|
||||
record_arena_rule_candidates,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"RESULTS_FILE",
|
||||
"_atomic_write_json",
|
||||
"_detector_accepts",
|
||||
"_load_results",
|
||||
"_matches_rule",
|
||||
"add_record",
|
||||
"add_records",
|
||||
"add_records_unique",
|
||||
"clear_command_results",
|
||||
"delete_record",
|
||||
"get_all_records",
|
||||
"get_record",
|
||||
"list_command_results",
|
||||
"record_arena_rule_candidates",
|
||||
"requests",
|
||||
]
|
||||
@@ -0,0 +1,13 @@
|
||||
"""
|
||||
app/services/config - 配置管理模块
|
||||
|
||||
职责:
|
||||
- 导出配置引擎单例(保持向后兼容)
|
||||
"""
|
||||
|
||||
from .engine import ConfigEngine, ConfigConstants, DEFAULT_WORKFLOW
|
||||
|
||||
# 创建单例
|
||||
config_engine = ConfigEngine()
|
||||
|
||||
__all__ = ['config_engine', 'ConfigEngine', 'ConfigConstants', 'DEFAULT_WORKFLOW']
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -0,0 +1,48 @@
|
||||
"""
|
||||
app/services/config/cache.py - lightweight in-memory cache for config reads.
|
||||
"""
|
||||
|
||||
import threading
|
||||
import time
|
||||
from typing import Any, Dict, Tuple
|
||||
|
||||
|
||||
_CACHE_MISS = object()
|
||||
|
||||
|
||||
class ConfigCache:
|
||||
"""Small TTL cache used only for derived config read results."""
|
||||
|
||||
def __init__(self, ttl: float = 5.0):
|
||||
try:
|
||||
ttl_value = float(ttl)
|
||||
except (TypeError, ValueError):
|
||||
ttl_value = 5.0
|
||||
self._ttl = max(0.0, ttl_value)
|
||||
self._cache: Dict[str, Tuple[Any, float]] = {}
|
||||
self._lock = threading.RLock()
|
||||
|
||||
def get(self, key: str, default: Any = _CACHE_MISS) -> Any:
|
||||
now = time.monotonic()
|
||||
with self._lock:
|
||||
entry = self._cache.get(key)
|
||||
if entry is None:
|
||||
return default
|
||||
|
||||
value, created_at = entry
|
||||
if self._ttl and now - created_at > self._ttl:
|
||||
self._cache.pop(key, None)
|
||||
return default
|
||||
|
||||
return value
|
||||
|
||||
def set(self, key: str, value: Any) -> None:
|
||||
with self._lock:
|
||||
self._cache[key] = (value, time.monotonic())
|
||||
|
||||
def invalidate(self, key: str = None) -> None:
|
||||
with self._lock:
|
||||
if key is None:
|
||||
self._cache.clear()
|
||||
else:
|
||||
self._cache.pop(key, None)
|
||||
@@ -0,0 +1,298 @@
|
||||
"""
|
||||
app/services/config/engine.py - 配置引擎主类
|
||||
|
||||
职责:
|
||||
- 配置文件读写
|
||||
- 站点配置管理
|
||||
- 配置缓存与热重载
|
||||
- 图片配置、提取器管理
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import copy
|
||||
import re
|
||||
import threading
|
||||
import time
|
||||
from app.core.config import get_logger
|
||||
from typing import Dict, Optional, List, Any, Set, Union, Callable
|
||||
from urllib.parse import urljoin
|
||||
from app.core.parsers import ParserRegistry
|
||||
from app.models.schemas import (
|
||||
SiteConfig,
|
||||
WorkflowStep,
|
||||
SelectorDefinition,
|
||||
ADVANCED_FIELDS,
|
||||
PRESET_ADVANCED_FIELDS,
|
||||
SITE_ADVANCED_FIELDS,
|
||||
get_default_image_extraction_config,
|
||||
get_default_file_paste_config,
|
||||
get_default_prompt_padding_config,
|
||||
get_default_site_advanced_config,
|
||||
get_default_attachment_monitor_config,
|
||||
get_default_send_confirmation_config,
|
||||
get_modality_policy,
|
||||
get_enabled_modalities,
|
||||
is_modality_enabled,
|
||||
normalize_modalities_config,
|
||||
normalize_modality_policy,
|
||||
)
|
||||
from app.services.extractor_manager import extractor_manager
|
||||
from app.services.parser_manager import parser_manager
|
||||
from app.utils.site_rules import derive_site_card_id, get_site_rule
|
||||
from app.utils.site_discovery import automatic_discovery_allowed, specific_chat_selectors, selectors_match_chat_html
|
||||
from app.core.request_transport import (
|
||||
get_default_request_transport_config,
|
||||
normalize_request_transport_config,
|
||||
)
|
||||
from .cache import ConfigCache
|
||||
from .site_store import SiteStore
|
||||
from .managers import GlobalConfigManager, ImagePresetsManager
|
||||
from .processors import HTMLCleaner, SelectorValidator, AIAnalyzer
|
||||
|
||||
|
||||
# R2-3:模块级定义迁到 engine_common,这里重新导出以保持兼容(外部与测试都从 engine 导入这些名字)
|
||||
from .engine_common import ( # noqa: E402,F401
|
||||
ConfigConstants,
|
||||
DEFAULT_PRESET_NAME,
|
||||
DEFAULT_WORKFLOW,
|
||||
PRESET_FIELDS,
|
||||
_MISSING,
|
||||
_SITE_STARTUP_JS_PATTERNS,
|
||||
_coerce_bool,
|
||||
_merge_config_patch,
|
||||
get_default_network_config,
|
||||
get_default_stream_config,
|
||||
logger,
|
||||
)
|
||||
from .engine_local_overrides import ConfigLocalOverridesMixin # noqa: E402
|
||||
from .engine_migrations import ConfigMigrationsMixin # noqa: E402
|
||||
from .engine_presets import ConfigPresetsMixin # noqa: E402
|
||||
from .engine_sites import ConfigSitesMixin # noqa: E402
|
||||
from .engine_media import ConfigMediaMixin # noqa: E402
|
||||
from .engine_stream import ConfigStreamMixin # noqa: E402
|
||||
|
||||
# ================= 配置引擎主类 =================
|
||||
|
||||
class ConfigEngine(
|
||||
ConfigLocalOverridesMixin,
|
||||
ConfigMigrationsMixin,
|
||||
ConfigPresetsMixin,
|
||||
ConfigSitesMixin,
|
||||
ConfigMediaMixin,
|
||||
ConfigStreamMixin,
|
||||
):
|
||||
"""配置引擎主类"""
|
||||
def __init__(self):
|
||||
self.config_file = ConfigConstants.CONFIG_FILE # 旧版单文件,仅用于自动迁移
|
||||
self.sites_dir = ConfigConstants.SITES_DIR
|
||||
self.site_store = SiteStore(self.sites_dir, self.config_file)
|
||||
self.local_sites_file = ConfigConstants.SITES_LOCAL_FILE
|
||||
self._io_lock = threading.RLock()
|
||||
self.last_mtime = 0.0
|
||||
self.last_signature: tuple = ()
|
||||
self.last_local_mtime = 0.0
|
||||
self.sites: Dict[str, SiteConfig] = {}
|
||||
self._global_default_presets: Dict[str, str] = {}
|
||||
self._local_default_presets: Dict[str, str] = {}
|
||||
self._local_sites_payload: Dict[str, Any] = {}
|
||||
self._cache = ConfigCache(ttl=5.0)
|
||||
|
||||
# 子管理器
|
||||
self.global_config = GlobalConfigManager()
|
||||
self.image_presets = ImagePresetsManager(ConfigConstants.IMAGE_PRESETS_FILE)
|
||||
|
||||
# 加载配置
|
||||
self._load_config()
|
||||
self._migrate_global_commands()
|
||||
|
||||
# 处理器
|
||||
self.html_cleaner = HTMLCleaner()
|
||||
self.validator = SelectorValidator(self.global_config.get_fallback_selectors())
|
||||
self.ai_analyzer = AIAnalyzer(self.global_config)
|
||||
|
||||
# 迁移旧配置(顺序重要:先转预设格式,再补缺失字段,最后清理残留)
|
||||
self._migrate_loaded_config()
|
||||
self._apply_local_site_overrides()
|
||||
|
||||
logger.debug(f"配置引擎已初始化,已加载 {len(self.sites)} 个站点配置")
|
||||
def _load_config(self):
|
||||
with self._io_lock:
|
||||
return self._load_config_locked()
|
||||
def _load_config_locked(self):
|
||||
"""初始化加载站点配置(config/sites/ 目录;发现旧版 sites.json 时自动迁移)"""
|
||||
try:
|
||||
data = self.site_store.load()
|
||||
self.last_signature = self.site_store.signature()
|
||||
if data is None:
|
||||
logger.info(f"站点配置目录 {self.sites_dir} 为空,将在首次保存时创建")
|
||||
self._apply_local_site_overrides()
|
||||
return
|
||||
self.last_mtime = time.time()
|
||||
|
||||
# 提取并加载 _global;缺失时重置为默认全局配置。
|
||||
global_section = data.pop("_global", {})
|
||||
self.global_config.load(global_section if isinstance(global_section, dict) else {})
|
||||
|
||||
# 过滤内部键
|
||||
self.sites = {
|
||||
k: v for k, v in data.items()
|
||||
if not k.startswith('_')
|
||||
}
|
||||
self._refresh_global_default_presets_from_sites()
|
||||
self._apply_local_site_overrides()
|
||||
logger.debug(f"已加载站点配置: {self.sites_dir}({len(self.sites)} 个站点)")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"加载配置失败: {e}")
|
||||
def refresh_if_changed(self):
|
||||
"""检查文件是否变化,如果变化则重载"""
|
||||
if not self.site_store.exists() and not os.path.exists(self.local_sites_file):
|
||||
return
|
||||
|
||||
try:
|
||||
current_signature = self.site_store.signature()
|
||||
current_local_mtime = os.path.getmtime(self.local_sites_file) if os.path.exists(self.local_sites_file) else 0.0
|
||||
if current_signature != self.last_signature or current_local_mtime != self.last_local_mtime:
|
||||
logger.debug("⚡ 检测到站点配置变化,热重载")
|
||||
self.reload_config()
|
||||
except Exception as e:
|
||||
logger.error(f"检查文件变化失败: {e}")
|
||||
def reload_config(self):
|
||||
with self._io_lock:
|
||||
return self._reload_config_locked()
|
||||
def _reload_config_locked(self):
|
||||
"""重新加载配置(Hot Reload)"""
|
||||
self._cache.invalidate()
|
||||
|
||||
if not self.site_store.exists():
|
||||
logger.warning("重载失败:站点配置目录为空")
|
||||
return
|
||||
|
||||
try:
|
||||
previous = {"_global": self.global_config.to_dict(), **self.sites}
|
||||
data = self.site_store.load(previous=previous)
|
||||
if data is None:
|
||||
logger.warning("重载失败:站点配置目录为空")
|
||||
return
|
||||
signature = self.site_store.signature()
|
||||
|
||||
# 提取并加载 _global;缺失时重置为默认全局配置。
|
||||
global_section = data.pop("_global", {})
|
||||
self.global_config.load(global_section if isinstance(global_section, dict) else {})
|
||||
self.validator.fallback_selectors = self.global_config.get_fallback_selectors()
|
||||
|
||||
# 过滤内部键
|
||||
self.sites = {
|
||||
k: v for k, v in data.items()
|
||||
if not k.startswith('_')
|
||||
}
|
||||
self.last_signature = signature
|
||||
self.last_mtime = time.time()
|
||||
self._refresh_global_default_presets_from_sites()
|
||||
self._migrate_loaded_config()
|
||||
self._apply_local_site_overrides()
|
||||
logger.debug(f"✅ 配置已热重载 (Sites: {len(self.sites)})")
|
||||
|
||||
except json.JSONDecodeError as e:
|
||||
logger.error(f"❌ 重载配置失败(JSON格式错误),保留旧配置: {e}")
|
||||
except Exception as e:
|
||||
logger.error(f"❌ 重载配置失败: {e}")
|
||||
def save_config(self):
|
||||
"""公开的保存方法(供 API 调用)"""
|
||||
return self._save_config()
|
||||
def _save_config(self) -> bool:
|
||||
with self._io_lock:
|
||||
return self._save_config_locked()
|
||||
def _save_config_locked(self) -> bool:
|
||||
"""保存站点配置(只重写有变化的站点文件,每个文件原子替换)"""
|
||||
local_snapshot: Optional[tuple[bool, bytes, Dict[str, Any], Dict[str, str]]] = None
|
||||
local_overrides_written = False
|
||||
default_maps_snapshot = (
|
||||
copy.deepcopy(self._global_default_presets),
|
||||
copy.deepcopy(self._local_default_presets),
|
||||
)
|
||||
|
||||
try:
|
||||
self._prune_default_preset_maps()
|
||||
persisted_sites = {}
|
||||
for domain, site in self.sites.items():
|
||||
if domain.startswith('_') or not isinstance(site, dict):
|
||||
continue
|
||||
|
||||
site_copy = copy.deepcopy(site)
|
||||
from app.utils.site_url import route_domain_matches
|
||||
if route_domain_matches("arena.ai", domain):
|
||||
try:
|
||||
from app.services.arena_model_catalog import load_arena_model_catalog_data
|
||||
cat_data = load_arena_model_catalog_data()
|
||||
d_key = str(domain).strip().lower().strip(".")
|
||||
domain_cat = cat_data.get(d_key, {})
|
||||
except Exception:
|
||||
domain_cat = None
|
||||
|
||||
presets_dict = site_copy.get("presets")
|
||||
if isinstance(presets_dict, dict):
|
||||
for p_key, p_val in presets_dict.items():
|
||||
if isinstance(p_val, dict) and "model_catalog" in p_val:
|
||||
if domain_cat is not None and str(p_key).strip() in domain_cat:
|
||||
p_val.pop("model_catalog", None)
|
||||
# 同步清理内存中的残留
|
||||
mem_presets = site.get("presets")
|
||||
if isinstance(mem_presets, dict):
|
||||
for p_key, p_val in mem_presets.items():
|
||||
if isinstance(p_val, dict) and "model_catalog" in p_val:
|
||||
if domain_cat is not None and str(p_key).strip() in domain_cat:
|
||||
p_val.pop("model_catalog", None)
|
||||
|
||||
persisted_default = self._get_persisted_default_preset(domain, site_copy)
|
||||
if persisted_default:
|
||||
site_copy["default_preset"] = persisted_default
|
||||
else:
|
||||
site_copy.pop("default_preset", None)
|
||||
persisted_sites[domain] = site_copy
|
||||
|
||||
# 构建完整配置(包含 _global)
|
||||
full_config = {
|
||||
"_global": self.global_config.to_dict(),
|
||||
**persisted_sites
|
||||
}
|
||||
|
||||
local_snapshot = self._snapshot_local_site_overrides_locked()
|
||||
if local_snapshot is None:
|
||||
self._global_default_presets, self._local_default_presets = default_maps_snapshot
|
||||
return False
|
||||
if not self._save_local_site_overrides():
|
||||
self._restore_local_site_overrides_locked(local_snapshot)
|
||||
self._global_default_presets, self._local_default_presets = default_maps_snapshot
|
||||
return False
|
||||
local_overrides_written = True
|
||||
|
||||
written = self.site_store.save(full_config)
|
||||
self.last_signature = self.site_store.signature()
|
||||
self.last_mtime = time.time()
|
||||
|
||||
self._cache.invalidate()
|
||||
logger.info(f"配置已保存: {self.sites_dir}(重写 {len(written)} 个站点文件)")
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"保存配置失败: {e}")
|
||||
if local_overrides_written:
|
||||
self._restore_local_site_overrides_locked(local_snapshot)
|
||||
self._global_default_presets, self._local_default_presets = default_maps_snapshot
|
||||
return False
|
||||
def _migrate_loaded_config(self):
|
||||
"""Run migrations that apply to freshly loaded site config data."""
|
||||
self._migrate_to_presets()
|
||||
self.migrate_site_configs()
|
||||
self._migrate_site_advanced_to_presets()
|
||||
self._cleanup_preset_residuals()
|
||||
try:
|
||||
from app.services.arena_model_catalog import migrate_and_cleanup_sites_model_catalog
|
||||
migrate_and_cleanup_sites_model_catalog(self)
|
||||
except Exception as exc:
|
||||
logger.warning(f"Arena 模型目录自动迁移失败: {exc}")
|
||||
|
||||
|
||||
__all__ = ['ConfigEngine', 'ConfigConstants', 'DEFAULT_WORKFLOW', 'DEFAULT_PRESET_NAME']
|
||||
@@ -0,0 +1,128 @@
|
||||
"""ConfigEngine 及其 mixin 共用的模块级定义(R2-3 拆分时从 engine.py 原样迁出,engine.py 仍重新导出)。"""
|
||||
|
||||
import os
|
||||
import copy
|
||||
import re
|
||||
from app.core.config import get_logger
|
||||
from typing import Dict, List, Any
|
||||
from app.models.schemas import (
|
||||
WorkflowStep,
|
||||
get_default_attachment_monitor_config,
|
||||
get_default_send_confirmation_config,
|
||||
)
|
||||
from app.core.request_transport import get_default_request_transport_config
|
||||
|
||||
|
||||
logger = get_logger("CFG_ENG")
|
||||
|
||||
|
||||
# ================= 常量配置 =================
|
||||
|
||||
class ConfigConstants:
|
||||
"""配置引擎常量"""
|
||||
_PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
|
||||
# R1-2:站点配置的事实来源是 config/sites/ 目录(每站点一个文件);CONFIG_FILE 只作为旧版单文件的迁移来源
|
||||
CONFIG_FILE = os.getenv("SITES_CONFIG_FILE", os.path.join(_PROJECT_ROOT, "config", "sites.json"))
|
||||
SITES_DIR = os.getenv("SITES_CONFIG_DIR", os.path.join(_PROJECT_ROOT, "config", "sites"))
|
||||
SITES_LOCAL_FILE = os.getenv("SITES_LOCAL_FILE", os.path.join(_PROJECT_ROOT, "config", "sites.local.json"))
|
||||
COMMANDS_FILE = os.getenv("COMMANDS_CONFIG_FILE", os.path.join(_PROJECT_ROOT, "config", "commands.json"))
|
||||
COMMANDS_LOCAL_FILE = os.getenv("COMMANDS_LOCAL_FILE", os.path.join(_PROJECT_ROOT, "config", "commands.local.json"))
|
||||
IMAGE_PRESETS_FILE = os.path.join(_PROJECT_ROOT, "config", "image_presets.json")
|
||||
|
||||
MAX_HTML_CHARS = int(os.getenv("MAX_HTML_CHARS", "120000"))
|
||||
TEXT_TRUNCATE_LENGTH = 80
|
||||
|
||||
AI_MAX_RETRIES = 3
|
||||
AI_RETRY_BASE_DELAY = 1.0
|
||||
AI_RETRY_MAX_DELAY = 10.0
|
||||
AI_REQUEST_TIMEOUT = 120
|
||||
|
||||
|
||||
_MISSING = object()
|
||||
|
||||
|
||||
def _coerce_bool(value: Any, default: bool = False) -> bool:
|
||||
"""Parse bool-like config values without treating every non-empty string as true."""
|
||||
if isinstance(value, bool):
|
||||
return value
|
||||
if value is None:
|
||||
return default
|
||||
if isinstance(value, str):
|
||||
lowered = value.strip().lower()
|
||||
if lowered in {"1", "true", "yes", "y", "on"}:
|
||||
return True
|
||||
if lowered in {"0", "false", "no", "n", "off"}:
|
||||
return False
|
||||
return default
|
||||
if isinstance(value, (int, float)):
|
||||
return value != 0
|
||||
return bool(value)
|
||||
|
||||
|
||||
# ================= 预设常量 =================
|
||||
|
||||
DEFAULT_PRESET_NAME = "主预设"
|
||||
|
||||
# 预设内包含的配置字段(用于迁移和校验)
|
||||
PRESET_FIELDS = [
|
||||
"selectors", "workflow", "stream_config",
|
||||
"image_extraction", "file_paste", "prompt_padding", "stealth",
|
||||
"extractor_id", "extractor_verified"
|
||||
]
|
||||
|
||||
# 默认工作流
|
||||
DEFAULT_WORKFLOW: List[WorkflowStep] = [
|
||||
{"action": "CLICK", "target": "new_chat_btn", "optional": True, "value": None},
|
||||
{"action": "WAIT", "target": "", "optional": False, "value": "0.5"},
|
||||
{"action": "FILL_INPUT", "target": "input_box", "optional": False, "value": None},
|
||||
{"action": "CLICK", "target": "send_btn", "optional": True, "value": None},
|
||||
{"action": "KEY_PRESS", "target": "Enter", "optional": True, "value": None},
|
||||
{"action": "STREAM_WAIT", "target": "result_container", "optional": False, "value": None}
|
||||
]
|
||||
|
||||
def get_default_stream_config() -> Dict[str, Any]:
|
||||
"""获取默认流式配置"""
|
||||
return {
|
||||
"mode": "dom", # dom / network
|
||||
"request_transport": get_default_request_transport_config(),
|
||||
"hard_timeout": 300, # 全局硬超时(秒)
|
||||
"send_confirmation": get_default_send_confirmation_config(),
|
||||
"attachment_monitor": get_default_attachment_monitor_config(),
|
||||
|
||||
# 网络监听配置(可选)
|
||||
"network": None
|
||||
}
|
||||
|
||||
|
||||
def get_default_network_config() -> Dict[str, Any]:
|
||||
"""获取默认网络监听配置"""
|
||||
return {
|
||||
"listen_pattern": "", # URL 匹配模式(必填)
|
||||
"stream_match_mode": "keyword", # 流目标匹配模式(keyword / regex)
|
||||
"stream_match_pattern": "", # 流目标匹配表达式(为空时回退到 listen_pattern)
|
||||
"parser": "", # 解析器 ID(必填)
|
||||
"first_response_timeout": 300.0, # 等待首个匹配目标流量的最大时间
|
||||
"silence_threshold": 3.0, # 静默超时(秒)
|
||||
"response_interval": 0.5 # 轮询间隔(秒)
|
||||
}
|
||||
|
||||
|
||||
def _merge_config_patch(base: Any, patch: Any) -> Any:
|
||||
"""Deep-merge object patches while letting scalars/lists replace existing values."""
|
||||
if not isinstance(base, dict) or not isinstance(patch, dict):
|
||||
return copy.deepcopy(patch)
|
||||
|
||||
merged = dict(base)
|
||||
for key, value in patch.items():
|
||||
existing = base.get(key)
|
||||
if isinstance(value, dict) and isinstance(existing, dict):
|
||||
merged[key] = _merge_config_patch(existing, value)
|
||||
else:
|
||||
merged[key] = copy.deepcopy(value)
|
||||
return merged
|
||||
|
||||
|
||||
_SITE_STARTUP_JS_PATTERNS = (
|
||||
re.compile(r"""location\.(?:assign|replace)\(\s*['"]([^'"]+)['"]\s*\)""", re.IGNORECASE),
|
||||
re.compile(r"""location\.href\s*=\s*['"]([^'"]+)['"]""", re.IGNORECASE),
|
||||
)
|
||||
@@ -0,0 +1,226 @@
|
||||
"""ConfigEngine 的一部分(R2-3 拆分):本地站点覆盖(sites.local.json)与默认预设状态。
|
||||
|
||||
方法原样从 engine.py 迁出,行为不变;通过 mixin 组合回 ConfigEngine,
|
||||
实例状态都在 ConfigEngine.__init__(engine.py)里初始化。
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import copy
|
||||
from typing import Dict, Optional, Any
|
||||
from .engine_common import logger
|
||||
|
||||
|
||||
class ConfigLocalOverridesMixin:
|
||||
"""本地站点覆盖(sites.local.json)与默认预设状态"""
|
||||
|
||||
def _load_local_site_overrides(self) -> Dict[str, str]:
|
||||
with self._io_lock:
|
||||
return self._load_local_site_overrides_locked()
|
||||
def _load_local_site_overrides_locked(self) -> Dict[str, str]:
|
||||
"""加载本地站点覆盖配置,并保留未识别字段。"""
|
||||
if not os.path.exists(self.local_sites_file):
|
||||
self.last_local_mtime = 0.0
|
||||
self._local_sites_payload = {}
|
||||
return {}
|
||||
|
||||
try:
|
||||
with open(self.local_sites_file, "r", encoding="utf-8-sig") as f:
|
||||
data = json.load(f)
|
||||
|
||||
self.last_local_mtime = os.path.getmtime(self.local_sites_file)
|
||||
self._local_sites_payload = data if isinstance(data, dict) else {}
|
||||
defaults = self._local_sites_payload.get("default_presets", {})
|
||||
if not isinstance(defaults, dict):
|
||||
return {}
|
||||
|
||||
return {
|
||||
str(domain).strip(): str(preset).strip()
|
||||
for domain, preset in defaults.items()
|
||||
if str(domain).strip() and str(preset).strip()
|
||||
}
|
||||
except json.JSONDecodeError as e:
|
||||
logger.error(f"本地站点覆盖配置格式错误: {e}")
|
||||
return copy.deepcopy(self._local_default_presets or {})
|
||||
except Exception as e:
|
||||
logger.error(f"加载本地站点覆盖配置失败: {e}")
|
||||
return copy.deepcopy(self._local_default_presets or {})
|
||||
def _prune_default_preset_maps(self) -> None:
|
||||
active_domains = {
|
||||
domain for domain, site in self.sites.items()
|
||||
if not domain.startswith('_') and isinstance(site, dict)
|
||||
}
|
||||
self._global_default_presets = {
|
||||
domain: preset
|
||||
for domain, preset in self._global_default_presets.items()
|
||||
if domain in active_domains and str(preset or "").strip()
|
||||
}
|
||||
self._local_default_presets = {
|
||||
domain: preset
|
||||
for domain, preset in self._local_default_presets.items()
|
||||
if domain in active_domains and str(preset or "").strip()
|
||||
}
|
||||
def _refresh_global_default_presets_from_sites(self) -> None:
|
||||
next_defaults: Dict[str, str] = {}
|
||||
for domain, site in self.sites.items():
|
||||
if domain.startswith('_') or not isinstance(site, dict):
|
||||
continue
|
||||
resolved = self._resolve_default_preset_name(site)
|
||||
if resolved:
|
||||
next_defaults[domain] = resolved
|
||||
self._global_default_presets = next_defaults
|
||||
def _get_persisted_default_preset(self, domain: str, site: Dict[str, Any]) -> Optional[str]:
|
||||
presets = site.get("presets", {})
|
||||
if not isinstance(presets, dict) or not presets:
|
||||
self._global_default_presets.pop(domain, None)
|
||||
return None
|
||||
|
||||
candidate = str(self._global_default_presets.get(domain, "") or "").strip()
|
||||
if candidate and candidate in presets:
|
||||
return candidate
|
||||
|
||||
resolved = self._resolve_default_preset_name(site)
|
||||
if resolved:
|
||||
self._global_default_presets[domain] = resolved
|
||||
return resolved
|
||||
|
||||
self._global_default_presets.pop(domain, None)
|
||||
return None
|
||||
def _sync_site_default_preset_state(self, domain: str, site: Dict[str, Any]) -> bool:
|
||||
presets = site.get("presets", {})
|
||||
if not isinstance(presets, dict) or not presets:
|
||||
self._global_default_presets.pop(domain, None)
|
||||
self._local_default_presets.pop(domain, None)
|
||||
if "default_preset" in site:
|
||||
del site["default_preset"]
|
||||
return True
|
||||
return False
|
||||
|
||||
persisted_default = self._get_persisted_default_preset(domain, site)
|
||||
|
||||
local_default = str(self._local_default_presets.get(domain, "") or "").strip()
|
||||
if local_default and local_default not in presets:
|
||||
self._local_default_presets.pop(domain, None)
|
||||
local_default = ""
|
||||
|
||||
if local_default and persisted_default and local_default == persisted_default:
|
||||
self._local_default_presets.pop(domain, None)
|
||||
local_default = ""
|
||||
|
||||
effective_default = local_default or persisted_default
|
||||
if not effective_default:
|
||||
effective_default = self._resolve_default_preset_name(site)
|
||||
|
||||
if effective_default and site.get("default_preset") != effective_default:
|
||||
site["default_preset"] = effective_default
|
||||
return True
|
||||
|
||||
if not effective_default and "default_preset" in site:
|
||||
del site["default_preset"]
|
||||
return True
|
||||
|
||||
return False
|
||||
def _apply_local_site_overrides(self):
|
||||
"""将本地默认预设选择覆盖到当前站点配置。"""
|
||||
self._local_default_presets = self._load_local_site_overrides()
|
||||
applied = 0
|
||||
for domain, site in self.sites.items():
|
||||
if not isinstance(site, dict):
|
||||
continue
|
||||
self._sync_site_default_preset_state(domain, site)
|
||||
local_default = self._local_default_presets.get(domain)
|
||||
if local_default and site.get("default_preset") == local_default:
|
||||
applied += 1
|
||||
|
||||
if applied > 0:
|
||||
logger.debug(f"已应用 {applied} 个本地默认预设覆盖")
|
||||
def _save_local_site_overrides(self) -> bool:
|
||||
with self._io_lock:
|
||||
return self._save_local_site_overrides_locked()
|
||||
def _save_local_site_overrides_locked(self) -> bool:
|
||||
"""保存本地站点覆盖配置。"""
|
||||
tmp_file = self.local_sites_file + ".tmp"
|
||||
self._prune_default_preset_maps()
|
||||
defaults = {}
|
||||
for domain, preset_name in self._local_default_presets.items():
|
||||
site = self.sites.get(domain)
|
||||
if not isinstance(site, dict):
|
||||
continue
|
||||
presets = site.get("presets", {})
|
||||
preset_value = str(preset_name or "").strip()
|
||||
if isinstance(presets, dict) and preset_value and preset_value in presets:
|
||||
defaults[domain] = preset_value
|
||||
|
||||
payload = dict(self._local_sites_payload or {})
|
||||
if os.path.exists(self.local_sites_file):
|
||||
try:
|
||||
with open(self.local_sites_file, "r", encoding="utf-8-sig") as f:
|
||||
latest_payload = json.load(f)
|
||||
if isinstance(latest_payload, dict):
|
||||
payload.update(latest_payload)
|
||||
except Exception:
|
||||
pass
|
||||
payload["default_presets"] = defaults
|
||||
|
||||
try:
|
||||
os.makedirs(os.path.dirname(self.local_sites_file), exist_ok=True)
|
||||
with open(tmp_file, "w", encoding="utf-8", newline="\n") as f:
|
||||
json.dump(payload, f, indent=2, ensure_ascii=False)
|
||||
f.flush()
|
||||
os.fsync(f.fileno())
|
||||
|
||||
os.replace(tmp_file, self.local_sites_file)
|
||||
try:
|
||||
self.last_local_mtime = os.path.getmtime(self.local_sites_file)
|
||||
except Exception as e:
|
||||
logger.warning(f"本地站点覆盖已保存但更新时间戳失败: {e}")
|
||||
self._local_sites_payload = payload
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.error(f"保存本地站点覆盖配置失败: {e}")
|
||||
try:
|
||||
if os.path.exists(tmp_file):
|
||||
os.remove(tmp_file)
|
||||
except Exception:
|
||||
pass
|
||||
return False
|
||||
def _snapshot_local_site_overrides_locked(self) -> Optional[tuple[bool, bytes, Dict[str, Any], Dict[str, str]]]:
|
||||
try:
|
||||
payload_snapshot = copy.deepcopy(self._local_sites_payload or {})
|
||||
defaults_snapshot = copy.deepcopy(self._local_default_presets or {})
|
||||
if not os.path.exists(self.local_sites_file):
|
||||
return (False, b"", payload_snapshot, defaults_snapshot)
|
||||
with open(self.local_sites_file, "rb") as f:
|
||||
return (True, f.read(), payload_snapshot, defaults_snapshot)
|
||||
except Exception as e:
|
||||
logger.error(f"读取本地站点覆盖快照失败: {e}")
|
||||
return None
|
||||
def _restore_local_site_overrides_locked(self, snapshot: Optional[tuple[bool, bytes, Dict[str, Any], Dict[str, str]]]) -> None:
|
||||
if snapshot is None:
|
||||
return
|
||||
|
||||
tmp_file = self.local_sites_file + ".restore.tmp"
|
||||
existed, payload, payload_snapshot, defaults_snapshot = snapshot
|
||||
try:
|
||||
if existed:
|
||||
os.makedirs(os.path.dirname(self.local_sites_file), exist_ok=True)
|
||||
with open(tmp_file, "wb") as f:
|
||||
f.write(payload)
|
||||
f.flush()
|
||||
os.fsync(f.fileno())
|
||||
os.replace(tmp_file, self.local_sites_file)
|
||||
elif os.path.exists(self.local_sites_file):
|
||||
os.remove(self.local_sites_file)
|
||||
try:
|
||||
self.last_local_mtime = os.path.getmtime(self.local_sites_file) if os.path.exists(self.local_sites_file) else 0.0
|
||||
except Exception as e:
|
||||
logger.warning(f"本地站点覆盖已恢复但更新时间戳失败: {e}")
|
||||
self._local_sites_payload = copy.deepcopy(payload_snapshot)
|
||||
self._local_default_presets = copy.deepcopy(defaults_snapshot)
|
||||
except Exception as e:
|
||||
logger.error(f"恢复本地站点覆盖失败: {e}")
|
||||
try:
|
||||
if os.path.exists(tmp_file):
|
||||
os.remove(tmp_file)
|
||||
except Exception:
|
||||
pass
|
||||
@@ -0,0 +1,685 @@
|
||||
"""ConfigEngine 的一部分(R2-3 拆分):图片提取、文件粘贴、提示词填充配置与图片预设。
|
||||
|
||||
方法原样从 engine.py 迁出,行为不变;通过 mixin 组合回 ConfigEngine,
|
||||
实例状态都在 ConfigEngine.__init__(engine.py)里初始化。
|
||||
"""
|
||||
|
||||
import copy
|
||||
from typing import Dict, Optional, Any
|
||||
from app.models.schemas import (
|
||||
get_default_image_extraction_config,
|
||||
get_default_file_paste_config,
|
||||
get_default_prompt_padding_config,
|
||||
get_default_attachment_monitor_config,
|
||||
get_default_send_confirmation_config,
|
||||
get_enabled_modalities,
|
||||
is_modality_enabled,
|
||||
normalize_modalities_config,
|
||||
normalize_modality_policy,
|
||||
)
|
||||
from .engine_common import DEFAULT_PRESET_NAME, _coerce_bool, _merge_config_patch, logger
|
||||
|
||||
|
||||
class ConfigMediaMixin:
|
||||
"""图片提取、文件粘贴、提示词填充配置与图片预设"""
|
||||
|
||||
def get_site_image_config(self, domain: str, preset_name: str = None) -> Dict:
|
||||
"""获取站点的图片提取配置"""
|
||||
self.refresh_if_changed()
|
||||
|
||||
default_config = get_default_image_extraction_config()
|
||||
|
||||
data = self._get_site_data(domain, preset_name)
|
||||
if data is None:
|
||||
return copy.deepcopy(default_config)
|
||||
|
||||
image_config = data.get("image_extraction", {})
|
||||
return self._validate_image_config(image_config)
|
||||
def set_site_image_config(self, domain: str, config: Dict,
|
||||
preset_name: str = None) -> bool:
|
||||
"""设置站点的图片提取配置"""
|
||||
self.refresh_if_changed()
|
||||
|
||||
data = self._get_site_data(domain, preset_name)
|
||||
if data is None:
|
||||
logger.warning(f"站点或预设不存在: {domain}/{preset_name}")
|
||||
return False
|
||||
|
||||
current_config = self._validate_image_config(data.get("image_extraction", {}))
|
||||
merged_config = _merge_config_patch(current_config, config if isinstance(config, dict) else {})
|
||||
validated = self._validate_image_config(merged_config)
|
||||
|
||||
if not self._assign_preset_field_and_save(data, "image_extraction", validated):
|
||||
return False
|
||||
|
||||
logger.info(f"站点 {domain} [{preset_name or DEFAULT_PRESET_NAME}] 多模态提取配置已更新")
|
||||
return True
|
||||
def _validate_image_config(self, config: Dict) -> Dict:
|
||||
"""验证并规范化多模态提取配置"""
|
||||
default = get_default_image_extraction_config()
|
||||
result = copy.deepcopy(default)
|
||||
|
||||
if not config:
|
||||
return result
|
||||
|
||||
raw_modalities = config.get("modalities")
|
||||
if isinstance(raw_modalities, dict):
|
||||
result["modalities"] = normalize_modalities_config(raw_modalities)
|
||||
|
||||
legacy_enabled = None
|
||||
if "enabled" in config:
|
||||
legacy_enabled = _coerce_bool(config.get("enabled"), False)
|
||||
result["enabled"] = legacy_enabled
|
||||
|
||||
if (
|
||||
legacy_enabled is not None
|
||||
and not isinstance(raw_modalities, dict)
|
||||
):
|
||||
result["modalities"]["image"] = normalize_modality_policy("image", legacy_enabled)
|
||||
elif legacy_enabled is not None and isinstance(raw_modalities, dict):
|
||||
if not legacy_enabled:
|
||||
for key in ("image", "audio", "video"):
|
||||
result["modalities"][key] = normalize_modality_policy(key, False)
|
||||
elif not get_enabled_modalities(result.get("modalities")):
|
||||
result["modalities"]["image"] = normalize_modality_policy("image", True)
|
||||
|
||||
if "selector" in config and config["selector"]:
|
||||
result["selector"] = str(config["selector"]).strip()
|
||||
if not result["selector"]:
|
||||
result["selector"] = "img"
|
||||
|
||||
if "audio_selector" in config and config["audio_selector"]:
|
||||
result["audio_selector"] = str(config["audio_selector"]).strip()
|
||||
if not result["audio_selector"]:
|
||||
result["audio_selector"] = default["audio_selector"]
|
||||
|
||||
if "video_selector" in config and config["video_selector"]:
|
||||
result["video_selector"] = str(config["video_selector"]).strip()
|
||||
if not result["video_selector"]:
|
||||
result["video_selector"] = default["video_selector"]
|
||||
|
||||
if "container_selector" in config:
|
||||
val = config["container_selector"]
|
||||
result["container_selector"] = str(val).strip() if val else None
|
||||
|
||||
if "final_target_strategy" in config:
|
||||
val = str(config["final_target_strategy"] or "").strip().lower()
|
||||
if val in ("container", "latest_reply", "latest_visual_reply"):
|
||||
result["final_target_strategy"] = val
|
||||
|
||||
if "latest_visual_column" in config:
|
||||
val = str(config["latest_visual_column"] or "").strip().lower()
|
||||
if val in ("left", "right"):
|
||||
result["latest_visual_column"] = val
|
||||
|
||||
if "allow_container_fallback" in config:
|
||||
result["allow_container_fallback"] = _coerce_bool(config.get("allow_container_fallback"), False)
|
||||
|
||||
if "force_postprocess" in config:
|
||||
result["force_postprocess"] = _coerce_bool(config.get("force_postprocess"), False)
|
||||
|
||||
if "arena_image_generation" in config:
|
||||
result["arena_image_generation"] = _coerce_bool(
|
||||
config.get("arena_image_generation"), False
|
||||
)
|
||||
|
||||
if "direct_postprocess_modalities" in config:
|
||||
raw_direct_modalities = config.get("direct_postprocess_modalities")
|
||||
if isinstance(raw_direct_modalities, (list, tuple, set)):
|
||||
allowed_direct_modalities = []
|
||||
for item in raw_direct_modalities:
|
||||
media_type = str(item or "").strip().lower()
|
||||
if (
|
||||
media_type in {"image", "audio", "video"}
|
||||
and is_modality_enabled(result.get("modalities"), media_type)
|
||||
and media_type not in allowed_direct_modalities
|
||||
):
|
||||
allowed_direct_modalities.append(media_type)
|
||||
if allowed_direct_modalities:
|
||||
result["direct_postprocess_modalities"] = allowed_direct_modalities
|
||||
|
||||
if "debounce_seconds" in config:
|
||||
try:
|
||||
val = float(config["debounce_seconds"])
|
||||
result["debounce_seconds"] = max(0, min(val, 30))
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
|
||||
if "wait_for_load" in config:
|
||||
result["wait_for_load"] = _coerce_bool(config.get("wait_for_load"), True)
|
||||
|
||||
if "load_timeout_seconds" in config:
|
||||
try:
|
||||
val = float(config["load_timeout_seconds"])
|
||||
result["load_timeout_seconds"] = max(1, min(val, 60))
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
|
||||
if "download_blobs" in config:
|
||||
result["download_blobs"] = _coerce_bool(config.get("download_blobs"), False)
|
||||
|
||||
if "max_size_mb" in config:
|
||||
try:
|
||||
val = int(config["max_size_mb"])
|
||||
result["max_size_mb"] = max(1, min(val, 100))
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
|
||||
if "canvas_export_mime" in config:
|
||||
val = str(config["canvas_export_mime"] or "").strip().lower()
|
||||
if val in {"image/jpeg", "image/webp", "image/png"}:
|
||||
result["canvas_export_mime"] = val
|
||||
|
||||
if "canvas_export_quality" in config:
|
||||
try:
|
||||
val = float(config["canvas_export_quality"])
|
||||
result["canvas_export_quality"] = max(0.1, min(val, 1.0))
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
|
||||
if "src_allow_patterns" in config:
|
||||
raw_patterns = config.get("src_allow_patterns")
|
||||
if isinstance(raw_patterns, str):
|
||||
raw_patterns = raw_patterns.replace("\r\n", "\n").replace(";", "\n").split("\n")
|
||||
if isinstance(raw_patterns, (list, tuple, set)):
|
||||
patterns = []
|
||||
seen = set()
|
||||
for item in raw_patterns:
|
||||
text = str(item or "").strip()
|
||||
if not text or text in seen:
|
||||
continue
|
||||
seen.add(text)
|
||||
patterns.append(text)
|
||||
result["src_allow_patterns"] = patterns
|
||||
|
||||
if "mode" in config:
|
||||
val = str(config["mode"]).lower()
|
||||
if val in ("all", "first", "last"):
|
||||
result["mode"] = val
|
||||
|
||||
if "audio_capture_enabled" in config:
|
||||
result["audio_capture_enabled"] = _coerce_bool(config.get("audio_capture_enabled"), False)
|
||||
|
||||
if "audio_capture_mute_playback" in config:
|
||||
result["audio_capture_mute_playback"] = _coerce_bool(config.get("audio_capture_mute_playback"), False)
|
||||
|
||||
if "audio_capture_preload_enabled" in config:
|
||||
result["audio_capture_preload_enabled"] = _coerce_bool(config.get("audio_capture_preload_enabled"), False)
|
||||
|
||||
if "audio_capture_reload_before_workflow" in config:
|
||||
result["audio_capture_reload_before_workflow"] = _coerce_bool(config.get("audio_capture_reload_before_workflow"), False)
|
||||
|
||||
if "audio_capture_preserve_graph" in config:
|
||||
result["audio_capture_preserve_graph"] = _coerce_bool(config.get("audio_capture_preserve_graph"), False)
|
||||
|
||||
if "audio_capture_terminal_settle_seconds" in config:
|
||||
try:
|
||||
val = float(config["audio_capture_terminal_settle_seconds"])
|
||||
result["audio_capture_terminal_settle_seconds"] = max(0.0, min(val, 5.0))
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
|
||||
if "audio_trigger_selector" in config:
|
||||
result["audio_trigger_selector"] = str(config["audio_trigger_selector"] or "").strip()
|
||||
|
||||
if "audio_trigger_labels" in config:
|
||||
raw_labels = config.get("audio_trigger_labels")
|
||||
if isinstance(raw_labels, (list, tuple)):
|
||||
labels = [
|
||||
str(item).strip()
|
||||
for item in raw_labels
|
||||
if str(item).strip()
|
||||
]
|
||||
if labels:
|
||||
result["audio_trigger_labels"] = labels
|
||||
|
||||
if "audio_capture_max_wait_seconds" in config:
|
||||
try:
|
||||
val = float(config["audio_capture_max_wait_seconds"])
|
||||
result["audio_capture_max_wait_seconds"] = max(1.0, min(val, 120.0))
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
|
||||
if "audio_capture_min_wait_seconds" in config:
|
||||
try:
|
||||
val = float(config["audio_capture_min_wait_seconds"])
|
||||
result["audio_capture_min_wait_seconds"] = max(0.2, min(val, 30.0))
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
|
||||
if "audio_capture_hard_max_wait_seconds" in config:
|
||||
try:
|
||||
val = float(config["audio_capture_hard_max_wait_seconds"])
|
||||
result["audio_capture_hard_max_wait_seconds"] = max(1.0, min(val, 180.0))
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
|
||||
if "audio_capture_estimated_chars_per_second" in config:
|
||||
try:
|
||||
val = float(config["audio_capture_estimated_chars_per_second"])
|
||||
result["audio_capture_estimated_chars_per_second"] = max(1.0, min(val, 20.0))
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
|
||||
if "audio_capture_wait_padding_seconds" in config:
|
||||
try:
|
||||
val = float(config["audio_capture_wait_padding_seconds"])
|
||||
result["audio_capture_wait_padding_seconds"] = max(0.0, min(val, 10.0))
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
|
||||
network_capture = dict(result.get("audio_network_capture") or {})
|
||||
raw_network_capture = config.get("audio_network_capture")
|
||||
if isinstance(raw_network_capture, dict):
|
||||
if "enabled" in raw_network_capture:
|
||||
network_capture["enabled"] = _coerce_bool(raw_network_capture.get("enabled"), False)
|
||||
if "timeout_seconds" in raw_network_capture:
|
||||
try:
|
||||
val = float(raw_network_capture["timeout_seconds"])
|
||||
network_capture["timeout_seconds"] = max(0.1, min(val, 15.0))
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
if "transport" in raw_network_capture:
|
||||
val = str(raw_network_capture["transport"] or "").strip()
|
||||
if val in {"page_websocket_probe"}:
|
||||
network_capture["transport"] = val
|
||||
if "extractor" in raw_network_capture:
|
||||
val = str(raw_network_capture["extractor"] or "").strip()
|
||||
if val in {"voicegenie_ogg_pages", "voicegenie_binary_stream"}:
|
||||
network_capture["extractor"] = val
|
||||
if "settle_seconds" in raw_network_capture:
|
||||
try:
|
||||
val = float(raw_network_capture["settle_seconds"])
|
||||
network_capture["settle_seconds"] = max(0.05, min(val, 5.0))
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
if "url_patterns" in raw_network_capture:
|
||||
raw_patterns = raw_network_capture.get("url_patterns")
|
||||
if isinstance(raw_patterns, (list, tuple)):
|
||||
patterns = [
|
||||
str(item).strip()
|
||||
for item in raw_patterns
|
||||
if str(item).strip()
|
||||
]
|
||||
if patterns:
|
||||
network_capture["url_patterns"] = patterns
|
||||
if "max_payload_bytes" in raw_network_capture:
|
||||
# 修复:此前漏处理该键,导致前端设置的单条载荷上限被静默重置为默认 10MB
|
||||
try:
|
||||
val = int(raw_network_capture["max_payload_bytes"])
|
||||
network_capture["max_payload_bytes"] = max(65536, min(val, 104857600))
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
|
||||
# 兼容旧平铺字段,最终统一收口到新对象
|
||||
if "audio_network_capture_enabled" in config:
|
||||
network_capture["enabled"] = _coerce_bool(config.get("audio_network_capture_enabled"), False)
|
||||
|
||||
if "audio_network_capture_timeout_seconds" in config:
|
||||
try:
|
||||
val = float(config["audio_network_capture_timeout_seconds"])
|
||||
network_capture["timeout_seconds"] = max(0.1, min(val, 15.0))
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
|
||||
if "audio_network_url_patterns" in config:
|
||||
raw_patterns = config.get("audio_network_url_patterns")
|
||||
if isinstance(raw_patterns, (list, tuple)):
|
||||
patterns = [
|
||||
str(item).strip()
|
||||
for item in raw_patterns
|
||||
if str(item).strip()
|
||||
]
|
||||
if patterns:
|
||||
network_capture["url_patterns"] = patterns
|
||||
|
||||
result["audio_network_capture"] = network_capture
|
||||
|
||||
browser_tts_fallback = dict(result.get("audio_browser_tts_fallback") or {})
|
||||
raw_browser_tts_fallback = config.get("audio_browser_tts_fallback")
|
||||
if isinstance(raw_browser_tts_fallback, dict):
|
||||
if "enabled" in raw_browser_tts_fallback:
|
||||
browser_tts_fallback["enabled"] = _coerce_bool(raw_browser_tts_fallback.get("enabled"), False)
|
||||
if "provider" in raw_browser_tts_fallback:
|
||||
val = str(raw_browser_tts_fallback["provider"] or "").strip()
|
||||
if val in {"doubao_samantha"}:
|
||||
browser_tts_fallback["provider"] = val
|
||||
if "speaker" in raw_browser_tts_fallback:
|
||||
val = str(raw_browser_tts_fallback["speaker"] or "").strip()
|
||||
if val:
|
||||
browser_tts_fallback["speaker"] = val
|
||||
if "speech_rate" in raw_browser_tts_fallback:
|
||||
try:
|
||||
val = int(raw_browser_tts_fallback["speech_rate"])
|
||||
browser_tts_fallback["speech_rate"] = max(-100, min(val, 100))
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
if "pitch" in raw_browser_tts_fallback:
|
||||
try:
|
||||
val = int(raw_browser_tts_fallback["pitch"])
|
||||
browser_tts_fallback["pitch"] = max(-100, min(val, 100))
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
if "format" in raw_browser_tts_fallback:
|
||||
val = str(raw_browser_tts_fallback["format"] or "").strip().lower()
|
||||
if val in {"aac"}:
|
||||
browser_tts_fallback["format"] = val
|
||||
if "timeout_seconds" in raw_browser_tts_fallback:
|
||||
try:
|
||||
val = float(raw_browser_tts_fallback["timeout_seconds"])
|
||||
browser_tts_fallback["timeout_seconds"] = max(3.0, min(val, 120.0))
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
for key in (
|
||||
"pc_version",
|
||||
"aid",
|
||||
"real_aid",
|
||||
"language",
|
||||
"device_platform",
|
||||
"pkg_type",
|
||||
"region",
|
||||
"sys_region",
|
||||
"use_olympus_account",
|
||||
"samantha_web",
|
||||
):
|
||||
if key in raw_browser_tts_fallback:
|
||||
val = str(raw_browser_tts_fallback[key] or "").strip()
|
||||
if val:
|
||||
browser_tts_fallback[key] = val
|
||||
|
||||
result["audio_browser_tts_fallback"] = browser_tts_fallback
|
||||
|
||||
if "audio_capture_poll_seconds" in config:
|
||||
try:
|
||||
val = float(config["audio_capture_poll_seconds"])
|
||||
result["audio_capture_poll_seconds"] = max(0.05, min(val, 5.0))
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
|
||||
if "audio_capture_silence_seconds" in config:
|
||||
try:
|
||||
val = float(config["audio_capture_silence_seconds"])
|
||||
result["audio_capture_silence_seconds"] = max(0.2, min(val, 30.0))
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
|
||||
if "audio_capture_activity_threshold" in config:
|
||||
try:
|
||||
val = float(config["audio_capture_activity_threshold"])
|
||||
result["audio_capture_activity_threshold"] = max(0.0001, min(val, 0.2))
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
|
||||
if "audio_capture_activity_silence_seconds" in config:
|
||||
try:
|
||||
val = float(config["audio_capture_activity_silence_seconds"])
|
||||
result["audio_capture_activity_silence_seconds"] = max(0.2, min(val, 10.0))
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
|
||||
result["enabled"] = bool(get_enabled_modalities(result.get("modalities")))
|
||||
|
||||
return result
|
||||
def get_site_file_paste_config(self, domain: str, preset_name: str = None) -> dict:
|
||||
"""获取站点的文件粘贴配置"""
|
||||
self.refresh_if_changed()
|
||||
|
||||
default_config = get_default_file_paste_config()
|
||||
|
||||
data = self._get_site_data(domain, preset_name)
|
||||
if data is None:
|
||||
result = copy.deepcopy(default_config)
|
||||
result["send_confirmation"] = get_default_send_confirmation_config()
|
||||
result["attachment_monitor"] = get_default_attachment_monitor_config()
|
||||
return result
|
||||
|
||||
return self._validate_file_paste_config(
|
||||
data.get("file_paste", {}),
|
||||
legacy_stream_config=data.get("stream_config"),
|
||||
include_attachment_defaults=True,
|
||||
)
|
||||
def set_site_file_paste_config(self, domain: str, config: dict,
|
||||
preset_name: str = None) -> bool:
|
||||
"""设置站点的文件粘贴配置"""
|
||||
self.refresh_if_changed()
|
||||
|
||||
data = self._get_site_data(domain, preset_name)
|
||||
if data is None:
|
||||
logger.warning(f"站点或预设不存在: {domain}/{preset_name}")
|
||||
return False
|
||||
|
||||
current_config = self._validate_file_paste_config(
|
||||
data.get("file_paste", {}),
|
||||
legacy_stream_config=data.get("stream_config"),
|
||||
)
|
||||
merged_config = _merge_config_patch(current_config, config if isinstance(config, dict) else {})
|
||||
validated = self._validate_file_paste_config(
|
||||
merged_config,
|
||||
legacy_stream_config=data.get("stream_config"),
|
||||
)
|
||||
|
||||
if not self._assign_preset_field_and_save(data, "file_paste", validated):
|
||||
return False
|
||||
|
||||
logger.info(f"站点 {domain} [{preset_name or DEFAULT_PRESET_NAME}] 文件粘贴配置已更新")
|
||||
return True
|
||||
def get_all_file_paste_configs(self) -> dict:
|
||||
"""获取所有站点的文件粘贴配置(使用各站点当前默认预设)"""
|
||||
self.refresh_if_changed()
|
||||
result = {}
|
||||
|
||||
for domain in self.sites:
|
||||
if domain.startswith('_'):
|
||||
continue
|
||||
|
||||
data = self._get_site_data(domain)
|
||||
if data is None:
|
||||
continue
|
||||
|
||||
result[domain] = self._validate_file_paste_config(
|
||||
data.get("file_paste", {}),
|
||||
legacy_stream_config=data.get("stream_config"),
|
||||
include_attachment_defaults=True,
|
||||
)
|
||||
|
||||
return result
|
||||
def get_site_prompt_padding_config(self, domain: str, preset_name: str = None) -> Dict[str, Any]:
|
||||
"""获取站点的提示词开头注入配置"""
|
||||
self.refresh_if_changed()
|
||||
|
||||
data = self._get_site_data(domain, preset_name)
|
||||
if data is None:
|
||||
return copy.deepcopy(get_default_prompt_padding_config())
|
||||
|
||||
return self._validate_prompt_padding_config(data.get("prompt_padding", {}))
|
||||
def set_site_prompt_padding_config(
|
||||
self,
|
||||
domain: str,
|
||||
config: Dict[str, Any],
|
||||
preset_name: str = None,
|
||||
) -> bool:
|
||||
"""设置站点的提示词开头注入配置"""
|
||||
self.refresh_if_changed()
|
||||
|
||||
data = self._get_site_data(domain, preset_name)
|
||||
if data is None:
|
||||
logger.warning(f"站点或预设不存在: {domain}/{preset_name}")
|
||||
return False
|
||||
|
||||
current_config = self._validate_prompt_padding_config(data.get("prompt_padding", {}))
|
||||
merged_config = _merge_config_patch(current_config, config if isinstance(config, dict) else {})
|
||||
validated = self._validate_prompt_padding_config(merged_config)
|
||||
if not self._assign_preset_field_and_save(data, "prompt_padding", validated):
|
||||
return False
|
||||
|
||||
logger.info(f"站点 {domain} [{preset_name or DEFAULT_PRESET_NAME}] 提示词开头注入配置已更新")
|
||||
return True
|
||||
def _validate_file_paste_config(
|
||||
self,
|
||||
config: dict,
|
||||
*,
|
||||
legacy_stream_config: Optional[Dict[str, Any]] = None,
|
||||
include_attachment_defaults: bool = False,
|
||||
) -> dict:
|
||||
"""验证并规范化文件粘贴配置"""
|
||||
default = get_default_file_paste_config()
|
||||
result = copy.deepcopy(default)
|
||||
|
||||
if not isinstance(config, dict):
|
||||
config = {}
|
||||
|
||||
if "enabled" in config:
|
||||
result["enabled"] = _coerce_bool(config.get("enabled"), False)
|
||||
|
||||
if "threshold" in config:
|
||||
try:
|
||||
val = int(config["threshold"])
|
||||
result["threshold"] = max(1000, min(val, 10000000))
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
|
||||
if "temp_file_type" in config:
|
||||
val = str(config.get("temp_file_type") or "").strip().lower().lstrip(".")
|
||||
if val in {"txt", "pdf", "chunk", "error"}:
|
||||
result["temp_file_type"] = val
|
||||
|
||||
if "hint_text" in config:
|
||||
val = str(config["hint_text"]).strip()
|
||||
# 限制长度,避免过长的引导文本
|
||||
hint_val = val[:500] if val else ""
|
||||
result["hint_text"] = hint_val
|
||||
|
||||
# 智能向后兼容:结合老配置的策略类型,防止新字段被无关的旧数据污染
|
||||
old_temp_type = config.get("temp_file_type", "txt")
|
||||
|
||||
if "txt_hint_text" not in config:
|
||||
result["txt_hint_text"] = hint_val if old_temp_type == "txt" else "完全专注于文件内容"
|
||||
if "pdf_hint_text" not in config:
|
||||
result["pdf_hint_text"] = hint_val if old_temp_type == "pdf" else "完全专注于文件内容"
|
||||
if "error_hint_text" not in config:
|
||||
result["error_hint_text"] = hint_val if old_temp_type == "error" else "输入文本长度超过限制,已中止发送"
|
||||
|
||||
if "txt_hint_text" in config:
|
||||
val = str(config["txt_hint_text"]).strip()
|
||||
result["txt_hint_text"] = val[:500] if val else ""
|
||||
|
||||
if "pdf_hint_text" in config:
|
||||
val = str(config["pdf_hint_text"]).strip()
|
||||
result["pdf_hint_text"] = val[:500] if val else ""
|
||||
|
||||
if "error_hint_text" in config:
|
||||
val = str(config["error_hint_text"]).strip()
|
||||
result["error_hint_text"] = val[:500] if val else ""
|
||||
|
||||
if "reacquire_input_after_upload" in config:
|
||||
result["reacquire_input_after_upload"] = _coerce_bool(config.get("reacquire_input_after_upload"), False)
|
||||
|
||||
if "post_upload_input_selector" in config:
|
||||
val = str(config["post_upload_input_selector"] or "").strip()
|
||||
result["post_upload_input_selector"] = val[:500] if val else ""
|
||||
|
||||
if "post_upload_settle" in config:
|
||||
try:
|
||||
val = float(config["post_upload_settle"])
|
||||
result["post_upload_settle"] = max(0.0, min(val, 30.0))
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
|
||||
if "upload_signal_timeout" in config:
|
||||
try:
|
||||
val = float(config["upload_signal_timeout"])
|
||||
result["upload_signal_timeout"] = max(0.5, min(val, 120.0))
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
|
||||
if "upload_signal_grace" in config:
|
||||
try:
|
||||
val = float(config["upload_signal_grace"])
|
||||
result["upload_signal_grace"] = max(0.0, min(val, 120.0))
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
|
||||
default_state_probe = copy.deepcopy(default.get("state_probe") or {})
|
||||
raw_state_probe = config.get("state_probe")
|
||||
state_probe = copy.deepcopy(default_state_probe)
|
||||
if isinstance(raw_state_probe, dict):
|
||||
if "enabled" in raw_state_probe:
|
||||
state_probe["enabled"] = _coerce_bool(raw_state_probe.get("enabled"), False)
|
||||
if "code" in raw_state_probe:
|
||||
code = str(raw_state_probe["code"] or "").strip()
|
||||
state_probe["code"] = code[:20000] if code else ""
|
||||
if state_probe:
|
||||
result["state_probe"] = state_probe
|
||||
|
||||
legacy_send_confirmation = {}
|
||||
if isinstance(legacy_stream_config, dict) and isinstance(legacy_stream_config.get("send_confirmation"), dict):
|
||||
legacy_send_confirmation.update(legacy_stream_config.get("send_confirmation") or {})
|
||||
raw_send_confirmation = config.get("send_confirmation")
|
||||
if isinstance(raw_send_confirmation, dict):
|
||||
legacy_send_confirmation.update(raw_send_confirmation)
|
||||
if legacy_send_confirmation or include_attachment_defaults:
|
||||
result["send_confirmation"] = self._validate_send_confirmation_config(legacy_send_confirmation)
|
||||
|
||||
legacy_attachment_monitor = {}
|
||||
if isinstance(legacy_stream_config, dict) and isinstance(legacy_stream_config.get("attachment_monitor"), dict):
|
||||
legacy_attachment_monitor.update(legacy_stream_config.get("attachment_monitor") or {})
|
||||
raw_attachment_monitor = config.get("attachment_monitor")
|
||||
if isinstance(raw_attachment_monitor, dict):
|
||||
legacy_attachment_monitor.update(raw_attachment_monitor)
|
||||
if legacy_attachment_monitor or include_attachment_defaults:
|
||||
result["attachment_monitor"] = self._validate_attachment_monitor_config(legacy_attachment_monitor)
|
||||
|
||||
from app.utils.attachments import attachment_config
|
||||
if "attachments" in config or include_attachment_defaults:
|
||||
result["attachments"] = attachment_config(config.get("attachments"))
|
||||
|
||||
return result
|
||||
def _validate_prompt_padding_config(self, config: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"""验证并规范化提示词开头注入配置。"""
|
||||
result = copy.deepcopy(get_default_prompt_padding_config())
|
||||
|
||||
if not isinstance(config, dict):
|
||||
return result
|
||||
|
||||
if "enabled" in config:
|
||||
result["enabled"] = _coerce_bool(config.get("enabled"), False)
|
||||
|
||||
if "marker_text" in config:
|
||||
marker_text = str(config.get("marker_text") or "").strip()
|
||||
result["marker_text"] = marker_text
|
||||
|
||||
if "segments_per_side" in config:
|
||||
try:
|
||||
segments_per_side = int(config.get("segments_per_side"))
|
||||
except (TypeError, ValueError):
|
||||
segments_per_side = int(result["segments_per_side"])
|
||||
result["segments_per_side"] = max(0, min(segments_per_side, 64))
|
||||
|
||||
if "random_insert_enabled" in config:
|
||||
result["random_insert_enabled"] = _coerce_bool(
|
||||
config.get("random_insert_enabled"), False
|
||||
)
|
||||
|
||||
if "random_insert_chars" in config:
|
||||
result["random_insert_chars"] = str(
|
||||
config.get("random_insert_chars") or ""
|
||||
)
|
||||
|
||||
return result
|
||||
def list_image_presets(self):
|
||||
"""列出所有可用的图片配置预设"""
|
||||
return self.image_presets.list_presets()
|
||||
def get_image_preset(self, domain: str):
|
||||
"""获取指定站点的预设信息"""
|
||||
return self.image_presets.get_preset_for_display(domain)
|
||||
def apply_image_preset(self, domain: str, preset_domain: str):
|
||||
"""将预设配置应用到站点"""
|
||||
preset_config = self.image_presets.get_preset(preset_domain)
|
||||
|
||||
if not preset_config:
|
||||
raise ValueError(f"找不到预设: {preset_domain}")
|
||||
|
||||
return self.set_site_image_config(domain, preset_config)
|
||||
def reload_presets(self):
|
||||
"""重新加载图片预设"""
|
||||
self.image_presets.reload()
|
||||
@@ -0,0 +1,374 @@
|
||||
"""ConfigEngine 的一部分(R2-3 拆分):历史配置迁移与命令文件读写。
|
||||
|
||||
方法原样从 engine.py 迁出,行为不变;通过 mixin 组合回 ConfigEngine,
|
||||
实例状态都在 ConfigEngine.__init__(engine.py)里初始化。
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import copy
|
||||
from typing import Dict, List, Any
|
||||
from app.models.schemas import (
|
||||
PRESET_ADVANCED_FIELDS,
|
||||
SITE_ADVANCED_FIELDS,
|
||||
get_default_image_extraction_config,
|
||||
get_default_file_paste_config,
|
||||
get_default_prompt_padding_config,
|
||||
)
|
||||
from app.utils.site_rules import get_site_rule
|
||||
from .engine_common import ConfigConstants, DEFAULT_PRESET_NAME, PRESET_FIELDS, logger
|
||||
|
||||
|
||||
class ConfigMigrationsMixin:
|
||||
"""历史配置迁移与命令文件读写"""
|
||||
|
||||
def _load_commands_file(self) -> List[Dict[str, Any]]:
|
||||
"""加载独立命令配置文件"""
|
||||
commands_file = ConfigConstants.COMMANDS_FILE
|
||||
if not os.path.exists(commands_file):
|
||||
return []
|
||||
|
||||
try:
|
||||
with open(commands_file, "r", encoding="utf-8") as f:
|
||||
data = json.load(f)
|
||||
|
||||
if isinstance(data, dict):
|
||||
data = data.get("commands", [])
|
||||
|
||||
if isinstance(data, list):
|
||||
return data
|
||||
|
||||
logger.warning(f"命令配置文件格式无效: {commands_file}")
|
||||
return []
|
||||
except json.JSONDecodeError as e:
|
||||
logger.error(f"命令配置文件格式错误: {e}")
|
||||
return []
|
||||
except Exception as e:
|
||||
logger.error(f"加载命令配置失败: {e}")
|
||||
return []
|
||||
def _save_commands_file(self, commands: List[Dict[str, Any]]) -> bool:
|
||||
"""保存独立命令配置文件"""
|
||||
commands_file = ConfigConstants.COMMANDS_FILE
|
||||
tmp_file = commands_file + ".tmp"
|
||||
|
||||
try:
|
||||
os.makedirs(os.path.dirname(commands_file), exist_ok=True)
|
||||
payload = {"commands": commands}
|
||||
|
||||
with open(tmp_file, "w", encoding="utf-8", newline="\n") as f:
|
||||
json.dump(payload, f, indent=2, ensure_ascii=False)
|
||||
f.flush()
|
||||
os.fsync(f.fileno())
|
||||
|
||||
os.replace(tmp_file, commands_file)
|
||||
logger.info(f"命令配置已保存: {commands_file}")
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.error(f"保存命令配置失败: {e}")
|
||||
try:
|
||||
if os.path.exists(tmp_file):
|
||||
os.remove(tmp_file)
|
||||
except Exception:
|
||||
pass
|
||||
return False
|
||||
@staticmethod
|
||||
def _merge_commands(existing: List[Dict[str, Any]], incoming: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
|
||||
"""保留已有命令,仅追加不存在的命令"""
|
||||
merged = list(existing or [])
|
||||
seen_ids = {
|
||||
str(cmd.get("id", "")).strip()
|
||||
for cmd in merged
|
||||
if isinstance(cmd, dict) and str(cmd.get("id", "")).strip()
|
||||
}
|
||||
seen_names = {
|
||||
str(cmd.get("name", "")).strip()
|
||||
for cmd in merged
|
||||
if isinstance(cmd, dict) and str(cmd.get("name", "")).strip()
|
||||
}
|
||||
|
||||
for cmd in incoming or []:
|
||||
if not isinstance(cmd, dict):
|
||||
continue
|
||||
|
||||
command_id = str(cmd.get("id", "")).strip()
|
||||
command_name = str(cmd.get("name", "")).strip()
|
||||
|
||||
if command_id and command_id in seen_ids:
|
||||
continue
|
||||
if command_name and command_name in seen_names:
|
||||
continue
|
||||
|
||||
merged.append(cmd)
|
||||
if command_id:
|
||||
seen_ids.add(command_id)
|
||||
if command_name:
|
||||
seen_names.add(command_name)
|
||||
|
||||
return merged
|
||||
def _migrate_global_commands(self):
|
||||
"""将旧版 _global.commands 迁移到独立文件"""
|
||||
legacy_commands = self.global_config.get("commands", [])
|
||||
existing_commands = self._load_commands_file()
|
||||
|
||||
if not isinstance(legacy_commands, list):
|
||||
legacy_commands = []
|
||||
|
||||
merged_commands = self._merge_commands(existing_commands, legacy_commands)
|
||||
|
||||
if legacy_commands or not os.path.exists(ConfigConstants.COMMANDS_FILE):
|
||||
self._save_commands_file(merged_commands)
|
||||
|
||||
if self.global_config.remove("commands"):
|
||||
self._save_config()
|
||||
def _migrate_to_presets(self):
|
||||
"""
|
||||
将旧格式(扁平)站点配置迁移为预设格式
|
||||
|
||||
旧格式: { "selectors": {...}, "workflow": [...], ... }
|
||||
新格式: { "presets": { "主预设": { "selectors": {...}, ... } } }
|
||||
"""
|
||||
migrated_count = 0
|
||||
|
||||
for domain in list(self.sites.keys()):
|
||||
if domain.startswith('_'):
|
||||
continue
|
||||
|
||||
site_config = self.sites[domain]
|
||||
|
||||
# 已经是预设格式,跳过
|
||||
if "presets" in site_config:
|
||||
continue
|
||||
|
||||
# 将所有已知配置字段提取到主预设中
|
||||
preset_data = {}
|
||||
remaining = {}
|
||||
|
||||
for key, value in site_config.items():
|
||||
if key in PRESET_FIELDS:
|
||||
preset_data[key] = value
|
||||
elif key == "advanced":
|
||||
remaining[key] = value
|
||||
else:
|
||||
# 未知字段也放入预设(保留用户自定义数据)
|
||||
preset_data[key] = value
|
||||
|
||||
# 构建新格式
|
||||
self.sites[domain] = {
|
||||
"default_preset": DEFAULT_PRESET_NAME,
|
||||
"presets": {
|
||||
DEFAULT_PRESET_NAME: preset_data
|
||||
},
|
||||
**remaining,
|
||||
}
|
||||
|
||||
migrated_count += 1
|
||||
logger.debug(f"迁移站点配置: {domain} → 预设格式")
|
||||
|
||||
if migrated_count > 0:
|
||||
self._save_config()
|
||||
logger.info(f"✅ 已迁移 {migrated_count} 个站点配置为预设格式")
|
||||
def _cleanup_preset_residuals(self):
|
||||
"""
|
||||
清理站点配置中预设外的残留字段
|
||||
|
||||
当站点已有 presets 结构时,顶层不应再有 selectors/workflow/file_paste 等字段。
|
||||
这些残留通常由旧版 bug 或手动编辑产生。
|
||||
"""
|
||||
cleaned_count = 0
|
||||
default_fixed_count = 0
|
||||
|
||||
for domain in list(self.sites.keys()):
|
||||
if domain.startswith('_'):
|
||||
continue
|
||||
|
||||
site_config = self.sites[domain]
|
||||
|
||||
# 只处理已有 presets 结构的站点
|
||||
if "presets" not in site_config:
|
||||
continue
|
||||
|
||||
# 找出预设外的残留字段
|
||||
residual_keys = []
|
||||
for key in list(site_config.keys()):
|
||||
if key == "presets":
|
||||
continue
|
||||
if key in PRESET_FIELDS:
|
||||
residual_keys.append(key)
|
||||
|
||||
# 删除残留
|
||||
for key in residual_keys:
|
||||
del site_config[key]
|
||||
cleaned_count += 1
|
||||
logger.debug(f"清理残留: {domain}.{key}")
|
||||
|
||||
if self._normalize_site_default_preset(domain, site_config):
|
||||
default_fixed_count += 1
|
||||
logger.debug(f"修正默认预设: {domain} -> {site_config.get('default_preset')}")
|
||||
|
||||
if cleaned_count > 0 or default_fixed_count > 0:
|
||||
self._save_config()
|
||||
logger.info(
|
||||
f"✅ 已清理 {cleaned_count} 个预设外残留字段,"
|
||||
f"修正 {default_fixed_count} 个站点默认预设"
|
||||
)
|
||||
def _migrate_site_advanced_to_presets(self):
|
||||
"""
|
||||
规范化已有预设级 advanced,避免启动时改写站点级 advanced 语义。
|
||||
|
||||
站点根级 advanced 仍作为所有预设的共享基线;预设级 advanced 只保存
|
||||
覆盖项。因此这里不再把站点级时序字段复制到所有预设,也不从站点级
|
||||
删除这些字段,只清理明显放错位置的 Cookie 字段并规范化已有覆盖项。
|
||||
"""
|
||||
cleaned_count = 0
|
||||
changed = False
|
||||
|
||||
for domain, site_config in self.sites.items():
|
||||
if domain.startswith("_") or not isinstance(site_config, dict):
|
||||
continue
|
||||
|
||||
presets = site_config.get("presets", {})
|
||||
if not isinstance(presets, dict) or not presets:
|
||||
continue
|
||||
|
||||
for preset_name, preset_data in presets.items():
|
||||
if not isinstance(preset_data, dict):
|
||||
continue
|
||||
|
||||
preset_advanced = preset_data.get("advanced")
|
||||
if not isinstance(preset_advanced, dict):
|
||||
continue
|
||||
|
||||
for key in list(SITE_ADVANCED_FIELDS):
|
||||
if key in preset_advanced:
|
||||
del preset_advanced[key]
|
||||
cleaned_count += 1
|
||||
changed = True
|
||||
|
||||
normalized_preset_advanced = self._normalize_site_advanced_config(
|
||||
preset_advanced
|
||||
)
|
||||
for key in list(preset_advanced.keys()):
|
||||
if key in PRESET_ADVANCED_FIELDS:
|
||||
value = normalized_preset_advanced[key]
|
||||
if preset_advanced.get(key) != value:
|
||||
preset_advanced[key] = value
|
||||
cleaned_count += 1
|
||||
changed = True
|
||||
|
||||
if preset_advanced != preset_data.get("advanced"):
|
||||
logger.debug(f"规范化预设高级配置: {domain}/{preset_name}")
|
||||
|
||||
if not preset_advanced:
|
||||
del preset_data["advanced"]
|
||||
cleaned_count += 1
|
||||
changed = True
|
||||
|
||||
if changed:
|
||||
self._save_config()
|
||||
logger.info(
|
||||
f"✅ 已规范化预设高级配置,清理 {cleaned_count} 个残留项"
|
||||
)
|
||||
def _guess_stealth(self, domain: str) -> bool:
|
||||
"""Guess whether stealth mode should default to enabled."""
|
||||
rule = get_site_rule(domain)
|
||||
if "stealth_default" in rule:
|
||||
enabled = bool(rule.get("stealth_default"))
|
||||
if enabled:
|
||||
logger.info(f"检测到默认启用低熵模式的域名: {domain}")
|
||||
return enabled
|
||||
return False
|
||||
def migrate_site_configs(self):
|
||||
"""迁移旧版站点配置,补充各预设中缺失的字段"""
|
||||
migrated_count = 0
|
||||
default_image_config = get_default_image_extraction_config()
|
||||
default_file_paste = get_default_file_paste_config()
|
||||
default_prompt_padding = get_default_prompt_padding_config()
|
||||
obsolete_stream_keys = {
|
||||
"silence_threshold",
|
||||
"initial_wait",
|
||||
"enable_wrapper_search",
|
||||
"rerender_wait",
|
||||
"content_shrink_tolerance",
|
||||
}
|
||||
obsolete_network_keys = set()
|
||||
|
||||
for domain, site_config in self.sites.items():
|
||||
if domain.startswith("_"):
|
||||
continue
|
||||
|
||||
presets = site_config.get("presets", {})
|
||||
|
||||
for preset_name, preset_data in presets.items():
|
||||
if "image_extraction" not in preset_data:
|
||||
preset_data["image_extraction"] = copy.deepcopy(default_image_config)
|
||||
migrated_count += 1
|
||||
logger.debug(f"迁移: {domain}/{preset_name} (添加 image_extraction)")
|
||||
else:
|
||||
normalized_image_config = self._validate_image_config(preset_data.get("image_extraction", {}))
|
||||
if normalized_image_config != preset_data.get("image_extraction"):
|
||||
preset_data["image_extraction"] = normalized_image_config
|
||||
migrated_count += 1
|
||||
logger.debug(f"迁移: {domain}/{preset_name} (规范化 image_extraction)")
|
||||
|
||||
if "file_paste" not in preset_data:
|
||||
preset_data["file_paste"] = copy.deepcopy(default_file_paste)
|
||||
migrated_count += 1
|
||||
logger.debug(f"迁移: {domain}/{preset_name} (添加 file_paste)")
|
||||
|
||||
if "prompt_padding" not in preset_data:
|
||||
preset_data["prompt_padding"] = copy.deepcopy(default_prompt_padding)
|
||||
migrated_count += 1
|
||||
logger.debug(f"迁移: {domain}/{preset_name} (添加 prompt_padding)")
|
||||
else:
|
||||
normalized_prompt_padding = self._validate_prompt_padding_config(
|
||||
preset_data.get("prompt_padding", {})
|
||||
)
|
||||
if normalized_prompt_padding != preset_data.get("prompt_padding"):
|
||||
preset_data["prompt_padding"] = normalized_prompt_padding
|
||||
migrated_count += 1
|
||||
logger.debug(f"迁移: {domain}/{preset_name} (规范化 prompt_padding)")
|
||||
|
||||
stream_config = preset_data.get("stream_config")
|
||||
moved_attachment_rules = False
|
||||
if not isinstance(stream_config, dict):
|
||||
stream_config = {}
|
||||
preset_data["stream_config"] = stream_config
|
||||
|
||||
normalized_file_paste = self._validate_file_paste_config(
|
||||
preset_data.get("file_paste", {}),
|
||||
legacy_stream_config=stream_config,
|
||||
)
|
||||
if normalized_file_paste != preset_data.get("file_paste"):
|
||||
preset_data["file_paste"] = normalized_file_paste
|
||||
migrated_count += 1
|
||||
logger.debug(f"迁移: {domain}/{preset_name} (规范化 file_paste)")
|
||||
|
||||
for legacy_key in ("send_confirmation", "attachment_monitor"):
|
||||
if legacy_key in stream_config:
|
||||
del stream_config[legacy_key]
|
||||
moved_attachment_rules = True
|
||||
|
||||
if isinstance(stream_config, dict):
|
||||
removed_stream = False
|
||||
for key in list(obsolete_stream_keys):
|
||||
if key in stream_config:
|
||||
del stream_config[key]
|
||||
removed_stream = True
|
||||
|
||||
network_config = stream_config.get("network")
|
||||
removed_network = False
|
||||
if isinstance(network_config, dict):
|
||||
for key in list(obsolete_network_keys):
|
||||
if key in network_config:
|
||||
del network_config[key]
|
||||
removed_network = True
|
||||
|
||||
if removed_stream or removed_network or moved_attachment_rules:
|
||||
migrated_count += 1
|
||||
logger.debug(f"迁移: {domain}/{preset_name} (清理废弃的流式配置字段)")
|
||||
|
||||
if migrated_count > 0:
|
||||
self._save_config()
|
||||
logger.info(f"已迁移 {migrated_count} 个预设配置")
|
||||
|
||||
return migrated_count
|
||||
@@ -0,0 +1,667 @@
|
||||
"""ConfigEngine 的一部分(R2-3 拆分):预设的解析、创建、改名、删除及引用同步。
|
||||
|
||||
方法原样从 engine.py 迁出,行为不变;通过 mixin 组合回 ConfigEngine,
|
||||
实例状态都在 ConfigEngine.__init__(engine.py)里初始化。
|
||||
"""
|
||||
|
||||
import json
|
||||
import copy
|
||||
from typing import Dict, Optional, List, Any
|
||||
from .engine_common import DEFAULT_PRESET_NAME, DEFAULT_WORKFLOW, _MISSING, logger
|
||||
|
||||
|
||||
class ConfigPresetsMixin:
|
||||
"""预设的解析、创建、改名、删除及引用同步"""
|
||||
|
||||
def _resolve_default_preset_name(self, site: Dict[str, Any]) -> Optional[str]:
|
||||
"""解析站点有效默认预设名(不修改原对象)"""
|
||||
presets = site.get("presets", {})
|
||||
if not presets:
|
||||
return None
|
||||
|
||||
configured_default = site.get("default_preset")
|
||||
if isinstance(configured_default, str):
|
||||
configured_default = configured_default.strip()
|
||||
else:
|
||||
configured_default = None
|
||||
|
||||
if configured_default and configured_default in presets:
|
||||
return configured_default
|
||||
|
||||
if DEFAULT_PRESET_NAME in presets:
|
||||
return DEFAULT_PRESET_NAME
|
||||
|
||||
return next(iter(presets))
|
||||
def _normalize_site_default_preset(self, domain: str, site: Dict[str, Any]) -> bool:
|
||||
"""
|
||||
规范化站点 default_preset 字段
|
||||
|
||||
Returns:
|
||||
是否发生修改
|
||||
"""
|
||||
return self._sync_site_default_preset_state(domain, site)
|
||||
def _get_site_data(self, domain: str, preset_name: str = None) -> Optional[Dict]:
|
||||
"""
|
||||
获取指定站点的预设配置数据(可变引用)
|
||||
|
||||
查找顺序:
|
||||
- preset_name 显式提供时:只匹配该预设(含别名)
|
||||
- preset_name 为空时:按站点默认预设 / 主预设 / 第一个可用预设回退
|
||||
|
||||
Args:
|
||||
domain: 站点域名
|
||||
preset_name: 预设名称,None 则使用默认
|
||||
|
||||
Returns:
|
||||
预设配置字典的引用(可直接修改),或 None
|
||||
"""
|
||||
if domain not in self.sites:
|
||||
return None
|
||||
|
||||
site = self.sites[domain]
|
||||
presets = site.get("presets", {})
|
||||
|
||||
if not presets:
|
||||
return None
|
||||
|
||||
requested_preset = str(preset_name or "").strip()
|
||||
if requested_preset:
|
||||
resolved_target = self._resolve_preset_alias_key(requested_preset, presets)
|
||||
if resolved_target != requested_preset:
|
||||
logger.debug(f"预设别名命中: '{requested_preset}' -> '{resolved_target}'")
|
||||
if resolved_target in presets:
|
||||
return presets[resolved_target]
|
||||
return None
|
||||
|
||||
resolved_default = self._resolve_default_preset_name(site)
|
||||
if resolved_default and resolved_default in presets:
|
||||
return presets[resolved_default]
|
||||
|
||||
if DEFAULT_PRESET_NAME in presets:
|
||||
return presets[DEFAULT_PRESET_NAME]
|
||||
|
||||
first_key = next(iter(presets))
|
||||
logger.warning(f"默认预设不存在,使用第一个预设: '{first_key}'")
|
||||
return presets[first_key]
|
||||
@staticmethod
|
||||
def _resolve_preset_alias_key(target: Any, presets: Dict[str, Any]) -> str:
|
||||
"""兼容历史命名:允许不带“预设_”前缀的名字命中真实预设键。"""
|
||||
normalized = str(target or "").strip()
|
||||
if not normalized or not isinstance(presets, dict):
|
||||
return normalized
|
||||
|
||||
if normalized in presets:
|
||||
return normalized
|
||||
|
||||
candidates = []
|
||||
if normalized.startswith("预设_"):
|
||||
stripped = normalized[len("预设_"):].strip()
|
||||
if stripped:
|
||||
candidates.append(stripped)
|
||||
else:
|
||||
candidates.append(f"预设_{normalized}")
|
||||
|
||||
for candidate in candidates:
|
||||
if candidate in presets:
|
||||
return candidate
|
||||
|
||||
return normalized
|
||||
def _get_site_data_readonly(self, domain: str, preset_name: str = None) -> Optional[Dict]:
|
||||
"""获取预设配置的深拷贝(只读用途)"""
|
||||
with self._io_lock:
|
||||
data = self._get_site_data(domain, preset_name)
|
||||
if data is None:
|
||||
return None
|
||||
return copy.deepcopy(data)
|
||||
def _get_preset_data_exact(self, domain: str, preset_name: str = None) -> Optional[Dict]:
|
||||
"""获取预设配置引用;显式预设不存在时不回退到默认预设。"""
|
||||
site = self.sites.get(domain)
|
||||
if not isinstance(site, dict):
|
||||
return None
|
||||
|
||||
presets = site.get("presets", {})
|
||||
if not isinstance(presets, dict) or not presets:
|
||||
return None
|
||||
|
||||
if preset_name is None or not str(preset_name).strip():
|
||||
target = self._resolve_default_preset_name(site)
|
||||
else:
|
||||
target = str(preset_name).strip()
|
||||
|
||||
resolved_target = self._resolve_preset_alias_key(target, presets)
|
||||
if resolved_target and resolved_target in presets:
|
||||
return presets[resolved_target]
|
||||
|
||||
return None
|
||||
def list_presets(self, domain: str) -> List[str]:
|
||||
"""获取指定站点的所有预设名称"""
|
||||
self.refresh_if_changed()
|
||||
|
||||
if domain not in self.sites:
|
||||
return []
|
||||
|
||||
site = self.sites[domain]
|
||||
presets = site.get("presets", {})
|
||||
return list(presets.keys())
|
||||
def get_default_preset(self, domain: str) -> Optional[str]:
|
||||
"""获取指定站点的默认预设名称(已解析回退)"""
|
||||
self.refresh_if_changed()
|
||||
|
||||
cache_key = f"default_preset:{domain}"
|
||||
cached = self._cache.get(cache_key, _MISSING)
|
||||
if cached is not _MISSING:
|
||||
return cached
|
||||
|
||||
site = self.sites.get(domain)
|
||||
if not site:
|
||||
self._cache.set(cache_key, None)
|
||||
return None
|
||||
|
||||
result = self._resolve_default_preset_name(site)
|
||||
self._cache.set(cache_key, result)
|
||||
return result
|
||||
def _assign_preset_field_and_save(
|
||||
self,
|
||||
preset_data: Dict[str, Any],
|
||||
field_name: str,
|
||||
value: Any,
|
||||
) -> bool:
|
||||
previous_value = preset_data.get(field_name, _MISSING)
|
||||
if previous_value is not _MISSING:
|
||||
previous_value = copy.deepcopy(previous_value)
|
||||
preset_data[field_name] = value
|
||||
if self._save_config():
|
||||
return True
|
||||
if previous_value is _MISSING:
|
||||
preset_data.pop(field_name, None)
|
||||
else:
|
||||
preset_data[field_name] = previous_value
|
||||
return False
|
||||
def set_default_preset(self, domain: str, preset_name: str) -> bool:
|
||||
"""设置指定站点的默认预设"""
|
||||
self.refresh_if_changed()
|
||||
|
||||
site = self.sites.get(domain)
|
||||
if not site:
|
||||
return False
|
||||
|
||||
presets = site.get("presets", {})
|
||||
if preset_name not in presets:
|
||||
logger.warning(f"默认预设设置失败,预设不存在: {domain}/{preset_name}")
|
||||
return False
|
||||
|
||||
previous_default = site.get("default_preset", _MISSING)
|
||||
if previous_default is not _MISSING:
|
||||
previous_default = copy.deepcopy(previous_default)
|
||||
previous_default_maps = (
|
||||
copy.deepcopy(getattr(self, "_global_default_presets", {})),
|
||||
copy.deepcopy(getattr(self, "_local_default_presets", {})),
|
||||
)
|
||||
persisted_default = self._get_persisted_default_preset(domain, site)
|
||||
if persisted_default == preset_name:
|
||||
self._local_default_presets.pop(domain, None)
|
||||
else:
|
||||
self._local_default_presets[domain] = preset_name
|
||||
site["default_preset"] = preset_name
|
||||
if not self._save_local_site_overrides():
|
||||
if previous_default is _MISSING:
|
||||
site.pop("default_preset", None)
|
||||
else:
|
||||
site["default_preset"] = previous_default
|
||||
self._global_default_presets, self._local_default_presets = previous_default_maps
|
||||
return False
|
||||
logger.info(f"✅ 站点 {domain} 默认预设已设置为: '{preset_name}'(仅本地覆盖)")
|
||||
return True
|
||||
def create_preset(self, domain: str, new_name: str,
|
||||
source_name: str = None) -> bool:
|
||||
"""
|
||||
创建新预设(克隆自现有预设)
|
||||
|
||||
Args:
|
||||
domain: 站点域名
|
||||
new_name: 新预设名称
|
||||
source_name: 要克隆的源预设名称,None 则克隆主预设
|
||||
|
||||
Returns:
|
||||
是否成功
|
||||
"""
|
||||
self.refresh_if_changed()
|
||||
|
||||
if domain not in self.sites:
|
||||
logger.warning(f"站点不存在: {domain}")
|
||||
return False
|
||||
|
||||
site = self.sites[domain]
|
||||
presets = site.get("presets", {})
|
||||
|
||||
if new_name in presets:
|
||||
logger.warning(f"预设已存在: {new_name}")
|
||||
return False
|
||||
|
||||
# 获取源预设
|
||||
if source_name is not None and str(source_name or "").strip():
|
||||
requested_source = str(source_name or "").strip()
|
||||
source = self._resolve_preset_alias_key(requested_source, presets)
|
||||
source_data = presets.get(source)
|
||||
if not source_data:
|
||||
logger.warning(f"源预设不存在: {requested_source}")
|
||||
return False
|
||||
else:
|
||||
source = DEFAULT_PRESET_NAME
|
||||
source_data = presets.get(source)
|
||||
if not source_data:
|
||||
# 未显式指定源时才尝试第一个可用预设
|
||||
if presets:
|
||||
source = next(iter(presets))
|
||||
source_data = presets[source]
|
||||
else:
|
||||
logger.warning(f"没有可克隆的源预设")
|
||||
return False
|
||||
|
||||
previous_default = site.get("default_preset", _MISSING)
|
||||
if previous_default is not _MISSING:
|
||||
previous_default = copy.deepcopy(previous_default)
|
||||
previous_presets = copy.deepcopy(presets)
|
||||
previous_global_default = copy.deepcopy(self._global_default_presets)
|
||||
previous_local_default = copy.deepcopy(self._local_default_presets)
|
||||
|
||||
# 深拷贝创建新预设
|
||||
presets[new_name] = copy.deepcopy(source_data)
|
||||
self._normalize_site_default_preset(domain, site)
|
||||
if not self._save_config():
|
||||
site["presets"] = previous_presets
|
||||
if previous_default is _MISSING:
|
||||
site.pop("default_preset", None)
|
||||
else:
|
||||
site["default_preset"] = previous_default
|
||||
self._global_default_presets = previous_global_default
|
||||
self._local_default_presets = previous_local_default
|
||||
return False
|
||||
|
||||
logger.info(f"✅ 站点 {domain} 创建预设: '{new_name}' (克隆自 '{source}')")
|
||||
return True
|
||||
@staticmethod
|
||||
def _build_preset_rename_map(old_name: Any, new_name: Any) -> Dict[str, str]:
|
||||
old_raw = str(old_name or "").strip()
|
||||
new_raw = str(new_name or "").strip()
|
||||
if not old_raw or not new_raw:
|
||||
return {}
|
||||
|
||||
def _strip_prefix(value: str) -> str:
|
||||
if value.startswith("预设_"):
|
||||
return value[len("预设_"):].strip()
|
||||
return value
|
||||
|
||||
def _add_prefix(value: str) -> str:
|
||||
return value if value.startswith("预设_") else f"预设_{value}"
|
||||
|
||||
mapping: Dict[str, str] = {}
|
||||
old_plain = _strip_prefix(old_raw)
|
||||
new_plain = _strip_prefix(new_raw)
|
||||
old_prefixed = _add_prefix(old_raw)
|
||||
new_prefixed = _add_prefix(new_raw)
|
||||
|
||||
for src, dst in (
|
||||
(old_raw, new_raw),
|
||||
(old_plain, new_plain),
|
||||
(old_prefixed, new_prefixed),
|
||||
):
|
||||
src_text = str(src or "").strip()
|
||||
dst_text = str(dst or "").strip()
|
||||
if src_text and dst_text:
|
||||
mapping[src_text] = dst_text
|
||||
|
||||
return mapping
|
||||
@staticmethod
|
||||
def _command_targets_domain(command: Dict[str, Any], domain: str) -> bool:
|
||||
normalized_domain = str(domain or "").strip().lower()
|
||||
if not normalized_domain or not isinstance(command, dict):
|
||||
return False
|
||||
|
||||
trigger = command.get("trigger", {}) or {}
|
||||
command_domain = str(trigger.get("domain", "") or "").strip().lower()
|
||||
if not command_domain:
|
||||
return False
|
||||
|
||||
return (
|
||||
command_domain == normalized_domain
|
||||
or command_domain.endswith(f".{normalized_domain}")
|
||||
or normalized_domain.endswith(f".{command_domain}")
|
||||
)
|
||||
def _rename_preset_references_in_commands(
|
||||
self,
|
||||
domain: str,
|
||||
rename_map: Dict[str, str],
|
||||
) -> int:
|
||||
if not rename_map:
|
||||
return 0
|
||||
|
||||
commands = self._load_commands_file()
|
||||
if not commands:
|
||||
return 0
|
||||
|
||||
updated = 0
|
||||
for command in commands:
|
||||
if not self._command_targets_domain(command, domain):
|
||||
continue
|
||||
|
||||
actions = command.get("actions", [])
|
||||
if not isinstance(actions, list):
|
||||
continue
|
||||
|
||||
for action in actions:
|
||||
if not isinstance(action, dict):
|
||||
continue
|
||||
current_preset = str(action.get("preset_name", "") or "").strip()
|
||||
replacement = rename_map.get(current_preset)
|
||||
if not replacement or replacement == current_preset:
|
||||
continue
|
||||
action["preset_name"] = replacement
|
||||
updated += 1
|
||||
|
||||
if updated > 0:
|
||||
self._save_commands_file(commands)
|
||||
|
||||
return updated
|
||||
def _rename_preset_references_in_active_tabs(
|
||||
self,
|
||||
domain: str,
|
||||
rename_map: Dict[str, str],
|
||||
) -> int:
|
||||
if not rename_map:
|
||||
return 0
|
||||
|
||||
try:
|
||||
from app.core import browser as browser_module
|
||||
|
||||
instance = getattr(browser_module, "_browser_instance", None)
|
||||
if instance is None:
|
||||
return 0
|
||||
|
||||
pool = getattr(instance, "_tab_pool", None)
|
||||
if pool is None or not hasattr(pool, "get_sessions_snapshot"):
|
||||
return 0
|
||||
|
||||
updated = 0
|
||||
for session in pool.get_sessions_snapshot():
|
||||
session_domain = str(getattr(session, "current_domain", "") or "").strip().lower()
|
||||
if session_domain != str(domain or "").strip().lower():
|
||||
continue
|
||||
|
||||
current_preset = str(getattr(session, "preset_name", "") or "").strip()
|
||||
replacement = rename_map.get(current_preset)
|
||||
if not replacement or replacement == current_preset:
|
||||
continue
|
||||
|
||||
if hasattr(session, "set_preset"):
|
||||
session.set_preset(replacement, source=getattr(session, "preset_override_source", None))
|
||||
else:
|
||||
session.preset_name = replacement
|
||||
updated += 1
|
||||
|
||||
return updated
|
||||
except Exception as e:
|
||||
logger.debug(f"同步活动标签页预设引用失败(忽略): {e}")
|
||||
return 0
|
||||
def _clear_preset_references_in_active_tabs(self, domain: str, preset_name: str) -> int:
|
||||
"""预设被删除后,把在用标签页里指向它的 preset_name 置为 None(回落站点默认预设)。"""
|
||||
target = str(preset_name or "").strip()
|
||||
if not target:
|
||||
return 0
|
||||
|
||||
# 兼容历史命名:带/不带“预设_”前缀都视为命中同一个预设
|
||||
targets = {target}
|
||||
if target.startswith("预设_"):
|
||||
stripped = target[len("预设_"):].strip()
|
||||
if stripped:
|
||||
targets.add(stripped)
|
||||
else:
|
||||
targets.add(f"预设_{target}")
|
||||
|
||||
try:
|
||||
from app.core import browser as browser_module
|
||||
|
||||
instance = getattr(browser_module, "_browser_instance", None)
|
||||
if instance is None:
|
||||
return 0
|
||||
|
||||
pool = getattr(instance, "_tab_pool", None)
|
||||
if pool is None or not hasattr(pool, "get_sessions_snapshot"):
|
||||
return 0
|
||||
|
||||
cleared = 0
|
||||
for session in pool.get_sessions_snapshot():
|
||||
session_domain = str(getattr(session, "current_domain", "") or "").strip().lower()
|
||||
if session_domain != str(domain or "").strip().lower():
|
||||
continue
|
||||
|
||||
current_preset = str(getattr(session, "preset_name", "") or "").strip()
|
||||
if current_preset not in targets:
|
||||
continue
|
||||
|
||||
if hasattr(session, "set_preset"):
|
||||
session.set_preset(None, source=None)
|
||||
else:
|
||||
session.preset_name = None
|
||||
cleared += 1
|
||||
|
||||
return cleared
|
||||
except Exception as e:
|
||||
logger.debug(f"清理活动标签页预设引用失败(忽略): {e}")
|
||||
return 0
|
||||
def _sync_preset_overrides_file(self, domain: str, old_preset: str, new_preset: Optional[str] = None) -> int:
|
||||
"""当预设重命名或删除时,同步清理/更新 preset_overrides.local.json"""
|
||||
target = str(old_preset or "").strip()
|
||||
if not target:
|
||||
return 0
|
||||
from pathlib import Path
|
||||
config_path = Path("config/preset_overrides.local.json")
|
||||
if not config_path.exists():
|
||||
return 0
|
||||
from app.utils.site_url import extract_remote_site_domain
|
||||
import json
|
||||
try:
|
||||
with open(config_path, "r", encoding="utf-8-sig") as f:
|
||||
payload = json.load(f)
|
||||
if not isinstance(payload, dict):
|
||||
return 0
|
||||
urls = payload.get("urls", {})
|
||||
if not isinstance(urls, dict):
|
||||
return 0
|
||||
|
||||
domain_norm = str(domain or "").strip().lower()
|
||||
changed = 0
|
||||
keys_to_delete = []
|
||||
for url_key, p_name in list(urls.items()):
|
||||
url_domain = str(extract_remote_site_domain(url_key) or "").strip().lower()
|
||||
if domain_norm and url_domain != domain_norm:
|
||||
continue
|
||||
if p_name == target or p_name == f"预设_{target}" or (target.startswith("预设_") and p_name == target[3:]):
|
||||
if new_preset:
|
||||
urls[url_key] = new_preset
|
||||
changed += 1
|
||||
else:
|
||||
keys_to_delete.append(url_key)
|
||||
changed += 1
|
||||
for k in keys_to_delete:
|
||||
urls.pop(k, None)
|
||||
|
||||
if changed > 0:
|
||||
from app.core.config import atomic_write_json
|
||||
atomic_write_json(config_path, payload)
|
||||
try:
|
||||
from app.core import browser as browser_module
|
||||
instance = getattr(browser_module, "_browser_instance", None)
|
||||
pool = getattr(instance, "_tab_pool", None)
|
||||
if pool is not None and hasattr(pool, "apply_runtime_config"):
|
||||
pool.apply_runtime_config(preset_overrides=payload)
|
||||
except Exception:
|
||||
pass
|
||||
return changed
|
||||
except Exception as e:
|
||||
logger.debug(f"同步预设本地配置失败: {e}")
|
||||
return 0
|
||||
def delete_preset(self, domain: str, preset_name: str) -> bool:
|
||||
"""
|
||||
删除预设(不允许删除最后一个预设)
|
||||
|
||||
Args:
|
||||
domain: 站点域名
|
||||
preset_name: 要删除的预设名称
|
||||
|
||||
Returns:
|
||||
是否成功
|
||||
"""
|
||||
self.refresh_if_changed()
|
||||
|
||||
if domain not in self.sites:
|
||||
return False
|
||||
|
||||
site = self.sites[domain]
|
||||
presets = site.get("presets", {})
|
||||
|
||||
resolved_preset_name = self._resolve_preset_alias_key(preset_name, presets)
|
||||
if resolved_preset_name not in presets:
|
||||
logger.warning(f"预设不存在: {preset_name}")
|
||||
return False
|
||||
|
||||
if len(presets) <= 1:
|
||||
logger.warning(f"不能删除最后一个预设")
|
||||
return False
|
||||
|
||||
previous_default = site.get("default_preset", _MISSING)
|
||||
if previous_default is not _MISSING:
|
||||
previous_default = copy.deepcopy(previous_default)
|
||||
removed_preset = copy.deepcopy(presets[resolved_preset_name])
|
||||
previous_global_default = copy.deepcopy(self._global_default_presets)
|
||||
previous_local_default = copy.deepcopy(self._local_default_presets)
|
||||
|
||||
del presets[resolved_preset_name]
|
||||
self._normalize_site_default_preset(domain, site)
|
||||
if not self._save_config():
|
||||
presets[resolved_preset_name] = removed_preset
|
||||
if previous_default is _MISSING:
|
||||
site.pop("default_preset", None)
|
||||
else:
|
||||
site["default_preset"] = previous_default
|
||||
self._global_default_presets = previous_global_default
|
||||
self._local_default_presets = previous_local_default
|
||||
return False
|
||||
|
||||
# 预设已删除并落盘:清理在用标签页对它的残留引用,否则 session.preset_name 会指向不存在的预设
|
||||
cleared_tab_refs = 0
|
||||
try:
|
||||
cleared_tab_refs = self._clear_preset_references_in_active_tabs(domain, resolved_preset_name)
|
||||
except Exception as e:
|
||||
logger.warning(f"清理活动标签页预设引用失败(忽略): {e}")
|
||||
|
||||
# 同步清理持久化 preset_overrides.local.json
|
||||
try:
|
||||
self._sync_preset_overrides_file(domain, resolved_preset_name, new_preset=None)
|
||||
except Exception as e:
|
||||
logger.warning(f"清理本地预设记忆文件失败(忽略): {e}")
|
||||
|
||||
logger.info(
|
||||
f"✅ 站点 {domain} 删除预设: '{resolved_preset_name}' "
|
||||
f"(活动标签页同步 {cleared_tab_refs} 处)"
|
||||
)
|
||||
return True
|
||||
def rename_preset(self, domain: str, old_name: str, new_name: str) -> bool:
|
||||
"""重命名预设"""
|
||||
self.refresh_if_changed()
|
||||
|
||||
if domain not in self.sites:
|
||||
return False
|
||||
|
||||
site = self.sites[domain]
|
||||
presets = site.get("presets", {})
|
||||
|
||||
resolved_old_name = self._resolve_preset_alias_key(old_name, presets)
|
||||
if resolved_old_name not in presets:
|
||||
return False
|
||||
|
||||
if new_name in presets:
|
||||
logger.warning(f"预设名已存在: {new_name}")
|
||||
return False
|
||||
|
||||
rename_map = self._build_preset_rename_map(resolved_old_name, new_name)
|
||||
default_preset = site.get("default_preset")
|
||||
previous_default = site.get("default_preset", _MISSING)
|
||||
if previous_default is not _MISSING:
|
||||
previous_default = copy.deepcopy(previous_default)
|
||||
previous_presets = copy.deepcopy(presets)
|
||||
previous_global_default = copy.deepcopy(self._global_default_presets)
|
||||
previous_local_default = copy.deepcopy(self._local_default_presets)
|
||||
local_default = str(self._local_default_presets.get(domain, "") or "").strip()
|
||||
|
||||
# 保持顺序:创建有序副本
|
||||
new_presets = {}
|
||||
for key, value in presets.items():
|
||||
if key == resolved_old_name:
|
||||
new_presets[new_name] = value
|
||||
else:
|
||||
new_presets[key] = value
|
||||
|
||||
site["presets"] = new_presets
|
||||
if default_preset == resolved_old_name:
|
||||
site["default_preset"] = new_name
|
||||
if local_default:
|
||||
local_replacement = rename_map.get(local_default)
|
||||
if local_replacement:
|
||||
self._local_default_presets[domain] = local_replacement
|
||||
self._normalize_site_default_preset(domain, site)
|
||||
if not self._save_config():
|
||||
site["presets"] = previous_presets
|
||||
if previous_default is _MISSING:
|
||||
site.pop("default_preset", None)
|
||||
else:
|
||||
site["default_preset"] = previous_default
|
||||
self._global_default_presets = previous_global_default
|
||||
self._local_default_presets = previous_local_default
|
||||
return False
|
||||
|
||||
updated_command_refs = self._rename_preset_references_in_commands(domain, rename_map)
|
||||
updated_tab_refs = self._rename_preset_references_in_active_tabs(domain, rename_map)
|
||||
try:
|
||||
self._sync_preset_overrides_file(domain, resolved_old_name, new_preset=new_name)
|
||||
except Exception as e:
|
||||
logger.warning(f"同步本地预设记忆文件失败(忽略): {e}")
|
||||
|
||||
logger.info(
|
||||
f"站点 {domain} 重命名预设: '{resolved_old_name}' → '{new_name}' "
|
||||
f"(命令引用同步 {updated_command_refs} 处, 活动标签页同步 {updated_tab_refs} 处)"
|
||||
)
|
||||
return True
|
||||
def get_preset_selectors(self, domain: str, preset_name: str = None) -> Dict:
|
||||
"""获取指定预设的选择器配置"""
|
||||
data = self._get_site_data_readonly(domain, preset_name)
|
||||
return data.get("selectors", {}) if data else {}
|
||||
def set_preset_selectors(self, domain: str, selectors: Dict,
|
||||
preset_name: str = None) -> bool:
|
||||
"""设置指定预设的选择器配置"""
|
||||
self.refresh_if_changed()
|
||||
data = self._get_site_data(domain, preset_name)
|
||||
if data is None:
|
||||
return False
|
||||
if not self._assign_preset_field_and_save(data, "selectors", selectors):
|
||||
return False
|
||||
logger.info(f"站点 {domain} [{preset_name or DEFAULT_PRESET_NAME}] 选择器已更新")
|
||||
return True
|
||||
def get_preset_workflow(self, domain: str, preset_name: str = None) -> List:
|
||||
"""获取指定预设的工作流配置"""
|
||||
data = self._get_site_data_readonly(domain, preset_name)
|
||||
return data.get("workflow", DEFAULT_WORKFLOW) if data else DEFAULT_WORKFLOW
|
||||
def set_preset_workflow(self, domain: str, workflow: List,
|
||||
preset_name: str = None) -> bool:
|
||||
"""设置指定预设的工作流配置"""
|
||||
from app.core.workflow.flow_runtime import has_control_flow, validate_workflow
|
||||
if has_control_flow(workflow):
|
||||
validate_workflow(workflow)
|
||||
self.refresh_if_changed()
|
||||
data = self._get_site_data(domain, preset_name)
|
||||
if data is None:
|
||||
return False
|
||||
if not self._assign_preset_field_and_save(data, "workflow", workflow):
|
||||
return False
|
||||
logger.info(f"站点 {domain} [{preset_name or DEFAULT_PRESET_NAME}] 工作流已更新")
|
||||
return True
|
||||
@@ -0,0 +1,535 @@
|
||||
"""ConfigEngine 的一部分(R2-3 拆分):站点目录、起始页推断、站点高级配置与选择器定义。
|
||||
|
||||
方法原样从 engine.py 迁出,行为不变;通过 mixin 组合回 ConfigEngine,
|
||||
实例状态都在 ConfigEngine.__init__(engine.py)里初始化。
|
||||
"""
|
||||
|
||||
import copy
|
||||
from typing import Dict, Optional, List, Any, Set, Union, Callable
|
||||
from urllib.parse import urljoin
|
||||
from app.models.schemas import (
|
||||
SiteConfig,
|
||||
SelectorDefinition,
|
||||
ADVANCED_FIELDS,
|
||||
PRESET_ADVANCED_FIELDS,
|
||||
SITE_ADVANCED_FIELDS,
|
||||
get_default_image_extraction_config,
|
||||
get_default_file_paste_config,
|
||||
get_default_prompt_padding_config,
|
||||
get_default_site_advanced_config,
|
||||
)
|
||||
from app.utils.site_rules import derive_site_card_id, get_site_rule
|
||||
from app.utils.site_discovery import (
|
||||
automatic_discovery_allowed,
|
||||
specific_chat_selectors,
|
||||
selectors_match_chat_html,
|
||||
)
|
||||
from .engine_common import (
|
||||
DEFAULT_PRESET_NAME,
|
||||
DEFAULT_WORKFLOW,
|
||||
_MISSING,
|
||||
_SITE_STARTUP_JS_PATTERNS,
|
||||
_coerce_bool,
|
||||
get_default_stream_config,
|
||||
logger,
|
||||
)
|
||||
|
||||
|
||||
class ConfigSitesMixin:
|
||||
"""站点目录、起始页推断、站点高级配置与选择器定义"""
|
||||
|
||||
def _extract_startup_url_from_script(self, script: str) -> str:
|
||||
text = str(script or "").strip()
|
||||
if not text:
|
||||
return ""
|
||||
|
||||
for pattern in _SITE_STARTUP_JS_PATTERNS:
|
||||
match = pattern.search(text)
|
||||
if match:
|
||||
return str(match.group(1) or "").strip()
|
||||
return ""
|
||||
def _normalize_startup_url(self, domain: str, raw_url: str) -> str:
|
||||
normalized_domain = str(domain or "").strip().lower().strip(".")
|
||||
text = str(raw_url or "").strip()
|
||||
if not normalized_domain:
|
||||
return text
|
||||
if not text:
|
||||
return f"https://{normalized_domain}"
|
||||
if text.startswith(("http://", "https://")):
|
||||
return text
|
||||
return urljoin(f"https://{normalized_domain}", text)
|
||||
def _infer_site_startup_url(self, domain: str, site: Dict[str, Any]) -> str:
|
||||
preset_name = self._resolve_default_preset_name(site)
|
||||
preset_data = self._get_site_data_readonly(domain, preset_name)
|
||||
workflow = preset_data.get("workflow", []) if isinstance(preset_data, dict) else []
|
||||
|
||||
if isinstance(workflow, list):
|
||||
for step in workflow:
|
||||
if str((step or {}).get("action") or "").strip().upper() != "JS_EXEC":
|
||||
continue
|
||||
startup_url = self._extract_startup_url_from_script((step or {}).get("value"))
|
||||
if startup_url:
|
||||
return self._normalize_startup_url(domain, startup_url)
|
||||
|
||||
return self._normalize_startup_url(domain, "")
|
||||
def get_site_catalog_entry(self, domain: str, fallback_order: int = 0) -> Optional[Dict[str, Any]]:
|
||||
self.refresh_if_changed()
|
||||
|
||||
site = self.sites.get(domain)
|
||||
if not isinstance(site, dict) or str(domain or "").startswith("_"):
|
||||
return None
|
||||
|
||||
rule = get_site_rule(domain)
|
||||
startup_url = rule.get("startup_url") or self._infer_site_startup_url(domain, site)
|
||||
display_name = str(rule.get("display_name") or domain).strip() or str(domain)
|
||||
card_id = str(rule.get("card_id") or derive_site_card_id(domain)).strip() or derive_site_card_id(domain)
|
||||
guide_priority = rule.get("guide_priority")
|
||||
if not isinstance(guide_priority, int):
|
||||
guide_priority = int(fallback_order)
|
||||
|
||||
entry = {
|
||||
"domain": str(domain),
|
||||
"display_name": display_name,
|
||||
"url": self._normalize_startup_url(domain, startup_url),
|
||||
"card_id": card_id,
|
||||
"guide_priority": guide_priority,
|
||||
}
|
||||
|
||||
if isinstance(rule.get("route_aliases"), list):
|
||||
entry["route_aliases"] = [str(item) for item in rule.get("route_aliases", [])]
|
||||
if "stealth_default" in rule:
|
||||
entry["stealth_default"] = bool(rule.get("stealth_default"))
|
||||
|
||||
return entry
|
||||
def list_site_catalog(self) -> List[Dict[str, Any]]:
|
||||
self.refresh_if_changed()
|
||||
|
||||
ordered_domains = [
|
||||
domain for domain, site in self.sites.items()
|
||||
if not domain.startswith("_") and isinstance(site, dict)
|
||||
]
|
||||
entries: List[Dict[str, Any]] = []
|
||||
|
||||
for index, domain in enumerate(ordered_domains):
|
||||
entry = self.get_site_catalog_entry(domain, fallback_order=index)
|
||||
if entry:
|
||||
entries.append(entry)
|
||||
|
||||
entries.sort(key=lambda item: (int(item.get("guide_priority", 0)), str(item.get("domain") or "")))
|
||||
return entries
|
||||
def _normalize_site_advanced_config(
|
||||
self,
|
||||
raw_config: Optional[Dict[str, Any]] = None,
|
||||
*,
|
||||
base_config: Optional[Dict[str, Any]] = None,
|
||||
) -> Dict[str, Any]:
|
||||
"""合并并规范化高级配置。"""
|
||||
normalized = {
|
||||
**get_default_site_advanced_config(),
|
||||
}
|
||||
if isinstance(base_config, dict):
|
||||
normalized.update(copy.deepcopy(base_config))
|
||||
if isinstance(raw_config, dict):
|
||||
normalized.update(copy.deepcopy(raw_config))
|
||||
|
||||
normalized["independent_cookies"] = _coerce_bool(normalized.get("independent_cookies"), False)
|
||||
normalized["independent_cookies_auto_takeover"] = _coerce_bool(
|
||||
normalized.get("independent_cookies_auto_takeover"), False
|
||||
)
|
||||
normalized["input_box_stability_wait_enabled"] = _coerce_bool(
|
||||
normalized.get("input_box_stability_wait_enabled"), False
|
||||
)
|
||||
normalized["input_box_stability_wait_after_new_chat_only"] = _coerce_bool(
|
||||
normalized.get("input_box_stability_wait_after_new_chat_only"), True
|
||||
)
|
||||
try:
|
||||
timeout_value = float(normalized.get("input_box_stability_wait_timeout", 1.5))
|
||||
except Exception:
|
||||
timeout_value = 1.5
|
||||
normalized["input_box_stability_wait_timeout"] = max(0.1, min(timeout_value, 10.0))
|
||||
normalized["url_transition_wait_on_new_chat"] = _coerce_bool(
|
||||
normalized.get("url_transition_wait_on_new_chat"), False
|
||||
)
|
||||
raw_patterns = normalized.get("url_transition_wait_patterns") or []
|
||||
if isinstance(raw_patterns, str):
|
||||
raw_patterns = raw_patterns.replace("\n", ",").replace(";", ",").split(",")
|
||||
if not isinstance(raw_patterns, (list, tuple, set)):
|
||||
raw_patterns = []
|
||||
normalized["url_transition_wait_patterns"] = [
|
||||
str(pattern or "").strip()
|
||||
for pattern in raw_patterns
|
||||
if str(pattern or "").strip()
|
||||
]
|
||||
normalized["send_confirmation_check_enabled"] = _coerce_bool(
|
||||
normalized.get("send_confirmation_check_enabled"), False
|
||||
)
|
||||
try:
|
||||
send_timeout_value = float(normalized.get("send_confirmation_check_timeout", 1.5))
|
||||
except Exception:
|
||||
send_timeout_value = 1.5
|
||||
normalized["send_confirmation_check_timeout"] = max(0.1, min(send_timeout_value, 10.0))
|
||||
normalized["skip_new_chat_on_retry"] = _coerce_bool(
|
||||
normalized.get("skip_new_chat_on_retry"), False
|
||||
)
|
||||
return normalized
|
||||
def get_site_advanced_config(self, domain: str, preset_name: str = None) -> Dict[str, Any]:
|
||||
"""获取站点高级配置;传入 preset_name 时叠加对应预设的覆盖项。"""
|
||||
self.refresh_if_changed()
|
||||
|
||||
site = self.sites.get(domain)
|
||||
if not site:
|
||||
return self._normalize_site_advanced_config()
|
||||
|
||||
raw_config = site.get("advanced") if isinstance(site, dict) else None
|
||||
normalized = self._normalize_site_advanced_config(
|
||||
raw_config if isinstance(raw_config, dict) else {}
|
||||
)
|
||||
|
||||
requested_preset = str(preset_name or "").strip()
|
||||
if requested_preset:
|
||||
preset_data = self._get_preset_data_exact(domain, requested_preset)
|
||||
if preset_data is None:
|
||||
logger.warning(f"站点高级配置预设不存在: {domain}/{requested_preset}")
|
||||
return normalized
|
||||
preset_advanced = (
|
||||
preset_data.get("advanced")
|
||||
if isinstance(preset_data, dict)
|
||||
else None
|
||||
)
|
||||
if isinstance(preset_advanced, dict):
|
||||
preset_advanced = {
|
||||
key: value
|
||||
for key, value in preset_advanced.items()
|
||||
if key in PRESET_ADVANCED_FIELDS
|
||||
}
|
||||
normalized = self._normalize_site_advanced_config(
|
||||
preset_advanced,
|
||||
base_config=normalized,
|
||||
)
|
||||
|
||||
return normalized
|
||||
def set_site_advanced_config(self, domain: str, config: Dict[str, Any]) -> bool:
|
||||
"""设置站点级高级配置。"""
|
||||
with self._io_lock:
|
||||
self.refresh_if_changed()
|
||||
|
||||
site = self.sites.get(domain)
|
||||
if not site:
|
||||
logger.warning(f"站点不存在: {domain}")
|
||||
return False
|
||||
|
||||
previous_advanced = site.get("advanced", _MISSING)
|
||||
if previous_advanced is not _MISSING:
|
||||
previous_advanced = copy.deepcopy(previous_advanced)
|
||||
|
||||
existing = site.get("advanced") if isinstance(site.get("advanced"), dict) else {}
|
||||
stored = copy.deepcopy(existing)
|
||||
normalized = self._normalize_site_advanced_config(config or {}, base_config=existing)
|
||||
|
||||
provided_keys = {
|
||||
key
|
||||
for key in ((config or {}).keys() if isinstance(config, dict) else set())
|
||||
if key in ADVANCED_FIELDS
|
||||
}
|
||||
if not provided_keys and previous_advanced is _MISSING:
|
||||
return True
|
||||
for key in ADVANCED_FIELDS:
|
||||
if key in provided_keys:
|
||||
stored[key] = normalized[key]
|
||||
|
||||
site["advanced"] = stored
|
||||
if self._save_config_locked():
|
||||
return True
|
||||
if previous_advanced is _MISSING:
|
||||
site.pop("advanced", None)
|
||||
else:
|
||||
site["advanced"] = previous_advanced
|
||||
return False
|
||||
def set_preset_advanced_config(
|
||||
self,
|
||||
domain: str,
|
||||
config: Dict[str, Any],
|
||||
preset_name: str = None,
|
||||
*,
|
||||
prune_inherited_fields: Optional[Set[str]] = None,
|
||||
) -> bool:
|
||||
"""设置当前预设的高级配置;拒绝混入站点级字段。"""
|
||||
with self._io_lock:
|
||||
self.refresh_if_changed()
|
||||
|
||||
site = self.sites.get(domain)
|
||||
if not isinstance(site, dict):
|
||||
logger.warning(f"站点不存在: {domain}")
|
||||
return False
|
||||
|
||||
requested_preset = str(preset_name or "").strip()
|
||||
if not requested_preset:
|
||||
logger.warning(f"预设级高级配置缺少 preset_name: {domain}")
|
||||
return False
|
||||
|
||||
data = self._get_preset_data_exact(domain, requested_preset)
|
||||
if data is None:
|
||||
logger.warning(f"站点或预设不存在: {domain}/{requested_preset}")
|
||||
return False
|
||||
|
||||
raw_config = config if isinstance(config, dict) else {}
|
||||
|
||||
invalid_site_keys = {
|
||||
key
|
||||
for key in raw_config.keys()
|
||||
if key in SITE_ADVANCED_FIELDS
|
||||
}
|
||||
if invalid_site_keys:
|
||||
joined = ", ".join(sorted(invalid_site_keys))
|
||||
logger.warning(f"预设级高级配置不能包含站点级字段: {domain}/{requested_preset} ({joined})")
|
||||
return False
|
||||
|
||||
previous_advanced = data.get("advanced", _MISSING)
|
||||
if previous_advanced is not _MISSING:
|
||||
previous_advanced = copy.deepcopy(previous_advanced)
|
||||
|
||||
stored = copy.deepcopy(data.get("advanced") or {})
|
||||
if not isinstance(stored, dict):
|
||||
stored = {}
|
||||
|
||||
for key in list(SITE_ADVANCED_FIELDS):
|
||||
stored.pop(key, None)
|
||||
|
||||
site_advanced = site.get("advanced") if isinstance(site.get("advanced"), dict) else {}
|
||||
inherited = self._normalize_site_advanced_config(site_advanced)
|
||||
|
||||
normalized = self._normalize_site_advanced_config(
|
||||
raw_config,
|
||||
base_config=stored,
|
||||
)
|
||||
|
||||
provided_keys = {
|
||||
key
|
||||
for key in raw_config.keys()
|
||||
if key in PRESET_ADVANCED_FIELDS
|
||||
}
|
||||
if not provided_keys and previous_advanced is _MISSING:
|
||||
return True
|
||||
prune_inherited_keys = set(prune_inherited_fields or set())
|
||||
for key in PRESET_ADVANCED_FIELDS:
|
||||
if key in provided_keys:
|
||||
if (
|
||||
key in prune_inherited_keys
|
||||
and key not in stored
|
||||
and normalized[key] == inherited.get(key)
|
||||
):
|
||||
stored.pop(key, None)
|
||||
else:
|
||||
stored[key] = normalized[key]
|
||||
|
||||
if not stored and previous_advanced is _MISSING:
|
||||
return True
|
||||
|
||||
data["advanced"] = stored
|
||||
if self._save_config_locked():
|
||||
return True
|
||||
if previous_advanced is _MISSING:
|
||||
data.pop("advanced", None)
|
||||
else:
|
||||
data["advanced"] = previous_advanced
|
||||
return False
|
||||
def list_sites(self) -> Dict[str, Any]:
|
||||
"""获取所有站点配置(过滤内部键)"""
|
||||
self.refresh_if_changed()
|
||||
|
||||
with self._io_lock:
|
||||
return copy.deepcopy({
|
||||
domain: config
|
||||
for domain, config in self.sites.items()
|
||||
if not domain.startswith('_')
|
||||
})
|
||||
def get_site_config(self, domain: str, html_content: Optional[Union[str, Callable[[], str]]] = None,
|
||||
preset_name: str = None) -> Optional[SiteConfig]:
|
||||
"""
|
||||
获取站点配置(缓存 + AI 分析)
|
||||
|
||||
Args:
|
||||
domain: 站点域名
|
||||
html_content: 页面 HTML 或用于延迟获取 HTML 的可调用对象(用于 AI 分析未知站点)
|
||||
preset_name: 预设名称,None 则使用默认预设
|
||||
"""
|
||||
self.refresh_if_changed()
|
||||
|
||||
if domain in self.sites:
|
||||
cache_key = f"site_config:{domain}:{preset_name or ''}"
|
||||
cached = self._cache.get(cache_key, _MISSING)
|
||||
if cached is not _MISSING:
|
||||
return copy.deepcopy(cached)
|
||||
|
||||
site = self.sites.get(domain, {})
|
||||
config = self._get_site_data(domain, preset_name)
|
||||
|
||||
if config is None:
|
||||
logger.warning(f"站点 {domain} 无可用预设")
|
||||
return None
|
||||
|
||||
used_preset = preset_name or self._resolve_default_preset_name(site) or DEFAULT_PRESET_NAME
|
||||
previous_config = None
|
||||
|
||||
# 补充缺失字段
|
||||
changed = False
|
||||
if "workflow" not in config:
|
||||
if previous_config is None:
|
||||
previous_config = copy.deepcopy(config)
|
||||
config["workflow"] = DEFAULT_WORKFLOW
|
||||
changed = True
|
||||
|
||||
if "image_extraction" not in config:
|
||||
if previous_config is None:
|
||||
previous_config = copy.deepcopy(config)
|
||||
config["image_extraction"] = get_default_image_extraction_config()
|
||||
changed = True
|
||||
|
||||
if "file_paste" not in config:
|
||||
if previous_config is None:
|
||||
previous_config = copy.deepcopy(config)
|
||||
config["file_paste"] = get_default_file_paste_config()
|
||||
changed = True
|
||||
else:
|
||||
normalized_file_paste = self._validate_file_paste_config(
|
||||
config.get("file_paste", {}),
|
||||
legacy_stream_config=config.get("stream_config"),
|
||||
)
|
||||
if normalized_file_paste != config.get("file_paste"):
|
||||
if previous_config is None:
|
||||
previous_config = copy.deepcopy(config)
|
||||
config["file_paste"] = normalized_file_paste
|
||||
changed = True
|
||||
|
||||
if "prompt_padding" not in config:
|
||||
if previous_config is None:
|
||||
previous_config = copy.deepcopy(config)
|
||||
config["prompt_padding"] = get_default_prompt_padding_config()
|
||||
changed = True
|
||||
else:
|
||||
normalized_prompt_padding = self._validate_prompt_padding_config(
|
||||
config.get("prompt_padding", {})
|
||||
)
|
||||
if normalized_prompt_padding != config.get("prompt_padding"):
|
||||
if previous_config is None:
|
||||
previous_config = copy.deepcopy(config)
|
||||
config["prompt_padding"] = normalized_prompt_padding
|
||||
changed = True
|
||||
|
||||
if changed:
|
||||
completed_config = copy.deepcopy(config)
|
||||
if not self._save_config():
|
||||
config.clear()
|
||||
config.update(previous_config)
|
||||
logger.warning(
|
||||
f"站点 {domain} [{used_preset}] 配置自动补全保存失败,"
|
||||
"已仅对当前请求返回补全配置"
|
||||
)
|
||||
return completed_config
|
||||
|
||||
logger.debug(f"使用缓存配置: {domain} [预设: {used_preset}]")
|
||||
result = copy.deepcopy(config)
|
||||
self._cache.set(cache_key, result)
|
||||
return copy.deepcopy(result)
|
||||
|
||||
if not automatic_discovery_allowed(domain):
|
||||
logger.info(f"跳过非聊天站点的自动收录: {domain}(已有手动配置不受影响)")
|
||||
return None
|
||||
|
||||
resolved_html = ""
|
||||
if callable(html_content):
|
||||
try:
|
||||
resolved_html = str(html_content() or "")
|
||||
except Exception as e:
|
||||
logger.warning(f"获取页面 HTML 失败: {e}")
|
||||
resolved_html = ""
|
||||
elif html_content is not None:
|
||||
resolved_html = str(html_content or "")
|
||||
|
||||
if resolved_html.strip():
|
||||
logger.info(f"🔍 未知域名 {domain},启动 AI 识别...")
|
||||
clean_html = self.html_cleaner.clean(resolved_html)
|
||||
selectors = self.ai_analyzer.analyze(clean_html)
|
||||
|
||||
if specific_chat_selectors(selectors):
|
||||
selectors = self.validator.validate(selectors)
|
||||
if selectors_match_chat_html(selectors, clean_html):
|
||||
new_preset: SiteConfig = {
|
||||
"selectors": selectors,
|
||||
"workflow": DEFAULT_WORKFLOW,
|
||||
"stealth": self._guess_stealth(domain),
|
||||
"stream_config": copy.deepcopy(get_default_stream_config()),
|
||||
"image_extraction": get_default_image_extraction_config(),
|
||||
"file_paste": get_default_file_paste_config(),
|
||||
"prompt_padding": get_default_prompt_padding_config(),
|
||||
}
|
||||
|
||||
self.sites[domain] = {
|
||||
"default_preset": DEFAULT_PRESET_NAME,
|
||||
"presets": {
|
||||
DEFAULT_PRESET_NAME: new_preset
|
||||
}
|
||||
}
|
||||
if self._save_config():
|
||||
logger.info(f"✅ 配置已生成并保存: {domain}")
|
||||
else:
|
||||
logger.warning(f"⚠️ 配置已生成但保存失败,仅保留在当前运行内存: {domain}")
|
||||
return copy.deepcopy(new_preset)
|
||||
|
||||
logger.warning(f"⚠️ AI 分析失败,使用通用回退配置: {domain}")
|
||||
else:
|
||||
logger.warning(f"⚠️ 未知域名 {domain} 且未提供网页 HTML,使用通用回退配置")
|
||||
|
||||
fallback_selectors = self.global_config.get_fallback_selectors()
|
||||
|
||||
fallback_preset: SiteConfig = {
|
||||
"selectors": fallback_selectors,
|
||||
"workflow": DEFAULT_WORKFLOW,
|
||||
"stealth": False,
|
||||
"stream_config": copy.deepcopy(get_default_stream_config()),
|
||||
"image_extraction": get_default_image_extraction_config(),
|
||||
"file_paste": get_default_file_paste_config(),
|
||||
"prompt_padding": get_default_prompt_padding_config(),
|
||||
}
|
||||
|
||||
# A generic fallback is only for this caller. It is not proof that this
|
||||
# domain hosts an AI chat product, and must never enter the site catalog.
|
||||
logger.info(f"通用回退仅用于当前请求,未自动添加站点: {domain}")
|
||||
return copy.deepcopy(fallback_preset)
|
||||
def delete_site_config(self, domain: str) -> bool:
|
||||
"""删除指定站点配置"""
|
||||
self.refresh_if_changed()
|
||||
|
||||
if domain in self.sites:
|
||||
removed_site = copy.deepcopy(self.sites[domain])
|
||||
previous_global_default = copy.deepcopy(self._global_default_presets)
|
||||
previous_local_default = copy.deepcopy(self._local_default_presets)
|
||||
del self.sites[domain]
|
||||
if not self._save_config():
|
||||
self.sites[domain] = removed_site
|
||||
self._global_default_presets = previous_global_default
|
||||
self._local_default_presets = previous_local_default
|
||||
return False
|
||||
logger.info(f"已删除配置: {domain}")
|
||||
return True
|
||||
return False
|
||||
def get_selector_definitions(self) -> List[SelectorDefinition]:
|
||||
"""获取元素定义列表"""
|
||||
return self.global_config.get_selector_definitions()
|
||||
def set_selector_definitions(self, definitions: List[SelectorDefinition]) -> bool:
|
||||
"""设置元素定义列表并保存"""
|
||||
previous_definitions = self.global_config.get_selector_definitions()
|
||||
previous_fallback_selectors = copy.deepcopy(self.validator.fallback_selectors)
|
||||
self.global_config.set_selector_definitions(definitions)
|
||||
|
||||
# 更新验证器的回退选择器
|
||||
self.validator.fallback_selectors = self.global_config.get_fallback_selectors()
|
||||
|
||||
# 保存配置
|
||||
if not self._save_config():
|
||||
self.global_config.set_selector_definitions(previous_definitions)
|
||||
self.validator.fallback_selectors = previous_fallback_selectors
|
||||
return False
|
||||
|
||||
logger.info(f"元素定义已更新: {len(definitions)} 个")
|
||||
return True
|
||||
@@ -0,0 +1,485 @@
|
||||
"""ConfigEngine 的一部分(R2-3 拆分):提取器、流式/网络监听配置及其校验、解析器列表。
|
||||
|
||||
方法原样从 engine.py 迁出,行为不变;通过 mixin 组合回 ConfigEngine,
|
||||
实例状态都在 ConfigEngine.__init__(engine.py)里初始化。
|
||||
"""
|
||||
|
||||
import copy
|
||||
from typing import Dict, Optional, List, Any
|
||||
from app.core.parsers import ParserRegistry
|
||||
from app.models.schemas import get_default_attachment_monitor_config, get_default_send_confirmation_config
|
||||
from app.services.extractor_manager import extractor_manager
|
||||
from app.services.parser_manager import parser_manager
|
||||
from app.core.request_transport import normalize_request_transport_config
|
||||
from .engine_common import (
|
||||
DEFAULT_PRESET_NAME,
|
||||
_MISSING,
|
||||
_coerce_bool,
|
||||
_merge_config_patch,
|
||||
get_default_network_config,
|
||||
get_default_stream_config,
|
||||
logger,
|
||||
)
|
||||
|
||||
|
||||
class ConfigStreamMixin:
|
||||
"""提取器、流式/网络监听配置及其校验、解析器列表"""
|
||||
|
||||
def get_site_extractor(self, domain: str, preset_name: str = None):
|
||||
"""获取站点的提取器实例"""
|
||||
self.refresh_if_changed()
|
||||
|
||||
data = self._get_site_data(domain, preset_name)
|
||||
if data is not None:
|
||||
return extractor_manager.get_extractor_for_site(data)
|
||||
|
||||
return extractor_manager.get_extractor()
|
||||
def set_site_extractor(self, domain: str, extractor_id: str,
|
||||
preset_name: str = None) -> bool:
|
||||
"""为站点设置提取器"""
|
||||
self.refresh_if_changed()
|
||||
|
||||
data = self._get_site_data(domain, preset_name)
|
||||
if data is None:
|
||||
logger.warning(f"站点或预设不存在: {domain}/{preset_name}")
|
||||
return False
|
||||
|
||||
from app.core.extractors import ExtractorRegistry
|
||||
if not ExtractorRegistry.exists(extractor_id):
|
||||
logger.error(f"提取器不存在: {extractor_id}")
|
||||
return False
|
||||
|
||||
previous_extractor_id = data.get("extractor_id", _MISSING)
|
||||
if previous_extractor_id is not _MISSING:
|
||||
previous_extractor_id = copy.deepcopy(previous_extractor_id)
|
||||
previous_verified = data.get("extractor_verified", _MISSING)
|
||||
if previous_verified is not _MISSING:
|
||||
previous_verified = copy.deepcopy(previous_verified)
|
||||
data["extractor_id"] = extractor_id
|
||||
data["extractor_verified"] = False
|
||||
if not self._save_config():
|
||||
if previous_extractor_id is _MISSING:
|
||||
data.pop("extractor_id", None)
|
||||
else:
|
||||
data["extractor_id"] = previous_extractor_id
|
||||
if previous_verified is _MISSING:
|
||||
data.pop("extractor_verified", None)
|
||||
else:
|
||||
data["extractor_verified"] = previous_verified
|
||||
return False
|
||||
|
||||
logger.info(f"站点 {domain} [{preset_name or DEFAULT_PRESET_NAME}] 已绑定提取器: {extractor_id}")
|
||||
return True
|
||||
def set_site_extractor_verified(self, domain: str, verified: bool = True,
|
||||
preset_name: str = None) -> bool:
|
||||
"""设置站点提取器验证状态"""
|
||||
self.refresh_if_changed()
|
||||
|
||||
data = self._get_site_data(domain, preset_name)
|
||||
if data is None:
|
||||
return False
|
||||
|
||||
if not self._assign_preset_field_and_save(data, "extractor_verified", verified):
|
||||
return False
|
||||
|
||||
return True
|
||||
def get_site_stream_config(self, domain: str, preset_name: str = None) -> Dict[str, Any]:
|
||||
"""
|
||||
获取站点的流式配置
|
||||
|
||||
Args:
|
||||
domain: 站点域名
|
||||
preset_name: 预设名称
|
||||
|
||||
Returns:
|
||||
完整的流式配置(包含默认值)
|
||||
"""
|
||||
self.refresh_if_changed()
|
||||
|
||||
cache_key = f"stream_config:{domain}:{preset_name or ''}"
|
||||
cached = self._cache.get(cache_key, _MISSING)
|
||||
if cached is not _MISSING:
|
||||
return copy.deepcopy(cached)
|
||||
|
||||
default_config = get_default_stream_config()
|
||||
|
||||
data = self._get_site_data(domain, preset_name)
|
||||
if data is None:
|
||||
result = copy.deepcopy(default_config)
|
||||
self._cache.set(cache_key, result)
|
||||
return copy.deepcopy(result)
|
||||
|
||||
stream_config = data.get("stream_config", {})
|
||||
|
||||
# 合并默认值
|
||||
result = copy.deepcopy(default_config)
|
||||
|
||||
# 更新顶层字段
|
||||
for key in ["mode", "hard_timeout"]:
|
||||
if key in stream_config:
|
||||
result[key] = stream_config[key]
|
||||
|
||||
if isinstance(stream_config.get("request_transport"), dict):
|
||||
result["request_transport"] = normalize_request_transport_config(
|
||||
stream_config.get("request_transport")
|
||||
)
|
||||
|
||||
# 处理 send_confirmation 配置
|
||||
if isinstance(stream_config.get("send_confirmation"), dict):
|
||||
result["send_confirmation"].update(stream_config["send_confirmation"])
|
||||
|
||||
# 处理 attachment_monitor 配置
|
||||
if isinstance(stream_config.get("attachment_monitor"), dict):
|
||||
result["attachment_monitor"].update(stream_config["attachment_monitor"])
|
||||
|
||||
file_paste_config = data.get("file_paste", {})
|
||||
if isinstance(file_paste_config, dict):
|
||||
if isinstance(file_paste_config.get("send_confirmation"), dict):
|
||||
result["send_confirmation"].update(file_paste_config["send_confirmation"])
|
||||
if isinstance(file_paste_config.get("attachment_monitor"), dict):
|
||||
result["attachment_monitor"].update(file_paste_config["attachment_monitor"])
|
||||
|
||||
# 处理 network 配置
|
||||
if stream_config.get("network"):
|
||||
network_default = get_default_network_config()
|
||||
network_config = stream_config["network"]
|
||||
|
||||
result["network"] = network_default.copy()
|
||||
for key in [
|
||||
"listen_pattern",
|
||||
"stream_match_mode",
|
||||
"stream_match_pattern",
|
||||
"parser",
|
||||
"hard_timeout",
|
||||
"first_response_timeout",
|
||||
"first_content_timeout",
|
||||
"initial_target_body_wait",
|
||||
"silence_threshold",
|
||||
"response_interval",
|
||||
]:
|
||||
if key in network_config:
|
||||
result["network"][key] = network_config[key]
|
||||
|
||||
self._cache.set(cache_key, copy.deepcopy(result))
|
||||
return result
|
||||
def set_site_stream_config(self, domain: str, config: Dict[str, Any],
|
||||
preset_name: str = None) -> bool:
|
||||
"""
|
||||
设置站点的流式配置
|
||||
|
||||
Args:
|
||||
domain: 站点域名
|
||||
config: 流式配置(部分或完整)
|
||||
preset_name: 预设名称
|
||||
|
||||
Returns:
|
||||
是否成功
|
||||
"""
|
||||
self.refresh_if_changed()
|
||||
|
||||
data = self._get_site_data(domain, preset_name)
|
||||
if data is None:
|
||||
logger.warning(f"站点或预设不存在: {domain}/{preset_name}")
|
||||
return False
|
||||
|
||||
previous_file_paste = data.get("file_paste", _MISSING)
|
||||
if previous_file_paste is not _MISSING:
|
||||
previous_file_paste = copy.deepcopy(previous_file_paste)
|
||||
previous_stream = data.get("stream_config", _MISSING)
|
||||
if previous_stream is not _MISSING:
|
||||
previous_stream = copy.deepcopy(previous_stream)
|
||||
|
||||
legacy_file_paste_updates: Dict[str, Any] = {}
|
||||
if isinstance(config.get("send_confirmation"), dict):
|
||||
legacy_file_paste_updates["send_confirmation"] = config.get("send_confirmation") or {}
|
||||
if isinstance(config.get("attachment_monitor"), dict):
|
||||
legacy_file_paste_updates["attachment_monitor"] = config.get("attachment_monitor") or {}
|
||||
|
||||
# 验证并规范化配置
|
||||
current_config = self.get_site_stream_config(domain, preset_name)
|
||||
merged_config = _merge_config_patch(current_config, config if isinstance(config, dict) else {})
|
||||
validated = self._validate_stream_config(merged_config)
|
||||
validated.pop("send_confirmation", None)
|
||||
validated.pop("attachment_monitor", None)
|
||||
|
||||
if legacy_file_paste_updates:
|
||||
existing_file_paste = self._validate_file_paste_config(
|
||||
data.get("file_paste", {}),
|
||||
legacy_stream_config=data.get("stream_config"),
|
||||
)
|
||||
merged_file_paste = _merge_config_patch(existing_file_paste, legacy_file_paste_updates)
|
||||
data["file_paste"] = self._validate_file_paste_config(
|
||||
merged_file_paste,
|
||||
legacy_stream_config=data.get("stream_config"),
|
||||
)
|
||||
|
||||
data["stream_config"] = validated
|
||||
if not self._save_config():
|
||||
if previous_stream is _MISSING:
|
||||
data.pop("stream_config", None)
|
||||
else:
|
||||
data["stream_config"] = previous_stream
|
||||
if previous_file_paste is _MISSING:
|
||||
data.pop("file_paste", None)
|
||||
else:
|
||||
data["file_paste"] = previous_file_paste
|
||||
return False
|
||||
|
||||
logger.info(f"站点 {domain} [{preset_name or DEFAULT_PRESET_NAME}] 流式配置已更新 (mode={validated.get('mode')})")
|
||||
return True
|
||||
def _validate_stream_config(self, config: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"""
|
||||
验证并规范化流式配置
|
||||
|
||||
Args:
|
||||
config: 原始配置
|
||||
|
||||
Returns:
|
||||
规范化后的配置
|
||||
"""
|
||||
result = get_default_stream_config()
|
||||
|
||||
if not config:
|
||||
return result
|
||||
|
||||
mode_explicitly_set = False
|
||||
|
||||
# 验证 mode
|
||||
if "mode" in config:
|
||||
mode = str(config["mode"]).lower()
|
||||
if mode in ("dom", "network"):
|
||||
result["mode"] = mode
|
||||
mode_explicitly_set = True
|
||||
|
||||
# 验证数值字段
|
||||
for key in ["hard_timeout"]:
|
||||
if key in config:
|
||||
try:
|
||||
val = float(config[key])
|
||||
if key == "hard_timeout":
|
||||
result[key] = max(10, min(val, 600))
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
|
||||
# 验证 send_confirmation 配置
|
||||
if isinstance(config.get("send_confirmation"), dict):
|
||||
result["send_confirmation"] = self._validate_send_confirmation_config(
|
||||
config["send_confirmation"]
|
||||
)
|
||||
|
||||
if isinstance(config.get("request_transport"), dict):
|
||||
result["request_transport"] = normalize_request_transport_config(
|
||||
config["request_transport"]
|
||||
)
|
||||
|
||||
# 验证 attachment_monitor 配置
|
||||
if isinstance(config.get("attachment_monitor"), dict):
|
||||
result["attachment_monitor"] = self._validate_attachment_monitor_config(
|
||||
config["attachment_monitor"]
|
||||
)
|
||||
|
||||
# 验证 network 配置
|
||||
if config.get("network"):
|
||||
network_config = self._validate_network_config(config["network"])
|
||||
if network_config:
|
||||
result["network"] = network_config
|
||||
# 仅在调用方没有显式指定 mode 时,才根据 network 配置自动切到 network。
|
||||
# 这样切回 DOM 时可以保留 parser/listen_pattern,而不会被后端强制改回 network。
|
||||
if (
|
||||
not mode_explicitly_set
|
||||
and network_config.get("parser")
|
||||
and network_config.get("listen_pattern")
|
||||
):
|
||||
result["mode"] = "network"
|
||||
|
||||
return result
|
||||
def _validate_send_confirmation_config(self, config: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"""验证发送成功判定配置。"""
|
||||
result = get_default_send_confirmation_config()
|
||||
|
||||
if not isinstance(config, dict):
|
||||
return result
|
||||
|
||||
numeric_ranges = {
|
||||
"post_click_observe_window": (0.0, 15.0),
|
||||
"pre_retry_probe_window": (0.0, 5.0),
|
||||
"retry_observe_window": (0.0, 15.0),
|
||||
"attachment_observe_window": (0.0, 30.0),
|
||||
"retry_interval": (0.0, 30.0),
|
||||
"retry_cooldown_window": (0.0, 30.0),
|
||||
}
|
||||
for key, (minimum, maximum) in numeric_ranges.items():
|
||||
if key not in config:
|
||||
continue
|
||||
try:
|
||||
value = float(config[key])
|
||||
except (TypeError, ValueError):
|
||||
continue
|
||||
result[key] = max(minimum, min(value, maximum))
|
||||
|
||||
if "max_retry_count" in config:
|
||||
try:
|
||||
value = int(config["max_retry_count"])
|
||||
except (TypeError, ValueError):
|
||||
value = None
|
||||
if value is not None:
|
||||
result["max_retry_count"] = max(0, min(value, 10))
|
||||
|
||||
retry_action = str(config.get("retry_action") or "").strip().lower()
|
||||
if retry_action in {"click_send_btn", "key_press"}:
|
||||
result["retry_action"] = retry_action
|
||||
|
||||
if "retry_key_combo" in config:
|
||||
retry_key_combo = str(config.get("retry_key_combo") or "").strip()
|
||||
if retry_key_combo:
|
||||
result["retry_key_combo"] = retry_key_combo[:64]
|
||||
|
||||
bool_fields = [
|
||||
"retry_on_unconfirmed_send",
|
||||
"accept_attachment_change",
|
||||
"accept_attachment_disappear",
|
||||
"accept_probe_confirmation",
|
||||
"retry_block_on_stop_button",
|
||||
"retry_block_if_generating",
|
||||
"trust_network_activity",
|
||||
"trust_generating_indicator",
|
||||
"trust_send_disabled_with_input_shrink",
|
||||
]
|
||||
for key in bool_fields:
|
||||
if key not in config:
|
||||
continue
|
||||
result[key] = _coerce_bool(config.get(key), bool(result.get(key, False)))
|
||||
|
||||
sensitivity = str(config.get("attachment_sensitivity") or "").strip().lower()
|
||||
if sensitivity in {"low", "medium", "high"}:
|
||||
result["attachment_sensitivity"] = sensitivity
|
||||
|
||||
return result
|
||||
def _validate_attachment_monitor_config(self, config: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"""验证附件上传/发送前判定规则。"""
|
||||
result = get_default_attachment_monitor_config()
|
||||
|
||||
if not isinstance(config, dict):
|
||||
return result
|
||||
|
||||
list_fields = [
|
||||
"root_selectors",
|
||||
"attachment_selectors",
|
||||
"pending_selectors",
|
||||
"status_selectors",
|
||||
"content_exclusion_selectors",
|
||||
"busy_text_markers",
|
||||
"ignored_busy_text_markers",
|
||||
"send_button_disabled_markers",
|
||||
]
|
||||
for key in list_fields:
|
||||
raw_value = config.get(key)
|
||||
if raw_value is None:
|
||||
continue
|
||||
if not isinstance(raw_value, list):
|
||||
continue
|
||||
cleaned = []
|
||||
for item in raw_value:
|
||||
value = str(item or "").strip()
|
||||
if value and value not in cleaned:
|
||||
cleaned.append(value)
|
||||
result[key] = cleaned
|
||||
|
||||
numeric_ranges = {
|
||||
"idle_timeout": (0.5, 60.0),
|
||||
"hard_max_wait": (1.0, 300.0),
|
||||
}
|
||||
for key, (minimum, maximum) in numeric_ranges.items():
|
||||
if key not in config:
|
||||
continue
|
||||
try:
|
||||
value = float(config[key])
|
||||
except (TypeError, ValueError):
|
||||
continue
|
||||
result[key] = max(minimum, min(value, maximum))
|
||||
|
||||
bool_fields = [
|
||||
"require_attachment_present",
|
||||
"require_upload_signal_before_ready",
|
||||
"continue_once_on_unconfirmed_send",
|
||||
]
|
||||
for key in bool_fields:
|
||||
if key not in config:
|
||||
continue
|
||||
result[key] = _coerce_bool(config.get(key), bool(result.get(key, False)))
|
||||
|
||||
return result
|
||||
def _validate_network_config(self, config: Dict[str, Any]) -> Optional[Dict[str, Any]]:
|
||||
"""
|
||||
验证网络监听配置
|
||||
|
||||
Args:
|
||||
config: 原始网络配置
|
||||
|
||||
Returns:
|
||||
规范化后的配置,无效则返回 None
|
||||
"""
|
||||
if not config:
|
||||
return None
|
||||
|
||||
result = get_default_network_config()
|
||||
|
||||
# listen_pattern(必填)
|
||||
if "listen_pattern" in config:
|
||||
pattern = str(config["listen_pattern"]).strip()
|
||||
if pattern:
|
||||
result["listen_pattern"] = pattern
|
||||
|
||||
if "stream_match_mode" in config:
|
||||
match_mode = str(config["stream_match_mode"]).strip().lower()
|
||||
if match_mode in {"keyword", "regex"}:
|
||||
result["stream_match_mode"] = match_mode
|
||||
|
||||
if "stream_match_pattern" in config:
|
||||
result["stream_match_pattern"] = str(config["stream_match_pattern"]).strip()
|
||||
|
||||
# parser(必填,需验证存在性)
|
||||
if "parser" in config:
|
||||
parser_id = str(config["parser"]).strip()
|
||||
if parser_id:
|
||||
# 验证解析器是否存在
|
||||
if ParserRegistry.exists(parser_id):
|
||||
result["parser"] = parser_id
|
||||
else:
|
||||
logger.warning(f"解析器不存在: {parser_id}")
|
||||
# 仍然保存,允许后续添加解析器
|
||||
result["parser"] = parser_id
|
||||
|
||||
# 验证数值字段
|
||||
for key in ["first_response_timeout", "silence_threshold", "response_interval"]:
|
||||
if key in config:
|
||||
try:
|
||||
val = float(config[key])
|
||||
if key == "first_response_timeout":
|
||||
result[key] = max(0.5, min(val, 300))
|
||||
elif key == "silence_threshold":
|
||||
result[key] = max(0.5, min(val, 30))
|
||||
elif key == "response_interval":
|
||||
result[key] = max(0.1, min(val, 5))
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
|
||||
# 检查是否有有效配置
|
||||
if not result["listen_pattern"] or not result["parser"]:
|
||||
return None
|
||||
|
||||
return result
|
||||
def list_available_parsers(self) -> List[Dict[str, str]]:
|
||||
"""
|
||||
列出所有可用的响应解析器
|
||||
|
||||
Returns:
|
||||
解析器信息列表
|
||||
"""
|
||||
return parser_manager.list_parsers()
|
||||
def get_extractor_manager(self):
|
||||
"""获取提取器管理器实例"""
|
||||
return extractor_manager
|
||||
def get_parser_manager(self):
|
||||
"""获取解析器管理器实例"""
|
||||
return parser_manager
|
||||
@@ -0,0 +1,440 @@
|
||||
"""
|
||||
app/services/config/managers.py - 配置子管理器
|
||||
|
||||
职责:
|
||||
- 全局配置管理(元素定义)
|
||||
- 图片预设管理
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import copy
|
||||
from app.core.config import get_logger
|
||||
from typing import Dict, List, Any, Optional
|
||||
|
||||
from app.models.schemas import (
|
||||
SelectorDefinition,
|
||||
get_default_selector_definitions,
|
||||
get_default_image_extraction_config
|
||||
)
|
||||
|
||||
|
||||
logger = get_logger("CFG_MGR")
|
||||
|
||||
|
||||
# ================= 全局配置管理器 =================
|
||||
|
||||
class GlobalConfigManager:
|
||||
"""
|
||||
全局配置管理器
|
||||
|
||||
管理 _global 节点中的配置,包括:
|
||||
- selector_definitions: 元素定义列表
|
||||
- 以及任意扩展字段(通过 get/set 访问)
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
self._selector_definitions: List[SelectorDefinition] = get_default_selector_definitions()
|
||||
self._extra: Dict[str, Any] = {} # 通用扩展存储
|
||||
|
||||
def load(self, global_section: Dict[str, Any]):
|
||||
"""从 _global 节点加载配置"""
|
||||
self._extra = {}
|
||||
self._selector_definitions = get_default_selector_definitions()
|
||||
if not global_section:
|
||||
return
|
||||
|
||||
if "selector_definitions" in global_section:
|
||||
defs = global_section["selector_definitions"]
|
||||
if isinstance(defs, list):
|
||||
self._selector_definitions = self._merge_selector_definitions(defs)
|
||||
logger.debug(f"已加载 {len(defs)} 个元素定义")
|
||||
|
||||
# 加载所有非 selector_definitions 的字段到通用存储
|
||||
for key, value in global_section.items():
|
||||
if key != "selector_definitions":
|
||||
self._extra[key] = copy.deepcopy(value)
|
||||
|
||||
if "commands" in self._extra:
|
||||
cmd_count = len(self._extra["commands"]) if isinstance(self._extra["commands"], list) else 0
|
||||
logger.debug(f"已加载 {cmd_count} 个自动化命令")
|
||||
|
||||
def get_selector_definitions(self) -> List[SelectorDefinition]:
|
||||
"""获取元素定义列表"""
|
||||
return copy.deepcopy(self._selector_definitions)
|
||||
|
||||
def set_selector_definitions(self, definitions: List[SelectorDefinition]):
|
||||
"""设置元素定义列表"""
|
||||
self._selector_definitions = self._merge_selector_definitions(definitions)
|
||||
|
||||
@staticmethod
|
||||
def _normalize_selector_definition(raw_item: Any) -> Optional[SelectorDefinition]:
|
||||
if not isinstance(raw_item, dict):
|
||||
return None
|
||||
key = raw_item.get("key")
|
||||
if not isinstance(key, str):
|
||||
return None
|
||||
key = key.strip()
|
||||
if not key:
|
||||
return None
|
||||
|
||||
description = raw_item.get("description", "")
|
||||
if not isinstance(description, str):
|
||||
description = str(description) if description is not None else ""
|
||||
|
||||
return {
|
||||
"key": key,
|
||||
"description": description,
|
||||
"enabled": bool(raw_item.get("enabled", True)),
|
||||
"required": bool(raw_item.get("required", False)),
|
||||
}
|
||||
|
||||
def _merge_selector_definitions(self, definitions: List[SelectorDefinition]) -> List[SelectorDefinition]:
|
||||
"""合并内置默认字段,确保升级后新增的选择器能自动出现。"""
|
||||
merged = []
|
||||
existing_keys = set()
|
||||
|
||||
for item in definitions or []:
|
||||
normalized = self._normalize_selector_definition(item)
|
||||
if not normalized:
|
||||
continue
|
||||
key = normalized["key"]
|
||||
if key in existing_keys:
|
||||
continue
|
||||
merged.append(normalized)
|
||||
existing_keys.add(key)
|
||||
|
||||
for default_def in get_default_selector_definitions():
|
||||
normalized_default = self._normalize_selector_definition(default_def)
|
||||
if not normalized_default:
|
||||
continue
|
||||
key = normalized_default.get("key")
|
||||
if key not in existing_keys:
|
||||
merged.append(normalized_default)
|
||||
existing_keys.add(key)
|
||||
|
||||
return merged
|
||||
|
||||
def get_enabled_definitions(self) -> List[SelectorDefinition]:
|
||||
"""获取启用的元素定义"""
|
||||
return [
|
||||
copy.deepcopy(d)
|
||||
for d in self._selector_definitions
|
||||
if isinstance(d, dict) and d.get("enabled", True)
|
||||
]
|
||||
|
||||
def get_fallback_selectors(self) -> Dict[str, Optional[str]]:
|
||||
"""
|
||||
生成回退选择器字典
|
||||
|
||||
基于元素定义生成,用于 SelectorValidator
|
||||
"""
|
||||
fallback_map = {
|
||||
"input_box": "textarea",
|
||||
"send_btn": 'button[type="submit"]',
|
||||
"result_container": "div",
|
||||
"new_chat_btn": None,
|
||||
"retry_send_btn": None,
|
||||
"message_wrapper": None,
|
||||
"generating_indicator": None,
|
||||
"upload_btn": None,
|
||||
"file_input": 'input[type="file"]',
|
||||
"drop_zone": None,
|
||||
}
|
||||
|
||||
result = {}
|
||||
for d in self._selector_definitions:
|
||||
if not isinstance(d, dict):
|
||||
continue
|
||||
key = d.get("key")
|
||||
if not isinstance(key, str) or not key:
|
||||
continue
|
||||
result[key] = fallback_map.get(key, None)
|
||||
|
||||
return result
|
||||
|
||||
def build_prompt_selector_list(self) -> str:
|
||||
"""
|
||||
生成 AI 提示词中的元素查找列表
|
||||
|
||||
只包含 enabled=True 的元素
|
||||
"""
|
||||
lines = []
|
||||
for d in self._selector_definitions:
|
||||
if not isinstance(d, dict):
|
||||
continue
|
||||
if not d.get("enabled", True):
|
||||
continue
|
||||
|
||||
key = str(d.get("key") or "").strip()
|
||||
if not key:
|
||||
continue
|
||||
desc = str(d.get("description") or "")
|
||||
required = d.get("required", False)
|
||||
|
||||
if required:
|
||||
tag = "[REQUIRED]"
|
||||
else:
|
||||
tag = "[OPTIONAL, return null if not found]"
|
||||
|
||||
lines.append(f"- `{key}`: {desc} {tag}")
|
||||
|
||||
return "\n".join(lines)
|
||||
|
||||
def build_prompt_json_keys(self) -> str:
|
||||
"""
|
||||
生成 AI 提示词中的 JSON 输出格式说明
|
||||
"""
|
||||
lines = []
|
||||
for d in self._selector_definitions:
|
||||
if not isinstance(d, dict):
|
||||
continue
|
||||
if not d.get("enabled", True):
|
||||
continue
|
||||
|
||||
key = str(d.get("key") or "").strip()
|
||||
if not key:
|
||||
continue
|
||||
desc = str(d.get("description") or "")
|
||||
lines.append(f'- `{key}`: {desc}')
|
||||
|
||||
return "\n".join(lines)
|
||||
|
||||
def get(self, key: str, default: Any = None) -> Any:
|
||||
"""获取通用配置值"""
|
||||
return copy.deepcopy(self._extra.get(key, default))
|
||||
|
||||
def set(self, key: str, value: Any):
|
||||
"""设置通用配置值"""
|
||||
self._extra[key] = copy.deepcopy(value)
|
||||
|
||||
def remove(self, key: str) -> bool:
|
||||
"""删除通用配置值"""
|
||||
if key not in self._extra:
|
||||
return False
|
||||
del self._extra[key]
|
||||
return True
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
"""导出为字典(用于保存)"""
|
||||
result = {
|
||||
"selector_definitions": self._selector_definitions
|
||||
}
|
||||
# 合并通用存储中的所有字段
|
||||
result.update(self._extra)
|
||||
return copy.deepcopy(result)
|
||||
|
||||
|
||||
# ================= 图片预设管理器 =================
|
||||
|
||||
class ImagePresetsManager:
|
||||
"""
|
||||
图片提取预设管理器
|
||||
|
||||
职责:
|
||||
- 加载预设配置文件
|
||||
- 提供站点预设查询
|
||||
- 应用预设到站点配置
|
||||
"""
|
||||
|
||||
def __init__(self, presets_file: str):
|
||||
self.presets_file = presets_file
|
||||
self.presets: Dict[str, Any] = {}
|
||||
self._load_presets()
|
||||
|
||||
@staticmethod
|
||||
def _default_presets_payload() -> Dict[str, Any]:
|
||||
return {
|
||||
"_default": {
|
||||
"name": "Default",
|
||||
"description": "Built-in fallback image extraction preset",
|
||||
"image_extraction": get_default_image_extraction_config(),
|
||||
}
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _get_preset_image_config(preset_data: Any) -> Optional[Dict[str, Any]]:
|
||||
if not isinstance(preset_data, dict):
|
||||
return None
|
||||
config = preset_data.get("image_extraction")
|
||||
return copy.deepcopy(config) if isinstance(config, dict) else None
|
||||
|
||||
@staticmethod
|
||||
def _get_preset_text(preset_data: Dict[str, Any], key: str, default: str = "") -> str:
|
||||
value = preset_data.get(key, default)
|
||||
if value is None:
|
||||
return default
|
||||
if isinstance(value, (str, int, float, bool)):
|
||||
return str(value)
|
||||
return default
|
||||
|
||||
def _find_matching_preset(self, domain: str) -> Optional[tuple[str, Dict[str, Any]]]:
|
||||
domain = str(domain or "").strip().lower().rstrip(".")
|
||||
if not domain:
|
||||
return None
|
||||
|
||||
normalized_presets = []
|
||||
for preset_domain, preset_data in self.presets.items():
|
||||
if preset_domain.startswith("_") or not isinstance(preset_data, dict):
|
||||
continue
|
||||
normalized_preset_domain = str(preset_domain).strip().lower().rstrip(".")
|
||||
if not normalized_preset_domain:
|
||||
continue
|
||||
normalized_presets.append((preset_domain, preset_data, normalized_preset_domain))
|
||||
|
||||
for preset_domain, preset_data, normalized_preset_domain in normalized_presets:
|
||||
if domain == normalized_preset_domain:
|
||||
return preset_domain, preset_data
|
||||
|
||||
suffix_matches = [
|
||||
item
|
||||
for item in normalized_presets
|
||||
if domain.endswith(f".{item[2]}")
|
||||
]
|
||||
if suffix_matches:
|
||||
preset_domain, preset_data, _ = max(
|
||||
suffix_matches,
|
||||
key=lambda item: len(item[2]),
|
||||
)
|
||||
logger.debug(f"使用模糊匹配预设: {domain} -> {preset_domain}")
|
||||
return preset_domain, preset_data
|
||||
|
||||
return None
|
||||
|
||||
def _use_default_presets_or_keep_existing(self, keep_existing: bool) -> None:
|
||||
if keep_existing and self.presets:
|
||||
logger.warning("图片预设加载失败,保留当前内存中的预设")
|
||||
return
|
||||
self.presets = self._default_presets_payload()
|
||||
|
||||
def _load_presets(self, keep_existing_on_error: bool = False):
|
||||
"""加载预设配置文件"""
|
||||
if not os.path.exists(self.presets_file):
|
||||
logger.warning(f"图片预设文件不存在: {self.presets_file}")
|
||||
self._use_default_presets_or_keep_existing(keep_existing_on_error)
|
||||
return
|
||||
|
||||
try:
|
||||
with open(self.presets_file, 'r', encoding='utf-8') as f:
|
||||
data = json.load(f)
|
||||
|
||||
if not isinstance(data, dict):
|
||||
logger.error("预设文件格式错误: 顶层必须是对象")
|
||||
self._use_default_presets_or_keep_existing(keep_existing_on_error)
|
||||
return
|
||||
|
||||
# 移除元数据
|
||||
loaded_presets = {
|
||||
str(k): v
|
||||
for k, v in data.items()
|
||||
if k != "_meta" and isinstance(v, dict)
|
||||
}
|
||||
skipped_count = len([k for k, v in data.items() if k != "_meta" and not isinstance(v, dict)])
|
||||
if skipped_count:
|
||||
logger.warning(f"已跳过 {skipped_count} 个格式无效的图片预设")
|
||||
if not loaded_presets:
|
||||
self._use_default_presets_or_keep_existing(keep_existing_on_error)
|
||||
logger.debug(f"已加载 {len(self.presets)} 个图片预设")
|
||||
return
|
||||
|
||||
self.presets = loaded_presets
|
||||
|
||||
logger.debug(f"已加载 {len(self.presets)} 个图片预设")
|
||||
|
||||
except json.JSONDecodeError as e:
|
||||
logger.error(f"预设文件格式错误: {e}")
|
||||
self._use_default_presets_or_keep_existing(keep_existing_on_error)
|
||||
except Exception as e:
|
||||
logger.error(f"加载预设失败: {e}")
|
||||
self._use_default_presets_or_keep_existing(keep_existing_on_error)
|
||||
|
||||
def get_preset(self, domain: str) -> Optional[Dict]:
|
||||
"""
|
||||
获取站点的预设配置
|
||||
|
||||
Args:
|
||||
domain: 站点域名
|
||||
|
||||
Returns:
|
||||
预设配置字典,不存在返回 None
|
||||
"""
|
||||
match = self._find_matching_preset(domain)
|
||||
if match is None:
|
||||
return None
|
||||
return self._get_preset_image_config(match[1])
|
||||
|
||||
def list_presets(self) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
列出所有可用预设
|
||||
|
||||
Returns:
|
||||
预设列表,每项包含 domain、name、description、enabled
|
||||
"""
|
||||
result = []
|
||||
|
||||
for domain, data in self.presets.items():
|
||||
if domain == "_meta":
|
||||
continue
|
||||
if not isinstance(data, dict):
|
||||
continue
|
||||
|
||||
config = self._get_preset_image_config(data) or {}
|
||||
|
||||
item = {
|
||||
"domain": domain,
|
||||
"name": self._get_preset_text(data, "name", domain),
|
||||
"description": self._get_preset_text(data, "description", ""),
|
||||
"enabled": config.get("enabled", False),
|
||||
"notes": self._get_preset_text(data, "notes", ""),
|
||||
"config": config
|
||||
}
|
||||
|
||||
result.append(item)
|
||||
|
||||
result.sort(key=lambda x: x["domain"])
|
||||
|
||||
return result
|
||||
|
||||
def get_preset_for_display(self, domain: str) -> Dict[str, Any]:
|
||||
"""
|
||||
获取用于显示的预设信息
|
||||
|
||||
Args:
|
||||
domain: 站点域名
|
||||
|
||||
Returns:
|
||||
包含 available、preset_domain、config 的字典
|
||||
"""
|
||||
match = self._find_matching_preset(domain)
|
||||
if match is None:
|
||||
preset_config = None
|
||||
else:
|
||||
preset_config = self._get_preset_image_config(match[1])
|
||||
|
||||
if preset_config:
|
||||
matched_domain, matched_data = match
|
||||
|
||||
return {
|
||||
"available": True,
|
||||
"preset_domain": matched_domain,
|
||||
"name": self._get_preset_text(
|
||||
matched_data,
|
||||
"name",
|
||||
"",
|
||||
),
|
||||
"config": preset_config
|
||||
}
|
||||
|
||||
return {
|
||||
"available": False,
|
||||
"preset_domain": None,
|
||||
"name": None,
|
||||
"config": None
|
||||
}
|
||||
|
||||
def reload(self):
|
||||
"""重新加载预设文件"""
|
||||
self._load_presets(keep_existing_on_error=True)
|
||||
|
||||
|
||||
__all__ = ['GlobalConfigManager', 'ImagePresetsManager']
|
||||
@@ -0,0 +1,661 @@
|
||||
"""
|
||||
app/services/config/processors.py - 配置处理器
|
||||
|
||||
职责:
|
||||
- HTML 清理
|
||||
- 选择器验证
|
||||
- AI 页面分析
|
||||
"""
|
||||
|
||||
import json
|
||||
import importlib.util
|
||||
import os
|
||||
import re
|
||||
import time
|
||||
from app.core.config import get_logger
|
||||
from app.utils.site_discovery import admitted_selectors
|
||||
from typing import Dict, Optional
|
||||
from urllib import request, error
|
||||
|
||||
import bs4
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
from app.core.config import AppConfig
|
||||
|
||||
from .managers import GlobalConfigManager
|
||||
|
||||
|
||||
logger = get_logger("CFG_PRC")
|
||||
|
||||
|
||||
_BS4_PARSER = "lxml" if importlib.util.find_spec("lxml") is not None else "html.parser"
|
||||
|
||||
|
||||
# ================= 常量 =================
|
||||
|
||||
# HTML 清理配置
|
||||
MAX_HTML_CHARS = int(os.getenv("MAX_HTML_CHARS", "120000"))
|
||||
TEXT_TRUNCATE_LENGTH = 80
|
||||
|
||||
# AI 请求配置
|
||||
AI_MAX_RETRIES = 3
|
||||
AI_RETRY_BASE_DELAY = 1.0
|
||||
AI_RETRY_MAX_DELAY = 10.0
|
||||
AI_REQUEST_TIMEOUT = 120
|
||||
|
||||
# 无效选择器语法模式
|
||||
INVALID_SYNTAX_PATTERNS = [
|
||||
(r'~\s*\.\.', '~ .. 无效语法'),
|
||||
(r'\.\.\s*$', '结尾 .. 无效'),
|
||||
(r'>>\s', '>> 无效语法'),
|
||||
(r':has\(', ':has() 兼容性差'),
|
||||
(r'\s~\s*$', '结尾 ~ 无效'),
|
||||
]
|
||||
|
||||
|
||||
# ================= HTML 清理器 =================
|
||||
|
||||
class HTMLCleaner:
|
||||
"""HTML 清理器"""
|
||||
|
||||
TAGS_TO_REMOVE = [
|
||||
'script', 'style', 'meta', 'link', 'noscript',
|
||||
'img', 'video', 'audio', 'iframe', 'canvas',
|
||||
'path', 'rect', 'circle', 'polygon', 'defs', 'clipPath',
|
||||
'header', 'footer', 'nav', 'aside',
|
||||
]
|
||||
|
||||
ALLOWED_ATTRS = [
|
||||
'id', 'class', 'name', 'placeholder', 'aria-label', 'role',
|
||||
'data-testid', 'type', 'disabled', 'value', 'title', 'tabindex',
|
||||
'contenteditable', 'href'
|
||||
]
|
||||
|
||||
INTERACTIVE_TAGS = ['input', 'textarea', 'button', 'form', 'a']
|
||||
|
||||
CORE_AREA_SELECTORS = [
|
||||
'[role="main"]',
|
||||
'main',
|
||||
'#app',
|
||||
'#root',
|
||||
'.chat',
|
||||
'.conversation',
|
||||
'.message',
|
||||
]
|
||||
|
||||
def __init__(self, max_chars: int = None, text_truncate: int = None):
|
||||
self.max_chars = max_chars or MAX_HTML_CHARS
|
||||
self.text_truncate = text_truncate or TEXT_TRUNCATE_LENGTH
|
||||
|
||||
def clean(self, html: str) -> str:
|
||||
"""深度清理 HTML"""
|
||||
logger.debug("开始 HTML 清理...")
|
||||
original_length = len(html)
|
||||
|
||||
soup = BeautifulSoup(html, _BS4_PARSER)
|
||||
try:
|
||||
interactive_elements = self._extract_interactive_elements(soup)
|
||||
|
||||
for tag in soup(self.TAGS_TO_REMOVE):
|
||||
tag.decompose()
|
||||
|
||||
for element in soup(text=lambda t: isinstance(t, bs4.element.Comment)):
|
||||
element.extract()
|
||||
|
||||
for tag in soup.find_all(True):
|
||||
if tag.string and len(tag.string) > self.text_truncate:
|
||||
tag.string = tag.string[:self.text_truncate] + "..."
|
||||
|
||||
attrs = dict(tag.attrs)
|
||||
for attr in attrs:
|
||||
if attr not in self.ALLOWED_ATTRS:
|
||||
del tag.attrs[attr]
|
||||
|
||||
if 'class' in tag.attrs and isinstance(tag.attrs['class'], list):
|
||||
tag.attrs['class'] = " ".join(tag.attrs['class'])
|
||||
|
||||
clean_html = str(soup.body) if soup.body else str(soup)
|
||||
clean_html = re.sub(r'\s+', ' ', clean_html).strip()
|
||||
|
||||
if len(clean_html) > self.max_chars:
|
||||
logger.warning(f"HTML 过长 ({len(clean_html)}),执行智能截断...")
|
||||
clean_html = self._smart_truncate(clean_html, interactive_elements)
|
||||
|
||||
final_length = len(clean_html)
|
||||
reduction = 100 - (final_length / original_length * 100) if original_length > 0 else 0
|
||||
logger.info(f"HTML 清理完成: {original_length} → {final_length} 字符 (减少 {reduction:.1f}%)")
|
||||
|
||||
return clean_html
|
||||
finally:
|
||||
soup.decompose()
|
||||
|
||||
def _extract_interactive_elements(self, soup: BeautifulSoup) -> str:
|
||||
elements = []
|
||||
|
||||
for tag_name in self.INTERACTIVE_TAGS:
|
||||
for element in soup.find_all(tag_name):
|
||||
context = self._get_element_with_context(element, levels=2)
|
||||
if context:
|
||||
elements.append(context)
|
||||
|
||||
unique_elements = list(dict.fromkeys(elements))
|
||||
return "\n".join(unique_elements)
|
||||
|
||||
def _get_element_with_context(self, element, levels: int = 2) -> str:
|
||||
try:
|
||||
current = element
|
||||
for _ in range(levels):
|
||||
if current.parent and current.parent.name not in ['body', 'html', '[document]']:
|
||||
current = current.parent
|
||||
else:
|
||||
break
|
||||
|
||||
html_str = str(current)
|
||||
if len(html_str) > 2000:
|
||||
html_str = html_str[:2000] + "..."
|
||||
return html_str
|
||||
except Exception:
|
||||
return str(element)
|
||||
|
||||
def _smart_truncate(self, html: str, interactive_html: str) -> str:
|
||||
interactive_budget = int(self.max_chars * 0.3)
|
||||
if len(interactive_html) > interactive_budget:
|
||||
interactive_html = interactive_html[:interactive_budget]
|
||||
|
||||
remaining_budget = self.max_chars - len(interactive_html) - 100
|
||||
|
||||
if remaining_budget <= 0:
|
||||
logger.warning("交互元素已占满预算")
|
||||
return interactive_html
|
||||
|
||||
core_html = self._extract_core_area(html)
|
||||
if core_html and len(core_html) <= remaining_budget:
|
||||
result = core_html + "\n<!-- INTERACTIVE ELEMENTS -->\n" + interactive_html
|
||||
return result[:self.max_chars]
|
||||
|
||||
head_budget = remaining_budget // 3
|
||||
tail_budget = remaining_budget // 3
|
||||
|
||||
head_part = self._truncate_at_tag_boundary(html[:head_budget * 2], head_budget, from_end=False)
|
||||
tail_part = self._truncate_at_tag_boundary(html[-tail_budget * 2:], tail_budget, from_end=True)
|
||||
|
||||
result = (
|
||||
head_part +
|
||||
"\n<!-- TRUNCATED: MIDDLE SECTION -->\n" +
|
||||
"<!-- INTERACTIVE ELEMENTS START -->\n" +
|
||||
interactive_html +
|
||||
"\n<!-- INTERACTIVE ELEMENTS END -->\n" +
|
||||
tail_part
|
||||
)
|
||||
|
||||
if len(result) > self.max_chars:
|
||||
result = result[:self.max_chars]
|
||||
|
||||
return result
|
||||
|
||||
def _extract_core_area(self, html: str) -> Optional[str]:
|
||||
soup = None
|
||||
try:
|
||||
soup = BeautifulSoup(html, _BS4_PARSER)
|
||||
|
||||
for selector in self.CORE_AREA_SELECTORS:
|
||||
try:
|
||||
element = soup.select_one(selector)
|
||||
if element:
|
||||
core_html = str(element)
|
||||
logger.debug(f"找到核心区域: {selector} ({len(core_html)} chars)")
|
||||
return core_html
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
return None
|
||||
except Exception as e:
|
||||
logger.debug(f"提取核心区域失败: {e}")
|
||||
return None
|
||||
finally:
|
||||
if soup is not None:
|
||||
soup.decompose()
|
||||
|
||||
def _truncate_at_tag_boundary(self, html: str, max_len: int, from_end: bool = False) -> str:
|
||||
if len(html) <= max_len:
|
||||
return html
|
||||
|
||||
if from_end:
|
||||
start_pos = len(html) - max_len
|
||||
tag_start = html.find('<', start_pos)
|
||||
if tag_start != -1 and tag_start < len(html) - 100:
|
||||
return html[tag_start:]
|
||||
return html[-max_len:]
|
||||
else:
|
||||
tag_end = html.rfind('>', 0, max_len)
|
||||
if tag_end != -1 and tag_end > 100:
|
||||
return html[:tag_end + 1]
|
||||
return html[:max_len]
|
||||
|
||||
|
||||
# ================= 选择器验证器 =================
|
||||
|
||||
class SelectorValidator:
|
||||
"""选择器验证器"""
|
||||
|
||||
def __init__(self, fallback_selectors: Dict[str, Optional[str]] = None):
|
||||
self.fallback_selectors = fallback_selectors or {}
|
||||
|
||||
def validate(self, selectors: Dict[str, Optional[str]]) -> Dict[str, Optional[str]]:
|
||||
"""验证并修复选择器"""
|
||||
fixed = {}
|
||||
|
||||
for key, selector in selectors.items():
|
||||
if selector is None:
|
||||
fixed[key] = self.fallback_selectors.get(key)
|
||||
continue
|
||||
|
||||
is_invalid = False
|
||||
invalid_reason = ""
|
||||
|
||||
for pattern, reason in INVALID_SYNTAX_PATTERNS:
|
||||
if re.search(pattern, selector):
|
||||
is_invalid = True
|
||||
invalid_reason = reason
|
||||
break
|
||||
|
||||
if is_invalid:
|
||||
logger.warning(f"❌ 无效选择器 [{key}]: {selector}")
|
||||
logger.warning(f" 原因: {invalid_reason}")
|
||||
|
||||
repaired = self._try_repair(selector)
|
||||
if repaired:
|
||||
logger.info(f" ✅ 修复为: {repaired}")
|
||||
fixed[key] = repaired
|
||||
else:
|
||||
fallback = self.fallback_selectors.get(key)
|
||||
logger.info(f" 🔄 回退为: {fallback}")
|
||||
fixed[key] = fallback
|
||||
else:
|
||||
if re.search(r'\._[a-f0-9]{5,}|^\.[a-f0-9]{6,}', selector):
|
||||
logger.info(f"ℹ️ 哈希类名 [{key}]: {selector} (可能不稳定,但保留)")
|
||||
|
||||
fixed[key] = selector
|
||||
|
||||
return fixed
|
||||
|
||||
def _try_repair(self, selector: str) -> Optional[str]:
|
||||
tag_match = re.match(r'^(\w+)', selector)
|
||||
if not tag_match:
|
||||
return None
|
||||
|
||||
tag = tag_match.group(1)
|
||||
|
||||
attr_patterns = [
|
||||
r'(\[name=["\']?\w+["\']?\])',
|
||||
r'(\[type=["\']?\w+["\']?\])',
|
||||
r'(\[role=["\']?\w+["\']?\])',
|
||||
r'(#[\w-]+)',
|
||||
]
|
||||
|
||||
for pattern in attr_patterns:
|
||||
match = re.search(pattern, selector)
|
||||
if match:
|
||||
return tag + match.group(1)
|
||||
|
||||
return tag
|
||||
|
||||
|
||||
# ================= AI 分析器 =================
|
||||
|
||||
class AIAnalyzer:
|
||||
"""AI 页面分析器(支持 OpenAI 兼容 / Gemini 官方 / Claude 官方)"""
|
||||
|
||||
# ===== Provider 注册表 =====
|
||||
# 每个 provider 定义如何构建请求和解析响应
|
||||
# 新增 provider 只需在此处注册,无需修改其他逻辑
|
||||
PROVIDERS = {
|
||||
"openai": {
|
||||
"build_url": "_build_url_openai",
|
||||
"build_headers": "_build_headers_openai",
|
||||
"build_body": "_build_body_openai",
|
||||
"parse_response": "_parse_response_openai",
|
||||
},
|
||||
"gemini": {
|
||||
"build_url": "_build_url_gemini",
|
||||
"build_headers": "_build_headers_gemini",
|
||||
"build_body": "_build_body_gemini",
|
||||
"parse_response": "_parse_response_gemini",
|
||||
},
|
||||
"claude": {
|
||||
"build_url": "_build_url_claude",
|
||||
"build_headers": "_build_headers_claude",
|
||||
"build_body": "_build_body_claude",
|
||||
"parse_response": "_parse_response_claude",
|
||||
},
|
||||
}
|
||||
|
||||
# URL 关键词 → provider 类型的映射(用于自动检测)
|
||||
URL_PROVIDER_HINTS = [
|
||||
("generativelanguage.googleapis.com", "gemini"),
|
||||
("api.anthropic.com", "claude"),
|
||||
("openai.azure.com", "azure"), # 未来可扩展
|
||||
]
|
||||
|
||||
def __init__(self, global_config: GlobalConfigManager = None):
|
||||
self.api_key = AppConfig.get_helper_api_key()
|
||||
|
||||
base_url = AppConfig.get_helper_base_url()
|
||||
self.base_url = base_url.rstrip('/') if base_url else "http://127.0.0.1:5104/v1"
|
||||
|
||||
self.model = AppConfig.get_helper_model()
|
||||
self.global_config = global_config
|
||||
|
||||
# 确定 provider 类型
|
||||
configured_provider = AppConfig.get_helper_api_provider()
|
||||
if configured_provider == "auto" or configured_provider not in self.PROVIDERS:
|
||||
self.provider = self._detect_provider(self.base_url)
|
||||
else:
|
||||
self.provider = configured_provider
|
||||
|
||||
if not self.api_key:
|
||||
logger.warning("⚠️ 未配置 HELPER_API_KEY,AI 分析功能将不可用")
|
||||
else:
|
||||
logger.debug(f"AI 分析器已配置: Provider={self.provider}, URL={self.base_url}, Model={self.model}")
|
||||
|
||||
# ================= 公共接口 =================
|
||||
|
||||
def analyze(self, html: str) -> Optional[Dict[str, str]]:
|
||||
"""分析 HTML 并返回选择器"""
|
||||
if not self.api_key:
|
||||
logger.error("API Key 未配置")
|
||||
return None
|
||||
|
||||
prompt = self._build_prompt(html)
|
||||
|
||||
for attempt in range(AI_MAX_RETRIES):
|
||||
try:
|
||||
logger.info(f"正在请求 AI 分析(尝试 {attempt + 1}/{AI_MAX_RETRIES},Provider: {self.provider})...")
|
||||
|
||||
response = self._request_ai(prompt)
|
||||
if response:
|
||||
analysis = self._extract_json(response)
|
||||
if analysis is not None:
|
||||
selectors = admitted_selectors(analysis)
|
||||
if selectors:
|
||||
logger.info("✅ 已确认 AI 聊天页并识别到专用定位器")
|
||||
return selectors
|
||||
logger.info("未确认 AI 聊天界面或缺少专用回复容器,不自动收录")
|
||||
return None
|
||||
|
||||
logger.warning(f"第 {attempt + 1} 次分析失败")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"AI 请求异常: {e}")
|
||||
|
||||
if attempt < AI_MAX_RETRIES - 1:
|
||||
delay = min(
|
||||
AI_RETRY_BASE_DELAY * (2 ** attempt),
|
||||
AI_RETRY_MAX_DELAY
|
||||
)
|
||||
jitter = delay * 0.1 * (0.5 - os.urandom(1)[0] / 255)
|
||||
sleep_time = delay + jitter
|
||||
|
||||
logger.info(f"等待 {sleep_time:.2f}s 后重试...")
|
||||
time.sleep(sleep_time)
|
||||
|
||||
logger.error("❌ AI 分析失败(已达最大重试次数)")
|
||||
return None
|
||||
|
||||
# ================= Provider 自动检测 =================
|
||||
|
||||
def _detect_provider(self, base_url: str) -> str:
|
||||
"""根据 URL 自动推断 API 类型"""
|
||||
if not base_url:
|
||||
return "openai"
|
||||
|
||||
url_lower = base_url.lower()
|
||||
for keyword, provider in self.URL_PROVIDER_HINTS:
|
||||
if keyword in url_lower:
|
||||
logger.info(f"自动检测到 API 类型: {provider}(基于 URL 包含 '{keyword}')")
|
||||
return provider
|
||||
|
||||
# 默认回退到 OpenAI 兼容格式
|
||||
return "openai"
|
||||
|
||||
# ================= 统一请求调度 =================
|
||||
|
||||
def _request_ai(self, prompt: str) -> Optional[str]:
|
||||
"""向 AI API 发送请求(根据 provider 自动适配格式)"""
|
||||
provider_config = self.PROVIDERS.get(self.provider)
|
||||
if not provider_config:
|
||||
logger.error(f"未知的 Provider: {self.provider},回退到 openai")
|
||||
provider_config = self.PROVIDERS["openai"]
|
||||
|
||||
# 通过注册表动态调用对应的构建/解析方法
|
||||
url = getattr(self, provider_config["build_url"])(prompt)
|
||||
headers = getattr(self, provider_config["build_headers"])()
|
||||
body = getattr(self, provider_config["build_body"])(prompt)
|
||||
parse_fn = getattr(self, provider_config["parse_response"])
|
||||
|
||||
try:
|
||||
req = request.Request(
|
||||
url,
|
||||
data=json.dumps(body).encode('utf-8'),
|
||||
headers=headers
|
||||
)
|
||||
|
||||
with request.urlopen(req, timeout=AI_REQUEST_TIMEOUT) as response:
|
||||
response_text = response.read().decode('utf-8')
|
||||
|
||||
try:
|
||||
json_resp = json.loads(response_text)
|
||||
return parse_fn(json_resp)
|
||||
except json.JSONDecodeError:
|
||||
logger.error(f"AI 响应 JSON 解析失败,原始响应前 500 字符: {response_text[:500]}")
|
||||
return None
|
||||
|
||||
except error.HTTPError as e:
|
||||
error_body = ""
|
||||
try:
|
||||
error_body = e.read().decode('utf-8')[:500]
|
||||
except Exception:
|
||||
pass
|
||||
logger.error(f"HTTP 错误 {e.code}: {e.reason}" + (f",响应: {error_body}" if error_body else ""))
|
||||
return None
|
||||
except error.URLError as e:
|
||||
logger.error(f"网络错误: {e.reason}")
|
||||
return None
|
||||
except Exception as e:
|
||||
# 兜底:捕获 socket.timeout 等未被上层覆盖的异常
|
||||
logger.error(f"请求异常: {type(e).__name__}: {e}")
|
||||
return None
|
||||
|
||||
# ================= OpenAI 兼容格式 =================
|
||||
|
||||
def _build_url_openai(self, prompt: str) -> str:
|
||||
return f"{self.base_url}/chat/completions"
|
||||
|
||||
def _build_headers_openai(self) -> Dict[str, str]:
|
||||
return {
|
||||
"Content-Type": "application/json",
|
||||
"Authorization": f"Bearer {self.api_key}",
|
||||
}
|
||||
|
||||
def _build_body_openai(self, prompt: str) -> Dict:
|
||||
return {
|
||||
"model": self.model,
|
||||
"messages": [{"role": "user", "content": prompt}],
|
||||
"temperature": 0.1,
|
||||
}
|
||||
|
||||
def _parse_response_openai(self, json_resp: Dict) -> Optional[str]:
|
||||
try:
|
||||
if "choices" in json_resp and len(json_resp["choices"]) > 0:
|
||||
return json_resp["choices"][0]["message"]["content"]
|
||||
except (KeyError, IndexError, TypeError) as e:
|
||||
logger.error(f"OpenAI 响应解析失败: {e}")
|
||||
return None
|
||||
|
||||
# ================= Gemini 官方格式 =================
|
||||
|
||||
def _build_url_gemini(self, prompt: str) -> str:
|
||||
# Gemini 官方 API: base_url 应为 https://generativelanguage.googleapis.com
|
||||
# 完整端点: /v1beta/models/{model}:generateContent?key={api_key}
|
||||
base = self.base_url.rstrip('/')
|
||||
# 如果用户配置的 base_url 已经包含 /v1beta 或 /v1,不再追加
|
||||
if "/v1beta" not in base and "/v1/" not in base:
|
||||
base = f"{base}/v1beta"
|
||||
return f"{base}/models/{self.model}:generateContent?key={self.api_key}"
|
||||
|
||||
def _build_headers_gemini(self) -> Dict[str, str]:
|
||||
# Gemini 官方通过 URL 参数传递 key,不需要 Authorization 头
|
||||
return {
|
||||
"Content-Type": "application/json",
|
||||
}
|
||||
|
||||
def _build_body_gemini(self, prompt: str) -> Dict:
|
||||
return {
|
||||
"contents": [
|
||||
{
|
||||
"parts": [
|
||||
{"text": prompt}
|
||||
]
|
||||
}
|
||||
],
|
||||
"generationConfig": {
|
||||
"temperature": 0.1,
|
||||
},
|
||||
}
|
||||
|
||||
def _parse_response_gemini(self, json_resp: Dict) -> Optional[str]:
|
||||
try:
|
||||
candidates = json_resp.get("candidates", [])
|
||||
if candidates:
|
||||
parts = candidates[0].get("content", {}).get("parts", [])
|
||||
if parts:
|
||||
return parts[0].get("text")
|
||||
except (KeyError, IndexError, TypeError) as e:
|
||||
logger.error(f"Gemini 响应解析失败: {e}")
|
||||
|
||||
# 检查是否有错误信息
|
||||
if "error" in json_resp:
|
||||
err = json_resp["error"]
|
||||
logger.error(f"Gemini API 错误: [{err.get('code')}] {err.get('message', '')}")
|
||||
|
||||
return None
|
||||
|
||||
# ================= Claude 官方格式 =================
|
||||
|
||||
def _build_url_claude(self, prompt: str) -> str:
|
||||
base = self.base_url.rstrip('/')
|
||||
# Claude 官方端点: /v1/messages
|
||||
if not base.endswith("/v1"):
|
||||
base = f"{base}/v1"
|
||||
return f"{base}/messages"
|
||||
|
||||
def _build_headers_claude(self) -> Dict[str, str]:
|
||||
return {
|
||||
"Content-Type": "application/json",
|
||||
"x-api-key": self.api_key,
|
||||
"anthropic-version": "2023-06-01",
|
||||
}
|
||||
|
||||
def _build_body_claude(self, prompt: str) -> Dict:
|
||||
return {
|
||||
"model": self.model,
|
||||
"max_tokens": 4096,
|
||||
"messages": [{"role": "user", "content": prompt}],
|
||||
"temperature": 0.1,
|
||||
}
|
||||
|
||||
def _parse_response_claude(self, json_resp: Dict) -> Optional[str]:
|
||||
try:
|
||||
content = json_resp.get("content", [])
|
||||
if content:
|
||||
# Claude 响应的 content 是列表,每项有 type 和 text
|
||||
text_parts = [block["text"] for block in content if block.get("type") == "text"]
|
||||
if text_parts:
|
||||
return "\n".join(text_parts)
|
||||
except (KeyError, IndexError, TypeError) as e:
|
||||
logger.error(f"Claude 响应解析失败: {e}")
|
||||
|
||||
# 检查是否有错误信息
|
||||
if "error" in json_resp:
|
||||
err = json_resp["error"]
|
||||
logger.error(f"Claude API 错误: [{err.get('type')}] {err.get('message', '')}")
|
||||
|
||||
return None
|
||||
|
||||
# ================= Prompt 构建(不变)=================
|
||||
|
||||
def _build_prompt(self, clean_html: str) -> str:
|
||||
"""构建 AI 提示词(动态版)"""
|
||||
if self.global_config:
|
||||
selector_list = self.global_config.build_prompt_selector_list()
|
||||
json_keys = self.global_config.build_prompt_json_keys()
|
||||
else:
|
||||
selector_list = self._build_default_selector_list()
|
||||
json_keys = self._build_default_json_keys()
|
||||
|
||||
lines = [
|
||||
"You are a web interface classifier. First determine whether this is an AI CONVERSATION interface; do not assume every page is a chat app.",
|
||||
"Return {\"is_chat_site\": false} for search engines, search results, login/consent pages, articles, ordinary forms, and pages with insufficient evidence.",
|
||||
"A textarea, submit button, generic div, or the word AI alone is NOT evidence of an AI conversation interface.",
|
||||
"Only for a genuine AI chat page with a composer and a specific assistant-reply container, return is_chat_site: true and the selector keys below. Do not invent selectors for missing elements.",
|
||||
"",
|
||||
"## CRITICAL RULES:",
|
||||
"1. **Uniqueness is Key**: Ensure selectors matches ONLY the intended element.",
|
||||
"2. **Distinguish AI vs User**: For `result_container`, specify the selector to target the **AI's response text** only. It MUST exclude user prompts, sidebars, or chat history.",
|
||||
"3. **Use Hierarchy**: If a class like `.prose` or `.markdown` is used for both User and AI, you MUST find a unique parent class to differentiate (e.g., `.bot-msg .prose`).",
|
||||
"4. **Syntax**: Use standard CSS selectors. Spaces for descendants (e.g., `div.bot p`) are encouraged for precision.",
|
||||
"5. **No Invalid Syntax**: Do NOT use `xpath`, `~`, `:has()`, or `text()`.",
|
||||
"",
|
||||
"## PREFERENCE ORDER:",
|
||||
"1. `id`, `name`, `data-testid` (Most preferred)",
|
||||
"2. `button[type=\"submit\"]`",
|
||||
"3. Unique parent class + target class (e.g., `.response-area .content`)",
|
||||
"4. Hashed classes (only if no other option exists)",
|
||||
"",
|
||||
"## ELEMENTS TO FIND:",
|
||||
selector_list,
|
||||
"",
|
||||
"## REQUIRED OUTPUT (JSON ONLY):",
|
||||
"Return a JSON object with boolean is_chat_site plus these keys ONLY when is_chat_site is true:",
|
||||
json_keys,
|
||||
"",
|
||||
"## HTML:",
|
||||
clean_html
|
||||
]
|
||||
return "\n".join(lines)
|
||||
|
||||
def _build_default_selector_list(self) -> str:
|
||||
"""默认的元素列表(回退用)"""
|
||||
return """- `input_box`: The text input area (textarea/input). [REQUIRED]
|
||||
- `send_btn`: The button that sends the message. [REQUIRED]
|
||||
- `result_container`: The container for the AI's generated text response. [REQUIRED]
|
||||
- `new_chat_btn`: Button to start a fresh conversation. [OPTIONAL, return null if not found]"""
|
||||
|
||||
def _build_default_json_keys(self) -> str:
|
||||
"""默认的 JSON 键说明(回退用)"""
|
||||
return """- `input_box`: The text input area
|
||||
- `send_btn`: The send button
|
||||
- `result_container`: AI response container
|
||||
- `new_chat_btn`: New chat button (or null)"""
|
||||
|
||||
# ================= JSON 提取(不变)=================
|
||||
|
||||
def _extract_json(self, text: str) -> Optional[Dict]:
|
||||
"""从 AI 响应中提取 JSON"""
|
||||
try:
|
||||
match = re.search(r'```(?:json)?\s*(\{[\s\S]*?\})\s*```', text)
|
||||
if match:
|
||||
return json.loads(match.group(1))
|
||||
|
||||
match = re.search(r'(\{[\s\S]*\})', text)
|
||||
if match:
|
||||
return json.loads(match.group(1))
|
||||
|
||||
return json.loads(text)
|
||||
|
||||
except json.JSONDecodeError as e:
|
||||
logger.error(f"JSON 解析失败: {e}")
|
||||
return None
|
||||
|
||||
|
||||
__all__ = ['HTMLCleaner', 'SelectorValidator', 'AIAnalyzer']
|
||||
@@ -0,0 +1,154 @@
|
||||
"""R1-1:站点适配器 Schema v1。
|
||||
|
||||
只校验“形状”:类型、必填项、嵌套关系,以及 default_preset 必须指向已有预设。
|
||||
语义校验(工作流 DSL、选择器能否命中等)仍由 ``flow_runtime.validate_workflow`` 和
|
||||
ConfigEngine 的各个 ``_validate_*`` 负责。为了不误伤现有配置和用户自建预设,未知字段一律放行,
|
||||
预设内部也不设必填项。
|
||||
|
||||
单站点文件(R1-2,``config/sites/<站点>.json``)的外层信封::
|
||||
|
||||
{"schema_version": 1, "site": "chat.deepseek.com", "adapter_version": "2026.09.26",
|
||||
"min_app_version": "3.0.0", "last_verified": "2026-09-26", "config": {...}}
|
||||
|
||||
``config`` 与旧版 ``config/sites.json`` 中对应节点的内容完全相同。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Dict, Iterable, List
|
||||
|
||||
from jsonschema import Draft202012Validator
|
||||
|
||||
SITE_FILE_SCHEMA_VERSION = 1
|
||||
GLOBAL_KEY = "_global"
|
||||
|
||||
_OBJECT = {"type": "object"}
|
||||
|
||||
WORKFLOW_STEP_SCHEMA: Dict[str, Any] = {
|
||||
"type": "object",
|
||||
"required": ["action"],
|
||||
"properties": {
|
||||
"action": {"type": "string", "minLength": 1},
|
||||
"target": {"type": ["string", "null"]},
|
||||
"optional": {"type": "boolean"},
|
||||
"label": {"type": "string"},
|
||||
"selector": {"type": "string"},
|
||||
"execution": _OBJECT,
|
||||
"flow_version": {"type": "integer"},
|
||||
"retry_safe": {"type": "boolean"},
|
||||
},
|
||||
}
|
||||
|
||||
PRESET_SCHEMA: Dict[str, Any] = {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"selectors": {"type": "object", "additionalProperties": {"type": ["string", "null"]}},
|
||||
"workflow": {"type": "array", "items": WORKFLOW_STEP_SCHEMA},
|
||||
"stealth": {"type": "boolean"},
|
||||
"stream_config": _OBJECT,
|
||||
"image_extraction": _OBJECT,
|
||||
"file_paste": _OBJECT,
|
||||
"prompt_padding": _OBJECT,
|
||||
"advanced": _OBJECT,
|
||||
},
|
||||
}
|
||||
|
||||
SITE_CONFIG_SCHEMA: Dict[str, Any] = {
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"type": "object",
|
||||
"required": ["presets"],
|
||||
"properties": {
|
||||
"presets": {"type": "object", "minProperties": 1, "additionalProperties": PRESET_SCHEMA},
|
||||
"default_preset": {"type": "string", "minLength": 1},
|
||||
"advanced": _OBJECT,
|
||||
},
|
||||
}
|
||||
|
||||
GLOBAL_CONFIG_SCHEMA: Dict[str, Any] = {
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"selector_definitions": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"required": ["key"],
|
||||
"properties": {
|
||||
"key": {"type": "string", "minLength": 1},
|
||||
"description": {"type": "string"},
|
||||
"enabled": {"type": "boolean"},
|
||||
"required": {"type": "boolean"},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
SITE_FILE_SCHEMA: Dict[str, Any] = {
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"type": "object",
|
||||
"required": ["schema_version", "site", "config"],
|
||||
"properties": {
|
||||
"schema_version": {"const": SITE_FILE_SCHEMA_VERSION},
|
||||
"site": {"type": "string", "minLength": 1},
|
||||
"adapter_version": {"type": "string"},
|
||||
"min_app_version": {"type": "string", "pattern": r"^\d+(\.\d+){0,3}$"},
|
||||
"last_verified": {"type": "string", "pattern": r"^\d{4}-\d{2}-\d{2}$"},
|
||||
"config": _OBJECT,
|
||||
},
|
||||
}
|
||||
|
||||
_VALIDATORS = {
|
||||
name: Draft202012Validator(schema)
|
||||
for name, schema in (
|
||||
("site", SITE_CONFIG_SCHEMA),
|
||||
("global", GLOBAL_CONFIG_SCHEMA),
|
||||
("file", SITE_FILE_SCHEMA),
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
def _format_path(prefix: str, parts: Iterable[Any]) -> str:
|
||||
path = prefix
|
||||
for part in parts:
|
||||
path += f"[{part}]" if isinstance(part, int) else f".{part}"
|
||||
return path
|
||||
|
||||
|
||||
def _schema_errors(kind: str, payload: Any, prefix: str) -> List[str]:
|
||||
errors = sorted(_VALIDATORS[kind].iter_errors(payload), key=lambda e: list(map(str, e.absolute_path)))
|
||||
return [f"{_format_path(prefix, error.absolute_path)}: {error.message}" for error in errors]
|
||||
|
||||
|
||||
def site_config_errors(site: str, config: Any) -> List[str]:
|
||||
"""校验单个站点(或 ``_global``)的配置,返回可读的路径级错误列表;空列表表示通过。"""
|
||||
prefix = f"sites[{site}]"
|
||||
if site == GLOBAL_KEY:
|
||||
return _schema_errors("global", config, prefix)
|
||||
errors = _schema_errors("site", config, prefix)
|
||||
if not errors and isinstance(config, dict):
|
||||
default = config.get("default_preset")
|
||||
presets = config.get("presets") or {}
|
||||
if default and default not in presets:
|
||||
errors.append(f"{prefix}.default_preset: '{default}' 不在 presets 中(可选:{', '.join(presets)})")
|
||||
return errors
|
||||
|
||||
|
||||
def site_file_errors(payload: Any) -> List[str]:
|
||||
"""校验单站点文件(信封 + config)。"""
|
||||
errors = _schema_errors("file", payload, "file")
|
||||
if errors:
|
||||
return errors
|
||||
return site_config_errors(payload["site"], payload["config"])
|
||||
|
||||
|
||||
def sites_errors(sites: Any) -> Dict[str, List[str]]:
|
||||
"""校验整份站点配置字典(旧 sites.json 结构),返回 {站点: [错误...]},只包含有错误的站点。"""
|
||||
if not isinstance(sites, dict):
|
||||
return {"*": [f"sites: 应为对象,实际为 {type(sites).__name__}"]}
|
||||
result: Dict[str, List[str]] = {}
|
||||
for site, config in sites.items():
|
||||
errors = site_config_errors(str(site), config)
|
||||
if errors:
|
||||
result[str(site)] = errors
|
||||
return result
|
||||
@@ -0,0 +1,312 @@
|
||||
"""R1-2:站点配置目录存储(``config/sites/<站点>.json``)。
|
||||
|
||||
旧版把全部站点放在一个 271KB 的 ``config/sites.json`` 里;现在每个站点(以及 ``_global``)一个文件,
|
||||
外层是 R1-1 定义的信封(见 ``site_schema.py``)。本模块负责:
|
||||
|
||||
- 读取目录并拼回与旧 ``sites.json`` **完全相同**的字典(``{"_global": {...}, "<域名>": {...}}``),
|
||||
所以 ConfigEngine 以外的代码无需改动;
|
||||
- 只重写内容有变化的站点文件(两阶段:先写全部临时文件,再逐个 ``os.replace``),保留信封元数据;
|
||||
被删除的站点移入 ``.trash/``,不直接删除;
|
||||
- 自动迁移旧的 ``config/sites.json``:目录为空时直接拆分;目录里已有发布版默认文件时(通过更新器升级的场景),
|
||||
按更新器 ``merge_site_records`` 的语义合并(同一站点优先保留本地值、补入发布版新增字段);
|
||||
迁移后旧文件改名为 ``sites.json.migrated-<时间>.bak``;
|
||||
- 读不懂的文件(JSON 损坏或信封不完整)只告警跳过,并且**永远不会被保存逻辑覆盖或移入回收站**;
|
||||
热重载时这类站点沿用上一次成功加载的配置。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
|
||||
from app.core.config import get_logger
|
||||
from app.services.config.site_schema import SITE_FILE_SCHEMA_VERSION, site_file_errors
|
||||
|
||||
logger = get_logger("SITE_STORE")
|
||||
|
||||
INDEX_FILE_NAME = "index.json"
|
||||
TRASH_DIR_NAME = ".trash"
|
||||
DEFAULT_MIN_APP_VERSION = "3.0.0"
|
||||
_ENVELOPE_ORDER = ("schema_version", "site", "adapter_version", "min_app_version", "last_verified")
|
||||
_WINDOWS_RESERVED = {"con", "prn", "aux", "nul", *(f"com{i}" for i in range(1, 10)), *(f"lpt{i}" for i in range(1, 10))}
|
||||
|
||||
|
||||
def canonical_config_text(config: Any) -> str:
|
||||
"""与格式、键顺序无关的规范化文本,用于变更检测与摘要(R1-3 的 config_sha256 也基于它)。"""
|
||||
return json.dumps(config, ensure_ascii=False, sort_keys=True, separators=(",", ":"))
|
||||
|
||||
|
||||
def site_filename(site: str) -> str:
|
||||
"""站点键 -> 文件名。保留 ``[A-Za-z0-9._-]``,其余字节百分号编码;避开 Windows 保留名与 index.json。"""
|
||||
encoded = "".join(
|
||||
ch if (ch.isascii() and (ch.isalnum() or ch in "._-")) else "".join(f"%{b:02X}" for b in ch.encode("utf-8"))
|
||||
for ch in str(site)
|
||||
)
|
||||
if not encoded or encoded.startswith(".") or encoded.split(".")[0].lower() in _WINDOWS_RESERVED | {"index"}:
|
||||
encoded = "%5F" + encoded
|
||||
return f"{encoded}.json"
|
||||
|
||||
|
||||
def build_envelope(site: str, config: Dict[str, Any], meta: Optional[Dict[str, Any]] = None) -> Dict[str, Any]:
|
||||
meta = dict(meta or {})
|
||||
payload: Dict[str, Any] = {"schema_version": SITE_FILE_SCHEMA_VERSION, "site": site}
|
||||
for key in _ENVELOPE_ORDER[2:]:
|
||||
if meta.get(key) not in (None, ""):
|
||||
payload[key] = meta[key]
|
||||
for key, value in meta.items(): # 其余元数据(例如 R1-3 的 origin)原样保留
|
||||
if key not in payload and key != "config":
|
||||
payload[key] = value
|
||||
payload["config"] = config
|
||||
return payload
|
||||
|
||||
|
||||
def _dump(payload: Dict[str, Any]) -> str:
|
||||
return json.dumps(payload, indent=2, ensure_ascii=False) + "\n"
|
||||
|
||||
|
||||
def _write_text_atomic(path: Path, text: str) -> None:
|
||||
tmp = path.with_name(path.name + ".tmp")
|
||||
with open(tmp, "w", encoding="utf-8", newline="\n") as handle:
|
||||
handle.write(text)
|
||||
handle.flush()
|
||||
os.fsync(handle.fileno())
|
||||
os.replace(tmp, path)
|
||||
|
||||
|
||||
def _timestamp() -> str:
|
||||
return time.strftime("%Y%m%d-%H%M%S")
|
||||
|
||||
|
||||
class SiteStore:
|
||||
"""``config/sites/`` 目录的读写与旧版迁移。线程安全由调用方(ConfigEngine 的 ``_io_lock``)保证。"""
|
||||
|
||||
def __init__(self, sites_dir: os.PathLike | str, legacy_file: os.PathLike | str):
|
||||
self.sites_dir = Path(sites_dir)
|
||||
self.legacy_file = Path(legacy_file)
|
||||
self._files: Dict[str, Path] = {}
|
||||
self._meta: Dict[str, Dict[str, Any]] = {}
|
||||
self._on_disk: Dict[str, str] = {}
|
||||
self._broken: Dict[Path, str] = {}
|
||||
self._broken_sites: Dict[Path, str] = {} # 损坏文件 -> 上次成功加载时对应的站点
|
||||
|
||||
# ------------------------------------------------------------------ 查询
|
||||
def site_files(self) -> List[Path]:
|
||||
if not self.sites_dir.is_dir():
|
||||
return []
|
||||
return sorted(
|
||||
path for path in self.sites_dir.glob("*.json")
|
||||
if path.is_file() and path.name != INDEX_FILE_NAME and not path.name.startswith(".")
|
||||
)
|
||||
|
||||
def exists(self) -> bool:
|
||||
return bool(self.site_files()) or self.legacy_file.is_file()
|
||||
|
||||
def signature(self) -> Tuple:
|
||||
"""目录内容签名(文件名、mtime、大小),用于热重载的变更检测。"""
|
||||
entries = []
|
||||
for path in self.site_files():
|
||||
try:
|
||||
stat = path.stat()
|
||||
except OSError:
|
||||
continue
|
||||
entries.append((path.name, stat.st_mtime_ns, stat.st_size))
|
||||
if self.legacy_file.is_file():
|
||||
stat = self.legacy_file.stat()
|
||||
entries.append(("<legacy>", stat.st_mtime_ns, stat.st_size))
|
||||
return tuple(entries)
|
||||
|
||||
def metadata(self, site: str) -> Dict[str, Any]:
|
||||
return dict(self._meta.get(site) or {})
|
||||
|
||||
def broken_files(self) -> Dict[str, str]:
|
||||
return {str(path): reason for path, reason in self._broken.items()}
|
||||
|
||||
def file_for(self, site: str) -> Path:
|
||||
return self._files.get(site) or (self.sites_dir / site_filename(site))
|
||||
|
||||
# ------------------------------------------------------------------ 读取
|
||||
def read_envelope(self, path: Path) -> Dict[str, Any]:
|
||||
payload = json.loads(path.read_text(encoding="utf-8-sig"))
|
||||
if not isinstance(payload, dict) or not isinstance(payload.get("site"), str) or not isinstance(payload.get("config"), dict):
|
||||
raise ValueError("不是有效的站点文件(缺少 site/config 字段)")
|
||||
return payload
|
||||
|
||||
def load(self, previous: Optional[Dict[str, Any]] = None) -> Optional[Dict[str, Any]]:
|
||||
"""读取全部站点,返回旧 sites.json 结构的字典;目录与旧文件都不存在时返回 None。
|
||||
|
||||
``previous`` 是上一次加载的结果:热重载时,暂时读不懂的文件沿用其中对应站点的配置。
|
||||
"""
|
||||
if self.legacy_file.is_file():
|
||||
self.migrate_legacy()
|
||||
files = self.site_files()
|
||||
if not files:
|
||||
return None
|
||||
|
||||
data: Dict[str, Any] = {}
|
||||
files_map: Dict[str, Path] = {}
|
||||
meta: Dict[str, Dict[str, Any]] = {}
|
||||
on_disk: Dict[str, str] = {}
|
||||
broken: Dict[Path, str] = {}
|
||||
previous_sites_by_path = {path: site for site, path in self._files.items()}
|
||||
previous_sites_by_path.update(self._broken_sites)
|
||||
broken_sites: Dict[Path, str] = {}
|
||||
|
||||
for path in files:
|
||||
try:
|
||||
payload = self.read_envelope(path)
|
||||
except (OSError, ValueError) as exc: # JSON 损坏、编码错误(均为 ValueError)、信封不完整、读文件失败
|
||||
broken[path] = str(exc)
|
||||
logger.error(f"站点配置文件无法读取,已跳过且不会被覆盖: {path.name}: {exc}")
|
||||
old_site = previous_sites_by_path.get(path)
|
||||
if old_site:
|
||||
broken_sites[path] = old_site
|
||||
if previous and old_site in previous:
|
||||
data[old_site] = previous[old_site]
|
||||
logger.warning(f"站点 {old_site} 暂时沿用上一次成功加载的配置")
|
||||
continue
|
||||
site = payload["site"]
|
||||
if site in files_map:
|
||||
broken[path] = f"与 {files_map[site].name} 重复声明站点 {site}"
|
||||
logger.error(f"站点配置文件重复声明 {site},已忽略: {path.name}")
|
||||
continue
|
||||
for error in site_file_errors(payload)[:5]:
|
||||
logger.warning(f"站点配置 Schema 告警({path.name}): {error}")
|
||||
data[site] = payload["config"]
|
||||
files_map[site] = path
|
||||
meta[site] = {key: value for key, value in payload.items() if key != "config"}
|
||||
on_disk[site] = canonical_config_text(payload["config"])
|
||||
|
||||
self._files, self._meta, self._on_disk = files_map, meta, on_disk
|
||||
self._broken, self._broken_sites = broken, broken_sites
|
||||
return data
|
||||
|
||||
# ------------------------------------------------------------------ 保存
|
||||
def save(self, full_config: Dict[str, Any]) -> List[str]:
|
||||
"""保存旧 sites.json 结构的字典,返回实际重写的站点列表。"""
|
||||
self.sites_dir.mkdir(parents=True, exist_ok=True)
|
||||
pending: List[Tuple[str, Path, Path, Dict[str, Any], str]] = []
|
||||
taken = {path for path in self._files.values()} | set(self._broken)
|
||||
try:
|
||||
for site, config in full_config.items():
|
||||
if not isinstance(config, dict):
|
||||
continue
|
||||
text = canonical_config_text(config)
|
||||
path = self._files.get(site)
|
||||
if path is not None and self._on_disk.get(site) == text and path.is_file():
|
||||
continue
|
||||
if path is None:
|
||||
path = self._unique_path(site, taken)
|
||||
taken.add(path)
|
||||
if path in self._broken:
|
||||
logger.error(f"站点 {site} 对应的文件 {path.name} 无法解析,为避免覆盖用户手改内容,本次跳过保存")
|
||||
continue
|
||||
payload = build_envelope(site, config, self._meta.get(site) or self._default_meta())
|
||||
tmp = path.with_name(path.name + ".tmp")
|
||||
with open(tmp, "w", encoding="utf-8", newline="\n") as handle:
|
||||
handle.write(_dump(payload))
|
||||
handle.flush()
|
||||
os.fsync(handle.fileno())
|
||||
pending.append((site, path, tmp, payload, text))
|
||||
except Exception: # broad-except: 清理已写出的临时文件后原样重新抛出
|
||||
for _, _, tmp, _, _ in pending:
|
||||
try:
|
||||
tmp.unlink()
|
||||
except OSError:
|
||||
pass
|
||||
raise
|
||||
|
||||
written = []
|
||||
for site, path, tmp, payload, text in pending:
|
||||
os.replace(tmp, path)
|
||||
self._files[site] = path
|
||||
self._on_disk[site] = text
|
||||
self._meta[site] = {key: value for key, value in payload.items() if key != "config"}
|
||||
written.append(site)
|
||||
|
||||
for site in [s for s in self._files if s not in full_config]:
|
||||
path = self._files.pop(site)
|
||||
self._on_disk.pop(site, None)
|
||||
self._meta.pop(site, None)
|
||||
if path in self._broken or not path.is_file():
|
||||
continue
|
||||
trash = self.sites_dir / TRASH_DIR_NAME
|
||||
trash.mkdir(exist_ok=True)
|
||||
target = trash / f"{path.stem}.{_timestamp()}.json"
|
||||
os.replace(path, target)
|
||||
logger.info(f"站点 {site} 已删除,配置文件移入回收站: {target}")
|
||||
return written
|
||||
|
||||
def _default_meta(self) -> Dict[str, Any]:
|
||||
return {"min_app_version": DEFAULT_MIN_APP_VERSION}
|
||||
|
||||
def _unique_path(self, site: str, taken: set) -> Path:
|
||||
base = self.sites_dir / site_filename(site)
|
||||
path, counter = base, 2
|
||||
while path in taken or (path.exists() and path not in self._files.values()):
|
||||
path = base.with_name(f"{base.stem}~{counter}.json")
|
||||
counter += 1
|
||||
return path
|
||||
|
||||
# ------------------------------------------------------------------ 迁移
|
||||
def migrate_legacy(self) -> Optional[Path]:
|
||||
"""把旧的 config/sites.json 迁移进目录,返回备份文件路径;旧文件无法解析时保持原样并返回 None。"""
|
||||
try:
|
||||
raw = self.legacy_file.read_text(encoding="utf-8-sig").strip()
|
||||
legacy = json.loads(raw) if raw else {}
|
||||
if not isinstance(legacy, dict):
|
||||
raise ValueError("顶层不是对象")
|
||||
except (OSError, ValueError) as exc:
|
||||
logger.error(f"旧版站点配置 {self.legacy_file} 无法解析,暂不迁移(原文件保持不变): {exc}")
|
||||
return None
|
||||
|
||||
from updater import merge_site_records # 与更新器合并 sites.json 的语义保持一致
|
||||
|
||||
self.sites_dir.mkdir(parents=True, exist_ok=True)
|
||||
existing: Dict[str, Tuple[Path, Dict[str, Any]]] = {}
|
||||
for path in self.site_files():
|
||||
try:
|
||||
payload = self.read_envelope(path)
|
||||
except (OSError, ValueError):
|
||||
continue
|
||||
existing[payload["site"]] = (path, payload)
|
||||
|
||||
taken = {path for path, _ in existing.values()}
|
||||
merged_count = created_count = 0
|
||||
for site, config in legacy.items():
|
||||
if not isinstance(config, dict) or (site.startswith("_") and site != "_global"):
|
||||
continue
|
||||
if site in existing:
|
||||
path, payload = existing[site]
|
||||
payload = dict(payload)
|
||||
merged_config = merge_site_records(config, payload["config"])
|
||||
if (
|
||||
site == "_global"
|
||||
and config.get("selector_definitions") == []
|
||||
and payload["config"].get("selector_definitions")
|
||||
):
|
||||
merged_config["selector_definitions"] = payload["config"]["selector_definitions"]
|
||||
payload["config"] = merged_config
|
||||
merged_count += 1
|
||||
else:
|
||||
path = self._unique_path(site, taken)
|
||||
taken.add(path)
|
||||
payload = build_envelope(site, config, self._default_meta())
|
||||
created_count += 1
|
||||
_write_text_atomic(path, _dump(payload))
|
||||
|
||||
backup = self.legacy_file.with_name(f"{self.legacy_file.name}.migrated-{_timestamp()}.bak")
|
||||
os.replace(self.legacy_file, backup)
|
||||
logger.warning(
|
||||
f"已把旧版站点配置迁移到 {self.sites_dir}:新建 {created_count} 个、与发布版合并 {merged_count} 个;"
|
||||
f"原文件已备份为 {backup.name}"
|
||||
)
|
||||
return backup
|
||||
|
||||
|
||||
def load_sites_dict(sites_dir: os.PathLike | str) -> Dict[str, Any]:
|
||||
"""只读地拼出目录里的全部站点(不迁移、不告警),供测试、脚本与导出使用。"""
|
||||
store = SiteStore(sites_dir, Path(sites_dir) / "__no_legacy__.json")
|
||||
return store.load() or {}
|
||||
@@ -0,0 +1,17 @@
|
||||
"""
|
||||
app/services/config_engine.py - 兼容性代理文件
|
||||
|
||||
向后兼容:保持原有导入路径有效
|
||||
实际实现已迁移到 app/services/config/ 模块
|
||||
"""
|
||||
|
||||
# 从新模块导入所有内容
|
||||
from app.services.config import (
|
||||
config_engine,
|
||||
ConfigEngine,
|
||||
ConfigConstants,
|
||||
DEFAULT_WORKFLOW
|
||||
)
|
||||
|
||||
# 保持原有导出
|
||||
__all__ = ['config_engine', 'ConfigEngine', 'ConfigConstants', 'DEFAULT_WORKFLOW']
|
||||
@@ -0,0 +1,428 @@
|
||||
"""
|
||||
app/services/error_metadata.py - 标准异常元数据与响应构建器
|
||||
|
||||
职责:
|
||||
- 解耦业务特定错误码(如 Arena / Gemini / Claude 拒绝与失败码)与主执行流
|
||||
- 提供标准化的 ErrorMetadata 数据结构
|
||||
- 提供基于元数据的 422 / 非可重试错误识别与统一 JSONResponse 构造
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import dataclasses
|
||||
from typing import Any, Dict, Optional, Set
|
||||
from fastapi.responses import JSONResponse
|
||||
|
||||
from app.services.arena_image_generation import (
|
||||
ARENA_NON_RETRYABLE_CODES,
|
||||
ARENA_PROMPT_REJECTED_CODE,
|
||||
)
|
||||
from app.core.workflow.arena_send_watchdog import (
|
||||
ARENA_PAGE_ERROR,
|
||||
ARENA_SEND_NO_TARGET_AFTER_RETRY,
|
||||
)
|
||||
|
||||
|
||||
@dataclasses.dataclass(frozen=True)
|
||||
class ErrorMetadata:
|
||||
code: str
|
||||
message: str
|
||||
status_code: int = 422
|
||||
retryable: bool = False
|
||||
error_type: str = "invalid_request_error"
|
||||
param: Optional[str] = None
|
||||
extra: Optional[Dict[str, Any]] = None
|
||||
|
||||
|
||||
_MANUAL_TERMINATE_REASONS: Set[str] = {
|
||||
"manual_terminate",
|
||||
"manual_terminate_from_tab_pool",
|
||||
}
|
||||
|
||||
_CANCEL_REASONS: Set[str] = {
|
||||
"manual_terminate",
|
||||
"manual_terminate_from_tab_pool",
|
||||
"client_disconnected",
|
||||
"coroutine_cancelled",
|
||||
"cleanup",
|
||||
"stream_done",
|
||||
"stop_sequence",
|
||||
"audio_media_fast_return",
|
||||
"request_cancelled",
|
||||
}
|
||||
|
||||
_PROMPT_REJECTION_CODES: Set[str] = {
|
||||
ARENA_PROMPT_REJECTED_CODE,
|
||||
"prompt_rejected",
|
||||
}
|
||||
|
||||
_KNOWN_NON_RETRYABLE_CODES: Set[str] = {
|
||||
*ARENA_NON_RETRYABLE_CODES,
|
||||
ARENA_SEND_NO_TARGET_AFTER_RETRY,
|
||||
ARENA_PAGE_ERROR,
|
||||
*_PROMPT_REJECTION_CODES,
|
||||
}
|
||||
|
||||
|
||||
def is_manual_terminate(reason_or_ctx: Any) -> bool:
|
||||
"""Check if the cancellation is due to manual termination."""
|
||||
if hasattr(reason_or_ctx, "cancel_reason"):
|
||||
reason = getattr(reason_or_ctx, "cancel_reason")
|
||||
else:
|
||||
reason = reason_or_ctx
|
||||
return str(reason or "").strip() in _MANUAL_TERMINATE_REASONS
|
||||
|
||||
|
||||
def is_prompt_rejection_code(code: Any) -> bool:
|
||||
"""Check if code matches known prompt rejection codes."""
|
||||
return str(code or "").strip() in _PROMPT_REJECTION_CODES
|
||||
|
||||
|
||||
def _parse_bool(val: Any, default: bool = False) -> bool:
|
||||
if val is None:
|
||||
return default
|
||||
if isinstance(val, bool):
|
||||
return val
|
||||
if isinstance(val, str):
|
||||
s = val.strip().lower()
|
||||
if s in {"1", "true", "yes"}:
|
||||
return True
|
||||
if s in {"0", "false", "no"}:
|
||||
return False
|
||||
return bool(val)
|
||||
|
||||
|
||||
def _parse_int(val: Any, default: int = 422) -> int:
|
||||
if val is None:
|
||||
return default
|
||||
try:
|
||||
return int(val)
|
||||
except (ValueError, TypeError):
|
||||
return default
|
||||
|
||||
|
||||
def _extract_json_dict_from_text(text: str) -> Optional[Dict[str, Any]]:
|
||||
"""尝试从文本中提取最外层的 JSON 字典对象。"""
|
||||
import json
|
||||
if not text or "{" not in text or "}" not in text:
|
||||
return None
|
||||
start_idx = text.find("{")
|
||||
end_idx = text.rfind("}")
|
||||
if start_idx == -1 or end_idx == -1 or end_idx <= start_idx:
|
||||
return None
|
||||
candidate = text[start_idx : end_idx + 1].strip()
|
||||
try:
|
||||
data = json.loads(candidate)
|
||||
if isinstance(data, dict):
|
||||
return data
|
||||
except Exception:
|
||||
pass
|
||||
return None
|
||||
|
||||
|
||||
def _extract_status_code_from_text(text: str) -> Optional[int]:
|
||||
"""从错误字符串中提取 HTTP 状态码。"""
|
||||
import re
|
||||
if not text:
|
||||
return None
|
||||
# 模式 1: 带有明确上下文前缀
|
||||
match = re.search(
|
||||
r"\b(?:http(?:\s+status)?\s+|status(?:\s*code)?\s*[:=]?\s*|failed\s+to\s+fetch(?:\s+image)?:\s*|code\s*[:=]\s*)([45]\d{2})\b",
|
||||
text,
|
||||
re.IGNORECASE,
|
||||
)
|
||||
if match:
|
||||
try:
|
||||
return int(match.group(1))
|
||||
except Exception:
|
||||
pass
|
||||
# 模式 2: 常见 HTTP 状态码 + 描述词组合(如 "400 Bad Request", "429 Too Many Requests", "502 Bad Gateway")
|
||||
match_status_desc = re.search(
|
||||
r"\b([45]\d{2})\s+(?:Bad\s+Request|Unauthorized|Forbidden|Not\s+Found|Unprocessable\s+Entity|Too\s+Many\s+Requests|Internal\s+Server\s+Error|Bad\s+Gateway|Service\s+Unavailable|Gateway\s+Timeout)\b",
|
||||
text,
|
||||
re.IGNORECASE,
|
||||
)
|
||||
if match_status_desc:
|
||||
try:
|
||||
return int(match_status_desc.group(1))
|
||||
except Exception:
|
||||
pass
|
||||
return None
|
||||
|
||||
|
||||
def resolve_error_metadata(
|
||||
reason_or_payload: Any,
|
||||
default_message: Optional[str] = None,
|
||||
) -> Optional[ErrorMetadata]:
|
||||
"""Extract standard ErrorMetadata from a context, cancel reason, exception, or payload dict."""
|
||||
if reason_or_payload is None:
|
||||
return None
|
||||
|
||||
# Handle Exception objects
|
||||
if isinstance(reason_or_payload, BaseException):
|
||||
code = getattr(reason_or_payload, "code", None) or getattr(reason_or_payload, "error_code", None)
|
||||
msg = (
|
||||
getattr(reason_or_payload, "message", None)
|
||||
or getattr(reason_or_payload, "error_msg", None)
|
||||
or str(reason_or_payload)
|
||||
)
|
||||
if code:
|
||||
return resolve_error_metadata(str(code), default_message=str(msg))
|
||||
return resolve_error_metadata(str(reason_or_payload), default_message=default_message)
|
||||
|
||||
# Handle RequestContext-like objects
|
||||
if hasattr(reason_or_payload, "cancel_reason"):
|
||||
raw_reason = str(getattr(reason_or_payload, "cancel_reason") or "").strip()
|
||||
if not raw_reason:
|
||||
return None
|
||||
if raw_reason in _CANCEL_REASONS:
|
||||
return None
|
||||
if raw_reason in _PROMPT_REJECTION_CODES:
|
||||
return ErrorMetadata(
|
||||
code=raw_reason,
|
||||
message=default_message or "Arena 拒绝了该提示词:内容违反 Terms of Use",
|
||||
status_code=422,
|
||||
retryable=False,
|
||||
error_type="invalid_request_error",
|
||||
)
|
||||
if raw_reason in _KNOWN_NON_RETRYABLE_CODES:
|
||||
return ErrorMetadata(
|
||||
code=raw_reason,
|
||||
message=default_message or "Arena 图片生成失败,响应不可重试",
|
||||
status_code=422,
|
||||
retryable=False,
|
||||
error_type="invalid_request_error",
|
||||
)
|
||||
return resolve_error_metadata(raw_reason, default_message=default_message)
|
||||
|
||||
# Handle payload dict like {"error": {...}} or SSE payload dict
|
||||
if isinstance(reason_or_payload, dict):
|
||||
error = reason_or_payload.get("error") if "error" in reason_or_payload else reason_or_payload
|
||||
if isinstance(error, dict):
|
||||
code = str(error.get("code") or "").strip()
|
||||
msg = str(error.get("message") or "").strip()
|
||||
raw_status_code = error.get("status_code")
|
||||
raw_retryable = error.get("retryable")
|
||||
param = error.get("param")
|
||||
extra = {
|
||||
k: v for k, v in error.items()
|
||||
if k not in {"message", "type", "code", "status_code", "retryable", "param"}
|
||||
}
|
||||
status_code = _parse_int(raw_status_code, 422) if raw_status_code is not None else 422
|
||||
retryable = _parse_bool(raw_retryable, False) if raw_retryable is not None else False
|
||||
error_type = str(error.get("type") or "invalid_request_error").strip()
|
||||
|
||||
# If inner message itself contains a serialized JSON error or terminal error prefix, resolve it
|
||||
if msg and ("{" in msg or "stream_terminal_error:" in msg):
|
||||
nested_meta = resolve_error_metadata(msg, default_message=default_message)
|
||||
if nested_meta:
|
||||
merged_extra = dict(nested_meta.extra or {})
|
||||
if extra:
|
||||
merged_extra.update(extra)
|
||||
return ErrorMetadata(
|
||||
code=code if (code and code not in {"error", "workflow_failed", "arena_page_error"}) else nested_meta.code,
|
||||
message=nested_meta.message,
|
||||
status_code=status_code if raw_status_code is not None else nested_meta.status_code,
|
||||
retryable=retryable if raw_retryable is not None else nested_meta.retryable,
|
||||
error_type=error_type if error_type not in {"execution_error", "invalid_request_error"} else nested_meta.error_type,
|
||||
param=param if param is not None else nested_meta.param,
|
||||
extra=merged_extra or None,
|
||||
)
|
||||
|
||||
if code in _PROMPT_REJECTION_CODES:
|
||||
return ErrorMetadata(
|
||||
code=code or ARENA_PROMPT_REJECTED_CODE,
|
||||
message=msg or default_message or "Arena 拒绝了该提示词:内容违反 Terms of Use",
|
||||
status_code=status_code,
|
||||
retryable=retryable,
|
||||
error_type=error_type,
|
||||
param=param,
|
||||
extra=extra or None,
|
||||
)
|
||||
if code in _KNOWN_NON_RETRYABLE_CODES:
|
||||
return ErrorMetadata(
|
||||
code=code,
|
||||
message=msg or default_message or "Arena 图片生成失败,响应不可重试",
|
||||
status_code=status_code,
|
||||
retryable=retryable,
|
||||
error_type=error_type,
|
||||
param=param,
|
||||
extra=extra or None,
|
||||
)
|
||||
if (raw_retryable is not None and not retryable) or (raw_status_code is not None and status_code in {400, 401, 403, 422}):
|
||||
return ErrorMetadata(
|
||||
code=code or "invalid_request_error",
|
||||
message=msg or default_message or "请求不可重试",
|
||||
status_code=status_code,
|
||||
retryable=False,
|
||||
error_type=error_type,
|
||||
param=param,
|
||||
extra=extra or None,
|
||||
)
|
||||
if msg or code:
|
||||
return ErrorMetadata(
|
||||
code=code or "error",
|
||||
message=msg or default_message or "请求处理失败",
|
||||
status_code=status_code if raw_status_code is not None else 500,
|
||||
retryable=retryable,
|
||||
error_type=error_type,
|
||||
param=param,
|
||||
extra=extra or None,
|
||||
)
|
||||
return None
|
||||
|
||||
# Handle string code / error message
|
||||
if isinstance(reason_or_payload, str):
|
||||
raw_str = reason_or_payload.strip()
|
||||
if not raw_str:
|
||||
return None
|
||||
if raw_str in _CANCEL_REASONS:
|
||||
return None
|
||||
|
||||
# 剥离内部标识前缀(支持多层嵌套剥离)
|
||||
changed = True
|
||||
while changed:
|
||||
changed = False
|
||||
for prefix in ("stream_terminal_error:", "execution_error:", "执行错误:", "[错误]", "[error]"):
|
||||
if raw_str.lower().startswith(prefix.lower()):
|
||||
raw_str = raw_str[len(prefix):].strip()
|
||||
changed = True
|
||||
break
|
||||
|
||||
# 1. 尝试从文本中解析嵌套的 JSON 对象
|
||||
json_obj = _extract_json_dict_from_text(raw_str)
|
||||
if json_obj:
|
||||
nested_error = json_obj.get("error") if isinstance(json_obj.get("error"), dict) else json_obj
|
||||
if isinstance(nested_error, dict):
|
||||
extracted_msg = str(nested_error.get("message") or "").strip()
|
||||
extracted_code = str(nested_error.get("code") or "").strip()
|
||||
extracted_type = str(nested_error.get("type") or "").strip()
|
||||
extracted_param = nested_error.get("param")
|
||||
extra = {
|
||||
k: v for k, v in nested_error.items()
|
||||
if k not in {"message", "type", "code", "status_code", "retryable", "param"}
|
||||
}
|
||||
detected_status = _extract_status_code_from_text(raw_str) or _parse_int(nested_error.get("status_code"), 400)
|
||||
is_retryable = detected_status >= 500 and extracted_code not in {"moderation_blocked", "content_policy_violation"}
|
||||
|
||||
return ErrorMetadata(
|
||||
code=extracted_code or "upstream_error",
|
||||
message=extracted_msg or raw_str or (default_message or "上游服务报错"),
|
||||
status_code=detected_status or 400,
|
||||
retryable=is_retryable,
|
||||
error_type=extracted_type or "invalid_request_error",
|
||||
param=str(extracted_param) if extracted_param is not None else None,
|
||||
extra=extra or None,
|
||||
)
|
||||
|
||||
# 2. 检查已知错误代码
|
||||
if raw_str in _PROMPT_REJECTION_CODES:
|
||||
return ErrorMetadata(
|
||||
code=raw_str,
|
||||
message=default_message or "Arena 拒绝了该提示词:内容违反 Terms of Use",
|
||||
status_code=422,
|
||||
retryable=False,
|
||||
error_type="invalid_request_error",
|
||||
)
|
||||
if raw_str in _KNOWN_NON_RETRYABLE_CODES:
|
||||
return ErrorMetadata(
|
||||
code=raw_str,
|
||||
message=default_message or "Arena 图片生成失败,响应不可重试",
|
||||
status_code=422,
|
||||
retryable=False,
|
||||
error_type="invalid_request_error",
|
||||
)
|
||||
|
||||
# 3. 检查是否含有 HTTP 状态码
|
||||
status_code = _extract_status_code_from_text(raw_str)
|
||||
if status_code is not None:
|
||||
if status_code == 429:
|
||||
return ErrorMetadata(
|
||||
code="rate_limit_exceeded",
|
||||
message=raw_str,
|
||||
status_code=429,
|
||||
retryable=False,
|
||||
error_type="rate_limit_error",
|
||||
)
|
||||
if status_code in {400, 422}:
|
||||
return ErrorMetadata(
|
||||
code="bad_request" if status_code == 400 else "unprocessable_entity",
|
||||
message=raw_str,
|
||||
status_code=status_code,
|
||||
retryable=False,
|
||||
error_type="invalid_request_error",
|
||||
)
|
||||
if status_code in {401, 403}:
|
||||
return ErrorMetadata(
|
||||
code="permission_denied",
|
||||
message=raw_str,
|
||||
status_code=status_code,
|
||||
retryable=False,
|
||||
error_type="permission_error",
|
||||
)
|
||||
if status_code >= 500:
|
||||
return ErrorMetadata(
|
||||
code="upstream_service_error",
|
||||
message=raw_str,
|
||||
status_code=status_code,
|
||||
retryable=True,
|
||||
error_type="api_error",
|
||||
)
|
||||
|
||||
# 4. 普通纯字符串
|
||||
return ErrorMetadata(
|
||||
code="workflow_failed",
|
||||
message=raw_str or (default_message or "请求处理失败"),
|
||||
status_code=500,
|
||||
retryable=False,
|
||||
error_type="execution_error",
|
||||
)
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def is_non_retryable_error(reason_or_payload: Any) -> bool:
|
||||
"""Check if a given reason or error payload represents a non-retryable 4xx error."""
|
||||
meta = resolve_error_metadata(reason_or_payload)
|
||||
return meta is not None and not meta.retryable and meta.status_code in {400, 401, 403, 422}
|
||||
|
||||
|
||||
def build_error_response(
|
||||
metadata: ErrorMetadata,
|
||||
headers: Optional[Dict[str, str]] = None,
|
||||
) -> JSONResponse:
|
||||
"""Build a standard JSONResponse from ErrorMetadata."""
|
||||
response_headers = dict(headers or {})
|
||||
if not metadata.retryable:
|
||||
response_headers["x-should-retry"] = "false"
|
||||
|
||||
error_body: Dict[str, Any] = {
|
||||
"message": metadata.message,
|
||||
"type": metadata.error_type,
|
||||
"code": metadata.code,
|
||||
"status_code": metadata.status_code,
|
||||
"retryable": metadata.retryable,
|
||||
}
|
||||
if metadata.param is not None:
|
||||
error_body["param"] = metadata.param
|
||||
if metadata.extra:
|
||||
for k, v in metadata.extra.items():
|
||||
if k not in error_body:
|
||||
error_body[k] = v
|
||||
|
||||
return JSONResponse(
|
||||
content={
|
||||
"error": error_body
|
||||
},
|
||||
status_code=metadata.status_code,
|
||||
headers=response_headers,
|
||||
)
|
||||
|
||||
|
||||
class BrowserExecutionError(RuntimeError):
|
||||
"""An upstream browser failure is not assistant text for tool parsing."""
|
||||
def __init__(self, error):
|
||||
error = error if isinstance(error, dict) else {"message": str(error)}
|
||||
self.code = str(error.get("code") or "browser_execution_failed")
|
||||
self.message = str(error.get("message") or self.code)
|
||||
super().__init__(self.message)
|
||||
@@ -0,0 +1,391 @@
|
||||
"""
|
||||
app/services/extractor_manager.py - 提取器配置管理器
|
||||
|
||||
职责:
|
||||
- 加载 config/extractors.json
|
||||
- 管理提取器配置
|
||||
- 根据站点配置返回提取器实例
|
||||
- 支持导入/导出提取器配置
|
||||
"""
|
||||
|
||||
import os
|
||||
import json
|
||||
import copy
|
||||
from app.core.config import atomic_write_json, get_logger
|
||||
from typing import Dict, Optional, List, Any
|
||||
|
||||
from app.core.extractors import ExtractorRegistry, BaseExtractor
|
||||
|
||||
|
||||
logger = get_logger("EXTRACT")
|
||||
|
||||
|
||||
class ExtractorConfigManager:
|
||||
"""
|
||||
提取器配置管理器
|
||||
|
||||
负责:
|
||||
1. 加载和保存 extractors.json
|
||||
2. 根据配置动态加载提取器
|
||||
3. 为站点提供提取器实例
|
||||
"""
|
||||
|
||||
# 配置文件路径
|
||||
_PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
CONFIG_FILE = os.path.join(_PROJECT_ROOT, "config", "extractors.json")
|
||||
|
||||
def __init__(self):
|
||||
self._config: Dict[str, Any] = {}
|
||||
self._extractors_config: Dict[str, Dict] = {}
|
||||
self._default_id: str = "deep_mode_v1"
|
||||
self._last_mtime: float = 0.0
|
||||
|
||||
# 加载配置
|
||||
self._load_config()
|
||||
|
||||
def _load_config(self, *, preserve_on_error: bool = False):
|
||||
"""加载 extractors.json 配置文件"""
|
||||
if not os.path.exists(self.CONFIG_FILE):
|
||||
logger.warning(f"提取器配置文件不存在: {self.CONFIG_FILE},使用默认配置")
|
||||
if not preserve_on_error:
|
||||
self._use_defaults()
|
||||
return
|
||||
|
||||
try:
|
||||
next_mtime = os.path.getmtime(self.CONFIG_FILE)
|
||||
|
||||
with open(self.CONFIG_FILE, "r", encoding="utf-8") as f:
|
||||
loaded_config = json.load(f)
|
||||
|
||||
if not isinstance(loaded_config, dict):
|
||||
raise ValueError("提取器配置必须是对象")
|
||||
extractors_config = loaded_config.get("extractors", {})
|
||||
if not isinstance(extractors_config, dict):
|
||||
raise ValueError("提取器配置 extractors 必须是对象")
|
||||
invalid_entries = [
|
||||
extractor_id
|
||||
for extractor_id, extractor_config in extractors_config.items()
|
||||
if not isinstance(extractor_config, dict)
|
||||
]
|
||||
if invalid_entries:
|
||||
raise ValueError(
|
||||
"提取器配置项必须是对象: "
|
||||
+ ", ".join(str(extractor_id) for extractor_id in invalid_entries)
|
||||
)
|
||||
|
||||
self._config = loaded_config
|
||||
self._extractors_config = extractors_config
|
||||
self._default_id = str(loaded_config.get("default") or "deep_mode_v1").strip() or "deep_mode_v1"
|
||||
self._config["extractors"] = self._extractors_config
|
||||
self._config["default"] = self._default_id
|
||||
self._config.setdefault("version", "1.0")
|
||||
self._last_mtime = next_mtime
|
||||
|
||||
# 动态加载配置中定义的提取器
|
||||
self._register_from_config()
|
||||
|
||||
logger.debug(
|
||||
f"已加载提取器配置: {len(self._extractors_config)} 个提取器,"
|
||||
f"默认: {self._default_id}"
|
||||
)
|
||||
|
||||
except json.JSONDecodeError as e:
|
||||
logger.error(f"提取器配置文件格式错误: {e}")
|
||||
if not preserve_on_error:
|
||||
self._use_defaults()
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"加载提取器配置失败: {e}")
|
||||
if not preserve_on_error:
|
||||
self._use_defaults()
|
||||
|
||||
def _use_defaults(self):
|
||||
"""使用默认配置"""
|
||||
self._extractors_config = {
|
||||
"deep_mode_v1": {
|
||||
"id": "deep_mode_v1",
|
||||
"name": "深度模式 (JS注入)",
|
||||
"description": "通过 JavaScript 注入深度提取内容",
|
||||
"class": "DeepBrowserExtractor",
|
||||
"module": "app.core.extractors.deep_mode",
|
||||
"enabled": True,
|
||||
"config": {}
|
||||
}
|
||||
}
|
||||
self._default_id = "deep_mode_v1"
|
||||
self._config = {
|
||||
"extractors": self._extractors_config,
|
||||
"default": self._default_id,
|
||||
"version": "1.0",
|
||||
}
|
||||
|
||||
def _register_from_config(self):
|
||||
"""根据配置文件动态注册提取器"""
|
||||
for extractor_id, config in self._extractors_config.items():
|
||||
if not isinstance(config, dict):
|
||||
logger.warning(f"跳过无效提取器配置 [{extractor_id}]: 配置项必须是对象")
|
||||
continue
|
||||
if not config.get("enabled", True):
|
||||
logger.debug(f"跳过禁用的提取器: {extractor_id}")
|
||||
continue
|
||||
|
||||
# 检查是否已注册
|
||||
if ExtractorRegistry.exists(extractor_id):
|
||||
logger.debug(f"提取器已注册: {extractor_id}")
|
||||
continue
|
||||
|
||||
# 尝试动态加载
|
||||
module_path = config.get("module")
|
||||
class_name = config.get("class")
|
||||
|
||||
if module_path and class_name:
|
||||
try:
|
||||
ExtractorRegistry.load_from_module(module_path, class_name, extractor_id)
|
||||
except Exception as e:
|
||||
logger.warning(f"动态加载提取器失败 [{extractor_id}]: {e}")
|
||||
|
||||
def refresh_if_changed(self):
|
||||
"""检查配置文件是否变化,如果变化则重载"""
|
||||
if not os.path.exists(self.CONFIG_FILE):
|
||||
return
|
||||
|
||||
try:
|
||||
current_mtime = os.path.getmtime(self.CONFIG_FILE)
|
||||
if current_mtime != self._last_mtime:
|
||||
logger.info("检测到提取器配置变化,重新加载")
|
||||
self._load_config(preserve_on_error=True)
|
||||
except Exception as e:
|
||||
logger.error(f"检查配置变化失败: {e}")
|
||||
|
||||
def reload_config(self) -> None:
|
||||
"""强制重新加载提取器配置,供配置备份恢复后热重载使用。"""
|
||||
self._load_config(preserve_on_error=False)
|
||||
|
||||
# ================= 提取器获取 =================
|
||||
|
||||
def get_extractor(self, extractor_id: Optional[str] = None) -> BaseExtractor:
|
||||
"""
|
||||
获取提取器实例
|
||||
|
||||
Args:
|
||||
extractor_id: 提取器 ID(None 则返回默认)
|
||||
|
||||
Returns:
|
||||
提取器实例
|
||||
"""
|
||||
eid = extractor_id or self._default_id
|
||||
|
||||
try:
|
||||
return ExtractorRegistry.get(eid)
|
||||
except ValueError:
|
||||
logger.warning(f"提取器不存在: {eid},使用默认: {self._default_id}")
|
||||
return ExtractorRegistry.get(self._default_id)
|
||||
|
||||
def get_extractor_for_site(self, site_config: Dict) -> BaseExtractor:
|
||||
"""
|
||||
根据站点配置获取提取器
|
||||
|
||||
Args:
|
||||
site_config: 站点配置字典(包含 extractor_id 字段)
|
||||
|
||||
Returns:
|
||||
提取器实例
|
||||
"""
|
||||
extractor_id = site_config.get("extractor_id")
|
||||
return self.get_extractor(extractor_id)
|
||||
|
||||
def get_default_id(self) -> str:
|
||||
"""获取默认提取器 ID"""
|
||||
return self._default_id
|
||||
|
||||
def set_default(self, extractor_id: str) -> bool:
|
||||
"""
|
||||
设置默认提取器
|
||||
|
||||
Args:
|
||||
extractor_id: 提取器 ID
|
||||
|
||||
Returns:
|
||||
是否成功
|
||||
"""
|
||||
if not ExtractorRegistry.exists(extractor_id):
|
||||
logger.error(f"无法设置默认提取器:{extractor_id} 不存在")
|
||||
return False
|
||||
|
||||
previous_default = self._default_id
|
||||
previous_config = copy.deepcopy(self._config)
|
||||
previous_extractors_config = copy.deepcopy(self._extractors_config)
|
||||
|
||||
try:
|
||||
self._default_id = extractor_id
|
||||
self._config["extractors"] = self._extractors_config
|
||||
self._config["default"] = extractor_id
|
||||
self._save_config()
|
||||
except Exception as exc:
|
||||
self._default_id = previous_default
|
||||
self._config = previous_config
|
||||
self._extractors_config = self._config.get("extractors", previous_extractors_config)
|
||||
logger.error(f"设置默认提取器失败,已回滚: {exc}")
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
# ================= 配置查询 =================
|
||||
|
||||
def list_extractors(self) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
列出所有可用的提取器
|
||||
|
||||
Returns:
|
||||
提取器信息列表
|
||||
"""
|
||||
result = []
|
||||
|
||||
# 从注册表获取(确保包含运行时注册的)
|
||||
for info in ExtractorRegistry.list_all():
|
||||
extractor_id = info["id"]
|
||||
|
||||
# 合并配置文件中的信息
|
||||
config = self._extractors_config.get(extractor_id, {})
|
||||
if not isinstance(config, dict):
|
||||
config = {}
|
||||
|
||||
result.append({
|
||||
"id": extractor_id,
|
||||
"name": config.get("name") or info.get("name", extractor_id),
|
||||
"description": config.get("description") or info.get("description", ""),
|
||||
"enabled": config.get("enabled", True),
|
||||
"is_default": extractor_id == self._default_id
|
||||
})
|
||||
|
||||
return result
|
||||
|
||||
def get_extractor_config(self, extractor_id: str) -> Optional[Dict]:
|
||||
"""获取提取器配置"""
|
||||
config = self._extractors_config.get(extractor_id)
|
||||
return copy.deepcopy(config) if isinstance(config, dict) else None
|
||||
|
||||
def has_extractor(self, extractor_id: str) -> bool:
|
||||
"""检查提取器是否已经注册。"""
|
||||
return ExtractorRegistry.exists(extractor_id)
|
||||
|
||||
# ================= 导入/导出 =================
|
||||
|
||||
def export_config(self) -> Dict[str, Any]:
|
||||
"""
|
||||
导出提取器配置
|
||||
|
||||
Returns:
|
||||
完整配置字典(可保存为 JSON)
|
||||
"""
|
||||
return {
|
||||
"extractors": copy.deepcopy(self._extractors_config),
|
||||
"default": self._default_id,
|
||||
"version": self._config.get("version", "1.0")
|
||||
}
|
||||
|
||||
def import_config(self, config: Dict[str, Any]) -> bool:
|
||||
"""
|
||||
导入提取器配置
|
||||
|
||||
Args:
|
||||
config: 配置字典
|
||||
|
||||
Returns:
|
||||
是否成功
|
||||
"""
|
||||
try:
|
||||
# 验证格式
|
||||
if not isinstance(config, dict):
|
||||
logger.error("导入失败:配置必须是对象")
|
||||
return False
|
||||
extractors = config.get("extractors")
|
||||
if "extractors" not in config:
|
||||
logger.error("导入失败:缺少 extractors 字段")
|
||||
return False
|
||||
if not isinstance(extractors, dict):
|
||||
logger.error("导入失败:extractors 必须是对象")
|
||||
return False
|
||||
invalid_entries = [
|
||||
extractor_id
|
||||
for extractor_id, extractor_config in extractors.items()
|
||||
if not isinstance(extractor_config, dict)
|
||||
]
|
||||
if invalid_entries:
|
||||
logger.error(
|
||||
"导入失败:提取器配置必须是对象: "
|
||||
+ ", ".join(str(extractor_id) for extractor_id in invalid_entries)
|
||||
)
|
||||
return False
|
||||
default_id = str(config.get("default") or "deep_mode_v1").strip() or "deep_mode_v1"
|
||||
default_entry = extractors.get(default_id)
|
||||
if isinstance(default_entry, dict) and not default_entry.get("enabled", True):
|
||||
logger.error(f"导入失败:默认提取器已禁用: {default_id}")
|
||||
return False
|
||||
if default_entry is None and not ExtractorRegistry.exists(default_id):
|
||||
logger.error(f"导入失败:默认提取器不存在: {default_id}")
|
||||
return False
|
||||
|
||||
previous_config = copy.deepcopy(self._config)
|
||||
previous_extractors_config = copy.deepcopy(self._extractors_config)
|
||||
previous_default_id = self._default_id
|
||||
previous_registry = copy.copy(getattr(ExtractorRegistry, "_extractors", {}))
|
||||
|
||||
next_config = copy.deepcopy(config)
|
||||
self._extractors_config = copy.deepcopy(next_config["extractors"])
|
||||
self._default_id = default_id
|
||||
next_config["extractors"] = self._extractors_config
|
||||
next_config["default"] = self._default_id
|
||||
self._config = next_config
|
||||
|
||||
# 保存到文件
|
||||
saved_config = False
|
||||
try:
|
||||
self._save_config()
|
||||
saved_config = True
|
||||
self._register_from_config()
|
||||
if not ExtractorRegistry.exists(self._default_id):
|
||||
raise ValueError(f"默认提取器无法注册: {self._default_id}")
|
||||
except Exception:
|
||||
self._config = previous_config
|
||||
self._extractors_config = self._config.get("extractors", previous_extractors_config)
|
||||
self._default_id = previous_default_id
|
||||
registry = getattr(ExtractorRegistry, "_extractors", None)
|
||||
if isinstance(registry, dict):
|
||||
registry.clear()
|
||||
registry.update(previous_registry)
|
||||
if saved_config:
|
||||
try:
|
||||
self._save_config()
|
||||
except Exception as rollback_error:
|
||||
logger.error(f"导入失败后恢复提取器配置文件失败: {rollback_error}")
|
||||
raise
|
||||
|
||||
logger.info(f"成功导入提取器配置: {len(self._extractors_config)} 个")
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"导入配置失败: {e}")
|
||||
return False
|
||||
|
||||
def _save_config(self):
|
||||
"""保存配置到文件"""
|
||||
try:
|
||||
atomic_write_json(self.CONFIG_FILE, self._config)
|
||||
try:
|
||||
self._last_mtime = os.path.getmtime(self.CONFIG_FILE)
|
||||
except Exception as mtime_error:
|
||||
logger.warning(f"提取器配置已保存但更新时间戳失败: {mtime_error}")
|
||||
logger.info("提取器配置已保存")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"保存配置失败: {e}")
|
||||
raise
|
||||
|
||||
|
||||
# ================= 单例 =================
|
||||
|
||||
extractor_manager = ExtractorConfigManager()
|
||||
|
||||
|
||||
__all__ = ['ExtractorConfigManager', 'extractor_manager']
|
||||
@@ -0,0 +1,214 @@
|
||||
"""R2-7:可观测性——无第三方依赖的 Prometheus 指标与请求 ID。
|
||||
|
||||
- ``RequestMetricsMiddleware``(纯 ASGI 中间件,不缓冲流式响应):
|
||||
- 为每个 HTTP 请求分配或透传 ``X-Request-ID``,写入响应头与 ``REQUEST_ID`` 上下文变量;
|
||||
- 记录请求数、耗时直方图(含流式响应体发送完的总耗时),以及进行中的请求数。
|
||||
- 标签用路由模板(如 ``/v1/chat/completions``),未匹配到路由的统一记为 ``<unmatched>``,避免标签基数爆炸。
|
||||
- ``render_prometheus()``:Prometheus 文本格式(text/plain; version=0.0.4)。
|
||||
抓取时顺带读取标签页池状态,但只在浏览器**已连接**时读取,抓取本身绝不触发连接。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import contextvars
|
||||
import re
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
from typing import Dict, Iterable, List, Optional, Tuple
|
||||
|
||||
REQUEST_ID: contextvars.ContextVar[str] = contextvars.ContextVar("uwapi_http_request_id", default="")
|
||||
_REQUEST_ID_PATTERN = re.compile(r"^[A-Za-z0-9._:-]{1,64}$")
|
||||
_STARTED_AT = time.time()
|
||||
|
||||
Labels = Tuple[Tuple[str, str], ...]
|
||||
|
||||
|
||||
def _escape(value: str) -> str:
|
||||
return str(value).replace("\\", "\\\\").replace("\n", "\\n").replace('"', '\\"')
|
||||
|
||||
|
||||
def _format_labels(labels: Labels, extra: Iterable[Tuple[str, str]] = ()) -> str:
|
||||
items = list(labels) + list(extra)
|
||||
if not items:
|
||||
return ""
|
||||
return "{" + ",".join(f'{key}="{_escape(value)}"' for key, value in items) + "}"
|
||||
|
||||
|
||||
class _Metric:
|
||||
kind = ""
|
||||
|
||||
def __init__(self, name: str, help_text: str):
|
||||
self.name = name
|
||||
self.help = help_text
|
||||
self._lock = threading.Lock()
|
||||
|
||||
def header(self) -> List[str]:
|
||||
return [f"# HELP {self.name} {self.help}", f"# TYPE {self.name} {self.kind}"]
|
||||
|
||||
|
||||
class Counter(_Metric):
|
||||
kind = "counter"
|
||||
|
||||
def __init__(self, name: str, help_text: str):
|
||||
super().__init__(name, help_text)
|
||||
self._values: Dict[Labels, float] = {}
|
||||
|
||||
def inc(self, amount: float = 1.0, **labels: str) -> None:
|
||||
key = tuple(sorted((k, str(v)) for k, v in labels.items()))
|
||||
with self._lock:
|
||||
self._values[key] = self._values.get(key, 0.0) + amount
|
||||
|
||||
def value(self, **labels: str) -> float:
|
||||
return self._values.get(tuple(sorted((k, str(v)) for k, v in labels.items())), 0.0)
|
||||
|
||||
def render(self) -> List[str]:
|
||||
with self._lock:
|
||||
items = sorted(self._values.items())
|
||||
return self.header() + [f"{self.name}{_format_labels(k)} {v:g}" for k, v in items]
|
||||
|
||||
|
||||
class Gauge(Counter):
|
||||
kind = "gauge"
|
||||
|
||||
def set(self, value: float, **labels: str) -> None:
|
||||
key = tuple(sorted((k, str(v)) for k, v in labels.items()))
|
||||
with self._lock:
|
||||
self._values[key] = float(value)
|
||||
|
||||
def dec(self, amount: float = 1.0, **labels: str) -> None:
|
||||
self.inc(-amount, **labels)
|
||||
|
||||
def clear(self) -> None:
|
||||
with self._lock:
|
||||
self._values.clear()
|
||||
|
||||
|
||||
class Histogram(_Metric):
|
||||
kind = "histogram"
|
||||
|
||||
def __init__(self, name: str, help_text: str, buckets: Iterable[float]):
|
||||
super().__init__(name, help_text)
|
||||
self.buckets = tuple(sorted(buckets))
|
||||
self._data: Dict[Labels, List[float]] = {} # [bucket counts..., sum, count]
|
||||
|
||||
def observe(self, value: float, **labels: str) -> None:
|
||||
key = tuple(sorted((k, str(v)) for k, v in labels.items()))
|
||||
with self._lock:
|
||||
data = self._data.setdefault(key, [0.0] * (len(self.buckets) + 2))
|
||||
for index, bound in enumerate(self.buckets):
|
||||
if value <= bound:
|
||||
data[index] += 1
|
||||
data[-2] += value
|
||||
data[-1] += 1
|
||||
|
||||
def render(self) -> List[str]:
|
||||
lines = self.header()
|
||||
with self._lock:
|
||||
items = sorted((k, list(v)) for k, v in self._data.items())
|
||||
for labels, data in items:
|
||||
for index, bound in enumerate(self.buckets):
|
||||
lines.append(f"{self.name}_bucket{_format_labels(labels, [('le', f'{bound:g}')])} {data[index]:g}")
|
||||
lines.append(f"{self.name}_bucket{_format_labels(labels, [('le', '+Inf')])} {data[-1]:g}")
|
||||
lines.append(f"{self.name}_sum{_format_labels(labels)} {data[-2]:.6f}")
|
||||
lines.append(f"{self.name}_count{_format_labels(labels)} {data[-1]:g}")
|
||||
return lines
|
||||
|
||||
|
||||
HTTP_REQUESTS = Counter("uwapi_http_requests_total", "HTTP 请求数(按方法、路由模板、状态码)")
|
||||
HTTP_DURATION = Histogram(
|
||||
"uwapi_http_request_duration_seconds",
|
||||
"HTTP 请求耗时(秒,流式响应计到响应体发送完)",
|
||||
buckets=(0.01, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60, 120, 300, 600),
|
||||
)
|
||||
HTTP_IN_FLIGHT = Gauge("uwapi_http_requests_in_flight", "进行中的 HTTP 请求数")
|
||||
CHAT_REQUESTS = Counter("uwapi_chat_requests_total", "聊天请求数(按协议 openai.chat / anthropic.messages / openai.responses 与最终状态)")
|
||||
CHAT_REQUEST_DURATION = Histogram(
|
||||
"uwapi_chat_request_duration_seconds",
|
||||
"聊天请求从创建到结束的耗时(秒,按协议)",
|
||||
buckets=(0.5, 1, 2.5, 5, 10, 20, 30, 60, 120, 300, 600),
|
||||
)
|
||||
TAB_POOL_TABS = Gauge("uwapi_tab_pool_tabs", "标签页池中的标签页数(按状态;浏览器未连接时不输出)")
|
||||
BUILD_INFO = Gauge("uwapi_build_info", "构建信息(值恒为 1)")
|
||||
UPTIME = Gauge("uwapi_uptime_seconds", "进程已运行秒数")
|
||||
REGISTRY: List[_Metric] = [
|
||||
HTTP_REQUESTS, HTTP_DURATION, HTTP_IN_FLIGHT, CHAT_REQUESTS, CHAT_REQUEST_DURATION, TAB_POOL_TABS, BUILD_INFO, UPTIME,
|
||||
]
|
||||
|
||||
|
||||
def new_request_id() -> str:
|
||||
return uuid.uuid4().hex[:16]
|
||||
|
||||
|
||||
def _incoming_request_id(scope) -> Optional[str]:
|
||||
for key, value in scope.get("headers") or []:
|
||||
if key == b"x-request-id":
|
||||
candidate = value.decode("latin-1").strip()
|
||||
return candidate if _REQUEST_ID_PATTERN.fullmatch(candidate) else None
|
||||
return None
|
||||
|
||||
|
||||
class RequestMetricsMiddleware:
|
||||
def __init__(self, app):
|
||||
self.app = app
|
||||
|
||||
async def __call__(self, scope, receive, send):
|
||||
if scope.get("type") != "http":
|
||||
await self.app(scope, receive, send)
|
||||
return
|
||||
request_id = _incoming_request_id(scope) or new_request_id()
|
||||
token = REQUEST_ID.set(request_id)
|
||||
status = {"code": 500}
|
||||
started = time.perf_counter()
|
||||
|
||||
async def send_with_request_id(message):
|
||||
if message.get("type") == "http.response.start":
|
||||
status["code"] = int(message.get("status") or 500)
|
||||
headers = [(k, v) for k, v in (message.get("headers") or []) if k.lower() != b"x-request-id"]
|
||||
headers.append((b"x-request-id", request_id.encode("latin-1")))
|
||||
message = {**message, "headers": headers}
|
||||
await send(message)
|
||||
|
||||
HTTP_IN_FLIGHT.inc()
|
||||
try:
|
||||
await self.app(scope, receive, send_with_request_id)
|
||||
finally:
|
||||
HTTP_IN_FLIGHT.dec()
|
||||
route = getattr(scope.get("route"), "path", None) or "<unmatched>"
|
||||
method = str(scope.get("method") or "GET")
|
||||
HTTP_REQUESTS.inc(method=method, route=route, status=str(status["code"]))
|
||||
HTTP_DURATION.observe(time.perf_counter() - started, method=method, route=route)
|
||||
REQUEST_ID.reset(token)
|
||||
|
||||
|
||||
def _collect_runtime_gauges() -> None:
|
||||
from app import __version__
|
||||
|
||||
BUILD_INFO.set(1, version=__version__)
|
||||
UPTIME.set(time.time() - _STARTED_AT)
|
||||
TAB_POOL_TABS.clear()
|
||||
try:
|
||||
from app.core.browser import main as browser_main
|
||||
|
||||
browser = getattr(browser_main, "_browser_instance", None) # 不调用 get_browser:抓取不能创建或连接浏览器
|
||||
if browser is None or not getattr(browser, "_connected", False):
|
||||
return
|
||||
pool = getattr(browser, "tab_pool", None)
|
||||
if pool is None:
|
||||
return
|
||||
counts: Dict[str, int] = {}
|
||||
for session in pool.get_sessions_snapshot():
|
||||
state = str(getattr(getattr(session, "status", None), "value", "unknown") or "unknown")
|
||||
counts[state] = counts.get(state, 0) + 1
|
||||
for state, count in counts.items():
|
||||
TAB_POOL_TABS.set(count, status=state)
|
||||
except Exception: # broad-except: 指标抓取不能因为浏览器状态异常而失败
|
||||
return
|
||||
|
||||
|
||||
def render_prometheus() -> str:
|
||||
_collect_runtime_gauges()
|
||||
lines: List[str] = []
|
||||
for metric in REGISTRY:
|
||||
lines.extend(metric.render())
|
||||
return "\n".join(lines) + "\n"
|
||||
@@ -0,0 +1,522 @@
|
||||
"""
|
||||
app/services/parser_manager.py - Runtime parser installation manager.
|
||||
|
||||
Parser packages can be written into app/core/parsers and then
|
||||
registered through ParserRegistry via config/parsers.json.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import ast
|
||||
import copy
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import tempfile
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from app.core.config import get_logger
|
||||
from app.core.config import atomic_write_json
|
||||
from app.core.parsers import ParserRegistry
|
||||
|
||||
logger = get_logger("PARSER_MGR")
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# S3:运行时安装解析器 = 把一段 Python 写进 app/core/parsers 并立即 import。
|
||||
# 1) 默认关闭,只有可信管理员显式设置 PARSER_INSTALL_ENABLED=true 才允许;
|
||||
# 2) 导入期「无副作用」静态约束:模块顶层 / 类体只允许 import、函数/类定义、
|
||||
# 字面量赋值,以及极少数纯函数调用(re.compile 等,且参数必须是字面量);
|
||||
# 装饰器只允许 staticmethod/classmethod/property 等;默认参数必须是字面量。
|
||||
# 这样 import 本身不会执行任意代码——解析逻辑只会在解析器被调用时运行;
|
||||
# 3) 配置中的模块路径必须位于 app.core.parsers 包内。
|
||||
# 这不是沙箱:解析器方法仍以服务进程权限运行,所以安装入口必须只对可信管理员开放。
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
PARSER_INSTALL_ENV = "PARSER_INSTALL_ENABLED"
|
||||
PARSER_MODULE_PREFIX = "app.core.parsers."
|
||||
|
||||
_IMPORT_TIME_SAFE_CALLS = frozenset({
|
||||
"re.compile", "set", "frozenset", "tuple", "list", "dict", "str.maketrans",
|
||||
"get_logger", "logging.getLogger",
|
||||
})
|
||||
_SAFE_DECORATORS = frozenset({
|
||||
"staticmethod", "classmethod", "property", "abstractmethod", "abc.abstractmethod",
|
||||
"dataclass", "dataclasses.dataclass", "functools.cached_property", "cached_property",
|
||||
})
|
||||
|
||||
|
||||
def parser_install_enabled() -> bool:
|
||||
value = str(os.getenv(PARSER_INSTALL_ENV, "") or "").strip().lower()
|
||||
return value in ("true", "1", "yes", "on")
|
||||
|
||||
|
||||
def _dotted_name(node: ast.AST) -> str:
|
||||
if isinstance(node, ast.Name):
|
||||
return node.id
|
||||
if isinstance(node, ast.Attribute):
|
||||
base = _dotted_name(node.value)
|
||||
return f"{base}.{node.attr}" if base else ""
|
||||
return ""
|
||||
|
||||
|
||||
def _import_time_value_ok(node: Optional[ast.AST]) -> bool:
|
||||
"""导入期会被求值的表达式:只允许字面量/名字引用/白名单纯函数调用。"""
|
||||
if node is None:
|
||||
return True
|
||||
if isinstance(node, (ast.Constant, ast.Name)):
|
||||
return True
|
||||
if isinstance(node, ast.Attribute):
|
||||
return not node.attr.startswith("__") and _import_time_value_ok(node.value)
|
||||
if isinstance(node, (ast.Tuple, ast.List, ast.Set)):
|
||||
return all(_import_time_value_ok(item) for item in node.elts)
|
||||
if isinstance(node, ast.Dict):
|
||||
return all(_import_time_value_ok(k) for k in node.keys if k is not None) and all(
|
||||
_import_time_value_ok(v) for v in node.values
|
||||
)
|
||||
if isinstance(node, ast.UnaryOp):
|
||||
return _import_time_value_ok(node.operand)
|
||||
if isinstance(node, ast.BinOp):
|
||||
return _import_time_value_ok(node.left) and _import_time_value_ok(node.right)
|
||||
if isinstance(node, ast.JoinedStr):
|
||||
return all(_import_time_value_ok(v) for v in node.values)
|
||||
if isinstance(node, ast.FormattedValue):
|
||||
return _import_time_value_ok(node.value)
|
||||
if isinstance(node, ast.Call):
|
||||
if _dotted_name(node.func) not in _IMPORT_TIME_SAFE_CALLS:
|
||||
return False
|
||||
return all(_import_time_value_ok(a) for a in node.args) and all(
|
||||
_import_time_value_ok(k.value) for k in node.keywords
|
||||
)
|
||||
return False
|
||||
|
||||
|
||||
def _decorator_ok(node: ast.AST) -> bool:
|
||||
name = _dotted_name(node)
|
||||
if name in _SAFE_DECORATORS:
|
||||
return True
|
||||
# @x.setter / @x.getter / @x.deleter(property 访问器)
|
||||
return isinstance(node, ast.Attribute) and node.attr in {"setter", "getter", "deleter"} and bool(name)
|
||||
|
||||
|
||||
def _function_signature_ok(node: ast.AST) -> bool:
|
||||
args = node.args # type: ignore[attr-defined]
|
||||
defaults = list(args.defaults) + [d for d in args.kw_defaults if d is not None]
|
||||
annotations = [a.annotation for a in (*args.posonlyargs, *args.args, *args.kwonlyargs) if a.annotation]
|
||||
if args.vararg and args.vararg.annotation:
|
||||
annotations.append(args.vararg.annotation)
|
||||
if args.kwarg and args.kwarg.annotation:
|
||||
annotations.append(args.kwarg.annotation)
|
||||
returns = getattr(node, "returns", None)
|
||||
if returns is not None:
|
||||
annotations.append(returns)
|
||||
return all(_import_time_value_ok(d) for d in defaults) and all(
|
||||
_annotation_ok(a) for a in annotations
|
||||
)
|
||||
|
||||
|
||||
def _annotation_ok(node: ast.AST) -> bool:
|
||||
# 注解在没有 `from __future__ import annotations` 时也会在导入期求值;
|
||||
# 允许 Name/Attribute/字符串/下标(Optional[str]、Dict[str, Any])/ | 联合。
|
||||
if isinstance(node, (ast.Constant, ast.Name)):
|
||||
return True
|
||||
if isinstance(node, ast.Attribute):
|
||||
return not node.attr.startswith("__") and _annotation_ok(node.value)
|
||||
if isinstance(node, ast.Subscript):
|
||||
return _annotation_ok(node.value) and _annotation_ok(node.slice)
|
||||
if isinstance(node, (ast.Tuple, ast.List)):
|
||||
return all(_annotation_ok(item) for item in node.elts)
|
||||
if isinstance(node, ast.BinOp) and isinstance(node.op, ast.BitOr):
|
||||
return _annotation_ok(node.left) and _annotation_ok(node.right)
|
||||
return False
|
||||
|
||||
|
||||
def check_import_time_side_effects(tree: ast.Module) -> List[str]:
|
||||
"""返回导入期可能执行任意代码的位置描述;空列表表示通过。"""
|
||||
problems: List[str] = []
|
||||
|
||||
def visit_body(body: List[ast.stmt], where: str) -> None:
|
||||
for index, stmt in enumerate(body):
|
||||
line = getattr(stmt, "lineno", "?")
|
||||
if isinstance(stmt, (ast.Import, ast.ImportFrom, ast.Pass)):
|
||||
continue
|
||||
if isinstance(stmt, ast.Expr) and isinstance(stmt.value, ast.Constant) and isinstance(stmt.value.value, str):
|
||||
continue # docstring
|
||||
if isinstance(stmt, (ast.FunctionDef, ast.AsyncFunctionDef)):
|
||||
if not all(_decorator_ok(d) for d in stmt.decorator_list):
|
||||
problems.append(f"{where}:{line} 不允许的装饰器")
|
||||
if not _function_signature_ok(stmt):
|
||||
problems.append(f"{where}:{line} 默认参数/注解必须是字面量")
|
||||
continue
|
||||
if isinstance(stmt, ast.ClassDef):
|
||||
if not all(_decorator_ok(d) for d in stmt.decorator_list):
|
||||
problems.append(f"{where}:{line} 不允许的类装饰器")
|
||||
if stmt.keywords or not all(isinstance(b, (ast.Name, ast.Attribute)) for b in stmt.bases):
|
||||
problems.append(f"{where}:{line} 类继承列表只允许名字引用")
|
||||
visit_body(stmt.body, f"class {stmt.name}")
|
||||
continue
|
||||
if isinstance(stmt, ast.Assign):
|
||||
if all(isinstance(t, (ast.Name, ast.Tuple)) for t in stmt.targets) and _import_time_value_ok(stmt.value):
|
||||
continue
|
||||
if isinstance(stmt, ast.AnnAssign):
|
||||
if isinstance(stmt.target, ast.Name) and _annotation_ok(stmt.annotation) and _import_time_value_ok(stmt.value):
|
||||
continue
|
||||
if isinstance(stmt, ast.If) and index == len(body) - 1 and where == "module":
|
||||
test = stmt.test
|
||||
# 允许末尾的 `if __name__ == "__main__":`(被 import 时不会执行)
|
||||
if (
|
||||
isinstance(test, ast.Compare)
|
||||
and isinstance(test.left, ast.Name)
|
||||
and test.left.id == "__name__"
|
||||
and len(test.comparators) == 1
|
||||
and isinstance(test.comparators[0], ast.Constant)
|
||||
and test.comparators[0].value == "__main__"
|
||||
):
|
||||
continue
|
||||
problems.append(f"{where}:{line} 导入期会执行的语句 ({type(stmt).__name__})")
|
||||
|
||||
visit_body(tree.body, "module")
|
||||
return problems
|
||||
|
||||
|
||||
class ParserConfigManager:
|
||||
"""Manage runtime-installed response parsers."""
|
||||
|
||||
_PROJECT_ROOT = Path(__file__).resolve().parents[2]
|
||||
CONFIG_FILE = Path(os.getenv("PARSERS_CONFIG_FILE", _PROJECT_ROOT / "config" / "parsers.json"))
|
||||
PARSER_DIR = _PROJECT_ROOT / "app" / "core" / "parsers"
|
||||
|
||||
_ID_PATTERN = re.compile(r"^[A-Za-z0-9._-]+$")
|
||||
_NAME_PATTERN = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$")
|
||||
|
||||
def __init__(self):
|
||||
self._config: Dict[str, Any] = {}
|
||||
self._parsers_config: Dict[str, Dict[str, Any]] = {}
|
||||
self._last_mtime: float = 0.0
|
||||
self._load_config()
|
||||
|
||||
def _load_config(self) -> None:
|
||||
if not self.CONFIG_FILE.exists():
|
||||
self._use_defaults()
|
||||
return
|
||||
|
||||
try:
|
||||
next_mtime = self.CONFIG_FILE.stat().st_mtime
|
||||
with self.CONFIG_FILE.open("r", encoding="utf-8") as handle:
|
||||
loaded_config = json.load(handle)
|
||||
if not isinstance(loaded_config, dict):
|
||||
logger.warning("Parsers config file must be an object; using defaults")
|
||||
loaded_config = {"version": "1.0", "parsers": {}}
|
||||
parsers_config = loaded_config.get("parsers", {})
|
||||
if not isinstance(parsers_config, dict):
|
||||
logger.warning("Parsers config field must be an object; ignoring invalid value")
|
||||
parsers_config = {}
|
||||
loaded_config["parsers"] = parsers_config
|
||||
loaded_config.setdefault("version", "1.0")
|
||||
self._config = loaded_config
|
||||
self._parsers_config = parsers_config
|
||||
self._last_mtime = next_mtime
|
||||
self._register_from_config()
|
||||
except json.JSONDecodeError as exc:
|
||||
logger.error(f"Failed to decode parsers config: {exc}")
|
||||
self._use_defaults()
|
||||
except Exception as exc:
|
||||
logger.error(f"Failed to load parsers config: {exc}")
|
||||
self._use_defaults()
|
||||
|
||||
def _use_defaults(self) -> None:
|
||||
self._config = {
|
||||
"version": "1.0",
|
||||
"parsers": {},
|
||||
}
|
||||
self._parsers_config = {}
|
||||
|
||||
def _register_from_config(self) -> None:
|
||||
for parser_id, config in self._parsers_config.items():
|
||||
if not isinstance(config, dict):
|
||||
logger.warning(f"Skip invalid parser config [{parser_id}]: entry must be an object")
|
||||
continue
|
||||
if not config.get("enabled", True):
|
||||
continue
|
||||
try:
|
||||
self._load_parser_entry(parser_id, config)
|
||||
except Exception as exc:
|
||||
logger.warning(f"Failed to register parser [{parser_id}]: {exc}")
|
||||
|
||||
def _save_config(self) -> None:
|
||||
atomic_write_json(self.CONFIG_FILE, self._config)
|
||||
try:
|
||||
self._last_mtime = self.CONFIG_FILE.stat().st_mtime
|
||||
except Exception as exc:
|
||||
logger.warning(f"Parsers config saved but failed to update mtime: {exc}")
|
||||
|
||||
@staticmethod
|
||||
def _atomic_write_text(path: Path, content: str) -> None:
|
||||
target = Path(path)
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
fd = None
|
||||
tmp_path: Optional[Path] = None
|
||||
|
||||
try:
|
||||
fd, tmp_name = tempfile.mkstemp(
|
||||
prefix=f".{target.name}.",
|
||||
suffix=".tmp",
|
||||
dir=str(target.parent),
|
||||
)
|
||||
tmp_path = Path(tmp_name)
|
||||
with os.fdopen(fd, "w", encoding="utf-8", newline="\n") as handle:
|
||||
fd = None
|
||||
handle.write(content)
|
||||
if not content.endswith("\n"):
|
||||
handle.write("\n")
|
||||
handle.flush()
|
||||
os.fsync(handle.fileno())
|
||||
os.replace(tmp_path, target)
|
||||
except Exception:
|
||||
if fd is not None:
|
||||
try:
|
||||
os.close(fd)
|
||||
except Exception:
|
||||
pass
|
||||
if tmp_path is not None:
|
||||
try:
|
||||
tmp_path.unlink(missing_ok=True)
|
||||
except Exception:
|
||||
pass
|
||||
raise
|
||||
|
||||
def _load_parser_entry(self, parser_id: str, config: Dict[str, Any]) -> None:
|
||||
module_path = str(config.get("module") or "").strip()
|
||||
class_name = str(config.get("class") or "").strip()
|
||||
if not module_path or not class_name:
|
||||
raise ValueError("Parser config is missing module/class")
|
||||
# S3:配置里的模块只能来自解析器包本身,不能借 parsers.json 导入任意模块
|
||||
suffix = module_path[len(PARSER_MODULE_PREFIX):] if module_path.startswith(PARSER_MODULE_PREFIX) else ""
|
||||
if not suffix or not self._NAME_PATTERN.match(suffix):
|
||||
raise ValueError(f"Parser module must be inside {PARSER_MODULE_PREFIX[:-1]}: {module_path}")
|
||||
ParserRegistry.load_from_module(module_path, class_name, parser_id)
|
||||
|
||||
def list_parsers(self) -> List[Dict[str, Any]]:
|
||||
result: List[Dict[str, Any]] = []
|
||||
for info in ParserRegistry.list_all():
|
||||
parser_id = str(info.get("id") or "").strip()
|
||||
config = self._parsers_config.get(parser_id, {})
|
||||
if not isinstance(config, dict):
|
||||
config = {}
|
||||
result.append({
|
||||
**info,
|
||||
"managed": bool(config),
|
||||
"module": str(config.get("module") or ""),
|
||||
"class": str(config.get("class") or ""),
|
||||
"filename": str(config.get("filename") or ""),
|
||||
"installed_at": str(config.get("installed_at") or ""),
|
||||
})
|
||||
result.sort(key=lambda item: str(item.get("id") or ""))
|
||||
return result
|
||||
|
||||
def get_parser_config(self, parser_id: str) -> Optional[Dict[str, Any]]:
|
||||
config = self._parsers_config.get(str(parser_id or "").strip())
|
||||
return copy.deepcopy(config) if isinstance(config, dict) else None
|
||||
|
||||
def install_parser_package(self, payload: Dict[str, Any], overwrite: bool = False) -> Dict[str, Any]:
|
||||
if not parser_install_enabled():
|
||||
raise PermissionError(
|
||||
f"运行时安装解析器默认关闭(会以服务进程权限执行源码);"
|
||||
f"仅可信管理员可设置 {PARSER_INSTALL_ENV}=true 后使用"
|
||||
)
|
||||
normalized = self._normalize_parser_package(payload)
|
||||
parser_id = normalized["parser_id"]
|
||||
module_name = normalized["module_name"]
|
||||
target_path = self.PARSER_DIR / normalized["filename"]
|
||||
had_previous_entry = parser_id in self._parsers_config
|
||||
previous_entry = copy.deepcopy(self._parsers_config.get(parser_id)) if had_previous_entry else None
|
||||
existing_entry = previous_entry if isinstance(previous_entry, dict) else None
|
||||
registry_snapshot = dict(getattr(ParserRegistry, "_parsers", {}))
|
||||
|
||||
if ParserRegistry.exists(parser_id) and existing_entry is None:
|
||||
raise ValueError(f"解析器 ID 已被内置解析器占用: {parser_id}")
|
||||
|
||||
for other_id, other_entry in self._parsers_config.items():
|
||||
if other_id == parser_id:
|
||||
continue
|
||||
if not isinstance(other_entry, dict):
|
||||
logger.warning(f"Skip invalid parser config [{other_id}]: entry must be an object")
|
||||
continue
|
||||
if str(other_entry.get("filename") or "") == normalized["filename"]:
|
||||
raise ValueError(f"解析器模块名已被 {other_id} 占用: {module_name}")
|
||||
|
||||
if target_path.exists():
|
||||
owned_filename = str((existing_entry or {}).get("filename") or "")
|
||||
if not owned_filename or owned_filename != normalized["filename"]:
|
||||
raise ValueError(f"解析器文件已存在,且不受市场安装器管理: {target_path.name}")
|
||||
|
||||
if existing_entry and str(existing_entry.get("filename") or "") != normalized["filename"]:
|
||||
raise ValueError("暂不支持修改已安装解析器的模块名")
|
||||
|
||||
if existing_entry and not overwrite:
|
||||
raise FileExistsError(f"解析器已存在: {parser_id}")
|
||||
|
||||
previous_source: Optional[str] = None
|
||||
if target_path.exists():
|
||||
previous_source = target_path.read_text(encoding="utf-8")
|
||||
|
||||
self._validate_parser_source(
|
||||
source_code=normalized["source_code"],
|
||||
class_name=normalized["class_name"],
|
||||
filename=target_path.name,
|
||||
)
|
||||
|
||||
self.PARSER_DIR.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
entry = {
|
||||
"id": parser_id,
|
||||
"name": normalized["name"],
|
||||
"description": normalized["description"],
|
||||
"module": normalized["module_path"],
|
||||
"class": normalized["class_name"],
|
||||
"module_name": module_name,
|
||||
"filename": normalized["filename"],
|
||||
"enabled": True,
|
||||
"installed_at": datetime.now(timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z"),
|
||||
"supported_patterns": normalized["supported_patterns"],
|
||||
}
|
||||
|
||||
try:
|
||||
self._atomic_write_text(target_path, normalized["source_code"])
|
||||
|
||||
self._parsers_config[parser_id] = entry
|
||||
self._config["parsers"] = self._parsers_config
|
||||
self._config.setdefault("version", "1.0")
|
||||
self._save_config()
|
||||
self._load_parser_entry(parser_id, entry)
|
||||
except Exception:
|
||||
self._rollback_install(
|
||||
parser_id=parser_id,
|
||||
target_path=target_path,
|
||||
previous_entry=previous_entry,
|
||||
previous_source=previous_source,
|
||||
registry_snapshot=registry_snapshot,
|
||||
had_previous_entry=had_previous_entry,
|
||||
)
|
||||
raise
|
||||
|
||||
return {
|
||||
"id": parser_id,
|
||||
"name": normalized["name"],
|
||||
"description": normalized["description"],
|
||||
"class_name": normalized["class_name"],
|
||||
"module_name": module_name,
|
||||
"module": normalized["module_path"],
|
||||
"filename": normalized["filename"],
|
||||
"supported_patterns": copy.deepcopy(normalized["supported_patterns"]),
|
||||
}
|
||||
|
||||
def _rollback_install(
|
||||
self,
|
||||
parser_id: str,
|
||||
target_path: Path,
|
||||
previous_entry: Optional[Any],
|
||||
previous_source: Optional[str],
|
||||
registry_snapshot: Optional[Dict[str, Any]] = None,
|
||||
had_previous_entry: bool = False,
|
||||
) -> None:
|
||||
try:
|
||||
if registry_snapshot is not None:
|
||||
ParserRegistry._parsers = dict(registry_snapshot)
|
||||
if previous_source is None:
|
||||
if target_path.exists():
|
||||
target_path.unlink()
|
||||
else:
|
||||
self._atomic_write_text(target_path, previous_source)
|
||||
if previous_entry is None and not had_previous_entry:
|
||||
self._parsers_config.pop(parser_id, None)
|
||||
else:
|
||||
self._parsers_config[parser_id] = previous_entry
|
||||
self._config["parsers"] = self._parsers_config
|
||||
self._save_config()
|
||||
if isinstance(previous_entry, dict) and previous_entry:
|
||||
self._load_parser_entry(parser_id, previous_entry)
|
||||
except Exception as rollback_exc:
|
||||
logger.error(f"Failed to rollback parser install [{parser_id}]: {rollback_exc}")
|
||||
|
||||
def _normalize_parser_package(self, payload: Dict[str, Any]) -> Dict[str, Any]:
|
||||
if not isinstance(payload, dict):
|
||||
raise ValueError("解析器包必须是对象")
|
||||
|
||||
nested = payload.get("parser_package")
|
||||
if isinstance(nested, dict):
|
||||
return self._normalize_parser_package(nested)
|
||||
|
||||
parser_id = str(payload.get("parser_id") or payload.get("id") or "").strip()
|
||||
class_name = str(payload.get("class_name") or payload.get("class") or "").strip()
|
||||
module_name = str(payload.get("module_name") or payload.get("module") or payload.get("filename") or "").strip()
|
||||
source_code = str(payload.get("source_code") or payload.get("content") or payload.get("code") or "")
|
||||
name = str(payload.get("name") or parser_id or class_name or "").strip()
|
||||
description = str(payload.get("description") or "").strip()
|
||||
supported_patterns = payload.get("supported_patterns") or payload.get("patterns") or []
|
||||
|
||||
if module_name.endswith(".py"):
|
||||
module_name = module_name[:-3]
|
||||
module_name = module_name.replace("-", "_").strip()
|
||||
|
||||
if not parser_id:
|
||||
raise ValueError("缺少 parser_id")
|
||||
if not class_name:
|
||||
raise ValueError("缺少 class_name")
|
||||
if not module_name:
|
||||
raise ValueError("缺少 module_name")
|
||||
if not source_code.strip():
|
||||
raise ValueError("缺少 source_code")
|
||||
if not self._ID_PATTERN.match(parser_id):
|
||||
raise ValueError(f"解析器 ID 不合法: {parser_id}")
|
||||
if not self._NAME_PATTERN.match(class_name):
|
||||
raise ValueError(f"解析器类名不合法: {class_name}")
|
||||
if not self._NAME_PATTERN.match(module_name):
|
||||
raise ValueError(f"解析器模块名不合法: {module_name}")
|
||||
|
||||
normalized_patterns = [
|
||||
str(pattern).strip()
|
||||
for pattern in (supported_patterns if isinstance(supported_patterns, list) else [])
|
||||
if str(pattern).strip()
|
||||
]
|
||||
|
||||
return {
|
||||
"parser_id": parser_id,
|
||||
"class_name": class_name,
|
||||
"module_name": module_name,
|
||||
"module_path": f"app.core.parsers.{module_name}",
|
||||
"filename": f"{module_name}.py",
|
||||
"name": name or parser_id,
|
||||
"description": description,
|
||||
"supported_patterns": normalized_patterns,
|
||||
"source_code": source_code,
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _validate_parser_source(source_code: str, class_name: str, filename: str) -> None:
|
||||
try:
|
||||
tree = ast.parse(source_code, filename=filename)
|
||||
except SyntaxError as exc:
|
||||
raise ValueError(f"解析器源码存在语法错误: {exc}") from exc
|
||||
|
||||
class_names = {node.name for node in tree.body if isinstance(node, ast.ClassDef)}
|
||||
if class_name not in class_names:
|
||||
raise ValueError(f"解析器源码里没有找到类: {class_name}")
|
||||
|
||||
problems = check_import_time_side_effects(tree)
|
||||
if problems:
|
||||
raise ValueError(
|
||||
"解析器源码在导入期会执行代码,已拒绝安装: " + "; ".join(problems[:5])
|
||||
)
|
||||
|
||||
try:
|
||||
compile(source_code, filename, "exec")
|
||||
except SyntaxError as exc:
|
||||
raise ValueError(f"解析器源码编译失败: {exc}") from exc
|
||||
|
||||
|
||||
parser_manager = ParserConfigManager()
|
||||
|
||||
|
||||
__all__ = ["ParserConfigManager", "parser_manager"]
|
||||
@@ -0,0 +1,559 @@
|
||||
"""
|
||||
Helpers for request execution lifetime, worker cleanup, and tab retirement.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import anyio
|
||||
from contextlib import contextmanager
|
||||
import os
|
||||
import queue
|
||||
import threading
|
||||
import time
|
||||
from typing import Any, Callable, Dict, Optional
|
||||
|
||||
from app.core.config import BrowserConstants, _request_context, get_logger
|
||||
from app.services.request_manager import RequestContext, RequestStatus, request_manager
|
||||
|
||||
|
||||
logger = get_logger("REQUEST.LIFECYCLE")
|
||||
|
||||
DEFAULT_MAX_REQUEST_EXECUTE_TIME_SEC = 300.0
|
||||
MIN_MAX_REQUEST_EXECUTE_TIME_SEC = 5.0
|
||||
WORKER_QUEUE_FINAL_GRACE_SEC = 5.0
|
||||
WORKER_QUEUE_MAX_PUT_BLOCK_SEC = 0.1
|
||||
WORKER_QUEUE_CANCEL_PUT_BLOCK_SEC = 0.01
|
||||
WORKER_QUEUE_WAIT_BACKOFF_FACTOR = 1.8
|
||||
COMPLETED_WORKER_FAST_JOIN_SEC = 0.5
|
||||
COMPLETED_WORKER_RETIRE_GRACE_SEC = 30.0
|
||||
|
||||
|
||||
@contextmanager
|
||||
def request_finalization(ctx: RequestContext, context_token=None):
|
||||
"""Shield only bounded cleanup, never browser execution, from ASGI cancellation.
|
||||
|
||||
Synchronous finalization is guaranteed even if cleanup raises or a second raw
|
||||
asyncio Task.cancel() interrupts it. A request is not retried here.
|
||||
"""
|
||||
try:
|
||||
with anyio.CancelScope(shield=True):
|
||||
yield
|
||||
finally:
|
||||
try:
|
||||
with ctx._lock:
|
||||
already_finalized = getattr(ctx, '_lifecycle_finalized', False)
|
||||
ctx._lifecycle_finalized = True
|
||||
if not already_finalized:
|
||||
request_manager.finish_request(ctx, success=(ctx.status == RequestStatus.COMPLETED))
|
||||
finally:
|
||||
if context_token is not None:
|
||||
try:
|
||||
_request_context.reset(context_token)
|
||||
except (ValueError, RuntimeError):
|
||||
# Explicit close usually runs in the original response context;
|
||||
# generator finalization during shutdown may use another one.
|
||||
logger.debug("Request context already released or finalized in another context")
|
||||
|
||||
|
||||
class TrackedWorkerExecutionCancelled(Exception):
|
||||
"""Raised when a tracked blocking worker keeps running after cancellation."""
|
||||
|
||||
|
||||
def _coerce_max_request_execute_time_sec(raw: Any, default: float, source: str) -> float:
|
||||
try:
|
||||
value = float(raw)
|
||||
except (TypeError, ValueError, OverflowError):
|
||||
logger.debug(f"Invalid {source} MAX_REQUEST_EXECUTE_TIME_SEC={raw!r}, using default {default}s")
|
||||
return max(MIN_MAX_REQUEST_EXECUTE_TIME_SEC, float(default))
|
||||
|
||||
if value <= 0:
|
||||
return 0.0
|
||||
return max(MIN_MAX_REQUEST_EXECUTE_TIME_SEC, value)
|
||||
|
||||
|
||||
def get_max_request_execute_time_sec(default: float = DEFAULT_MAX_REQUEST_EXECUTE_TIME_SEC) -> float:
|
||||
"""Return the request hard timeout; set MAX_REQUEST_EXECUTE_TIME_SEC<=0 to disable."""
|
||||
env_raw = os.getenv("MAX_REQUEST_EXECUTE_TIME_SEC")
|
||||
if env_raw is not None and str(env_raw).strip() != "":
|
||||
return _coerce_max_request_execute_time_sec(env_raw, default, "env")
|
||||
|
||||
try:
|
||||
config_raw = BrowserConstants.get("MAX_REQUEST_EXECUTE_TIME_SEC")
|
||||
except Exception as e:
|
||||
logger.debug(f"Read browser config MAX_REQUEST_EXECUTE_TIME_SEC failed: {e}")
|
||||
config_raw = default
|
||||
|
||||
if config_raw is None or str(config_raw).strip() == "":
|
||||
config_raw = default
|
||||
return _coerce_max_request_execute_time_sec(config_raw, default, "browser_config")
|
||||
|
||||
|
||||
def mark_request_hard_timeout(
|
||||
ctx: RequestContext,
|
||||
started_at: float,
|
||||
max_execute_time_sec: Optional[float],
|
||||
*,
|
||||
label: str = "",
|
||||
) -> bool:
|
||||
"""Cancel ctx when an application-level hard timeout is reached."""
|
||||
try:
|
||||
max_seconds = (
|
||||
get_max_request_execute_time_sec()
|
||||
if max_execute_time_sec is None
|
||||
else float(max_execute_time_sec)
|
||||
)
|
||||
except Exception:
|
||||
max_seconds = DEFAULT_MAX_REQUEST_EXECUTE_TIME_SEC
|
||||
|
||||
if max_seconds <= 0:
|
||||
return False
|
||||
ctx_started_at = getattr(ctx, "started_at_monotonic", None)
|
||||
if ctx_started_at is None:
|
||||
ctx_started_at = float(started_at or time.monotonic())
|
||||
elapsed = time.monotonic() - ctx_started_at
|
||||
if elapsed < max_seconds:
|
||||
return False
|
||||
|
||||
if not ctx.should_stop():
|
||||
logger.warning(
|
||||
f"[{ctx.request_id}] request hard timeout reached "
|
||||
f"(elapsed={elapsed:.1f}s, limit={max_seconds:.1f}s, {label or '-'})"
|
||||
)
|
||||
ctx.request_cancel("absolute_request_timeout")
|
||||
return True
|
||||
|
||||
|
||||
def put_worker_queue_item(
|
||||
chunk_queue: queue.Queue,
|
||||
ctx: RequestContext,
|
||||
item: Any,
|
||||
*,
|
||||
final: bool = False,
|
||||
poll_timeout: float = 0.5,
|
||||
) -> bool:
|
||||
"""Put worker output with bounded backpressure and cancellation awareness."""
|
||||
deadline = time.monotonic() + WORKER_QUEUE_FINAL_GRACE_SEC if final else None
|
||||
|
||||
while final or not ctx.should_stop():
|
||||
try:
|
||||
chunk_queue.put(
|
||||
item,
|
||||
timeout=_coerce_worker_queue_put_timeout(
|
||||
poll_timeout,
|
||||
cancelled=ctx.should_stop(),
|
||||
),
|
||||
)
|
||||
return True
|
||||
except queue.Full:
|
||||
if final:
|
||||
if ctx.should_stop() or (deadline is not None and time.monotonic() >= deadline):
|
||||
return False
|
||||
continue
|
||||
if ctx.should_stop():
|
||||
return False
|
||||
|
||||
return False
|
||||
|
||||
|
||||
def _coerce_worker_queue_put_timeout(raw_timeout: Any, *, cancelled: bool = False) -> float:
|
||||
try:
|
||||
value = float(raw_timeout)
|
||||
except (TypeError, ValueError, OverflowError):
|
||||
value = 0.5
|
||||
if cancelled:
|
||||
return WORKER_QUEUE_CANCEL_PUT_BLOCK_SEC
|
||||
return max(
|
||||
WORKER_QUEUE_CANCEL_PUT_BLOCK_SEC,
|
||||
min(value if value > 0 else 0.5, WORKER_QUEUE_MAX_PUT_BLOCK_SEC),
|
||||
)
|
||||
|
||||
|
||||
async def wait_worker_queue_item(
|
||||
chunk_queue: queue.Queue,
|
||||
*,
|
||||
timeout: float = 0.5,
|
||||
poll_interval: float = 0.01,
|
||||
max_poll_interval: float = 0.05,
|
||||
) -> Any:
|
||||
"""Wait for a worker queue item without occupying the default executor."""
|
||||
try:
|
||||
timeout_value = float(timeout)
|
||||
except (TypeError, ValueError, OverflowError):
|
||||
timeout_value = 0.5
|
||||
timeout_value = max(0.0, timeout_value)
|
||||
|
||||
try:
|
||||
interval = float(poll_interval)
|
||||
except (TypeError, ValueError, OverflowError):
|
||||
interval = 0.01
|
||||
interval = max(0.001, min(interval, 0.05))
|
||||
|
||||
try:
|
||||
max_interval = float(max_poll_interval)
|
||||
except (TypeError, ValueError, OverflowError):
|
||||
max_interval = 0.05
|
||||
max_interval = max(interval, min(max_interval, 0.05))
|
||||
|
||||
deadline = time.monotonic() + timeout_value
|
||||
current_interval = min(interval, timeout_value) if timeout_value > 0 else interval
|
||||
while True:
|
||||
try:
|
||||
return chunk_queue.get_nowait()
|
||||
except queue.Empty:
|
||||
remaining = deadline - time.monotonic()
|
||||
if remaining <= 0:
|
||||
raise
|
||||
sleep_for = min(current_interval, remaining)
|
||||
await asyncio.sleep(sleep_for)
|
||||
current_interval = _next_worker_queue_wait_interval(
|
||||
current_interval,
|
||||
max_interval,
|
||||
remaining,
|
||||
)
|
||||
|
||||
|
||||
def _next_worker_queue_wait_interval(current: float, max_interval: float, remaining: float) -> float:
|
||||
try:
|
||||
current_value = float(current)
|
||||
except (TypeError, ValueError, OverflowError):
|
||||
current_value = 0.001
|
||||
try:
|
||||
max_value = float(max_interval)
|
||||
except (TypeError, ValueError, OverflowError):
|
||||
max_value = 0.05
|
||||
try:
|
||||
remaining_value = float(remaining)
|
||||
except (TypeError, ValueError, OverflowError):
|
||||
remaining_value = max_value
|
||||
|
||||
max_value = max(0.001, min(max_value, 0.05))
|
||||
current_value = max(0.001, current_value)
|
||||
remaining_value = max(0.0, remaining_value)
|
||||
|
||||
next_value = current_value * WORKER_QUEUE_WAIT_BACKOFF_FACTOR
|
||||
return max(0.001, min(next_value, max_value, remaining_value or max_value))
|
||||
|
||||
|
||||
def _set_worker_future_result(
|
||||
future: "asyncio.Future[Any]",
|
||||
*,
|
||||
result: Any = None,
|
||||
error: Optional[BaseException] = None,
|
||||
) -> None:
|
||||
if future.done():
|
||||
return
|
||||
if error is not None:
|
||||
future.set_exception(error)
|
||||
return
|
||||
future.set_result(result)
|
||||
|
||||
|
||||
def _consume_worker_future_exception(future: "asyncio.Future[Any]") -> None:
|
||||
"""Retrieve late worker failures after the awaiting request has been cancelled."""
|
||||
if future.cancelled():
|
||||
return
|
||||
try:
|
||||
future.exception()
|
||||
except (asyncio.CancelledError, Exception):
|
||||
pass
|
||||
|
||||
|
||||
def _get_cancel_reason(ctx: RequestContext, fallback: str = "worker_cancelled") -> str:
|
||||
reason = str(getattr(ctx, "cancel_reason", "") or "").strip()
|
||||
return reason or fallback
|
||||
|
||||
|
||||
def run_in_request_context(ctx: RequestContext, worker_fn: Callable[[], Any]) -> Any:
|
||||
"""Run a callable with the request id attached to all logs it emits."""
|
||||
token = _request_context.set(str(getattr(ctx, "request_id", "") or "SYSTEM"))
|
||||
try:
|
||||
return worker_fn()
|
||||
finally:
|
||||
_request_context.reset(token)
|
||||
|
||||
|
||||
async def run_tracked_blocking_call(
|
||||
worker_fn: Callable[[], Any],
|
||||
*,
|
||||
ctx: RequestContext,
|
||||
worker_state: Dict[str, Any],
|
||||
label: str,
|
||||
poll_timeout: float = 0.5,
|
||||
max_execute_time_sec: Optional[float] = None,
|
||||
) -> Any:
|
||||
"""Run a blocking callable on a daemon thread and keep it visible for cleanup."""
|
||||
loop = asyncio.get_running_loop()
|
||||
result_future: "asyncio.Future[Any]" = loop.create_future()
|
||||
started_at = time.monotonic()
|
||||
|
||||
def worker() -> None:
|
||||
try:
|
||||
result = run_in_request_context(ctx, worker_fn)
|
||||
except Exception as exc:
|
||||
try:
|
||||
loop.call_soon_threadsafe(
|
||||
lambda exc=exc: _set_worker_future_result(result_future, error=exc)
|
||||
)
|
||||
except RuntimeError:
|
||||
pass
|
||||
else:
|
||||
try:
|
||||
loop.call_soon_threadsafe(
|
||||
lambda result=result: _set_worker_future_result(result_future, result=result)
|
||||
)
|
||||
except RuntimeError:
|
||||
pass
|
||||
|
||||
worker_thread = threading.Thread(
|
||||
target=worker,
|
||||
daemon=True,
|
||||
name=f"tracked-worker-{label}",
|
||||
)
|
||||
worker_state["thread"] = worker_thread
|
||||
worker_state["label"] = label
|
||||
ctx.worker_thread = worker_thread
|
||||
|
||||
def _clear_finished_worker_state() -> None:
|
||||
if worker_state.get("thread") is worker_thread:
|
||||
worker_state["thread"] = None
|
||||
worker_state["label"] = None
|
||||
|
||||
def _handle_finished_worker(future: "asyncio.Future[Any]") -> None:
|
||||
_clear_finished_worker_state()
|
||||
_consume_worker_future_exception(future)
|
||||
|
||||
result_future.add_done_callback(_handle_finished_worker)
|
||||
worker_thread.start()
|
||||
|
||||
try:
|
||||
while True:
|
||||
if mark_request_hard_timeout(
|
||||
ctx,
|
||||
started_at,
|
||||
max_execute_time_sec,
|
||||
label=f"worker={label}",
|
||||
):
|
||||
raise TrackedWorkerExecutionCancelled("absolute_request_timeout")
|
||||
if ctx.should_stop():
|
||||
raise TrackedWorkerExecutionCancelled(_get_cancel_reason(ctx))
|
||||
try:
|
||||
return await asyncio.wait_for(
|
||||
asyncio.shield(result_future),
|
||||
timeout=max(0.05, float(poll_timeout or 0.5)),
|
||||
)
|
||||
except asyncio.TimeoutError:
|
||||
continue
|
||||
finally:
|
||||
if result_future.done():
|
||||
_clear_finished_worker_state()
|
||||
|
||||
|
||||
def _resolve_raw_tab_id(pool: Any, session: Any) -> str:
|
||||
raw_tab_id = ""
|
||||
if pool is not None:
|
||||
try:
|
||||
persistent_index = int(getattr(session, "persistent_index", 0) or 0)
|
||||
for candidate_raw_id, candidate_index in getattr(pool, "_raw_id_to_persistent", {}).items():
|
||||
if int(candidate_index or 0) == persistent_index:
|
||||
raw_tab_id = str(candidate_raw_id or "").strip()
|
||||
break
|
||||
except Exception:
|
||||
raw_tab_id = ""
|
||||
if raw_tab_id:
|
||||
return raw_tab_id
|
||||
|
||||
try:
|
||||
return str(getattr(getattr(session, "tab", None), "tab_id", "") or "").strip()
|
||||
except Exception:
|
||||
return ""
|
||||
|
||||
|
||||
def retire_bound_tab_after_worker_leak(ctx: RequestContext, reason: str) -> None:
|
||||
"""Mark the tab bound to ctx as unhealthy without closing its browser target."""
|
||||
tab_id = str(getattr(ctx, "tab_id", "") or "").strip()
|
||||
if not tab_id:
|
||||
return
|
||||
|
||||
try:
|
||||
from app.core import get_browser
|
||||
|
||||
browser = get_browser(auto_connect=False)
|
||||
pool = getattr(browser, "_tab_pool", None)
|
||||
session = getattr(pool, "_tabs", {}).get(tab_id) if pool is not None else None
|
||||
if session is None:
|
||||
return
|
||||
|
||||
current_task = str(getattr(session, "current_task_id", "") or "").strip()
|
||||
bound_request_id = str(getattr(session, "_bound_request_id", "") or "").strip()
|
||||
if current_task and current_task != ctx.request_id:
|
||||
logger.warning(
|
||||
f"[{tab_id}] leaked worker found but tab has another task; skip retire "
|
||||
f"(request={ctx.request_id}, current_task={current_task})"
|
||||
)
|
||||
return
|
||||
if bound_request_id and bound_request_id != ctx.request_id:
|
||||
logger.warning(
|
||||
f"[{tab_id}] leaked worker found but bound request changed; skip retire "
|
||||
f"(request={ctx.request_id}, bound={bound_request_id})"
|
||||
)
|
||||
return
|
||||
|
||||
if hasattr(session, "mark_error"):
|
||||
session.mark_error(reason)
|
||||
|
||||
raw_tab_id = _resolve_raw_tab_id(pool, session)
|
||||
|
||||
if pool is not None:
|
||||
try:
|
||||
monitor = getattr(pool, "_global_network_monitor", None)
|
||||
if monitor is not None and hasattr(monitor, "request_stop_for_session"):
|
||||
monitor.request_stop_for_session(tab_id, reason=reason, detach=True)
|
||||
except Exception as e:
|
||||
logger.debug(f"[{tab_id}] stop leaked tab global monitor failed: {e}")
|
||||
|
||||
if pool is not None and hasattr(pool, "_condition"):
|
||||
try:
|
||||
with pool._condition:
|
||||
pool._condition.notify_all()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
logger.warning(
|
||||
f"[{tab_id}] worker did not exit in time; marked tab ERROR "
|
||||
f"(request={ctx.request_id}, reason={reason}, raw={raw_tab_id or '-'}, "
|
||||
"browser_tab_retained=True)"
|
||||
)
|
||||
except Exception as e:
|
||||
logger.debug(f"retire leaked worker tab failed: {e}")
|
||||
|
||||
|
||||
def cleanup_worker_thread(
|
||||
worker_thread: Optional[threading.Thread],
|
||||
ctx: RequestContext,
|
||||
*,
|
||||
cancel_reason: str = "cleanup",
|
||||
join_timeout: float = 5.0,
|
||||
retire_reason: str = "worker_cleanup_timeout",
|
||||
) -> bool:
|
||||
"""Cancel, join, and retire a worker thread that outlives its request."""
|
||||
if not isinstance(worker_thread, threading.Thread) or not worker_thread.is_alive():
|
||||
return False
|
||||
|
||||
ctx.request_cancel(cancel_reason)
|
||||
worker_thread.join(timeout=max(0.0, float(join_timeout or 0.0)))
|
||||
if worker_thread.is_alive():
|
||||
retire_bound_tab_after_worker_leak(ctx, retire_reason)
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def schedule_completed_worker_retire_check(
|
||||
worker_thread: threading.Thread,
|
||||
ctx: RequestContext,
|
||||
reason: str,
|
||||
*,
|
||||
delay_sec: float = COMPLETED_WORKER_RETIRE_GRACE_SEC,
|
||||
) -> None:
|
||||
"""Watch a completed request's slow worker cleanup without blocking response finalization."""
|
||||
try:
|
||||
delay = max(0.1, float(delay_sec or COMPLETED_WORKER_RETIRE_GRACE_SEC))
|
||||
except Exception:
|
||||
delay = COMPLETED_WORKER_RETIRE_GRACE_SEC
|
||||
|
||||
def _wait_and_retire_if_still_alive() -> None:
|
||||
started_at = time.monotonic()
|
||||
worker_thread.join(timeout=delay)
|
||||
elapsed = time.monotonic() - started_at
|
||||
if worker_thread.is_alive():
|
||||
logger.warning(
|
||||
f"[{ctx.request_id}] completed request worker still alive after cleanup grace; "
|
||||
f"retiring bound tab (tab={ctx.tab_id or '-'}, reason={reason}, "
|
||||
f"grace={delay:.1f}s)"
|
||||
)
|
||||
retire_bound_tab_after_worker_leak(ctx, reason)
|
||||
return
|
||||
logger.debug(
|
||||
f"[{ctx.request_id}] completed request worker cleanup finished in background "
|
||||
f"(elapsed={elapsed:.2f}s, tab={ctx.tab_id or '-'})"
|
||||
)
|
||||
|
||||
threading.Thread(
|
||||
target=_wait_and_retire_if_still_alive,
|
||||
daemon=True,
|
||||
name=f"completed-worker-cleanup-{ctx.request_id[:8]}",
|
||||
).start()
|
||||
|
||||
|
||||
async def _wait_for_worker_exit(worker_thread: threading.Thread, timeout: float) -> None:
|
||||
"""Bound cleanup wall time without queueing joins in the shared executor.
|
||||
|
||||
A saturated default executor must not postpone cleanup or starve the next
|
||||
request's route discovery. is_alive() is nonblocking; cancellation is still
|
||||
allowed here, and callers shield only this bounded cleanup window.
|
||||
"""
|
||||
deadline = time.monotonic() + max(0.0, timeout)
|
||||
while worker_thread.is_alive():
|
||||
remaining = deadline - time.monotonic()
|
||||
if remaining <= 0:
|
||||
break
|
||||
await asyncio.sleep(min(.02, remaining))
|
||||
|
||||
|
||||
async def cleanup_worker_thread_after_request(
|
||||
worker_thread: Optional[threading.Thread],
|
||||
ctx: RequestContext,
|
||||
*,
|
||||
completed: bool = False,
|
||||
cancel_reason: str = "cleanup",
|
||||
join_timeout: float = 5.0,
|
||||
retire_reason: str = "worker_cleanup_timeout",
|
||||
completed_join_timeout: float = COMPLETED_WORKER_FAST_JOIN_SEC,
|
||||
completed_retire_delay: float = COMPLETED_WORKER_RETIRE_GRACE_SEC,
|
||||
) -> bool:
|
||||
"""Cleanup a request worker, deferring leak retirement for already-completed requests.
|
||||
|
||||
Returns True when the worker was still alive after the immediate cleanup window.
|
||||
"""
|
||||
if not isinstance(worker_thread, threading.Thread) or not worker_thread.is_alive():
|
||||
return False
|
||||
|
||||
if completed:
|
||||
try:
|
||||
fast_join = max(0.0, float(completed_join_timeout or 0.0))
|
||||
except Exception:
|
||||
fast_join = COMPLETED_WORKER_FAST_JOIN_SEC
|
||||
try:
|
||||
await _wait_for_worker_exit(worker_thread, fast_join)
|
||||
except asyncio.CancelledError:
|
||||
if worker_thread.is_alive():
|
||||
schedule_completed_worker_retire_check(worker_thread, ctx, retire_reason, delay_sec=completed_retire_delay)
|
||||
raise
|
||||
if worker_thread.is_alive():
|
||||
logger.warning(
|
||||
f"[{ctx.request_id}] completed request worker cleanup is still running; "
|
||||
f"defer tab retirement check (tab={ctx.tab_id or '-'}, "
|
||||
f"fast_join={fast_join:.1f}s, grace={completed_retire_delay:.1f}s)"
|
||||
)
|
||||
schedule_completed_worker_retire_check(
|
||||
worker_thread,
|
||||
ctx,
|
||||
retire_reason,
|
||||
delay_sec=completed_retire_delay,
|
||||
)
|
||||
return True
|
||||
return False
|
||||
|
||||
ctx.request_cancel(cancel_reason)
|
||||
try:
|
||||
timeout = max(0.0, float(join_timeout or 0.0))
|
||||
except Exception:
|
||||
timeout = 5.0
|
||||
try:
|
||||
await _wait_for_worker_exit(worker_thread, timeout)
|
||||
except asyncio.CancelledError:
|
||||
if worker_thread.is_alive():
|
||||
schedule_completed_worker_retire_check(worker_thread, ctx, retire_reason, delay_sec=timeout)
|
||||
raise
|
||||
if worker_thread.is_alive():
|
||||
retire_bound_tab_after_worker_leak(ctx, retire_reason)
|
||||
return True
|
||||
return False
|
||||
File diff suppressed because it is too large.
Load diff
Loaded 100 of 7311 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user