Initial commit

This commit is contained in:
Alex committed 2026-11-02 22:54:14 +08:00
commit 57fc3525eb
7311 files changed
+1291398

No files matched your search

+22
View File
@@ -0,0 +1,22 @@
"""
app/services - 业务服务层
职责:
- 请求管理
- 配置引擎
"""
from .request_manager import request_manager, RequestManager, RequestContext, RequestStatus
from .config_engine import config_engine, ConfigEngine
__all__ = [
# 请求管理
'request_manager',
'RequestManager',
'RequestContext',
'RequestStatus',
# 配置引擎
'config_engine',
'ConfigEngine',
]
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
+147
View File
@@ -0,0 +1,147 @@
"""R1-6:站点适配器健康巡检——只查询页面 DOM,绝不输入、点击或发送消息。
对站点当前预设里配置的每个选择器,在该站点已打开的空闲标签页上统计命中数量,给出:
- ``broken``:输入框找不到(常见原因:未登录、页面改版),或存在语法无效的选择器;
- ``degraded``:其他必需元素找不到。发送按钮通常要输入内容后才出现,回复容器要有对话后才出现,
所以这里只作提示,不算坏;
- ``healthy``:输入框能找到,且没有无效选择器;
- ``unreachable``:浏览器里没有该站点的空闲标签页(巡检不会自行打开页面)。
``evaluate_preset`` 是与浏览器无关的纯函数(便于单测);``DrissionProbe`` 把 DrissionPage 标签页适配成探针。
"""
from __future__ import annotations
import time
from typing import Any, Dict, Iterable, List, Optional, Protocol
from app.core.config import get_logger
from app.core.driver.locators import to_locator as _driver_to_locator
from app.services.selector_quality import fallback_suggestions, lint_selector
logger = get_logger("ADAPTER_HEALTH")
ALWAYS_PRESENT_KEYS = ("input_box",)
_HINTS = {
"send_btn": "发送按钮通常在输入内容后才出现或可点击",
"result_container": "回复容器通常在已有对话时才出现",
}
class Probe(Protocol):
url: str
def count(self, selector: str) -> int:
"""返回命中数量;选择器语法无效时返回 -1。"""
# R2-1:定位规则统一到驱动层,这里保留旧名以兼容已有调用与测试
to_locator = _driver_to_locator
class DrissionProbe:
"""把标签页适配成探针。R2-1 起经由统一驱动查询元素(参数既可以是 DrissionPage 标签页,也可以是驱动对象)。"""
def __init__(self, tab: Any, url: str = "", timeout: float = 0.3):
from app.core.driver import driver_for_tab
self.driver = driver_for_tab(tab)
self.tab = self.driver.raw
self.url = url or self.driver.url
self.timeout = timeout
def count(self, selector: str) -> int:
from app.core.driver import DriverError
try:
# 配置里的选择器默认是 CSS:先规范化,再交给驱动(驱动按 DrissionPage 原生语法原样传递)
return len(self.driver.find_all(to_locator(selector), timeout=self.timeout))
except DriverError as exc:
logger.debug(f"选择器无效或查询失败: {selector!r}: {exc}")
return -1
def evaluate_preset(probe: Probe, preset: Dict[str, Any], definitions: Iterable[Dict[str, Any]]) -> Dict[str, Any]:
required = {str(d.get("key")) for d in definitions or [] if isinstance(d, dict) and d.get("required")}
rows: List[Dict[str, Any]] = []
for key, selector in (preset.get("selectors") or {}).items():
if not isinstance(selector, str) or not selector.strip():
continue
count = probe.count(selector)
status = "invalid" if count < 0 else ("missing" if count == 0 else "ok")
row = {"key": key, "selector": selector, "count": max(count, 0), "status": status, "required": key in required,
"quality": lint_selector(selector)}
if status == "missing" and key in _HINTS:
row["hint"] = _HINTS[key]
if status != "ok" and key in required:
# R1-7:运行时找不到配置的元素时会尝试这些回退选择器;命中的可作为修复起点
row["suggestions"] = fallback_suggestions(key, probe)
rows.append(row)
by_key = {row["key"]: row for row in rows}
problems = []
if any(row["status"] == "invalid" for row in rows):
problems.append("存在语法无效的选择器")
for key in ALWAYS_PRESENT_KEYS:
if key not in by_key:
problems.append(f"预设没有配置 {key}")
elif by_key[key]["status"] != "ok":
problems.append(f"找不到 {key}(可能未登录或页面已改版)")
if problems:
overall = "broken"
elif any(row["required"] and row["status"] != "ok" for row in rows):
overall = "degraded"
else:
overall = "healthy"
return {"status": overall, "url": probe.url, "problems": problems, "selectors": rows}
def check_site_health(domain: str, preset_name: Optional[str] = None, *, timeout: float = 0.3) -> Dict[str, Any]:
"""在浏览器里该站点的空闲标签页上巡检(与 /api/debug/test-selector 相同的借用/归还方式)。"""
from app.core.browser import get_browser
from app.services.config_engine import config_engine
from app.utils.site_url import extract_remote_site_domain, route_domain_matches
site = config_engine.sites.get(domain)
if not isinstance(site, dict):
raise KeyError(f"未知站点: {domain}")
presets = site.get("presets") or {}
name = preset_name or site.get("default_preset") or next(iter(presets), "")
if name not in presets:
raise KeyError(f"站点 {domain} 没有预设 {name!r}")
definitions = config_engine.global_config.to_dict().get("selector_definitions") or []
browser = get_browser()
pool = browser.tab_pool
pool.refresh_tabs()
reports, busy = [], 0
for session in pool.get_sessions_snapshot():
cached = str(getattr(session, "current_domain", "") or "")
if cached and not route_domain_matches(domain, cached):
continue
if str(getattr(getattr(session, "status", None), "value", "")) != "idle":
busy += 1 if cached else 0
continue
raw_tab_id = str(getattr(getattr(session, "tab", None), "tab_id", "") or "")
if not raw_tab_id:
continue
task_id = f"adapter_health_{time.time_ns()}"
acquired = pool.acquire_by_raw_tab_id(raw_tab_id, task_id, timeout=1.0, count_request=False, activate=False)
if acquired is None:
continue
try:
url = str(getattr(acquired.tab, "url", "") or "")
if not route_domain_matches(domain, extract_remote_site_domain(url) or ""):
continue
reports.append(evaluate_preset(DrissionProbe(acquired.tab, url, timeout), presets[name], definitions))
finally:
pool.release(acquired.id, check_triggers=False, expected_task_id=task_id)
if not reports:
return {"site": domain, "preset": name, "status": "unreachable", "busy_tabs": busy,
"problems": [f"浏览器里没有 {domain} 的空闲标签页;巡检不会自行打开页面"], "tabs": []}
order = {"healthy": 0, "degraded": 1, "broken": 2}
best = min(reports, key=lambda report: order[report["status"]])
return {"site": domain, "preset": name, "status": best["status"], "busy_tabs": busy,
"problems": best["problems"], "tabs": reports}
+231
View File
@@ -0,0 +1,231 @@
"""R1-3:站点适配器在线更新(不发整包也能热更新站点配置)。
依据三份信息逐站点比较:
- 官方 ``config/sites/index.json``:官方当前版本的规范化摘要(config_sha256)、文件摘要、版本号;
- 本地 ``config/sites/index.json``:本地**安装时**附带的清单,代表“用户拿到的原始版本”;
- 本地站点文件的当前内容:与安装清单不同,说明用户改过。
状态:
- ``up_to_date``:本地内容与官方一致;
- ``update_available``:官方有变化,且用户没改过本地文件,可以安全自动应用;
- ``conflict``:官方有变化,且用户也改过(或来源未知),需要显式确认,按“本地优先”合并;
- ``new_site``:官方新增、本地没有;
- ``local_only``:只在本地存在;
- ``requires_newer_app``:官方版本要求更高的 min_app_version。
应用更新时逐个下载并按 ``file_sha256`` 校验、Schema 校验,写入后更新本地安装清单中对应条目,最后热重载。
"""
from __future__ import annotations
import hashlib
import json
import os
import re
from pathlib import Path
from typing import Any, Callable, Dict, Iterable, List, Optional
from app.core.config import get_logger
from app.services.config.site_schema import site_file_errors
from app.services.config.site_store import SiteStore, canonical_config_text
logger = get_logger("ADAPTER_UPD")
INDEX_NAME = "index.json"
SAFE_STATUSES = ("update_available", "new_site")
_FILE_PATTERN = re.compile(r"^[A-Za-z0-9._%~-]+\.json$")
_SHA_PATTERN = re.compile(r"^[0-9a-f]{64}$")
FetchBytes = Callable[[str], bytes]
def config_sha256(config: Any) -> str:
return hashlib.sha256(canonical_config_text(config).encode("utf-8")).hexdigest()
def _version_tuple(value: str) -> tuple:
return tuple(int(part) for part in re.findall(r"\d+", str(value or ""))[:4]) or (0,)
def _atomic_write(path: Path, text: str) -> None:
tmp = path.with_name(path.name + ".tmp")
with open(tmp, "w", encoding="utf-8", newline="\n") as handle:
handle.write(text)
handle.flush()
os.fsync(handle.fileno())
os.replace(tmp, path)
class AdapterUpdater:
def __init__(
self,
sites_dir: os.PathLike | str,
fetch_bytes: FetchBytes,
*,
app_version: str,
reload: Optional[Callable[[], None]] = None,
lock: Any = None,
):
"""``fetch_bytes(relative_path)`` 返回官方仓库中该路径的原始字节(由调用方负责安全抓取与大小限制)。"""
self.sites_dir = Path(sites_dir)
self.fetch_bytes = fetch_bytes
self.app_version = str(app_version or "0")
self.reload = reload
self.lock = lock
# ------------------------------------------------------------------ 读取
def _local_index(self) -> Dict[str, Any]:
try:
payload = json.loads((self.sites_dir / INDEX_NAME).read_text(encoding="utf-8-sig"))
return payload if isinstance(payload, dict) else {}
except (OSError, ValueError):
return {}
def _local_envelopes(self) -> Dict[str, Dict[str, Any]]:
store = SiteStore(self.sites_dir, self.sites_dir / "__no_legacy__.json")
result = {}
for path in store.site_files():
try:
payload = store.read_envelope(path)
except (OSError, ValueError):
continue
result[payload["site"]] = {"path": path, "payload": payload}
return result
def _official_index(self) -> Dict[str, Any]:
raw = self.fetch_bytes(f"config/sites/{INDEX_NAME}")
index = json.loads(raw.decode("utf-8-sig"))
entries = index.get("sites") if isinstance(index, dict) else None
if not isinstance(entries, dict):
raise ValueError("官方 index.json 缺少 sites 清单")
for site, entry in entries.items():
if not isinstance(entry, dict) or not _FILE_PATTERN.fullmatch(str(entry.get("file") or "")) \
or not _SHA_PATTERN.fullmatch(str(entry.get("file_sha256") or "")) \
or not _SHA_PATTERN.fullmatch(str(entry.get("config_sha256") or "")):
raise ValueError(f"官方 index.json 中站点 {site} 的条目无效")
return index
# ------------------------------------------------------------------ 检查
def check(self) -> Dict[str, Any]:
official = self._official_index()["sites"]
installed = (self._local_index().get("sites") or {})
local = self._local_envelopes()
rows: List[Dict[str, Any]] = []
for site in sorted(set(official) | set(local)):
official_entry = official.get(site)
local_item = local.get(site)
row = {
"site": site,
"local_version": (local_item or {}).get("payload", {}).get("adapter_version", ""),
"official_version": (official_entry or {}).get("adapter_version", ""),
"min_app_version": (official_entry or {}).get("min_app_version", ""),
}
if official_entry is None:
row["status"] = "local_only"
else:
too_new = _version_tuple(official_entry.get("min_app_version")) > _version_tuple(self.app_version)
if local_item is None:
row["status"] = "requires_newer_app" if too_new else "new_site"
else:
local_hash = config_sha256(local_item["payload"]["config"])
installed_hash = (installed.get(site) or {}).get("config_sha256")
if local_hash == official_entry["config_sha256"]:
row["status"] = "up_to_date"
elif too_new:
row["status"] = "requires_newer_app"
elif installed_hash and local_hash == installed_hash:
row["status"] = "update_available"
else:
row["status"] = "conflict"
row["locally_modified"] = not (installed_hash and local_hash == installed_hash)
rows.append(row)
summary: Dict[str, int] = {}
for row in rows:
summary[row["status"]] = summary.get(row["status"], 0) + 1
return {"app_version": self.app_version, "sites": rows, "summary": summary}
# ------------------------------------------------------------------ 应用
def apply(self, sites: Optional[Iterable[str]] = None, *, include_conflicts: bool = False) -> Dict[str, Any]:
"""应用更新。``sites`` 为空时应用全部安全更新(update_available / new_site)。"""
from updater import merge_site_records # 与更新器、迁移使用同一合并语义
official_index = self._official_index()
report = self.check()
status_by_site = {row["site"]: row["status"] for row in report["sites"]}
allowed = set(SAFE_STATUSES) | ({"conflict"} if include_conflicts else set())
targets = list(sites) if sites else [s for s, st in status_by_site.items() if st in SAFE_STATUSES]
applied, skipped = [], []
pending_writes: List[tuple] = []
local = self._local_envelopes()
for site in targets:
status = status_by_site.get(site)
if status not in allowed:
skipped.append({"site": site, "status": status or "unknown"})
continue
entry = official_index["sites"][site]
raw = self.fetch_bytes(f"config/sites/{entry['file']}")
if hashlib.sha256(raw).hexdigest() != entry["file_sha256"]:
raise ValueError(f"官方站点文件 {entry['file']} 的 sha256 与 index.json 不一致,已中止")
payload = json.loads(raw.decode("utf-8-sig"))
errors = site_file_errors(payload)
if errors or payload.get("site") != site:
raise ValueError(f"官方站点文件 {entry['file']} 未通过校验: {errors[:3] or 'site 字段不符'}")
if status == "conflict":
merged = dict(payload)
merged["config"] = merge_site_records(local[site]["payload"]["config"], payload["config"])
text = json.dumps(merged, indent=2, ensure_ascii=False) + "\n"
else:
text = raw.decode("utf-8-sig")
target = local[site]["path"] if site in local else self.sites_dir / entry["file"]
pending_writes.append((site, target, text, entry, status))
def write_all():
if not pending_writes:
return
local_index = self._local_index() or {"schema_version": 1, "sites": {}}
local_index.setdefault("sites", {})
for site, target, text, entry, status in pending_writes:
_atomic_write(target, text)
local_index["sites"][site] = dict(entry)
applied.append({"site": site, "status": status, "version": entry.get("adapter_version", "")})
local_index["sites"] = dict(sorted(local_index["sites"].items()))
_atomic_write(self.sites_dir / INDEX_NAME, json.dumps(local_index, indent=2, ensure_ascii=False) + "\n")
if self.lock is not None:
with self.lock:
write_all()
else:
write_all()
if applied and self.reload is not None:
self.reload()
logger.info(f"站点适配器更新:应用 {len(applied)} 个,跳过 {len(skipped)} 个")
return {"applied": applied, "skipped": skipped}
def default_updater(branch: str = "main") -> AdapterUpdater:
"""使用全局 config_engine 与官方仓库(GITHUB_REPO)的更新器。"""
from app import __version__ as app_version
from app.api import config_compare_support as support
from app.services.config_engine import config_engine
from app.utils.remote_resource import get_public_remote_resource
repo = support._resolve_official_repo()
def fetch_bytes(relative_path: str) -> bytes:
url = support._official_config_url(repo, branch, relative_path)
response = get_public_remote_resource(
url, headers={"Accept": "application/json", "User-Agent": "Universal-Web-API-Adapter-Update/1.0"},
timeout=(5, 15), stream=True,
)
try:
response.raise_for_status()
return support._read_limited_response(response)
finally:
response.close()
return AdapterUpdater(
config_engine.sites_dir, fetch_bytes, app_version=app_version,
reload=config_engine.reload_config, lock=config_engine._io_lock,
)
+105
View File
@@ -0,0 +1,105 @@
"""
app/services/arena_cf_solver.py - Arena Cloudflare Turnstile / 5s 盾自动过盾服务(兼容适配层)
说明:
本模块已重构并委托底层通用过盾引擎 `app.services.cf_turnstile_solver`,
在此处保留 Arena 专有的 URL 校验规则与兼容别名。
"""
from __future__ import annotations
import sys
from typing import Any, Callable, Dict, Optional, Tuple
from app.core.config import logger as default_logger
from app.utils.human_mouse import cdp_precise_click, smooth_move_mouse
from app.services.cf_turnstile_solver import (
DEFAULT_FIND_CLICK_POINT_JS as _FIND_CLICK_POINT_JS,
DEFAULT_PAGE_READY_PROBE_JS as _PAGE_READY_PROBE_JS,
_interruptible_sleep,
_urls_match,
_wait_for_challenge_resolved,
_wait_for_clickable_point,
solve_turnstile_challenge,
)
_DEFAULT_MAX_ATTEMPTS = 3
def _is_valid_arena_url(url: Optional[str]) -> bool:
if not url or not isinstance(url, str):
return False
u = url.lower().strip()
if not (u.startswith("http://") or u.startswith("https://")):
return False
if "challenges.cloudflare.com" in u or "challenge-platform" in u:
return False
if "arena.ai" in u or "lmarena.ai" in u:
return True
return False
def _get_occupied_url(tab: Any, session: Any = None) -> str:
"""提取本次请求在过盾前绑定的实际占用 URL。"""
if session is not None:
occupied = getattr(session, "_request_occupied_url", None)
if occupied and _is_valid_arena_url(occupied):
return str(occupied).strip()
last_known = getattr(session, "last_known_url", None)
if last_known and _is_valid_arena_url(last_known):
return str(last_known).strip()
try:
tab_url = str(getattr(tab, "url", "") or "").strip()
if tab_url and _is_valid_arena_url(tab_url):
return tab_url
except Exception:
pass
return "https://arena.ai/code"
def solve_arena_turnstile_challenge(
tab: Any,
session: Any = None,
logger: Any = None,
max_attempts: int = _DEFAULT_MAX_ATTEMPTS,
raise_if_cancelled: Optional[Callable[[], None]] = None,
fallback_coords: Optional[Tuple[int, int]] = None,
click_timeout_sec: float = 6.0,
resolve_timeout_sec: float = 8.0,
) -> Dict[str, Any]:
"""
自动处理 Arena 页面 Cloudflare Turnstile / 5s 盾的核心流程。
"""
# Fetch module-level smooth_move_mouse and cdp_precise_click in case patched by tests
current_module = sys.modules.get(__name__)
cur_smooth_move = getattr(current_module, "smooth_move_mouse", smooth_move_mouse)
cur_cdp_click = getattr(current_module, "cdp_precise_click", cdp_precise_click)
return solve_turnstile_challenge(
tab=tab,
session=session,
is_valid_url_fn=_is_valid_arena_url,
max_attempts=max_attempts,
raise_if_cancelled=raise_if_cancelled,
fallback_coords=fallback_coords,
click_timeout_sec=click_timeout_sec,
resolve_timeout_sec=resolve_timeout_sec,
smooth_move_fn=cur_smooth_move,
cdp_click_fn=cur_cdp_click,
logger=logger or default_logger,
)
__all__ = [
"_FIND_CLICK_POINT_JS",
"_PAGE_READY_PROBE_JS",
"_get_occupied_url",
"_interruptible_sleep",
"_is_valid_arena_url",
"_urls_match",
"_wait_for_challenge_resolved",
"_wait_for_clickable_point",
"cdp_precise_click",
"smooth_move_mouse",
"solve_arena_turnstile_challenge",
]
+816
View File
@@ -0,0 +1,816 @@
"""Read Arena Direct's model catalog from the already-loaded page state."""
from __future__ import annotations
import copy
import json
import os
import re
import threading
import time
from pathlib import Path
from typing import Any, Dict, List, Optional
from urllib.parse import urlparse
from app.core.config import logger
from app.utils.site_url import extract_remote_site_domain, route_domain_matches
from app.core.driver import driver_for_tab
ARENA_DIRECT_MODEL_PREFIX = "arena.ai/direct/"
ARENA_DIRECT_MODEL_CACHE_TTL = 300.0
MODEL_CATALOG_SOURCE = "arena_direct"
ARENA_MODEL_ALIAS_OVERRIDES_PATH = Path(
os.getenv(
"ARENA_MODEL_ALIAS_OVERRIDES_PATH",
str(Path(__file__).resolve().parents[2] / "config" / "arena_model_aliases.local.json"),
)
)
_ARENA_DIRECT_MODEL_EXTRACT_JS = r"""
return (() => {
const pathname = (typeof window !== 'undefined' && window.location && window.location.pathname) ? String(window.location.pathname).toLowerCase() : '';
const isImagePage = pathname.includes('/image');
const isCodePage = pathname.includes('/code');
const defaultPageModality = isImagePage ? 'image' : (isCodePage ? 'code' : 'text');
// 1. 优先从 React Fiber 提取前端真实加载的可用模型
const all = Array.from(document.querySelectorAll('*'));
let fiberModels = null;
for (const el of all) {
const fiberKey = Object.keys(el).find(k => k.startsWith('__reactFiber') || k.startsWith('__reactInternalInstance'));
if (!fiberKey) continue;
let cur = el[fiberKey];
let d = 0;
while (cur && d < 25) {
if (cur.memoizedProps) {
for (const [k, v] of Object.entries(cur.memoizedProps)) {
if (Array.isArray(v) && v.length >= 30 && v.length <= 300 && v[0] && (v[0].id && (v[0].displayName || v[0].name || v[0].publicName))) {
fiberModels = v;
break;
}
}
}
if (fiberModels) break;
cur = cur.return;
d++;
}
if (fiberModels) break;
}
const seenIds = new Set();
const results = [];
const appendModel = (model, defaultModality) => {
if (!model || !model.id) return;
const idStr = String(model.id);
const idLower = idStr.toLowerCase();
if (seenIds.has(idLower)) return;
seenIds.add(idLower);
const caps = model.capabilities || {};
const outCaps = caps.outputCapabilities || model.outputCapabilities || {};
const ranks = model.rankByModality || {};
const rankIsFinite = (key) => (
typeof ranks[key] === 'number' &&
Number.isFinite(ranks[key]) &&
ranks[key] < Number.MAX_SAFE_INTEGER
);
const hasText = Boolean(outCaps.text);
const hasImage = Boolean(outCaps.image) || rankIsFinite('image');
const hasVideo = Boolean(outCaps.video) || rankIsFinite('video');
// `outputCapabilities.web` means the model can browse/use web data;
// it is not the WebDev/code modality. Only the modality rank marks
// a model as code-capable for catalog filtering.
const hasWebDev = rankIsFinite('webdev') || Boolean(outCaps.web);
const hasSearch = Boolean(outCaps.search) || rankIsFinite('search');
const dispName = String(model.displayName || model.publicName || model.name || idStr).trim();
const internalName = String(model.name || model.id || dispName).trim();
const pubName = String(model.publicName || dispName || internalName).trim();
const nameSearch = `${internalName} ${dispName} ${pubName} ${idStr}`.toLowerCase();
const isKnownImageModel = nameSearch.includes('gpt-image') ||
nameSearch.includes('mona-lisa') ||
nameSearch.includes('luna-lisa') ||
nameSearch.includes('lina-alpha') ||
nameSearch.includes('lina-f-alpha') ||
nameSearch.includes('silver_halide') ||
nameSearch.includes('flux') ||
nameSearch.includes('imagen') ||
nameSearch.includes('seedream') ||
nameSearch.includes('seededit') ||
nameSearch.includes('z-image') ||
nameSearch.includes('midjourney') ||
nameSearch.includes('dall-e') ||
nameSearch.includes('recraft') ||
nameSearch.includes('krea') ||
(nameSearch.includes('grok') && nameSearch.includes('image')) ||
(nameSearch.includes('imagine') && !nameSearch.includes('video'));
let modality = defaultModality === 'image' ? 'image' : (defaultModality === 'code' ? 'code' : 'text');
if (hasVideo && !hasText && !isKnownImageModel) {
modality = 'video';
} else if (isKnownImageModel || (hasImage && defaultModality === 'image') || (hasImage && !hasText)) {
modality = 'image';
} else if (defaultModality === 'code' || (hasWebDev && !hasText)) {
modality = 'code';
} else if (hasText) {
modality = 'text';
} else if (hasSearch) {
modality = 'search';
} else if (hasWebDev) {
modality = 'code';
} else if (hasImage) {
modality = 'image';
} else if (hasVideo) {
modality = 'video';
}
results.push({
arena_model_id: idStr,
name: internalName,
public_name: dispName,
display_name: dispName,
provider: String(model.provider || ''),
organization: String(model.organization || model.provider || 'arena.ai'),
modality
});
};
const isModelAvailable = (model) => {
if (!model || model.userSelectable === false) return false;
const ranks = model.rankByModality || {};
const rankValues = Object.values(ranks);
const hasVisibleRank = rankValues.some((rank) => (
typeof rank === 'number' &&
Number.isFinite(rank) &&
rank < Number.MAX_SAFE_INTEGER
));
const hasPublicRank = typeof model.rank === 'number' &&
Number.isFinite(model.rank) && model.rank < 1000;
const hasNoAvailabilityMetadata = rankValues.length === 0 &&
typeof model.rank === 'undefined';
// Arena can keep a user-selectable model at MAX_SAFE_INTEGER for a
// modality while still exposing that modality as a valid capability.
// userSelectable is the authoritative visibility signal in that case.
const hasCapabilityMetadata = Object.values(
model.capabilities?.outputCapabilities || model.outputCapabilities || {}
).some(Boolean);
return hasVisibleRank || hasPublicRank || hasNoAvailabilityMetadata ||
(model.userSelectable === true && hasCapabilityMetadata);
};
if (fiberModels && fiberModels.length > 0) {
for (const m of fiberModels) {
if (isModelAvailable(m)) appendModel(m, defaultPageModality);
}
} else {
// 2. 仅在未挂载 React Fiber 时才从 SSR initialModels 提取保底
const readArray = (text, marker) => {
const markerIndex = text.indexOf(marker);
if (markerIndex < 0) return null;
const start = text.indexOf('[', markerIndex + marker.length);
if (start < 0) return null;
let depth = 0;
let quoted = false;
let escaped = false;
for (let index = start; index < text.length; index += 1) {
const char = text[index];
if (quoted) {
if (escaped) escaped = false;
else if (char === '\\') escaped = true;
else if (char === '"') quoted = false;
continue;
}
if (char === '"') quoted = true;
else if (char === '[') depth += 1;
else if (char === ']') {
depth -= 1;
if (depth === 0) return text.slice(start, index + 1);
}
}
return null;
};
const payloadTexts = [];
for (const script of document.scripts) {
const source = String(script.textContent || '').trim();
const cleanSource = source.replace(/;\s*$/, '');
const prefix = 'self.__next_f.push(';
if (!cleanSource.startsWith(prefix) || !cleanSource.endsWith(')')) continue;
try {
const payload = JSON.parse(cleanSource.slice(prefix.length, -1));
if (Array.isArray(payload) && typeof payload[1] === 'string') {
payloadTexts.push(payload[1]);
}
} catch (_) {}
}
for (const text of payloadTexts) {
const rawModels = readArray(text, '"initialModels":');
if (!rawModels) continue;
try {
const models = JSON.parse(rawModels);
for (const model of models) {
if (!model) continue;
const effectiveName = String(model.displayName || model.publicName || model.name || '').trim();
const nameLower = effectiveName.toLowerCase();
const isUserSpecifiedImage = nameLower.includes('gpt-image') ||
nameLower.includes('mona-lisa') ||
nameLower.includes('luna-lisa') ||
nameLower.includes('lina-alpha') ||
nameLower.includes('lina-f-alpha') ||
nameLower.includes('silver_halide') ||
nameLower.includes('flux') ||
nameLower.includes('imagen') ||
nameLower.includes('seedream') ||
nameLower.includes('seededit') ||
nameLower.includes('z-image') ||
nameLower.includes('midjourney') ||
nameLower.includes('dall-e') ||
nameLower.includes('recraft') ||
nameLower.includes('krea') ||
nameLower.includes('grok-imagine') ||
(nameLower.includes('grok') && nameLower.includes('image'));
if (isUserSpecifiedImage && isModelAvailable(model)) {
appendModel(model, 'image');
} else if (isModelAvailable(model)) {
appendModel(model, defaultPageModality);
}
}
} catch (_) {}
}
}
return results;
})();
"""
_cache_lock = threading.RLock()
_refresh_lock = threading.Lock()
_cached_at = 0.0
_cached_models: List[Dict[str, Any]] = []
def _canonical_modality(value: Any) -> str:
"""Normalize Arena's page/API terminology to the catalog terminology."""
modality = str(value or "").strip().lower()
return {
"webdev": "code",
"web": "code",
"chat": "text",
}.get(modality, modality)
def build_arena_direct_model_id(arena_model_id: Any) -> str:
clean_id = str(arena_model_id or "").strip()
return f"{ARENA_DIRECT_MODEL_PREFIX}{clean_id}" if clean_id else ""
def parse_arena_direct_model_id(model_id: Any) -> str:
value = str(model_id or "").strip()
if not value.lower().startswith(ARENA_DIRECT_MODEL_PREFIX):
return ""
return value[len(ARENA_DIRECT_MODEL_PREFIX):].strip()
def is_arena_direct_model_id(model_id: Any) -> bool:
return bool(parse_arena_direct_model_id(model_id))
def get_arena_direct_model_public_id(model: Any) -> str:
if not isinstance(model, dict):
return ""
return str(
model.get("display_name")
or model.get("public_name")
or model.get("search_name")
or model.get("name")
or ""
).strip()
def match_arena_direct_model(
models: Any,
requested_model: Any,
) -> Optional[Dict[str, Any]]:
if not isinstance(models, (list, tuple, set)):
return None
requested_value = str(requested_model or "").strip()
if not requested_value:
return None
parsed_id = parse_arena_direct_model_id(requested_value)
candidates = {requested_value.casefold()}
if parsed_id:
candidates.add(parsed_id.casefold())
# 1. 优先匹配 UUID (arena_model_id)
for model in models or []:
if not isinstance(model, dict):
continue
mid = str(model.get("arena_model_id") or "").strip().casefold()
if mid and mid in candidates:
return model
# 2. 匹配标准名称 / 展示名称 / 搜索名称
for model in models or []:
if not isinstance(model, dict):
continue
for key in ("name", "public_name", "display_name", "search_name"):
val = str(model.get(key) or "").strip().casefold()
if val and val in candidates:
return model
# 3. 匹配别名
for model in models or []:
if not isinstance(model, dict):
continue
for alias in (model.get("aliases") or []):
alias_val = str(alias or "").strip().casefold()
if alias_val and alias_val in candidates:
return model
# 4. 匹配去除括号后缀的基础名 (例如 gpt-image-2 匹配 gpt-image-2 (medium)),以及 -high、-preview、尾部数字等基础名
for model in models or []:
if not isinstance(model, dict):
continue
for key in ("name", "public_name", "display_name", "search_name"):
raw_val = str(model.get(key) or "").strip().casefold()
if not raw_val:
continue
base_val = raw_val.split("(")[0].strip()
if base_val and base_val in candidates:
return model
if raw_val.endswith("-high"):
high_base = raw_val[:-5].strip()
if high_base and high_base in candidates:
return model
if raw_val.endswith("-preview"):
prev_base = raw_val[:-8].strip()
if prev_base and prev_base in candidates:
return model
if "-" in raw_val:
dash_base = raw_val.rsplit("-", 1)[0].strip()
if dash_base and dash_base in candidates:
return model
return None
def normalize_model_catalog_config(value: Any) -> Dict[str, Any]:
from app.services.arena_model_catalog import normalize_arena_model_catalog_config
return normalize_arena_model_catalog_config(value)
def get_model_catalog_preset(config_engine: Any, domain: Any) -> Optional[Dict[str, Any]]:
from app.services.arena_model_catalog import get_model_catalog_preset as _get_preset
return _get_preset(config_engine, domain)
def get_arena_direct_catalog_for_tab(
config_engine: Any,
tab: Any,
*,
preset_name: Any = None,
) -> Optional[Dict[str, Any]]:
from app.services.arena_model_catalog import get_arena_direct_catalog_for_tab as _get_tab_catalog
return _get_tab_catalog(config_engine, tab, preset_name=preset_name)
def _is_arena_direct_url(
value: Any,
catalog_preset: Optional[Dict[str, Any]] = None,
catalog_config: Optional[Dict[str, Any]] = None,
) -> bool:
current_url = str(value or "").strip()
if not current_url or current_url.lower() in {"about:blank", "javascript:void(0)"}:
return False
try:
actual_domain = extract_remote_site_domain(current_url) or ""
if not route_domain_matches("arena.ai", actual_domain):
return False
path = str(urlparse(current_url).path or "").rstrip("/").lower()
except Exception:
return False
if path in {"/direct", "/text/direct", "/image/direct", "/code/direct"} or path.startswith((
"/direct/",
"/text/direct/",
"/image/direct/",
"/code/direct/",
)) or path.endswith("/direct"):
return True
if path == "/c" or path.startswith("/c/"):
catalog = catalog_config
if catalog is None and catalog_preset is not None:
catalog = normalize_model_catalog_config(catalog_preset.get("model_catalog"))
if catalog is not None:
return bool(catalog.get("enabled") and catalog.get("source") == MODEL_CATALOG_SOURCE)
return True
return False
def _filter_models(
models: List[Dict[str, Any]],
catalog_config: Any,
) -> List[Dict[str, Any]]:
from app.services.arena_model_catalog import filter_arena_catalog_models
return filter_arena_catalog_models(models, catalog_config)
def _natural_sort_key(text: Any) -> list:
value = str(text or "").strip()
return [int(part) if part.isdecimal() else part.casefold() for part in re.split(r"(\d+)", value)]
def _normalize_models(raw_models: Any) -> List[Dict[str, Any]]:
if not isinstance(raw_models, list):
return []
normalized: List[Dict[str, Any]] = []
alias_overrides = _load_alias_overrides()
seen_ids = set()
seen_names = set()
for raw in raw_models:
if not isinstance(raw, dict):
continue
arena_model_id = str(raw.get("arena_model_id") or "").strip()
name = str(raw.get("name") or raw.get("display_name") or raw.get("public_name") or "").strip()
if not arena_model_id or not name:
continue
id_key = arena_model_id.lower()
name_key = name.lower()
if id_key in seen_ids or name_key in seen_names:
continue
seen_ids.add(id_key)
seen_names.add(name_key)
public_name = str(raw.get("public_name") or raw.get("display_name") or name).strip()
display_name = str(raw.get("display_name") or public_name or name).strip()
override = (
alias_overrides.get(name.casefold())
or alias_overrides.get(display_name.casefold())
or alias_overrides.get(public_name.casefold())
)
if not isinstance(override, dict):
override = {}
search_name = str(
override.get("search_name") or display_name or public_name or name
).strip()
aliases = []
for alias in (
name,
public_name,
display_name,
search_name,
*(override.get("aliases") or [] if isinstance(override.get("aliases"), list) else []),
):
alias_text = str(alias or "").strip()
if alias_text and alias_text.casefold() not in {item.casefold() for item in aliases}:
aliases.append(alias_text)
for candidate_name in (display_name, public_name, search_name, name):
if not candidate_name:
continue
if "(" in candidate_name:
base_no_paren = candidate_name.split("(")[0].strip()
if base_no_paren and base_no_paren.casefold() not in {item.casefold() for item in aliases}:
aliases.append(base_no_paren)
if candidate_name.endswith("-high"):
base_no_high = candidate_name[:-5].strip()
if base_no_high and base_no_high.casefold() not in {item.casefold() for item in aliases}:
aliases.append(base_no_high)
if candidate_name.endswith("-preview"):
base_no_prev = candidate_name[:-8].strip()
if base_no_prev and base_no_prev.casefold() not in {item.casefold() for item in aliases}:
aliases.append(base_no_prev)
raw_modality = _canonical_modality(raw.get("modality"))
name_check = f"{name} {display_name} {public_name} {arena_model_id}".lower()
if not raw_modality and any(
k in name_check
for k in (
"gpt-image",
"mona-lisa",
"luna-lisa",
"lina-alpha",
"lina-f-alpha",
"silver_halide",
"flux",
"seedream",
"seededit",
"imagen",
"z-image",
"midjourney",
"dall-e",
"recraft",
"krea",
"grok-imagine",
)
):
raw_modality = "image"
normalized.append(
{
"arena_model_id": arena_model_id,
"name": name,
"public_name": public_name or name,
"display_name": display_name or public_name or name,
"search_name": search_name or display_name or public_name or name,
"aliases": aliases,
"provider": str(raw.get("provider") or "").strip(),
"organization": str(raw.get("organization") or raw.get("provider") or "arena.ai").strip(),
"modality": raw_modality or "text",
}
)
return normalized
def _load_alias_overrides() -> Dict[str, Dict[str, Any]]:
try:
payload = json.loads(ARENA_MODEL_ALIAS_OVERRIDES_PATH.read_text(encoding="utf-8"))
except (FileNotFoundError, OSError, ValueError, TypeError):
return {}
models = payload.get("models") if isinstance(payload, dict) else None
return models if isinstance(models, dict) else {}
def read_arena_direct_models_from_tab(tab: Any) -> List[Dict[str, Any]]:
if tab is None:
return []
try:
try:
return _normalize_models(driver_for_tab(tab).run_js(_ARENA_DIRECT_MODEL_EXTRACT_JS, timeout=3.0))
except TypeError:
return _normalize_models(driver_for_tab(tab).run_js(_ARENA_DIRECT_MODEL_EXTRACT_JS))
except Exception as e:
logger.debug(f"从标签页读取 Arena 直连模型失败(已优雅降级): {e}")
return []
_session_cached_models: Dict[str, tuple[float, List[Dict[str, Any]]]] = {}
def _get_session_cache_key(session_or_tab: Any) -> str:
if session_or_tab is None:
return ""
if isinstance(session_or_tab, dict):
idx = session_or_tab.get("persistent_index")
tid = session_or_tab.get("tab_id")
return f"idx_{idx}" if idx is not None else (f"id_{tid}" if tid else "")
idx = getattr(session_or_tab, "persistent_index", None)
sid = getattr(session_or_tab, "id", None)
return f"idx_{idx}" if idx is not None else (f"id_{sid}" if sid else "")
def _session_cache_snapshot(key: str) -> tuple[float, List[Dict[str, Any]]]:
if not key:
return 0.0, []
with _cache_lock:
if key in _session_cached_models:
ts, models = _session_cached_models[key]
return ts, copy.deepcopy(models)
return 0.0, []
def _replace_session_cache(key: str, models: Any) -> List[Dict[str, Any]]:
normalized = _normalize_models(models)
if not key:
return normalized
with _cache_lock:
_session_cached_models[key] = (time.monotonic(), copy.deepcopy(normalized))
return copy.deepcopy(normalized)
def _cache_snapshot() -> tuple[float, List[Dict[str, Any]]]:
with _cache_lock:
return _cached_at, copy.deepcopy(_cached_models)
def _replace_cache(models: Any) -> List[Dict[str, Any]]:
global _cached_at, _cached_models
normalized = _normalize_models(models)
if not normalized:
return []
with _cache_lock:
_cached_at = time.monotonic()
_cached_models = copy.deepcopy(normalized)
return copy.deepcopy(normalized)
def _session_is_idle(session: Any) -> bool:
raw_status = getattr(session, "status", None)
status = str(getattr(raw_status, "value", None) or raw_status or "").strip().lower()
return status in {"", "idle"}
def _arena_sessions(browser: Any) -> List[Any]:
try:
sessions = browser.tab_pool.get_sessions_snapshot()
except Exception:
return []
from app.services.config_engine import config_engine
result = []
for session in sessions or []:
raw_status = getattr(session, "status", None)
status = str(getattr(raw_status, "value", None) or raw_status or "").strip().lower()
if status not in {"idle", "busy"}:
continue
try:
current_url, _domain = session.get_cached_route_snapshot()
except Exception:
current_url = str(getattr(getattr(session, "tab", None), "url", "") or "")
tab_dict = {
"status": status,
"url": current_url,
"preset_name": getattr(session, "preset_name", None),
"terminating": False,
}
if not get_arena_direct_catalog_for_tab(config_engine, tab_dict):
continue
result.append(session)
result.sort(key=lambda item: (not _session_is_idle(item), int(getattr(item, "persistent_index", 0) or 0)))
return result
def list_arena_direct_models(
browser: Any,
*,
force: bool = False,
catalog_config: Any = None,
tab: Any = None,
) -> List[Dict[str, Any]]:
# 指定单标签页时执行严格标签页隔离读取与缓存
if tab is not None:
target_session = None
target_tab_obj = None
if hasattr(tab, "tab"):
target_session = tab
target_tab_obj = getattr(tab, "tab", None)
elif hasattr(tab, "run_js"):
target_tab_obj = tab
elif isinstance(tab, dict):
target_tab_obj = tab.get("tab")
if browser and hasattr(browser, "tab_pool"):
try:
p_idx = tab.get("persistent_index")
t_id = tab.get("tab_id")
sessions = browser.tab_pool.get_sessions_snapshot()
for s in sessions or []:
if p_idx is not None and getattr(s, "persistent_index", None) == p_idx:
target_session = s
target_tab_obj = getattr(s, "tab", None)
break
if t_id and getattr(s, "id", None) == t_id:
target_session = s
target_tab_obj = getattr(s, "tab", None)
break
except Exception:
pass
key = _get_session_cache_key(target_session or tab)
cached_at, cached = _session_cache_snapshot(key)
if cached and not force and time.monotonic() - cached_at < ARENA_DIRECT_MODEL_CACHE_TTL:
filtered = _filter_models(cached, catalog_config)
if filtered:
return filtered
if target_tab_obj is not None:
if target_session is None or _session_is_idle(target_session):
try:
models = read_arena_direct_models_from_tab(target_tab_obj)
if models:
_replace_session_cache(key, models)
return _filter_models(models, catalog_config)
except Exception as exc:
logger.debug(f"从目标标签页读取 Arena 直连模型失败: {exc}")
if cached:
return _filter_models(cached, catalog_config)
return []
sessions = _arena_sessions(browser)
if not sessions:
return []
cached_at, cached = _cache_snapshot()
if cached and not force and time.monotonic() - cached_at < ARENA_DIRECT_MODEL_CACHE_TTL:
filtered = _filter_models(cached, catalog_config)
if filtered:
return filtered
if not _refresh_lock.acquire(blocking=False):
return _filter_models(cached, catalog_config)
try:
cached_at, cached = _cache_snapshot()
if cached and not force and time.monotonic() - cached_at < ARENA_DIRECT_MODEL_CACHE_TTL:
filtered = _filter_models(cached, catalog_config)
if filtered:
return filtered
all_extracted_models: List[Dict[str, Any]] = []
for session in sessions:
if not _session_is_idle(session):
continue
try:
models = read_arena_direct_models_from_tab(session.tab)
if models:
s_key = _get_session_cache_key(session)
_replace_session_cache(s_key, models)
all_extracted_models.extend(models)
except Exception as exc:
logger.debug(f"Arena Direct 模型目录读取失败(尝试下一标签页): {exc}")
continue
if all_extracted_models:
merged_models = list(cached) + all_extracted_models
updated = _replace_cache(merged_models)
logger.info(f"Arena Direct 模型目录已刷新: {len(updated)} 个模型")
return _filter_models(updated, catalog_config)
return _filter_models(cached, catalog_config)
finally:
_refresh_lock.release()
def resolve_arena_direct_model(
tab: Any,
requested_model: Any,
*,
catalog_config: Any = None,
) -> Optional[Dict[str, Any]]:
requested_value = str(requested_model or "").strip()
if not requested_value:
return None
_cached_at_value, cached = _cache_snapshot()
matched = match_arena_direct_model(_filter_models(cached, catalog_config), requested_value)
if matched:
return matched
if tab is None:
try:
from app.core.browser import get_browser
browser = get_browser(auto_connect=False)
if browser:
models = list_arena_direct_models(browser, catalog_config=catalog_config)
if models:
return match_arena_direct_model(_filter_models(models, catalog_config), requested_value)
except Exception:
pass
else:
models = read_arena_direct_models_from_tab(tab)
if models:
_cached_at_val, current_cached = _cache_snapshot()
merged = list(current_cached) + list(models)
updated = _replace_cache(merged)
return match_arena_direct_model(_filter_models(updated, catalog_config), requested_value)
return None
def build_openai_model_entries(models: List[Dict[str, Any]], *, created: int) -> List[Dict[str, Any]]:
entries = []
seen_ids = set()
for model in models or []:
if not isinstance(model, dict):
continue
model_id = get_arena_direct_model_public_id(model)
normalized_id = model_id.casefold()
if not model_id or normalized_id in seen_ids:
continue
seen_ids.add(normalized_id)
entries.append(
{
"id": model_id,
"object": "model",
"type": "model",
"created": created,
"owned_by": model.get("organization") or model.get("provider") or "arena.ai",
"display_name": model.get("display_name") or model.get("public_name") or model_id,
}
)
entries.sort(key=lambda e: _natural_sort_key(str(e.get("id") or "")))
return entries
__all__ = [
"ARENA_DIRECT_MODEL_PREFIX",
"build_arena_direct_model_id",
"build_openai_model_entries",
"get_arena_direct_catalog_for_tab",
"get_model_catalog_preset",
"get_arena_direct_model_public_id",
"is_arena_direct_model_id",
"list_arena_direct_models",
"match_arena_direct_model",
"normalize_model_catalog_config",
"parse_arena_direct_model_id",
"read_arena_direct_models_from_tab",
"resolve_arena_direct_model",
]
+891
View File
@@ -0,0 +1,891 @@
"""Dedicated Arena blind-test runner for GPT Image 2 C2PA detection."""
from __future__ import annotations
import os
import random
import time
from pathlib import Path
from typing import Any
from urllib.parse import urlsplit
from app.services.arena_image_generation import (
image_signatures as _image_signatures,
read_image_bytes as _read_image_bytes,
same_image as _same_image,
)
from app.core.driver import as_element, driver_for_tab
GPT_IMAGE_MARKERS = (
b"gpt-image",
b"OpenAI Media Service API",
b"OpenAI OpCo, LLC",
)
ARENA_RESULT_IMAGE_SELECTOR = (
"img.transition-opacity.duration-500.opacity-100.aspect-square.object-cover"
)
ARENA_AGENT_PAGE_PATH = "/agent"
def inspect_gpt_image2_png(payload: bytes) -> dict[str, Any]:
"""Return strong GPT Image 2 evidence found in a PNG C2PA caBX chunk."""
data = bytes(payload or b"")
if not data.startswith(b"\x89PNG\r\n\x1a\n"):
return {"matched": False, "reason": "not_png", "markers": []}
offset = 8
cabx_chunks: list[bytes] = []
while offset + 12 <= len(data):
length = int.from_bytes(data[offset : offset + 4], "big")
chunk_type = data[offset + 4 : offset + 8]
chunk_end = offset + 12 + length
if chunk_end > len(data):
return {"matched": False, "reason": "invalid_png", "markers": []}
if chunk_type == b"caBX":
cabx_chunks.append(data[offset + 8 : offset + 8 + length])
offset = chunk_end
if chunk_type == b"IEND":
break
if not cabx_chunks:
return {"matched": False, "reason": "c2pa_missing", "markers": []}
manifest = b"".join(cabx_chunks)
marker_hits = [marker.decode("ascii") for marker in GPT_IMAGE_MARKERS if marker in manifest]
# CBOR text values use one-byte length prefixes here: i=9 for gpt-image,
# c=3 for 2.0. Requiring the adjacent fields prevents a stray version
# elsewhere in the signed manifest from producing a false positive.
has_model = b"dnameigpt-imagegversionc2.0" in manifest
has_openai_claim = b"OpenAI Media Service API" in manifest
matched = has_model and has_openai_claim
return {
"matched": matched,
"reason": "gpt_image_2_c2pa" if matched else "c2pa_not_gpt_image_2",
"markers": marker_hits,
"caBX_bytes": len(manifest),
}
def _value(values: dict[str, Any], key: str, default: Any = "") -> Any:
value = values.get(key, default)
return default if value is None else value
def _sleep(ctx: dict[str, Any], seconds: float, step: float = 0.2) -> None:
deadline = time.time() + max(0.0, float(seconds))
while time.time() < deadline:
ctx["raise_if_cancelled"]()
ctx["raise_if_command_loop_cancelled"]()
agent_guard = ctx.get("_arena_agent_guard")
if agent_guard and agent_guard():
raise RuntimeError("arena_agent_redirected")
ctx["reset_timeout"]()
time.sleep(min(step, max(0.0, deadline - time.time())))
def _current_url(tab: Any) -> str:
try:
return str(driver_for_tab(tab).run_js("return location.href") or "").strip()
except Exception:
return str(getattr(tab, "url", "") or "").strip()
def _is_agent_page_url(url: str) -> bool:
"""Return whether Arena has landed on its agent page, which is abnormal for this runner."""
try:
parsed = urlsplit(str(url or "").strip())
return (
parsed.scheme.lower() == "https"
and parsed.netloc.lower() == "arena.ai"
and parsed.path.rstrip("/") == ARENA_AGENT_PAGE_PATH
)
except Exception:
return False
def _redirect_if_agent_page(ctx: dict[str, Any], redirect_url: str) -> bool:
"""Redirect an unexpected /agent page and report whether the guard fired."""
tab = ctx["tab"]
current_url = _current_url(tab)
if not _is_agent_page_url(current_url):
return False
logger = ctx["logger"]
logger.warning(
f"[GPT-IMAGE-2] unexpected Arena agent page: {current_url}; "
f"navigating to {redirect_url}"
)
try:
tab.get(redirect_url)
except Exception as navigate_error:
logger.warning(
"[GPT-IMAGE-2] agent-page redirect tab.get() failed, "
f"falling back to location.replace(): {navigate_error}"
)
driver_for_tab(tab).run_js("location.replace(arguments[0])", redirect_url)
return True
def _same_page_url(current_url: str, target_url: str) -> bool:
try:
current = urlsplit(str(current_url or "").strip())
target = urlsplit(str(target_url or "").strip())
return (
current.scheme.lower() == target.scheme.lower()
and current.netloc.lower() == target.netloc.lower()
and current.path.rstrip("/") == target.path.rstrip("/")
)
except Exception:
return False
def _new_image_chat_ready(tab: Any, redirect_url: str) -> bool:
if not _same_page_url(_current_url(tab), redirect_url):
return False
input_box = _find(tab, "css:textarea[name='message']", 1.0)
if not input_box:
return False
try:
return int(as_element(input_box).run_js("return String(this.value || '').length")) == 0
except Exception:
return False
def _visible_image_sources(tab: Any) -> list[str]:
script = f"""
const collect = (images) => Array.from(images)
.filter((img) => {{
const r = img.getBoundingClientRect();
const s = getComputedStyle(img);
return img.isConnected && img.complete && img.naturalWidth >= 256
&& img.naturalHeight >= 256 && r.width >= 120 && r.height >= 120
&& s.display !== 'none' && s.visibility !== 'hidden';
}});
const resultImages = collect(document.querySelectorAll(
{ARENA_RESULT_IMAGE_SELECTOR!r}
));
const images = resultImages.length ? resultImages : collect(document.images);
return images
.sort((a, b) => a.getBoundingClientRect().left - b.getBoundingClientRect().left)
.map((img) => img.currentSrc || img.src)
.filter(Boolean);
"""
try:
return list(dict.fromkeys(str(item) for item in (driver_for_tab(tab).run_js(script) or []) if item))
except Exception:
return []
def _log_image_url(url: str) -> str:
try:
parsed = urlsplit(str(url or ""))
return f"{parsed.scheme}://{parsed.netloc}{parsed.path}"[:200]
except Exception:
return str(url or "").split("?", 1)[0][:200]
def _find(tab: Any, selector: str, timeout: float = 1.0) -> Any:
try:
return driver_for_tab(tab).find(selector, timeout=timeout)
except Exception:
return None
def _upload_reference_image(ctx: dict[str, Any], path: str) -> bool:
tab = ctx["tab"]
logger = ctx["logger"]
image_path = Path(os.path.expandvars(os.path.expanduser(path))).resolve()
if not image_path.is_file():
raise RuntimeError(f"reference_image_missing:{image_path}")
inputs = []
try:
inputs = list(driver_for_tab(tab).find_all('css:input[type="file"]', timeout=1.5) or [])
except Exception:
pass
if not inputs:
for selector in (
"xpath://button[contains(., 'Upload') or contains(., '上传')]",
"css:button[aria-label*='upload' i]",
"css:button[aria-label*='image' i]",
):
button = _find(tab, selector, 0.5)
if button:
try:
button.click()
_sleep(ctx, 0.5)
except Exception:
pass
try:
inputs = list(driver_for_tab(tab).find_all('css:input[type="file"]', timeout=1.0) or [])
except Exception:
inputs = []
if inputs:
break
for file_input in inputs:
try:
file_input.input(str(image_path))
logger.info(f"[GPT-IMAGE-2] reference image uploaded: {image_path.name}")
_sleep(ctx, 2.0)
return True
except Exception as error:
logger.debug(f"[GPT-IMAGE-2] file input upload failed: {error}")
return False
def _refresh_page(ctx: dict[str, Any]) -> None:
tab = ctx["tab"]
try:
tab.refresh()
except Exception:
driver_for_tab(tab).run_js("location.reload()")
_sleep(ctx, 2.0)
def _is_generating(tab: Any) -> bool:
try:
result = driver_for_tab(tab).run_js(
"""
return (() => {
const visible = (element) => {
if (!(element instanceof HTMLElement) || !element.isConnected) return false;
const rect = element.getBoundingClientRect();
const style = getComputedStyle(element);
return rect.width >= 8 && rect.height >= 8
&& style.display !== 'none' && style.visibility !== 'hidden';
};
const hasStopButton = Array.from(document.querySelectorAll(
'button[aria-label="Stop generation"], button[aria-label*="Stop" i], button[data-testid*="stop" i]'
)).some(visible);
if (hasStopButton) return true;
return Array.from(document.querySelectorAll('main *, [role="main"] *'))
.filter(visible)
.some((element) => /^generating\\s+image(?:\\.\\.\\.)?$/i.test(
String(element.textContent || '').trim()
));
})();
"""
)
return result is True
except Exception:
return False
def _is_generation_stopped(tab: Any) -> bool:
try:
result = driver_for_tab(tab).run_js(
"""
return (() => {
const visible = (element) => {
if (!(element instanceof HTMLElement) || !element.isConnected) return false;
const rect = element.getBoundingClientRect();
const style = getComputedStyle(element);
return rect.width >= 8 && rect.height >= 8
&& rect.bottom > 0 && rect.right > 0
&& rect.top < innerHeight && rect.left < innerWidth
&& style.display !== 'none' && style.visibility !== 'hidden';
};
return Array.from(document.querySelectorAll('main *, [role="main"] *'))
.filter(visible)
.some((element) => /^generation\\s+stopped$/i.test(
String(element.textContent || '').trim()
));
})();
"""
)
return result is True
except Exception:
return False
def _stop_generation(ctx: dict[str, Any]) -> bool:
tab = ctx["tab"]
logger = ctx["logger"]
for selector in (
'css:button[aria-label="Stop generation"]',
"css:button[aria-label*='Stop' i]",
"css:button[data-testid*='stop' i]",
):
button = _find(tab, selector, 0.8)
if not button:
continue
try:
button.click()
logger.warning("[GPT-IMAGE-2] generation still active after recovery minute; clicked stop")
_sleep(ctx, 2.0)
return True
except Exception as error:
logger.warning(f"[GPT-IMAGE-2] stop generation click failed: {error}")
logger.error("[GPT-IMAGE-2] generation is active but stop button was not found")
return False
def _stop_generation_and_confirm(ctx: dict[str, Any]) -> bool:
if not _stop_generation(ctx):
return False
deadline = time.time() + 5.0
while time.time() < deadline:
if _is_generation_stopped(ctx["tab"]) or not _is_generating(ctx["tab"]):
return True
_sleep(ctx, 0.25, 0.05)
return False
def _vote_a_better(ctx: dict[str, Any]) -> bool:
tab = ctx["tab"]
logger = ctx["logger"]
script = r"""
return (() => {
const norm = (value) => String(value || '').replace(/\s+/g, ' ').trim().toLowerCase();
const visible = (element) => {
if (!(element instanceof HTMLElement) || !element.isConnected) return false;
const rect = element.getBoundingClientRect();
const style = getComputedStyle(element);
return rect.width >= 8 && rect.height >= 8 && rect.bottom > 0 && rect.right > 0
&& rect.top < innerHeight && rect.left < innerWidth
&& style.display !== 'none' && style.visibility !== 'hidden'
&& style.pointerEvents !== 'none' && Number(style.opacity || '1') > 0.2;
};
const matches = (text) => {
const value = norm(text);
return value.includes('a 更好') || value.includes('a is better')
|| value === 'a better' || value.includes('← a');
};
const elements = Array.from(document.querySelectorAll('button, [role="button"]'));
const button = elements.find((element) => {
const text = element.innerText || element.textContent || element.getAttribute('aria-label');
return visible(element) && !element.disabled
&& element.getAttribute('aria-disabled') !== 'true' && matches(text);
});
if (!button) return false;
button.scrollIntoView({block: 'center', inline: 'center'});
button.click();
return true;
})();
"""
try:
clicked = bool(driver_for_tab(tab).run_js(script))
except Exception as error:
logger.warning(f"[GPT-IMAGE-2] failed to click A better: {error}")
return False
if clicked:
logger.info("[GPT-IMAGE-2] non-target result; selected A better")
_sleep(ctx, 1.0)
else:
logger.warning("[GPT-IMAGE-2] non-target result; A better button was not found")
return clicked
def _start_new_image_chat(ctx: dict[str, Any], redirect_url: str) -> bool:
tab = ctx["tab"]
logger = ctx["logger"]
selectors = (
"css:a[href='/image']",
"xpath://a[contains(., 'New Chat') or contains(., 'New chat')]",
"xpath://button[contains(., 'New Chat') or contains(., 'New chat')]",
)
if _new_image_chat_ready(tab, redirect_url):
logger.stream(f"[GPT-IMAGE-2] new image chat ready: url={_current_url(tab)}")
return True
for selector in selectors:
element = _find(tab, selector, 0.8)
if not element:
continue
try:
element.click()
_sleep(ctx, 1.2)
if _new_image_chat_ready(tab, redirect_url):
logger.stream(f"[GPT-IMAGE-2] new image chat confirmed: url={_current_url(tab)}")
return True
except Exception:
continue
try:
logger.stream(f"[GPT-IMAGE-2] New Chat not confirmed; navigating to {redirect_url}")
tab.get(redirect_url)
_sleep(ctx, 2.0)
return _new_image_chat_ready(tab, redirect_url)
except Exception:
return False
def _send_prompt(
ctx: dict[str, Any], prompt: str, *, require_clear_confirmation: bool = False
) -> bool:
tab = ctx["tab"]
logger = ctx["logger"]
input_box = _find(tab, "css:textarea[name='message']", 5.0)
if not input_box:
return False
if require_clear_confirmation:
try:
replaced = as_element(input_box).run_js(
"""
const value = String(arguments[0] || '');
const setter = Object.getOwnPropertyDescriptor(
HTMLTextAreaElement.prototype, 'value'
).set;
this.focus();
setter.call(this, value);
this.dispatchEvent(new InputEvent('input', {
bubbles: true,
data: value,
inputType: 'insertFromPaste'
}));
this.dispatchEvent(new Event('change', {bubbles: true}));
this.setSelectionRange(value.length, value.length);
return String(this.value || '');
""",
prompt,
)
except Exception as error:
logger.error(f"[GPT-IMAGE-2] failed to replace composer text: {error}")
return False
if str(replaced or "") != prompt:
logger.error("[GPT-IMAGE-2] composer replacement did not match the prompt")
return False
stable_checks = 0
deadline = time.time() + 1.5
while time.time() < deadline:
current_input = _find(tab, "css:textarea[name='message']", 0.3)
try:
current_value = str(
as_element(current_input).run_js("return String(this.value || '')") if current_input else ""
)
except Exception:
current_value = ""
if current_value == prompt:
stable_checks += 1
if stable_checks >= 3:
break
else:
stable_checks = 0
_sleep(ctx, 0.15, 0.05)
if stable_checks < 3:
logger.error("[GPT-IMAGE-2] controlled composer restored or changed the prompt")
return False
def composer_is_empty() -> bool:
current_input = _find(tab, "css:textarea[name='message']", 0.3)
if not current_input:
return False
try:
return int(as_element(current_input).run_js("return String(this.value || '').length")) == 0
except Exception:
return False
def wait_until_sent(timeout: float = 3.0) -> bool:
deadline = time.time() + timeout
while time.time() < deadline:
if composer_is_empty():
return True
_sleep(ctx, 0.15, 0.05)
return False
button = _find(
tab,
"css:button[aria-label='Send message'][type='submit']:not(:disabled), "
"button[type='submit']:not([aria-label='Stop generation']):not(:disabled)",
1.0,
)
if button:
try:
button.click()
except Exception as error:
logger.warning(f"[GPT-IMAGE-2] send button click failed: {error}")
else:
logger.warning("[GPT-IMAGE-2] send button not found")
if wait_until_sent():
logger.stream("[GPT-IMAGE-2] send confirmed after button click")
return True
for enter_attempt in range(2):
current_input = _find(tab, "css:textarea[name='message']", 0.5)
if not current_input:
logger.warning("[GPT-IMAGE-2] composer missing before Enter fallback")
else:
try:
current_input.input("\n")
except Exception as error:
logger.warning(
f"[GPT-IMAGE-2] Enter fallback failed: "
f"attempt={enter_attempt + 1}/2 error={error}"
)
if wait_until_sent():
logger.stream(
f"[GPT-IMAGE-2] send confirmed after Enter: attempt={enter_attempt + 1}/2"
)
return True
logger.warning(
f"[GPT-IMAGE-2] composer still contains text after Enter: "
f"attempt={enter_attempt + 1}/2"
)
return False
try:
input_box.clear()
except Exception:
pass
input_box.input(prompt)
_sleep(ctx, 0.3)
before_url = _current_url(tab)
def submission_confirmed(timeout: float = 2.0) -> bool:
deadline = time.time() + timeout
while time.time() < deadline:
current_url = _current_url(tab)
if current_url and current_url != before_url:
return True
try:
if int(as_element(input_box).run_js("return String(this.value || '').length")) == 0:
return True
except Exception:
pass
for selector in (
"css:button[aria-label*='Stop' i]",
"css:button[data-testid*='stop' i]",
"css:[aria-busy='true']",
):
if _find(tab, selector, 0.15):
return True
_sleep(ctx, 0.15, 0.05)
return False
for attempt in range(2):
try:
input_box.input("\n")
except Exception as error:
if isinstance(error, RuntimeError) and "cancelled" in str(error).lower():
raise
logger.warning(
f"[GPT-IMAGE-2] Enter submit failed: attempt={attempt + 1}/2 error={error}"
)
if submission_confirmed(2.5):
logger.stream(f"[GPT-IMAGE-2] Enter submit confirmed: attempt={attempt + 1}/2")
return True
if attempt == 0:
logger.stream("[GPT-IMAGE-2] Enter submit not confirmed; retrying Enter")
_sleep(ctx, 0.5)
button = _find(tab, "css:button[type='submit']", 1.0)
if button:
try:
logger.stream("[GPT-IMAGE-2] Enter submit not confirmed; trying send button")
button.click()
if submission_confirmed(2.5):
logger.stream("[GPT-IMAGE-2] send button submit confirmed")
return True
except Exception as error:
logger.warning(f"[GPT-IMAGE-2] send button fallback failed: {error}")
return False
def _wait_for_gpt_image2(
ctx: dict[str, Any],
baseline: set[str],
timeout: float,
reference_signature: dict[str, Any] | None = None,
) -> tuple[list[dict[str, Any]], list[str]]:
tab = ctx["tab"]
logger = ctx["logger"]
deadline = time.time() + timeout
inspected: dict[str, str] = {}
matches: list[dict[str, Any]] = []
candidates: list[str] = []
last_new_image_at = 0.0
while time.time() < deadline:
_sleep(ctx, 0.5)
sources = [url for url in _visible_image_sources(tab) if url not in baseline]
if sources:
last_new_image_at = last_new_image_at or time.time()
for side_index, url in enumerate(sources):
if url in inspected:
continue
payload = _read_image_bytes(tab, url)
if not payload:
continue
signatures = _image_signatures(payload)
inspected[url] = signatures["sha256"]
if _same_image(signatures, reference_signature):
logger.info(
f"[GPT-IMAGE-2] skipped submitted reference image: "
f"side={'A' if side_index == 0 else 'B'} "
f"url={_log_image_url(url)}"
)
continue
if url not in candidates:
candidates.append(url)
evidence = inspect_gpt_image2_png(payload)
logger.info(
f"[GPT-IMAGE-2][C2PA] side={'A' if side_index == 0 else 'B'} "
f"matched={evidence['matched']} reason={evidence['reason']} "
f"bytes={len(payload)} url={_log_image_url(url)}"
)
if evidence["matched"]:
matches.append({"url": url, "side": "A" if side_index == 0 else "B", **evidence})
generation_done = _is_generation_stopped(tab) or not _is_generating(tab)
if (
last_new_image_at
and time.time() - last_new_image_at >= 4.0
and generation_done
):
# Arena can reuse a URL while replacing an early preview with the
# final PNG, so force one content-aware pass after generation ends.
final_sources = [
url for url in _visible_image_sources(tab) if url not in baseline
]
for side_index, url in enumerate(final_sources):
payload = _read_image_bytes(tab, url)
if not payload:
continue
signatures = _image_signatures(payload)
final_side = "A" if side_index == 0 else "B"
if inspected.get(url) == signatures["sha256"]:
for match in matches:
if match.get("url") == url:
match["side"] = final_side
continue
inspected[url] = signatures["sha256"]
if _same_image(signatures, reference_signature):
continue
if url not in candidates:
candidates.append(url)
evidence = inspect_gpt_image2_png(payload)
logger.info(
f"[GPT-IMAGE-2][C2PA][FINAL] "
f"side={final_side} "
f"matched={evidence['matched']} reason={evidence['reason']} "
f"bytes={len(payload)} url={_log_image_url(url)}"
)
if evidence["matched"]:
matches.append(
{
"url": url,
"side": final_side,
**evidence,
}
)
break
unique_matches = list(
{item.get("url", ""): item for item in matches if item.get("url")}.values()
)
return unique_matches, candidates
def _recover_timed_out_generation(
ctx: dict[str, Any],
initial_matches: list[dict[str, Any]],
initial_candidates: list[str],
reference_signature: dict[str, Any] | None = None,
) -> tuple[list[dict[str, Any]], list[str], bool]:
tab = ctx["tab"]
logger = ctx["logger"]
matches = list(initial_matches)
candidates = list(initial_candidates)
_refresh_page(ctx)
if _is_generating(tab):
logger.warning("[GPT-IMAGE-2] generation still active after refresh; stopping immediately")
if not _stop_generation_and_confirm(ctx):
raise _StopGenerationFailure(
"generation_still_active_after_stop; command loop terminated"
)
logger.info("[GPT-IMAGE-2] timed-out generation stopped; continuing next round")
return matches, candidates, True
else:
logger.info("[GPT-IMAGE-2] generation completed after refresh; inspecting current images")
recovered_matches, recovered_candidates = _wait_for_gpt_image2(
ctx, set(), 5.0, reference_signature
)
matches.extend(recovered_matches)
candidates.extend(recovered_candidates)
unique_matches = list({item.get("url", ""): item for item in matches if item.get("url")}.values())
unique_candidates = list(dict.fromkeys(url for url in candidates if url))
return unique_matches, unique_candidates, False
class _StopGenerationFailure(RuntimeError):
pass
def run(ctx: dict[str, Any], *, single_mode: bool = False) -> str:
tab = ctx["tab"]
logger = ctx["logger"]
session = ctx.get("session")
current_command_id = str(getattr(session, "_current_command_id", "") or "").strip()
current_command_name = str(getattr(session, "current_command_name", "") or "").strip()
single_mode = bool(
single_mode
or current_command_id == "cmd_15d91034"
or current_command_name == "ARENA自动盲测不翻牌 -gpt image 2 - 单"
)
ui = ctx.get("command_ui") if isinstance(ctx.get("command_ui"), dict) else {}
values = ui.get("values") if isinstance(ui.get("values"), dict) else ui
values = values if isinstance(values, dict) else {}
total_runs = max(1, int(_value(values, "total_runs", 20)))
prompt = str(_value(values, "prompt_text", "Create an image.")).strip()
random_chars = str(_value(values, "random_insert_chars", "abcdefghijklmnopqrstuvwxyz"))
reference_image = str(_value(values, "reference_image", "")).strip()
reference_signature = None
if reference_image:
reference_path = Path(
os.path.expandvars(os.path.expanduser(reference_image))
).resolve()
if reference_path.is_file():
reference_signature = _image_signatures(reference_path.read_bytes())
generation_mode = "image-to-image" if reference_image else "text-to-image"
timeout = 150.0 if single_mode else min(
100.0,
max(1.0, float(_value(values, "wait_reply_timeout_sec", 100))),
)
redirect_url = str(_value(values, "redirect_url", "https://arena.ai/image")).strip()
ctx["_arena_agent_guard"] = lambda: _redirect_if_agent_page(ctx, redirect_url)
hit_urls: list[str] = []
consecutive_send_failures = 0
logger.stream(
f"[GPT-IMAGE-2] starting {total_runs} rounds; "
f"mode={generation_mode}; timeout={timeout:g}s"
)
for index in range(total_runs):
status = "completed"
ctx["begin_command_loop"](index + 1, total_runs, "Arena GPT Image 2")
try:
if _redirect_if_agent_page(ctx, redirect_url):
raise RuntimeError("arena_agent_redirected")
if not single_mode:
logger.stream(
f"[GPT-IMAGE-2] round {index + 1}/{total_runs}: creating a new image chat"
)
if not _start_new_image_chat(ctx, redirect_url):
raise RuntimeError("new_image_chat_failed")
if reference_image and not _upload_reference_image(ctx, reference_image):
raise RuntimeError("reference_image_upload_failed")
baseline = set(_visible_image_sources(tab))
run_prompt = prompt
if random_chars:
position = random.randint(0, len(run_prompt))
run_prompt = run_prompt[:position] + random.choice(random_chars) + run_prompt[position:]
logger.stream(f"[GPT-IMAGE-2] round {index + 1}/{total_runs}: sending prompt")
if not _send_prompt(
ctx,
run_prompt,
require_clear_confirmation=single_mode,
):
if not single_mode:
raise RuntimeError("send_failed")
consecutive_send_failures += 1
status = "failed"
logger.error(
f"[GPT-IMAGE-2] send failed {consecutive_send_failures} consecutive rounds; "
"recovering the stuck page before the next round"
)
_refresh_page(ctx)
if _is_generating(tab):
logger.warning(
"[GPT-IMAGE-2] stale generation remained after failed send; stopping it"
)
if not _stop_generation_and_confirm(ctx):
raise _StopGenerationFailure(
"generation_still_active_after_stop; command loop terminated"
)
continue
consecutive_send_failures = 0
matches, candidates = _wait_for_gpt_image2(
ctx, baseline, timeout, reference_signature
)
if single_mode and not matches and _is_generation_stopped(tab):
logger.info(
f"[GPT-IMAGE-2] round {index + 1}/{total_runs} generation stopped "
"without a target match; continuing next round without voting"
)
continue
if single_mode and (_is_generating(tab) or not candidates):
logger.warning(
f"[GPT-IMAGE-2] round {index + 1}/{total_runs} reached {timeout:g}s "
"without a completed image response; refreshing within the same round"
)
matches, candidates, interrupted = _recover_timed_out_generation(
ctx, matches, candidates, reference_signature
)
if interrupted:
continue
page_url = _current_url(tab)
if matches:
info = {
"url": page_url,
"response_sides": [
f"gpt-image 2.0 OpenAI Media Service API mode:{generation_mode}"
if item["side"] == side else "other image model"
for side in ("A", "B")
for item in matches[:1]
],
}
record_result = ctx["record_arena_rule_candidates"](
info,
run_prompt,
source="image-c2pa/gpt-image-2",
)
if page_url and page_url not in hit_urls:
hit_urls.append(page_url)
logger.info(
f"[GPT-IMAGE-2][HIT] sides={','.join(item['side'] for item in matches)} "
f"url={page_url} drawer_matches={record_result.get('matched', 0)}"
)
if single_mode:
break
elif single_mode:
logger.info(
f"[GPT-IMAGE-2] no GPT Image 2 C2PA match; images={len(candidates)} url={page_url}"
)
if candidates:
_vote_a_better(ctx)
else:
logger.info(
f"[GPT-IMAGE-2] no GPT Image 2 C2PA match; images={len(candidates)} url={page_url}"
)
except _StopGenerationFailure:
status = "failed"
raise
except RuntimeError as error:
if str(error) in {"python_script_cancelled", "python_script_loop_cancelled"}:
status = "cancelled"
if str(error) == "python_script_cancelled":
break
elif str(error) == "arena_agent_redirected":
status = "failed"
logger.warning(
f"[GPT-IMAGE-2] round {index + 1} treated as abnormal because the page was /agent"
)
else:
status = "failed"
logger.error(f"[GPT-IMAGE-2] round {index + 1} failed: {error}")
if not single_mode:
try:
tab.get(redirect_url)
_sleep(ctx, 2.0)
except Exception:
pass
except Exception as error:
status = "failed"
logger.error(f"[GPT-IMAGE-2] round {index + 1} failed: {error}")
finally:
ctx["end_command_loop"](status)
if hit_urls:
return "GPT Image 2 C2PA hit URLs:\n" + "\n".join(
f"{index}. {url}" for index, url in enumerate(hit_urls, 1)
)
return "GPT Image 2 C2PA hits: 0"
__all__ = ["GPT_IMAGE_MARKERS", "inspect_gpt_image2_png", "run"]
+968
View File
@@ -0,0 +1,968 @@
"""Arena-only guards for browser image generation.
This module deliberately owns Arena's DOM error handling and image-to-image
validation. Callers outside Arena should never enable these rules.
"""
from __future__ import annotations
import base64
import hashlib
import json
import uuid
from dataclasses import dataclass
from io import BytesIO
from pathlib import Path
from typing import Any, Iterable, Optional
from urllib.parse import urlsplit
from app.core.config import logger
from app.utils.image_validation import (
extract_media_item_bytes as _common_extract_media_item_bytes,
get_current_page_url as current_page_url,
image_signatures,
read_image_bytes,
read_local_image_bytes,
read_uploaded_image_bytes as _common_read_uploaded_image_bytes,
same_image,
validate_generated_images as _common_validate_generated_images,
)
from app.core.driver import driver_for_tab
ARENA_NATIVE_STOP_SELECTOR = 'css:button[aria-label="Stop generation"]'
ARENA_PROMPT_REJECTED_CODE = "arena_prompt_rejected"
ARENA_IMAGE_GENERATION_FAILED_CODE = "arena_image_generation_failed"
ARENA_IMAGE_UNCHANGED_CODE = "arena_image_unchanged"
ARENA_RESULT_BASELINE_PROPERTY = "__universalProxyArenaResultBaseline"
ARENA_NON_RETRYABLE_CODES = frozenset(
{
ARENA_PROMPT_REJECTED_CODE,
ARENA_IMAGE_GENERATION_FAILED_CODE,
ARENA_IMAGE_UNCHANGED_CODE,
}
)
_IMAGE_PROMPT_MARKERS = (
"生成图片",
"生成图像",
"生成一张图",
"生成一张图片",
"画一张",
"画一幅",
"帮我画",
"请画",
"出图",
"做图",
"文生图",
"以图生图",
"image generation",
"generate image",
"generate an image",
"create image",
"create an image",
"draw an image",
"draw me",
"make an image",
"render an image",
"render image",
)
_INTERRUPTED_STREAM_MARKERS = (
"未完整结束",
"提前关闭",
"连接已关闭",
"未收到完成标志",
"缺少明确结束事件",
"未产出有效正文",
"响应正文未就绪",
"network listener timeout",
"stream ended without completion",
"connection closed",
)
class ArenaImageGenerationError(RuntimeError):
"""A terminal Arena image-generation error that must not be retried."""
status_code = 422
retryable = False
def __init__(self, code: str, message: str):
super().__init__(message)
self.code = str(code)
self.message = str(message)
@dataclass(frozen=True)
class ArenaImageGenerationObservation:
"""The state used to decide whether an Arena image response is final."""
stop_present: bool
has_new_image: bool
terminal_error: Optional[ArenaImageGenerationError] = None
has_stop: Optional[bool] = None
has_generating_text: bool = False
has_spin_canvas: bool = False
still_generating: Optional[bool] = None
def __post_init__(self):
if self.has_stop is None:
object.__setattr__(self, "has_stop", bool(self.stop_present))
if self.still_generating is None:
object.__setattr__(
self,
"still_generating",
bool(self.stop_present or (self.has_generating_text and self.has_spin_canvas)),
)
@property
def is_complete(self) -> bool:
if self.terminal_error is not None:
return True
return bool(not self.still_generating and self.has_new_image)
def is_arena_page_url(page_url: str) -> bool:
"""Return true only for Arena hosts, not lookalike host names."""
try:
hostname = str(urlsplit(str(page_url or "")).hostname or "").lower().rstrip(".")
except Exception:
return False
return hostname in {"arena.ai", "lmarena.ai"} or hostname.endswith(
(".arena.ai", ".lmarena.ai")
)
def looks_like_image_generation_request(text: str) -> bool:
"""Recognize an image generation request without binding it to a site."""
lowered = str(text or "").strip().lower()
if not lowered:
return False
if any(marker in lowered for marker in _IMAGE_PROMPT_MARKERS):
return True
english_actions = ("generate", "create", "draw", "make", "render", "design", "produce")
english_objects = (
"image",
"images",
"picture",
"pictures",
"photo",
"photos",
"illustration",
"artwork",
"poster",
"logo",
"icon",
"banner",
"wallpaper",
"portrait",
)
if any(action in lowered for action in english_actions) and any(
item in lowered for item in english_objects
):
return True
chinese_actions = ("画", "绘制", "生成", "创作", "设计", "修改", "改成", "编辑")
chinese_objects = ("图片", "图像", "照片", "插画", "海报", "logo", "图标", "头像", "封面", "壁纸")
return any(action in lowered for action in chinese_actions) and any(
item in lowered for item in chinese_objects
)
def _image_modality_enabled(image_config: Optional[dict[str, Any]]) -> bool:
config = image_config or {}
if bool(config.get("enabled", False)):
return True
image_setting = (config.get("modalities") or {}).get("image")
if isinstance(image_setting, dict):
return bool(image_setting.get("enabled", False))
return bool(image_setting)
def is_arena_image_generation_request(
page_url: str,
prompt: str,
image_config: Optional[dict[str, Any]] = None,
uploaded_images: Optional[Iterable[Any]] = None,
*,
preset_name: str = "",
) -> bool:
"""Enable Arena image-generation rules only for marked templates."""
config = image_config or {}
if not is_arena_page_url(page_url) or not _image_modality_enabled(config):
return False
marker = config.get("_arena_image_generation_active")
if marker is None:
marker = config.get("arena_image_generation", False)
return bool(marker)
def is_interrupted_stream_reason(reason: str) -> bool:
normalized = str(reason or "").strip().lower()
return bool(normalized) and any(marker in normalized for marker in _INTERRUPTED_STREAM_MARKERS)
def auto_skip_arena_direct_comparison(tab: Any) -> bool:
"""Detect and click the Skip button in Arena Direct mode's occasional image comparison prompt with debouncing."""
script = r"""
try {
const evalBars = Array.from(document.querySelectorAll('div')).filter(d => {
const text = (d.innerText || '');
return (text.includes('继续使用 A') || text.includes('Continue with A') || text.includes('Use A'))
&& (text.includes('跳过') || text.includes('Skip'));
});
const searchRoots = evalBars.length > 0 ? evalBars : [document.body];
for (const root of searchRoots) {
const buttons = Array.from(root.querySelectorAll('button:not([data-skip-triggered="true"])'));
const skipBtn = buttons.find(b => {
const text = (b.innerText || '').trim();
const hasSkipSvg = !!b.querySelector('svg path[d*="M18 7V17"]') || !!b.querySelector('svg path[d*="M18 6V18"]');
const isVisible = b.offsetWidth > 0 && b.offsetHeight > 0;
return isVisible && (text === '跳过' || text === 'Skip' || hasSkipSvg);
});
if (skipBtn) {
skipBtn.setAttribute('data-skip-triggered', 'true');
skipBtn.click();
return true;
}
}
} catch (_) {}
return false;
"""
try:
return bool(driver_for_tab(tab).run_js(script))
except Exception:
return False
def detect_arena_render_crash(tab: Any) -> bool:
"""Detect unstyled / FOUC layout crash where Tailwind styles failed to load."""
script = r"""
try {
const ol = document.querySelector('ol');
if (ol) {
const display = window.getComputedStyle(ol)?.display;
if (display === 'block') return true;
}
if (document.styleSheets.length === 0) return true;
} catch (_) {}
return false;
"""
try:
return bool(driver_for_tab(tab).run_js(script))
except Exception:
return False
def evaluate_arena_direct_generation_state(
tab: Any,
baseline_depth: int = 0,
current_prompt: str = "",
stop_selector: str = ARENA_NATIVE_STOP_SELECTOR,
) -> Optional[dict[str, Any]]:
"""Execute node-scoped five-state generation evaluation for Arena Direct image requests."""
script = r"""
return ((baselineDepth, currentPrompt, stopSelector) => {
const result = {
status: 'IDLE',
still_generating: false,
is_complete: false,
image_urls: [],
error_code: null,
error_msg: '',
skipped_comparison: false
};
// 0. Auto-skip occasional comparison bar if present (scoped & debounced)
try {
const evalBars = Array.from(document.querySelectorAll('div')).filter(d => {
const text = (d.innerText || '');
return (text.includes('继续使用 A') || text.includes('Continue with A') || text.includes('Use A'))
&& (text.includes('跳过') || text.includes('Skip'));
});
const searchRoots = evalBars.length > 0 ? evalBars : [document.body];
for (const root of searchRoots) {
const buttons = Array.from(root.querySelectorAll('button:not([data-skip-triggered="true"])'));
const skipBtn = buttons.find(b => {
const text = (b.innerText || '').trim();
const hasSkipSvg = !!b.querySelector('svg path[d*="M18 7V17"]') || !!b.querySelector('svg path[d*="M18 6V18"]');
return b.offsetWidth > 0 && (text === '跳过' || text === 'Skip' || hasSkipSvg);
});
if (skipBtn) {
skipBtn.setAttribute('data-skip-triggered', 'true');
skipBtn.click();
result.skipped_comparison = true;
break;
}
}
} catch (_) {}
const ol = document.querySelector('ol.flex-col-reverse') || document.querySelector('ol');
const textarea = document.querySelector('textarea, [contenteditable="true"]');
if (!ol) {
result.status = 'WAITING_HYDRATION';
result.still_generating = true;
return result;
}
const rawChildren = Array.from(ol.children);
const validChildren = rawChildren.filter(c => c.tagName === 'DIV' && !c.classList.contains('h-0'));
if (validChildren.length === 0) {
result.status = 'WAITING_HYDRATION';
result.still_generating = true;
return result;
}
// 1. Blocker Fix: Assistant-Only Depth Guard
const assistantNodes = validChildren.filter(c => !c.classList.contains('justify-end') && !c.querySelector('[data-role="user"]'));
if (baselineDepth > 0 && assistantNodes.length < baselineDepth + 1) {
result.status = 'WAITING_HYDRATION';
result.still_generating = true;
return result;
}
// 2. Blocker Fix: Physical Top Child Isolation
// In flex-col-reverse, index 0 is visually at the very bottom.
// If index 0 is a User node, the new Assistant node has NOT mounted yet!
const firstChild = validChildren[0];
const isFirstChildUser = firstChild.classList.contains('justify-end') || !!firstChild.querySelector('[data-role="user"]');
if (isFirstChildUser) {
result.status = 'WAITING_HYDRATION';
result.still_generating = true;
return result;
}
const latestAssistant = firstChild;
// 3. Prompt Echo Guard on the latest User node. Do not return
// early: active generation and a newly rendered image are stronger
// evidence than a UI-normalized prompt prefix mismatch.
let promptEchoMismatch = false;
if (currentPrompt && String(currentPrompt).trim()) {
const latestUser = validChildren.find(c => c.classList.contains('justify-end') || !!c.querySelector('[data-role="user"]'));
if (latestUser) {
const userText = (latestUser.innerText || '').replace(/\s+/g, ' ').trim();
const promptSnippet = String(currentPrompt).trim().slice(0, 15);
if (promptSnippet && !userText.includes(promptSnippet)) {
promptEchoMismatch = true;
}
}
}
// 4. Stop button & active generating status
let hasStop = false;
try {
if (window.__arenaHardStop && typeof window.__arenaHardStop.status === 'function') {
const st = window.__arenaHardStop.status();
if (st && (st.hasNativeStopButton || st.hasOverlayStopButton)) hasStop = true;
}
} catch (_) {}
if (!hasStop && stopSelector) {
try {
const sel = String(stopSelector).trim().replace(/^css:/i, '');
const stopEl = document.querySelector(sel);
if (stopEl && stopEl.offsetWidth > 0 && stopEl.offsetHeight > 0) hasStop = true;
} catch (_) {}
}
if (!hasStop) {
const stopOverlays = ['button[aria-label="Stop generation"]', '[data-arena-hard-stop-overlay="true"]'];
for (const s of stopOverlays) {
const el = document.querySelector(s);
if (el && el.offsetWidth > 0 && el.offsetHeight > 0) {
hasStop = true;
break;
}
}
}
const hasSpinner = !!latestAssistant.querySelector('.lucide-loader.animate-spin, .animate-spin, [class*="animate-spin"]');
const isTextareaBusy = textarea ? (textarea.disabled || textarea.getAttribute('aria-busy') === 'true') : false;
// Priority 1: Still generating
if (hasStop || hasSpinner || isTextareaBusy) {
result.status = 'GENERATING';
result.still_generating = true;
return result;
}
// Priority 2: Error detection inside latest Assistant container
const assistantText = (latestAssistant.innerText || '').toLowerCase();
if (assistantText.includes('something went wrong with this response, please try again') ||
assistantText.includes('an error occurred while generating')) {
result.status = 'ERROR';
result.error_code = 'arena_image_generation_failed';
result.error_msg = 'Arena 图片生成失败:Something went wrong with this response, please try again.';
result.is_complete = true;
return result;
}
// Priority 3: Safety / Terms of Use violation inside latest Assistant container
if (assistantText.includes('this content violates our terms of use')) {
result.status = 'SAFETY_VIOLATION';
result.error_code = 'arena_prompt_rejected';
result.error_msg = 'Arena 拒绝了该图片请求:内容违反 Terms of Use';
result.is_complete = true;
return result;
}
// Priority 4: Success - image URLs resolution inside latest Assistant container
const imgs = Array.from(latestAssistant.querySelectorAll('img')).filter(img => {
const src = String(img.src || img.currentSrc || '').trim();
const isAvatar = img.closest('[class*="avatar"], .size-6, .rounded-full') || (img.naturalWidth > 0 && img.naturalWidth <= 96);
return src && !isAvatar && (src.includes('r2.cloudflarestorage.com') || src.startsWith('blob:') || src.startsWith('http'));
});
if (imgs.length > 0) {
result.status = 'SUCCESS';
result.is_complete = true;
result.still_generating = false;
result.image_urls = imgs.map(img => img.src || img.currentSrc);
return result;
}
if (promptEchoMismatch) {
result.status = 'WAITING_HYDRATION';
result.still_generating = true;
return result;
}
// Priority 5: Empty abort fallback (Stop button gone, action bar mounted, but no image or error)
const hasActionBar = !!latestAssistant.querySelector('button[aria-label*="Like" i], button[aria-label*="Dislike" i]') ||
Array.from(latestAssistant.querySelectorAll('button')).some(b => b.innerText.includes('Edit'));
if (hasActionBar) {
result.status = 'EMPTY_ABORT';
result.error_code = 'arena_image_empty_response';
result.error_msg = 'Arena 图片生成结束但未产出有效图片或错误';
result.is_complete = true;
return result;
}
result.status = 'GENERATING';
result.still_generating = true;
return result;
})(arguments[0], arguments[1], arguments[2]);
"""
try:
data = _decode_js_result(
driver_for_tab(tab).run_js(_json_js(script), baseline_depth, current_prompt, stop_selector)
)
if isinstance(data, dict) and data.get("status"):
return data
except Exception as exc:
logger.debug(f"evaluate_arena_direct_generation_state 异常: {exc}")
return None
def _json_js(script: str) -> str:
"""P0-2: stringify a polled script's result in-page (no RemoteObject leak)."""
from app.core.cdp_hygiene import wrap_js_json
return wrap_js_json(script)
def _decode_js_result(raw: Any) -> Any:
"""Decode a JSON string from :func:`_json_js`; keep non-JSON values as-is."""
if isinstance(raw, str):
text = raw.strip()
if text[:1] in ("{", "[", '"') or text in ("null", "true", "false"):
try:
return json.loads(text)
except ValueError:
return raw
return raw
def get_arena_generation_status(
tab: Any,
selector: str = ARENA_NATIVE_STOP_SELECTOR,
) -> dict[str, bool]:
"""Return structured multi-modal generation indicators and fault-tolerant generation state."""
script = r"""
const selector = String(arguments[0] || '').trim().replace(/^css:/i, '');
const visible = (element) => {
if (!(element instanceof Element) || !element.isConnected) return false;
if (element.disabled || element.getAttribute('aria-disabled') === 'true' || element.getAttribute('data-disabled') === 'true') {
return false;
}
if (element.hidden || element.closest('[hidden], [inert], [aria-hidden="true"]')) {
return false;
}
if (element.closest('[data-message-author="user"], [data-role="user"]')) {
return false;
}
const style = getComputedStyle(element);
if (style.display === 'none' || style.visibility === 'hidden'
|| style.visibility === 'collapse' || Number(style.opacity) === 0) {
return false;
}
const rect = element.getBoundingClientRect();
return rect.width > 0 && rect.height > 0;
};
// 1. Indicator 1: Stop button & __arenaHardStop
let has_stop = false;
try {
if (window.__arenaHardStop && typeof window.__arenaHardStop.status === 'function') {
const st = window.__arenaHardStop.status();
if (st && (st.hasNativeStopButton || st.hasOverlayStopButton)) {
has_stop = true;
}
}
} catch (_) {}
if (!has_stop && selector) {
try {
if (Array.from(document.querySelectorAll(selector)).some(visible)) {
has_stop = true;
}
} catch (_) {}
}
if (!has_stop) {
try {
const overlaySelectors = [
'button[aria-label="Stop generation"]',
'[data-arena-hard-stop-overlay="true"]',
'button[aria-label="Hard stop Arena stream"]',
];
for (const sel of overlaySelectors) {
if (Array.from(document.querySelectorAll(sel)).some(visible)) {
has_stop = true;
break;
}
}
} catch (_) {}
}
// 2. Indicator 2: "Generating image..." text placeholder
let has_generating_text = false;
try {
const textPlaceholders = [
'generating image',
'generating images',
'creating image',
'creating your image',
'image is being generated',
'images are being generated',
'正在生成图片',
'正在生成图像',
'正在创建您的图片',
'图片正在生成',
'图像正在生成',
];
const textCandidates = Array.from(document.querySelectorAll(
'.text-shimmer, [class*="text-shimmer"], [class*="shimmer"], [data-testid*="generating"], div[role="status"]'
));
for (const el of textCandidates) {
if (el.closest('[data-message-author="user"], [data-role="user"]')) {
continue;
}
const text = (el.innerText || el.textContent || '').replace(/\s+/g, ' ').trim().toLowerCase();
if (text && text.length <= 160) {
if (textPlaceholders.some(p => text.includes(p))) {
if (visible(el)) {
has_generating_text = true;
break;
}
}
}
}
} catch (_) {}
// 3. Indicator 3: animate-spin canvas loading animation
let has_spin_canvas = false;
try {
const spinCanvases = Array.from(document.querySelectorAll(
'.animate-spin canvas, [class*="animate-spin"] canvas, canvas.animate-spin, canvas[class*="animate-spin"]'
));
for (const el of spinCanvases) {
if (visible(el)) {
has_spin_canvas = true;
break;
}
}
} catch (_) {}
const still_generating = Boolean(has_stop || (has_generating_text && has_spin_canvas));
return JSON.stringify({
has_stop: Boolean(has_stop),
has_generating_text: Boolean(has_generating_text),
has_spin_canvas: Boolean(has_spin_canvas),
still_generating: Boolean(still_generating),
});
"""
try:
result = driver_for_tab(tab).run_js(script, selector)
if isinstance(result, str):
# P0-2:页面内 JSON.stringify,避免 RemoteObject 泄漏
try:
result = json.loads(result)
except ValueError:
result = None
except Exception:
return {
"has_stop": False,
"has_generating_text": False,
"has_spin_canvas": False,
"still_generating": False,
}
if isinstance(result, dict):
has_stop = bool(result.get("has_stop", False))
has_generating_text = bool(result.get("has_generating_text", False))
has_spin_canvas = bool(result.get("has_spin_canvas", False))
still_generating = bool(
result.get(
"still_generating",
has_stop or (has_generating_text and has_spin_canvas),
)
)
return {
"has_stop": has_stop,
"has_generating_text": has_generating_text,
"has_spin_canvas": has_spin_canvas,
"still_generating": still_generating,
}
elif isinstance(result, bool):
return {
"has_stop": bool(result),
"has_generating_text": False,
"has_spin_canvas": False,
"still_generating": bool(result),
}
return {
"has_stop": False,
"has_generating_text": False,
"has_spin_canvas": False,
"still_generating": False,
}
def is_visible_arena_stop(tab: Any, selector: str = ARENA_NATIVE_STOP_SELECTOR) -> bool:
"""Return whether the Arena stop button / overlay is visible."""
status = get_arena_generation_status(tab, selector)
return bool(status.get("has_stop", False))
def capture_arena_result_baseline(
tab: Any,
result_selector: str,
) -> Optional[dict[str, Any]]:
"""Mark response nodes that existed immediately before request submission."""
token = uuid.uuid4().hex
script = r"""
return ((resultSelector, baselineToken, baselineProperty) => {
const selector = String(resultSelector || '').trim().replace(/^css:/i, '');
const normalize = (value) => String(value || '').replace(/\s+/g, ' ').trim();
let candidates = [];
if (selector) {
try {
candidates = Array.from(document.querySelectorAll(selector));
} catch (_) {
candidates = [];
}
}
if (!candidates.length) {
candidates = Array.from(document.querySelectorAll(
'[role="alert"], [role="status"]'
));
}
let markedCount = 0;
for (const element of candidates) {
const marker = {
token: String(baselineToken || ''),
text: normalize(element.innerText || element.textContent || ''),
};
try {
Object.defineProperty(element, baselineProperty, {
configurable: true,
writable: true,
value: marker,
});
markedCount += 1;
} catch (_) {
try {
element[baselineProperty] = marker;
markedCount += 1;
} catch (_) {}
}
}
return {ok: true, node_count: candidates.length, marked_count: markedCount};
})(arguments[0], arguments[1], arguments[2]);
"""
try:
result = _decode_js_result(driver_for_tab(tab).run_js(
_json_js(script),
result_selector,
token,
ARENA_RESULT_BASELINE_PROPERTY,
)) or {}
except Exception:
return None
if not isinstance(result, dict) or not bool(result.get("ok")):
return None
return {
"token": token,
"property": ARENA_RESULT_BASELINE_PROPERTY,
"node_count": int(result.get("node_count") or 0),
"marked_count": int(result.get("marked_count") or 0),
}
class ArenaImageGenerationGuard:
"""Arena-specific terminal-state and interrupted-stream recovery policy."""
DEFAULT_REFRESH_INTERVAL_SECONDS = 20.0
DEFAULT_MAX_REFRESHES = 16
def __init__(
self,
tab: Any,
*,
result_selector: str = "",
stop_selector: str = ARENA_NATIVE_STOP_SELECTOR,
baseline_token: str = "",
baseline_property: str = "",
):
self.tab = tab
self.result_selector = str(result_selector or "")
self.stop_selector = str(stop_selector or ARENA_NATIVE_STOP_SELECTOR)
self.baseline_token = str(baseline_token or "")
self.baseline_property = str(baseline_property or "")
def generation_status(self) -> dict[str, bool]:
return get_arena_generation_status(
self.tab,
self.stop_selector,
)
def native_stop_present(self) -> bool:
return bool(self.generation_status().get("has_stop", False))
def detect_terminal_error(self) -> Optional[ArenaImageGenerationError]:
"""Map the current Arena result/error node to a terminal failure.
Do not scan the whole conversation: an older turn can legitimately
contain the same error text and must not poison the current request.
"""
script = r"""
return ((resultSelector, baselineToken, baselineProperty) => {
const normalize = (value) => String(value || '')
.replace(/\s+/g, ' ').trim();
const visible = (element) => {
if (!(element instanceof Element) || !element.isConnected) return false;
const rect = element.getBoundingClientRect();
const style = getComputedStyle(element);
return rect.width > 0 && rect.height > 0
&& style.display !== 'none' && style.visibility !== 'hidden';
};
const selector = String(resultSelector || '').trim().replace(/^css:/i, '');
let candidates = [];
if (selector) {
try {
candidates = Array.from(document.querySelectorAll(selector)).filter(visible);
} catch (_) {
candidates = [];
}
}
if (!candidates.length) {
candidates = Array.from(document.querySelectorAll(
'[role="alert"], [role="status"]'
)).filter(visible);
}
if (baselineToken && baselineProperty) {
candidates = candidates.filter((element) => {
const marker = element[baselineProperty];
if (!marker || marker.token !== baselineToken) return true;
return normalize(element.innerText || element.textContent || '')
!== normalize(marker.text);
});
}
// Arena renders its conversation with flex-col-reverse, so the
// newest response is first in DOM order. Select by visual
// position instead: the current response is closest to the
// composer at the bottom of the conversation.
const current = candidates.reduce((lowest, element) => {
if (!lowest) return element;
return element.getBoundingClientRect().bottom
> lowest.getBoundingClientRect().bottom
? element : lowest;
}, null);
const text = current
? (current.innerText || current.textContent || '')
.replace(/\s+/g, ' ').trim()
: '';
const normalized = text.toLowerCase();
if (normalized.includes('this content violates our terms of use')) {
return {code: 'arena_prompt_rejected', text};
}
if (normalized.includes('something went wrong with this response, please try again')) {
return {code: 'arena_image_generation_failed', text};
}
return null;
})(arguments[0], arguments[1], arguments[2]);
"""
try:
if self.result_selector or self.baseline_token or self.baseline_property:
result = driver_for_tab(self.tab).run_js(
_json_js(script),
self.result_selector,
self.baseline_token,
self.baseline_property,
)
else:
# Preserve compatibility with lightweight tab doubles and
# older wrappers whose run_js accepts only the script.
result = driver_for_tab(self.tab).run_js(_json_js(script))
except Exception:
return None
result = _decode_js_result(result)
if isinstance(result, dict):
code = str(result.get("code") or "").strip()
text = str(result.get("text") or "").strip()
else:
code = ""
text = str(result or "").strip()
normalized = text.lower()
if code == ARENA_PROMPT_REJECTED_CODE or "this content violates our terms of use" in normalized:
return ArenaImageGenerationError(
ARENA_PROMPT_REJECTED_CODE,
"Arena 拒绝了该图片请求:内容违反 Terms of Use",
)
if code == ARENA_IMAGE_GENERATION_FAILED_CODE or (
"something went wrong with this response, please try again" in normalized
):
return ArenaImageGenerationError(
ARENA_IMAGE_GENERATION_FAILED_CODE,
"Arena 图片生成失败:Something went wrong with this response, please try again.",
)
return None
def observe(self, has_new_image: bool) -> ArenaImageGenerationObservation:
status = self.generation_status()
obs = ArenaImageGenerationObservation(
stop_present=bool(status.get("has_stop", False)),
has_new_image=bool(has_new_image),
terminal_error=self.detect_terminal_error(),
has_stop=bool(status.get("has_stop", False)),
has_generating_text=bool(status.get("has_generating_text", False)),
has_spin_canvas=bool(status.get("has_spin_canvas", False)),
still_generating=bool(status.get("still_generating", False)),
)
current_state = (
obs.still_generating,
obs.has_new_image,
obs.is_complete,
bool(obs.terminal_error),
)
if getattr(self, "_last_logged_state", None) != current_state:
self._last_logged_state = current_state
logger.debug(
f"[Arena Guard] 生图状态变化: still_generating={obs.still_generating}, "
f"has_new_image={obs.has_new_image}, complete={obs.is_complete}, "
f"stop={obs.has_stop}, text={obs.has_generating_text}, spin={obs.has_spin_canvas}, "
f"terminal_error={bool(obs.terminal_error)}"
)
return obs
@classmethod
def refresh_interval_seconds(cls, image_config: Optional[dict[str, Any]]) -> float:
config = image_config or {}
value = config.get("arena_image_refresh_interval_seconds")
# Do not inherit the old generic DOM recovery interval here. Arena image
# recovery has a separate contract: absent an explicit Arena override,
# its first and subsequent reloads happen every ten seconds.
try:
return max(1.0, float(value)) if value is not None else cls.DEFAULT_REFRESH_INTERVAL_SECONDS
except (TypeError, ValueError):
return cls.DEFAULT_REFRESH_INTERVAL_SECONDS
@classmethod
def max_refreshes(
cls,
image_config: Optional[dict[str, Any]],
hard_timeout_seconds: float,
) -> int:
config = image_config or {}
value = config.get("arena_image_max_refreshes")
if value is None:
value = config.get("dom_interrupted_max_refreshes")
if value is not None:
try:
return max(1, int(value))
except (TypeError, ValueError):
pass
interval = cls.refresh_interval_seconds(config)
return min(
cls.DEFAULT_MAX_REFRESHES,
max(12, int(max(1.0, float(hard_timeout_seconds)) / interval) + 1),
)
_read_local_image = read_local_image_bytes
def _read_uploaded_image(value: Any) -> bytes:
return _common_read_uploaded_image_bytes(value, local_reader=_read_local_image)
def _media_item_image_bytes(tab: Any, item: dict[str, Any]) -> bytes:
return _common_extract_media_item_bytes(tab, item, local_reader=_read_local_image)
def validate_generated_images(
uploaded_images: Iterable[Any],
generated_media_items: Iterable[dict[str, Any]],
*,
tab: Any = None,
) -> None:
"""Reject Arena image output that is the uploaded reference image itself."""
_common_validate_generated_images(
uploaded_images,
generated_media_items,
tab=tab,
max_dhash_distance=None,
local_reader=_read_local_image,
error_builder=lambda msg: ArenaImageGenerationError(
ARENA_IMAGE_UNCHANGED_CODE,
f"Arena 图片生成失败:{msg}",
),
)
from app.services.arena_image_stream_observer import ArenaImageStreamObserver
__all__ = [
"ARENA_IMAGE_GENERATION_FAILED_CODE",
"ARENA_IMAGE_UNCHANGED_CODE",
"ARENA_NATIVE_STOP_SELECTOR",
"ARENA_NON_RETRYABLE_CODES",
"ARENA_PROMPT_REJECTED_CODE",
"ARENA_RESULT_BASELINE_PROPERTY",
"ArenaImageGenerationError",
"ArenaImageGenerationGuard",
"ArenaImageGenerationObservation",
"ArenaImageStreamObserver",
"auto_skip_arena_direct_comparison",
"detect_arena_render_crash",
"evaluate_arena_direct_generation_state",
"current_page_url",
"capture_arena_result_baseline",
"get_arena_generation_status",
"image_signatures",
"is_arena_image_generation_request",
"is_arena_page_url",
"is_interrupted_stream_reason",
"is_visible_arena_stop",
"looks_like_image_generation_request",
"read_image_bytes",
"same_image",
"validate_generated_images",
]
+204
View File
@@ -0,0 +1,204 @@
"""
app/services/arena_image_stream_observer.py - Arena 生图守卫与流式监控观察者
将 Arena 专属的生图状态机、跳过比较逻辑与原生停止按钮守卫封装为 StreamObserver,
供 StreamMonitor 在生图会话中挂载。
"""
from typing import Any, Dict, List, Optional, Tuple
from app.core.config import logger
from app.core.stream_observer import (
StreamObserver,
register_stream_observer_factory,
)
from app.services.arena_image_generation import (
ARENA_NATIVE_STOP_SELECTOR as ARENA_IMAGE_NATIVE_STOP_SELECTOR,
ArenaImageGenerationError,
ArenaImageGenerationGuard,
auto_skip_arena_direct_comparison,
evaluate_arena_direct_generation_state,
is_arena_image_generation_request,
)
class ArenaImageStreamObserver(StreamObserver):
"""Arena 生图守卫与流式状态机观察者。"""
def __init__(
self,
guard: Optional[ArenaImageGenerationGuard] = None,
image_config: Optional[Dict[str, Any]] = None,
):
self.guard = guard
self.image_config = image_config or {}
self.last_observation = None
self.last_direct_eval: Optional[Dict[str, Any]] = None
def on_stream_start(self, monitor: Any, ctx: Any) -> None:
if self.guard is None and hasattr(monitor, "tab"):
selector = getattr(monitor, "selector", "")
token = self.image_config.get("arena_result_baseline_token", "")
prop = self.image_config.get("arena_result_baseline_property", "")
self.guard = ArenaImageGenerationGuard(
monitor.tab,
result_selector=selector,
baseline_token=token,
baseline_property=prop,
)
def _evaluate_direct_state(
self,
monitor: Any,
ctx: Any,
current_has_new_rendered_image: bool,
still_generating: bool,
) -> Tuple[bool, bool, Optional[Dict[str, Any]]]:
image_config = getattr(monitor, "_image_config", None) or self.image_config or {}
parser_id = str(image_config.get("_parser_id", "") or getattr(monitor, "_workflow_parser_id", "") or "").lower()
parser_side = str(image_config.get("_parser_target_side", "") or "").strip().lower()
is_battle = "battle" in parser_id or parser_side in {"left", "right"}
tab = getattr(monitor, "tab", None)
if is_battle or not tab:
return current_has_new_rendered_image, still_generating, None
try:
auto_skip_arena_direct_comparison(tab)
baseline_depth = int(ctx.baseline_snapshot.get("groups_count") or 0) if ctx.baseline_snapshot else 0
prompt_text = str(getattr(monitor, "_prompt", "") or getattr(monitor, "prompt", "") or getattr(ctx, "prompt", "") or "")
eval_data = evaluate_arena_direct_generation_state(
tab,
baseline_depth=baseline_depth,
current_prompt=prompt_text,
stop_selector=ARENA_IMAGE_NATIVE_STOP_SELECTOR,
)
if isinstance(eval_data, dict):
if eval_data.get("error_code"):
raise ArenaImageGenerationError(
str(eval_data.get("error_code")),
str(eval_data.get("error_msg") or "Arena image generation error"),
)
status = eval_data.get("status")
if status == "SUCCESS" and eval_data.get("image_urls"):
urls = [u for u in eval_data.get("image_urls") or [] if isinstance(u, str) and u.strip()]
if urls:
monitor._final_image_urls = list(urls)
if hasattr(monitor, "_prefetch_snapshot_image_urls"):
monitor._prefetch_snapshot_image_urls({"image_urls": urls, "image_references": urls})
return True, False, eval_data
if status in {"GENERATING", "WAITING_HYDRATION"}:
return current_has_new_rendered_image, True, eval_data
return current_has_new_rendered_image, still_generating, eval_data
return current_has_new_rendered_image, still_generating, None
except ArenaImageGenerationError:
raise
except Exception as exc:
logger.debug(f"Direct generation evaluation non-fatal error: {exc}")
return current_has_new_rendered_image, still_generating, None
def on_poll_tick(
self,
monitor: Any,
ctx: Any,
snapshot: Dict[str, Any],
state: Dict[str, Any],
) -> None:
current_has_new_rendered_image = state.get("current_has_new_rendered_image", False)
still_generating = state.get("still_generating", False)
interrupted_stream_recovery = state.get("interrupted_stream_recovery", False)
stream_recovery_refresh_done = state.get("stream_recovery_refresh_done", False)
post_refresh_settled = state.get("post_refresh_settled", True)
try:
(
current_has_new_rendered_image,
still_generating,
direct_eval,
) = self._evaluate_direct_state(
monitor,
ctx,
current_has_new_rendered_image,
still_generating,
)
except ArenaImageGenerationError as exc:
state["terminal_error"] = exc
state["terminal_break"] = True
return
self.last_direct_eval = direct_eval
state["current_has_new_rendered_image"] = current_has_new_rendered_image
if self.guard is not None:
arena_observation = self.guard.observe(current_has_new_rendered_image)
self.last_observation = arena_observation
if direct_eval is not None:
still_generating = bool(direct_eval.get("still_generating", arena_observation.still_generating))
else:
still_generating = arena_observation.still_generating
state["still_generating"] = still_generating
recovery_has_output = bool(arena_observation.has_new_image or arena_observation.terminal_error)
state["recovery_has_output"] = recovery_has_output
if direct_eval is not None:
recovery_confirmed = bool(
direct_eval.get("is_complete")
and (not stream_recovery_refresh_done or post_refresh_settled)
)
else:
recovery_confirmed = bool(
arena_observation.is_complete
and (
(
interrupted_stream_recovery
and stream_recovery_refresh_done
and (post_refresh_settled or arena_observation.has_new_image)
)
or not interrupted_stream_recovery
)
)
state["recovery_confirmed"] = recovery_confirmed
if arena_observation.terminal_error is not None:
state["terminal_error"] = arena_observation.terminal_error
if recovery_confirmed:
state["terminal_break"] = True
logger.info("[Arena Image] 原生停止按钮已消失且已确认新图片,生成完成")
def get_active_stop_recovery_grace_seconds(self, default: float) -> Optional[float]:
cfg = getattr(self.guard, "image_config", None) or self.image_config
val = cfg.get("arena_active_stop_recovery_grace_seconds", 60.0)
try:
return float(val)
except (TypeError, ValueError, OverflowError):
return 60.0
def get_stream_recovery_max_refreshes(self, default: int) -> Optional[int]:
cfg = getattr(self.guard, "image_config", None) or self.image_config
hard_timeout = default * 20.0
return ArenaImageGenerationGuard.max_refreshes(cfg, hard_timeout)
def get_interrupted_refresh_interval(self, default: float) -> Optional[float]:
cfg = getattr(self.guard, "image_config", None) or self.image_config
return ArenaImageGenerationGuard.refresh_interval_seconds(cfg)
def _arena_image_observer_factory(
tab: Any,
user_input: Any,
image_config: Dict[str, Any],
) -> Optional[ArenaImageStreamObserver]:
try:
url = str(getattr(tab, "url", "") or "")
uploaded = image_config.get("uploaded_image_paths") or []
if is_arena_image_generation_request(url, user_input, image_config, uploaded):
return ArenaImageStreamObserver(image_config=image_config)
except Exception as exc:
logger.debug(f"[StreamObserver] Arena 生图观察者工厂检查失败(忽略): {exc}")
return None
register_stream_observer_factory(_arena_image_observer_factory)
__all__ = ["ArenaImageStreamObserver"]
+751
View File
@@ -0,0 +1,751 @@
"""Arena direct model catalog persistence, migration, and dark pool filtering."""
from __future__ import annotations
import copy
import json
import os
import re
import threading
from datetime import datetime, timezone, timedelta
from pathlib import Path
from typing import Any, Dict, List, Optional, Set, Tuple
from app.core.config import logger
from app.utils.site_url import extract_remote_site_domain, route_domain_matches
ARENA_MODEL_CATALOG_CONFIG_PATH = Path(
os.getenv(
"ARENA_MODEL_CATALOG_CONFIG_PATH",
str(Path(__file__).resolve().parents[2] / "config" / "arena_model_catalog.local.json"),
)
)
ARENA_MODELS_CACHE_PATH = Path(
os.getenv(
"ARENA_MODELS_CACHE_PATH",
str(Path(__file__).resolve().parents[2] / "scripts" / "arena_models_cache.json"),
)
)
MODEL_CATALOG_SOURCE = "arena_direct"
ONLINE_PUBLIC_VISIBLE_MODELS: Set[str] = {
"gemini-3-flash", "glm-5.1", "qwen3.5-397b-a17b", "claude-sonnet-4-5-20250929",
"gemini-3.1-pro-preview", "qwen3.7-plus", "minimax-m3", "claude-haiku-4-5-20251001",
"gemini-2.5-pro", "glm-5v-turbo", "grok-4.20-beta-0309-reasoning", "gpt-5.2-high",
"gpt-5.5-instant", "gpt-5.1", "gpt-5.2", "gemini-3.6-flash", "claude-sonnet-4-6",
"grok-4.20-multi-agent-beta-0309", "qwen3.5-max-preview", "gemini-3.5-flash-lite",
"glm-5", "claude-sonnet-4-5-20250929-thinking-32k", "gpt-5.1-high", "gpt-5.4-mini-high",
"glm-4.7", "qwen3-max-preview", "gpt-5-high", "kimi-k2.5-instant", "o3-2025-04-16",
"kimi-k2-thinking-turbo", "gpt-5-chat", "qwen3-max-2025-09-23", "qwen3-235b-a22b-instruct-2507",
"kimi-k2-0711-preview", "kimi-k2-0905-preview", "qwen3.5-122b-a10b", "minimax-m2.7",
"qwen3-vl-235b-a22b-instruct", "mistral-large-3", "gpt-4.1-2025-04-14", "gemini-2.5-flash",
"mistral-medium-2508", "qwen3.5-27b", "inkling-small", "qwen3-235b-a22b-no-thinking",
"gpt-5.4-nano-high", "longcat-flash-chat", "qwen3-next-80b-a3b-instruct",
"claude-sonnet-4-20250514-thinking-32k", "qwen3-235b-a22b-thinking-2507", "qwen3.5-flash",
"qwen3.5-35b-a3b", "hunyuan-vision-1.5-thinking", "qwen3-vl-235b-a22b-thinking",
"step-3.5-flash", "minimax-m2.5", "o4-mini-2025-04-16", "gpt-5-mini-high",
"claude-sonnet-4-20250514", "qwen3-coder-480b-a35b-instruct", "minimax-m2.1-preview",
"qwen3-30b-a3b-instruct-2507", "gpt-4.1-mini-2025-04-14", "trinity-large-preview",
"qwen3-235b-a22b", "trinity-large-thinking", "qwen3-next-80b-a3b-thinking",
"gemma-3-27b-it", "minimax-m1", "gemini-2.0-flash-001", "intellect-3",
"gemma-3-12b-it", "o3-mini-high", "gemma-3-4b-it", "mistral-small-3.2",
"qwen3-vl-30b-a3b-instruct", "gemini-2.5-flash-lite", "qwen3-30b-a3b-thinking-2507",
"qwen3-30b-a3b", "deepseek-v3.2", "gemma-3-1b-it", "deepseek-v4-flash",
"o3-mini-2025-01-31", "glm-4-flash", "gemini-2.0-flash-thinking-exp-01-21",
"qwen2.5-max-preview", "claude-3.5-haiku-20241022", "claude-3.7-sonnet",
"step-2-16k-exp", "claude-3.7-sonnet-thinking", "deepseek-r1-distill-qwen-32b",
"deepseek-v3.1", "deepseek-v4", "claude-3.5-sonnet-20241022", "gpt-4o-2024-11-20",
"deepseek-v4-pro-max", "claude-sonnet-5-high", "gpt-5.4-mini", "gemini-3.7-pro",
"gemini-3.7-pro-high", "glm-5.2 (max)", "deepseek-v4-flash-20260731",
"gpt-5.4-high", "claude-sonnet-5-search", "gemini-3.5-pro-high",
"gemini-3.5-flash-high", "deepseek-v4-flash-high", "gemini-3.5-flash",
"gpt-5.4-turbo", "gpt-5.4", "gemini-3.5-pro", "gemini-2.5-pro-high",
"gemini-3-flash-high", "gemini-3-pro-high", "gemini-3-flash-search",
"gemini-2.5-ultra", "gemini-3-pro", "gemini-2.5-flash-thinking",
"gemini-2.5-flash-high", "gemini-3.7-flash-high", "qwen3.5-122b-a10b-thinking",
"qwen3.5-35b-a3b-thinking", "claude-opus-4.5", "claude-opus-4.5-thinking",
"claude-opus-4.6", "claude-opus-4.6-thinking", "claude-opus-4.1",
"claude-opus-4.1-thinking", "claude-opus-4", "claude-opus-4-thinking",
"o4-mini-high", "o4-mini", "o4-max", "o4-high", "o3-pro-max", "o3-pro",
"gpt-5.3-high", "gpt-5.3-mini", "gpt-5.3", "gpt-5.2-mini", "gpt-5.2-turbo",
"gpt-5.1-turbo", "gpt-5.1-mini", "gpt-5.1-codex-max", "gpt-5-turbo", "gpt-5-mini",
"inkling-low", "qwen3-omni-flash", "inkling-small-low", "inkling-medium",
"gemini-3.7-flash", "mistral-medium-3.5", "gemini-3-flash (thinking-minimal)",
"qwen3.7-plus-preview", "grok-4.6-medium", "grok-4.5"
}
_catalog_io_lock = threading.RLock()
def _canonical_modality(value: Any) -> str:
"""Normalize Arena's page/API terminology to the catalog terminology."""
modality = str(value or "").strip().lower()
return {
"webdev": "code",
"web": "code",
"chat": "text",
}.get(modality, modality)
def _keywords_list(source: Any) -> List[str]:
if isinstance(source, str):
source = re.split(r"[\n,]+", source)
if not isinstance(source, (list, tuple, set)):
return []
result: List[str] = []
seen = set()
for item in source:
keyword = str(item or "").strip()
folded = keyword.casefold()
if not keyword or folded in seen:
continue
seen.add(folded)
result.append(keyword)
return result
def normalize_arena_model_catalog_config(value: Any) -> Dict[str, Any]:
raw = value if isinstance(value, dict) else {}
result = {
"enabled": bool(raw.get("enabled", False)),
"source": str(raw.get("source") or MODEL_CATALOG_SOURCE).strip() or MODEL_CATALOG_SOURCE,
"include_keywords": _keywords_list(raw.get("include_keywords")),
"exclude_keywords": _keywords_list(raw.get("exclude_keywords")),
"enable_dark_pool": bool(raw.get("enable_dark_pool", False)),
"dark_pool_since": str(raw.get("dark_pool_since") or "").strip(),
"dark_pool_whitelist_keywords": _keywords_list(raw.get("dark_pool_whitelist_keywords")),
"dark_pool_blacklist_keywords": _keywords_list(raw.get("dark_pool_blacklist_keywords")),
}
if "modality" in raw and str(raw.get("modality") or "").strip():
result["modality"] = _canonical_modality(raw.get("modality"))
return result
def _default_arena_model_catalog_config(domain: str, preset_name: str) -> Optional[Dict[str, Any]]:
"""Return built-in catalog defaults for presets that should start enabled."""
normalized_domain = str(domain or "").strip().lower().strip(".")
normalized_preset = str(preset_name or "").strip()
if route_domain_matches("arena.ai", normalized_domain) and normalized_preset == "主预设-直连模式":
return normalize_arena_model_catalog_config(
{
"enabled": True,
"source": MODEL_CATALOG_SOURCE,
"modality": "text",
}
)
return None
def load_arena_model_catalog_data() -> Dict[str, Dict[str, Any]]:
"""加载独立配置文件,返回 { domain: { preset_name: catalog_config } }"""
with _catalog_io_lock:
if not ARENA_MODEL_CATALOG_CONFIG_PATH.exists():
return {}
try:
content = ARENA_MODEL_CATALOG_CONFIG_PATH.read_text(encoding="utf-8").strip()
if not content:
return {}
data = json.loads(content)
if not isinstance(data, dict):
return {}
normalized: Dict[str, Dict[str, Any]] = {}
for domain, presets in data.items():
if not isinstance(presets, dict):
continue
d_key = str(domain).strip().lower()
normalized[d_key] = {}
for preset_name, cat in presets.items():
p_key = str(preset_name).strip()
if isinstance(cat, dict):
normalized[d_key][p_key] = normalize_arena_model_catalog_config(cat)
return normalized
except Exception as e:
logger.warning(f"读取 Arena 独立模型目录配置失败: {e}")
return {}
def save_arena_model_catalog_data(data: Dict[str, Any]) -> bool:
"""原子保存独立配置文件"""
with _catalog_io_lock:
try:
ARENA_MODEL_CATALOG_CONFIG_PATH.parent.mkdir(parents=True, exist_ok=True)
tmp_file = ARENA_MODEL_CATALOG_CONFIG_PATH.with_suffix(".tmp")
with open(tmp_file, "w", encoding="utf-8", newline="\n") as f:
json.dump(data, f, indent=2, ensure_ascii=False)
f.flush()
os.fsync(f.fileno())
os.replace(tmp_file, ARENA_MODEL_CATALOG_CONFIG_PATH)
logger.info(f"Arena 独立模型目录配置已保存: {ARENA_MODEL_CATALOG_CONFIG_PATH}")
return True
except Exception as e:
logger.error(f"保存 Arena 独立模型目录配置失败: {e}")
return False
def get_arena_model_catalog(
domain: str = "arena.ai",
preset_name: Optional[str] = None,
config_engine: Any = None,
) -> Dict[str, Any]:
"""获取指定预设的 Arena 模型目录配置。支持旧配置兼容回退。"""
normalized_domain = str(domain or "arena.ai").strip().lower().strip(".")
catalog_data = load_arena_model_catalog_data()
domain_catalogs = catalog_data.get(normalized_domain, {})
target_preset = str(preset_name or "").strip()
if not target_preset and config_engine:
try:
target_preset = str(config_engine.get_default_preset(normalized_domain) or "").strip()
except Exception:
pass
if not target_preset:
target_preset = "主预设"
if target_preset in domain_catalogs:
return copy.deepcopy(domain_catalogs[target_preset])
# 兼容期回退:若独立配置中无该预设,检查 config_engine 站点预设中是否有旧 model_catalog
if config_engine:
try:
presets = None
if hasattr(config_engine, "sites") and isinstance(config_engine.sites, dict):
site = config_engine.sites.get(normalized_domain, {})
if isinstance(site, dict):
presets = site.get("presets")
if presets is None and hasattr(config_engine, "presets") and isinstance(config_engine.presets, dict):
presets = config_engine.presets
if isinstance(presets, dict):
resolved_key = config_engine._resolve_preset_alias_key(target_preset, presets) if hasattr(config_engine, "_resolve_preset_alias_key") else target_preset
preset_dict = presets.get(resolved_key)
if isinstance(preset_dict, dict) and "model_catalog" in preset_dict:
return normalize_arena_model_catalog_config(preset_dict.get("model_catalog"))
if hasattr(config_engine, "_get_site_data_readonly"):
preset_dict = config_engine._get_site_data_readonly(normalized_domain, target_preset)
if isinstance(preset_dict, dict) and "model_catalog" in preset_dict:
return normalize_arena_model_catalog_config(preset_dict.get("model_catalog"))
except Exception:
pass
default_catalog = _default_arena_model_catalog_config(normalized_domain, target_preset)
if default_catalog is not None:
return default_catalog
return normalize_arena_model_catalog_config({})
def set_arena_model_catalog(
domain: str = "arena.ai",
preset_name: Optional[str] = None,
catalog_config: Any = None,
) -> Dict[str, Any]:
"""设置并原子保存指定预设的 Arena 模型目录配置"""
normalized_domain = str(domain or "arena.ai").strip().lower().strip(".")
target_preset = str(preset_name or "主预设").strip()
normalized_cat = normalize_arena_model_catalog_config(catalog_config)
with _catalog_io_lock:
catalog_data = load_arena_model_catalog_data()
if normalized_domain not in catalog_data:
catalog_data[normalized_domain] = {}
catalog_data[normalized_domain][target_preset] = normalized_cat
if not save_arena_model_catalog_data(catalog_data):
raise IOError(f"保存 Arena 独立模型目录配置失败: {normalized_domain} [{target_preset}]")
return normalized_cat
def migrate_and_cleanup_sites_model_catalog(config_engine: Any) -> bool:
"""扫描 config_engine.sites,将各预设下的 model_catalog 迁移到独立文件,并从 sites.json 中清理旧字段"""
if not config_engine or not hasattr(config_engine, "sites") or not isinstance(config_engine.sites, dict):
return False
with _catalog_io_lock:
catalog_data = load_arena_model_catalog_data()
catalog_data_changed = False
to_cleanup = []
for domain, site_config in config_engine.sites.items():
if domain.startswith("_") or not isinstance(site_config, dict):
continue
d_key = str(domain).strip().lower().strip(".")
if not route_domain_matches("arena.ai", d_key):
continue
presets = site_config.get("presets")
if not isinstance(presets, dict):
continue
if d_key not in catalog_data:
catalog_data[d_key] = {}
for p_name, p_data in presets.items():
if not isinstance(p_data, dict):
continue
p_key = str(p_name).strip()
if "model_catalog" in p_data:
old_cat = p_data.get("model_catalog")
# 如果独立文件没有该预设配置,则补全迁移
if p_key not in catalog_data[d_key]:
catalog_data[d_key][p_key] = normalize_arena_model_catalog_config(old_cat)
catalog_data_changed = True
logger.info(f"已迁移站点 {domain} 预设 [{p_name}] 的 model_catalog 到独立配置文件")
to_cleanup.append((p_data, d_key, p_key))
if catalog_data_changed:
if not save_arena_model_catalog_data(catalog_data):
logger.error("保存独立模型目录配置文件失败,取消清理 sites.json 中的 model_catalog")
return False
# 只有在独立配置文件确认已包含该配置时,才安全清理内存与 sites.json
sites_changed = False
for p_data, d_key, p_key in to_cleanup:
if d_key in catalog_data and p_key in catalog_data[d_key]:
if "model_catalog" in p_data:
p_data.pop("model_catalog", None)
sites_changed = True
if sites_changed:
try:
config_engine.save_config()
logger.info("已从 sites.json 中清理已迁移的 model_catalog 残留字段")
except Exception as e:
logger.warning(f"清理 sites.json 中的 model_catalog 后保存失败: {e}")
return catalog_data_changed or sites_changed
def get_model_catalog_preset(config_engine: Any, domain: Any) -> Optional[Dict[str, Any]]:
"""读取指定域名下已启用 model_catalog 的预设信息"""
normalized_domain = str(domain or "").strip().lower().strip(".")
if not normalized_domain or not route_domain_matches("arena.ai", normalized_domain):
return None
try:
if hasattr(config_engine, "refresh_if_changed"):
config_engine.refresh_if_changed()
site = config_engine.sites.get(normalized_domain) if hasattr(config_engine, "sites") else None
except Exception:
site = None
presets = site.get("presets") if isinstance(site, dict) else (
getattr(config_engine, "presets", {}) if hasattr(config_engine, "presets") else {}
)
catalog_data = load_arena_model_catalog_data()
domain_catalogs = catalog_data.get(normalized_domain, {})
# 优先检查默认预设
default_preset = ""
if hasattr(config_engine, "get_default_preset"):
try:
default_preset = str(config_engine.get_default_preset(normalized_domain) or "").strip()
except Exception:
pass
all_preset_names = []
if default_preset:
all_preset_names.append(default_preset)
if isinstance(presets, dict):
for p_name in presets.keys():
if str(p_name) not in all_preset_names:
all_preset_names.append(str(p_name))
for p_name in domain_catalogs.keys():
if str(p_name) not in all_preset_names:
all_preset_names.append(str(p_name))
for preset_name in all_preset_names:
cat = domain_catalogs.get(preset_name)
if not cat:
# 兼容回退
cat = get_arena_model_catalog(normalized_domain, preset_name, config_engine=config_engine)
if cat and cat.get("enabled") and cat.get("source") == MODEL_CATALOG_SOURCE:
preset = presets.get(preset_name, {}) if isinstance(presets, dict) else {}
return {
"preset_name": str(preset_name),
"preset": preset,
"catalog": cat,
}
return None
def get_arena_direct_catalog_for_tab(
config_engine: Any,
tab: Any,
*,
preset_name: Any = None,
) -> Optional[Dict[str, Any]]:
"""获取当前标签页对应的 Arena 直连目录配置"""
if not isinstance(tab, dict):
return None
status = str(tab.get("status") or "").strip().lower()
if status not in {"idle", "busy"} or bool(tab.get("terminating")):
return None
current_url = str(tab.get("url") or tab.get("current_url") or "").strip()
effective_preset_name = str(preset_name or tab.get("preset_name") or "").strip()
try:
config_engine.refresh_if_changed()
if not effective_preset_name:
effective_preset_name = str(
config_engine.get_default_preset("arena.ai") or ""
).strip()
preset = config_engine._get_site_data_readonly(
"arena.ai",
effective_preset_name or None,
)
except Exception:
return None
if not isinstance(preset, dict):
return None
catalog = get_arena_model_catalog("arena.ai", effective_preset_name, config_engine=config_engine)
if not catalog.get("enabled") or catalog.get("source") != MODEL_CATALOG_SOURCE:
return None
from app.services.arena_direct_models import _is_arena_direct_url
if not _is_arena_direct_url(current_url, catalog_preset=preset, catalog_config=catalog):
return None
return {
"preset_name": effective_preset_name,
"preset": preset,
"catalog": catalog,
}
# ==============================================================================
# 暗池模型判定、时间戳解析与全量缓存加载
# ==============================================================================
def parse_uuidv7_timestamp(uuid_str: str) -> Optional[float]:
"""从 UUIDv7 中解析毫秒级时间戳"""
if not uuid_str or not isinstance(uuid_str, str):
return None
clean_hex = uuid_str.replace("-", "").strip().lower()
if len(clean_hex) != 32 or clean_hex[12] != "7":
return None
try:
ts_ms = int(clean_hex[:12], 16)
if 1577836800000 <= ts_ms <= 1893456000000:
return float(ts_ms)
except Exception:
pass
return None
def parse_date_to_timestamp(date_str: str) -> Optional[float]:
"""将 YYYY-MM-DD 字符串转为 UTC 00:00:00 毫秒时间戳"""
if not date_str or not isinstance(date_str, str):
return None
clean_str = date_str.strip()
m = re.match(r"^(\d{4})[-_/](\d{1,2})[-_/](\d{1,2})$", clean_str)
if m:
try:
year, month, day = int(m.group(1)), int(m.group(2)), int(m.group(3))
dt = datetime(year, month, day, 0, 0, 0, tzinfo=timezone.utc)
return dt.timestamp() * 1000.0
except Exception:
return None
return None
def get_model_timestamp(model: Dict[str, Any]) -> Optional[float]:
"""获取模型入库时间戳(ms)"""
mid = str(model.get("arena_model_id") or model.get("id") or "")
ts = parse_uuidv7_timestamp(mid)
if ts is not None:
return ts
name_str = " ".join(
str(model.get(k) or "") for k in ("name", "displayName", "display_name", "publicName", "public_name")
)
m = re.search(r"202[4-7][_-]?(0[1-9]|1[0-2])[_-]?([0-3][0-9])", name_str)
if m:
try:
match_full = m.group(0).replace("_", "-")
digits = re.sub(r"\D", "", match_full)
if len(digits) == 8:
year, month, day = int(digits[:4]), int(digits[4:6]), int(digits[6:])
dt = datetime(year, month, day, 0, 0, 0, tzinfo=timezone.utc)
return dt.timestamp() * 1000.0
except Exception:
pass
return None
def is_dark_pool_model(model: Dict[str, Any]) -> bool:
"""判定是否属于暗池(在前端模型选择器中隐藏或来自离线暗池缓存)"""
if "_is_dark_pool" in model:
return bool(model["_is_dark_pool"])
if model.get("is_dark_pool") is not None:
return bool(model.get("is_dark_pool"))
disp = str(model.get("display_name") or model.get("displayName") or "").strip().lower()
name = str(model.get("name") or "").strip().lower()
pub = str(model.get("public_name") or model.get("publicName") or "").strip().lower()
search_str = f"{disp} {name} {pub}"
# 包含明确内部/测试特征的直接归为暗池
if any(k in search_str for k in ["internal", "test", "dlp", "fireworks", "node"]):
return True
# 命中公开白名单的必定为明池
for pub_name in ONLINE_PUBLIC_VISIBLE_MODELS:
pub_lower = pub_name.lower()
if disp == pub_lower or pub == pub_lower or name == pub_lower:
return False
# 来自离线暗池全量缓存且不在公开白名单的归为暗池
if model.get("_is_offline_cache"):
return True
# 实时从页面抓取(或单测传入)的普通模型默认视为明池
return False
_offline_cache_lock = threading.Lock()
_offline_cached_models: Optional[List[Dict[str, Any]]] = None
def load_arena_offline_cache_models() -> List[Dict[str, Any]]:
"""加载本地 scripts/arena_models_cache.json 中的模型并规范化"""
global _offline_cached_models
with _offline_cache_lock:
if _offline_cached_models is not None:
return _offline_cached_models
if not ARENA_MODELS_CACHE_PATH.exists():
_offline_cached_models = []
return []
try:
with open(ARENA_MODELS_CACHE_PATH, "r", encoding="utf-8") as f:
raw_list = json.load(f)
if not isinstance(raw_list, list):
_offline_cached_models = []
return []
results = []
for item in raw_list:
if not isinstance(item, dict):
continue
mid = str(item.get("id") or "").strip()
if not mid:
continue
caps = item.get("capabilities") or {}
out_caps = caps.get("outputCapabilities") or {}
ranks = item.get("rankByModality") or {}
def rank_is_finite(key):
val = ranks.get(key)
return isinstance(val, (int, float)) and val < 9007199254740991
has_text = bool(out_caps.get("text"))
has_image = bool(out_caps.get("image")) or rank_is_finite("image")
has_video = bool(out_caps.get("video")) or rank_is_finite("video")
# `outputCapabilities.web` is browsing/search support, not
# the WebDev/code modality. Use only the webdev rank here.
has_web = rank_is_finite("webdev") or bool(out_caps.get("web"))
has_search = bool(out_caps.get("search")) or rank_is_finite("search")
disp_name = str(item.get("displayName") or item.get("publicName") or item.get("name") or mid).strip()
internal_name = str(item.get("name") or item.get("id") or disp_name).strip()
name_search = f"{internal_name} {disp_name} {mid}".lower()
is_known_image = any(k in name_search for k in [
"gpt-image", "mona-lisa", "luna-lisa", "lina-alpha", "lina-f-alpha", "silver_halide",
"flux", "seedream", "seededit", "imagine", "imagen", "z-image", "midjourney", "dall-e", "recraft", "krea"
])
modality = "text"
if has_video and not has_text and not is_known_image:
modality = "video"
elif is_known_image or (has_image and not has_text):
modality = "image"
elif has_web and not has_text:
modality = "code"
elif has_text:
modality = "text"
elif has_web:
modality = "code"
elif has_search:
modality = "search"
elif has_image:
modality = "image"
elif has_video:
modality = "video"
results.append({
"arena_model_id": mid,
"name": internal_name,
"public_name": disp_name,
"display_name": disp_name,
"provider": str(item.get("provider") or ""),
"organization": str(item.get("organization") or item.get("provider") or "arena.ai"),
"modality": modality,
"_is_offline_cache": True,
})
_offline_cached_models = results
return results
except Exception as e:
logger.warning(f"读取 Arena 离线模型缓存失败: {e}")
_offline_cached_models = []
return []
# ==============================================================================
# 模型过滤引擎(明池 + 暗池)
# ==============================================================================
def filter_arena_catalog_models(
models: List[Dict[str, Any]],
catalog_config: Any,
) -> List[Dict[str, Any]]:
"""对模型列表执行明池与暗池的分别过滤与合并去重排序"""
from app.services.arena_direct_models import (
_canonical_modality,
_natural_sort_key,
get_arena_direct_model_public_id,
)
catalog = normalize_arena_model_catalog_config(catalog_config)
target_modality = _canonical_modality(catalog.get("modality") or "text")
enable_dark_pool = bool(catalog.get("enable_dark_pool", False))
dark_since_ts = parse_date_to_timestamp(catalog.get("dark_pool_since", ""))
dark_whitelist = [item.casefold() for item in catalog.get("dark_pool_whitelist_keywords", [])]
dark_blacklist = [item.casefold() for item in catalog.get("dark_pool_blacklist_keywords", [])]
pub_includes = [item.casefold() for item in catalog.get("include_keywords", [])]
pub_excludes = [item.casefold() for item in catalog.get("exclude_keywords", [])]
# 合并来自页面实时抓取与离线缓存的模型(按 arena_model_id 去重)
all_candidates: List[Dict[str, Any]] = []
seen_ids = set()
for m in (models or []) + (load_arena_offline_cache_models() if enable_dark_pool else []):
if not isinstance(m, dict):
continue
mid = str(m.get("arena_model_id") or "").strip().lower()
if not mid or mid in seen_ids:
continue
seen_ids.add(mid)
all_candidates.append(m)
video_keywords = (
"video", "seedance", "dreamina", "sora", "veo", "gemini-omni", "kling",
"hailuo", "runway", "pika", "mochi", "ltx", "vidu", "luma", "wan2",
"wan-2", "wan_2", "wanx", "pixverse", "kandinsky-video", "kandinsky_video", "happyhorse"
)
image_keywords = (
"image", "mona-lisa", "luna-lisa", "lina-alpha", "lina-f-alpha",
"silver_halide", "flux", "imagine", "seedream", "seededit", "imagen", "z-image",
"midjourney", "dall-e", "recraft", "krea"
)
result = []
result_seen_ids = set()
for model in all_candidates:
search_str = " ".join(
str(model.get(k) or "") for k in ("name", "public_name", "display_name", "search_name")
).lower()
if target_modality != "video":
if any((k in search_str and (k != "kling" or "inkling" not in search_str)) for k in video_keywords):
continue
model_modality = _canonical_modality(model.get("modality"))
if target_modality:
if target_modality in {"code", "search", "video"}:
if model_modality != target_modality:
continue
elif target_modality == "image":
model_is_image = (model_modality == "image") or (
not model_modality and any(k in search_str for k in image_keywords)
)
if not model_is_image:
continue
elif target_modality == "text":
if model_modality and model_modality != "text":
continue
if not model_modality and any(k in search_str for k in image_keywords):
continue
else:
if model_modality and model_modality != target_modality:
continue
searchable = " ".join(
str(model.get(key) or "")
for key in (
"name", "public_name", "display_name", "search_name", "provider", "organization"
)
).casefold()
searchable += " " + " ".join(
str(alias or "") for alias in (model.get("aliases") or [])
).casefold()
is_dark = is_dark_pool_model(model)
if is_dark:
# 暗池处理规则
if not enable_dark_pool:
continue
# 1. 黑名单最终否决
if dark_blacklist and any(kw in searchable for kw in dark_blacklist):
continue
# 2. 白名单与日期条件检查(若设置了白名单或日期,必须满足其中之一)
if dark_whitelist or dark_since_ts is not None:
whitelisted = bool(dark_whitelist and any(kw in searchable for kw in dark_whitelist))
if not whitelisted:
if dark_since_ts is None:
continue
model_ts = get_model_timestamp(model)
if model_ts is None or model_ts < dark_since_ts:
continue
else:
# 明池处理规则(不受暗池规则影响)
if pub_includes and not any(keyword in searchable for keyword in pub_includes):
continue
if pub_excludes and any(keyword in searchable for keyword in pub_excludes):
continue
mid_lower = str(model.get("arena_model_id") or "").lower()
if mid_lower in result_seen_ids:
continue
result_seen_ids.add(mid_lower)
result.append(model)
result.sort(key=lambda m: _natural_sort_key(get_arena_direct_model_public_id(m)))
return result
__all__ = [
"ARENA_MODEL_CATALOG_CONFIG_PATH",
"MODEL_CATALOG_SOURCE",
"ONLINE_PUBLIC_VISIBLE_MODELS",
"filter_arena_catalog_models",
"get_arena_direct_catalog_for_tab",
"get_arena_model_catalog",
"get_model_catalog_preset",
"get_model_timestamp",
"is_dark_pool_model",
"load_arena_model_catalog_data",
"load_arena_offline_cache_models",
"migrate_and_cleanup_sites_model_catalog",
"normalize_arena_model_catalog_config",
"parse_date_to_timestamp",
"parse_uuidv7_timestamp",
"save_arena_model_catalog_data",
"set_arena_model_catalog",
]
+155
View File
@@ -0,0 +1,155 @@
"""Shared Clash proxy rotation for Arena and other browser automation commands."""
from __future__ import annotations
import json
import os
import time
import urllib.parse
from typing import Any, Callable, Dict, Iterable, Optional, Tuple
import requests
ARENA_CLASH_API = os.getenv("CLASH_API", os.getenv("ARENA_CLASH_API", "http://127.0.0.1:9097")).rstrip("/")
ARENA_CLASH_SECRET = os.getenv("CLASH_SECRET", os.getenv("ARENA_CLASH_SECRET", "1"))
ARENA_CLASH_SELECTOR = os.getenv("CLASH_SELECTOR", os.getenv("ARENA_CLASH_SELECTOR", "主代理"))
DEFAULT_ARENA_PROXY_POOL = (
"HK自动选择",
"JP自动选择",
"KR自动选择",
"SG自动选择",
"TW自动选择",
"US自动选择",
)
def _get_proxy_pool_from_env() -> Tuple[str, ...]:
raw = os.getenv("CLASH_PROXY_POOL", os.getenv("ARENA_PROXY_POOL", ""))
if not raw:
return DEFAULT_ARENA_PROXY_POOL
try:
if raw.startswith("["):
parsed = json.loads(raw)
if isinstance(parsed, list):
return tuple(str(x).strip() for x in parsed if str(x).strip())
except Exception:
pass
parts = [p.strip() for p in raw.split(",") if p.strip()]
return tuple(parts) if parts else DEFAULT_ARENA_PROXY_POOL
ARENA_PROXY_POOL = _get_proxy_pool_from_env()
def rotate_arena_proxy(
*,
tab: Any,
logger: Any,
raise_if_cancelled: Optional[Callable[[], None]] = None,
requests_module: Any = requests,
sleep: Callable[[float], None] = time.sleep,
proxy_pool: Optional[Iterable[str]] = None,
clash_api: Optional[str] = None,
clash_secret: Optional[str] = None,
clash_selector: Optional[str] = None,
) -> Dict[str, Any]:
"""Move the Clash selector to the next node in the pool."""
cancel = raise_if_cancelled or (lambda: None)
effective_pool = tuple(str(item) for item in (proxy_pool if proxy_pool is not None else ARENA_PROXY_POOL) if str(item).strip())
api_base = str(clash_api or os.getenv("CLASH_API", os.getenv("ARENA_CLASH_API", ARENA_CLASH_API))).rstrip("/")
secret = str(clash_secret if clash_secret is not None else os.getenv("CLASH_SECRET", os.getenv("ARENA_CLASH_SECRET", ARENA_CLASH_SECRET)))
selector_name = str(clash_selector or os.getenv("CLASH_SELECTOR", os.getenv("ARENA_CLASH_SELECTOR", ARENA_CLASH_SELECTOR)))
headers = {"Content-Type": "application/json"}
if secret:
headers["Authorization"] = f"Bearer {secret}"
selector_url = (
f"{api_base}/proxies/"
f"{urllib.parse.quote(selector_name, safe='')}"
)
try:
cancel()
response = requests_module.get(selector_url, headers=headers, timeout=5)
response.raise_for_status()
selector = response.json()
available = [name for name in effective_pool if name in (selector.get("all") or [])]
if len(available) != len(effective_pool):
return {
"ok": False,
"error": "arena_proxy_pool_incomplete",
"missing": [name for name in effective_pool if name not in available],
}
current = str(selector.get("now") or "")
next_proxy = (
available[(available.index(current) + 1) % len(available)]
if current in available
else available[0]
)
if next_proxy == current:
return {
"ok": True,
"switched": False,
"selector": selector_name,
"node": current,
"pool_size": len(available),
}
cancel()
switch_response = requests_module.put(
selector_url,
json={"name": next_proxy},
headers=headers,
timeout=5,
)
switch_response.raise_for_status()
try:
requests_module.delete(
f"{api_base}/connections",
headers=headers,
timeout=5,
).raise_for_status()
except Exception as close_error:
logger.warning(
f"[ARENA][IP-ROTATE] closing Clash connections failed: {close_error}"
)
logger.info(
f"[ARENA][IP-ROTATE] {selector_name}: "
f"{current or '-'} -> {next_proxy}"
)
sleep(1.0)
try:
tab.refresh()
sleep(2.0)
except Exception as refresh_error:
logger.warning(f"[ARENA][IP-ROTATE] page refresh failed: {refresh_error}")
return {
"ok": True,
"switched": True,
"selector": selector_name,
"from": current,
"to": next_proxy,
"pool_size": len(available),
}
except Exception as error:
logger.error(f"[ARENA][IP-ROTATE] Clash switch failed: {error}")
return {"ok": False, "error": str(error)}
# 通用别名
rotate_clash_proxy = rotate_arena_proxy
__all__ = [
"ARENA_CLASH_API",
"ARENA_CLASH_SECRET",
"ARENA_CLASH_SELECTOR",
"ARENA_PROXY_POOL",
"DEFAULT_ARENA_PROXY_POOL",
"rotate_arena_proxy",
"rotate_clash_proxy",
]
+359
View File
@@ -0,0 +1,359 @@
"""
app/services/arena_rule_service.py - Arena 规则匹配与候选记录业务服务
职责:
- 评估多规则匹配逻辑(必含/排除词/检测器评估);
- 整合 Link Drawer 本地书签抽屉写入;
- 生成标准化候选命中记录并调用 command_result_store 进行持久化。
"""
from __future__ import annotations
import json
import logging
import os
import re
import tempfile
import threading
import time
import uuid
from pathlib import Path
from typing import Any, Callable, Dict, Iterable, List, Optional
def _get_store() -> Any:
from app.services import command_result_store
return command_result_store
logger = logging.getLogger(__name__)
try:
import requests
except ImportError: # pragma: no cover
requests = None
_DRAWER_LOCK = threading.RLock()
def _split_terms(value: Any) -> List[str]:
if isinstance(value, list):
parts = value
else:
parts = re.split(r"[,,\n\r]+", str(value or ""))
return [str(part).strip() for part in parts if str(part).strip()]
def _matches_rule(text: str, rule: Dict[str, Any]) -> tuple[bool, str]:
folded = str(text or "").casefold()
excluded = _split_terms(rule.get("excluded"))
for token in excluded:
if token.casefold() in folded:
return False, f"excluded:{token}"
required_all = _split_terms(rule.get("required_all"))
missing = [token for token in required_all if token.casefold() not in folded]
if missing:
return False, f"missing:{','.join(missing)}"
required_any = _split_terms(rule.get("required_any"))
if required_any and not any(token.casefold() in folded for token in required_any):
return False, "missing_any"
return True, "matched"
def _detector_accepts(
prompt: str,
response_text: str,
rule: Dict[str, Any],
values: Dict[str, Any],
) -> tuple[bool, Dict[str, Any]]:
req_module = getattr(_get_store(), "requests", requests)
keyword = str(rule.get("detector_keyword") or "").strip()
if not keyword:
return True, {"skipped": True, "models": []}
detector_url = str(values.get("detector_url") or "http://127.0.0.1:8765/api/judge").strip()
if req_module is None:
return True, {"unavailable": True, "error": "requests unavailable", "models": []}
try:
response = req_module.post(
detector_url,
json={"prompt": str(prompt or ""), "response": str(response_text or "")},
timeout=20,
)
response.raise_for_status()
payload = response.json()
predictions = list(payload.get("predictions") or [])[:5] if isinstance(payload, dict) else []
models = [str(item.get("model") or "").strip() for item in predictions if isinstance(item, dict)]
accepted = any(keyword.casefold() in model.casefold() for model in models)
return accepted, {"models": models, "best_model": payload.get("best_model") if isinstance(payload, dict) else ""}
except Exception as error:
# Preserve a text-filter hit if the optional local detector is offline.
return True, {"unavailable": True, "error": str(error), "models": []}
def _resolve_drawer_file(raw_path: Any) -> Path | None:
text = os.path.expandvars(os.path.expanduser(str(raw_path or "").strip()))
if not text:
return None
path = Path(text)
return path / "drawer_data.json" if path.is_dir() or not path.suffix else path
def _write_link_drawer(
path: Path | None,
url: str,
title: str,
category: str,
controlled_browser: Optional[Dict[str, Any]] = None,
) -> Dict[str, Any]:
if path is None:
return {"status": "disabled"}
with _DRAWER_LOCK:
try:
if path.exists():
with path.open("r", encoding="utf-8") as handle:
payload = json.load(handle)
else:
payload = {"categories": ["默认分类"], "links": [], "settings": {}}
categories = payload.get("categories") if isinstance(payload.get("categories"), list) else []
links = payload.get("links") if isinstance(payload.get("links"), list) else []
category = str(category or "默认分类").strip() or "默认分类"
if category not in categories:
categories.append(category)
normalized_url = str(url or "").strip().rstrip("/")
for item in links:
if str(item.get("url") or "").strip().rstrip("/") == normalized_url:
return {"status": "duplicate", "category": item.get("category", "")}
link = {
"id": uuid.uuid4().hex[:8],
"url": str(url or "").strip(),
"title": title,
"category": category,
"dateAdded": int(time.time() * 1000),
}
if isinstance(controlled_browser, dict) and controlled_browser:
link["controlledBrowser"] = controlled_browser
links.append(link)
payload["categories"] = categories
payload["links"] = links
_get_store()._atomic_write_json(path, payload)
return {"status": "added", "category": category}
except Exception as error:
return {"status": "error", "error": str(error)}
def _clean_title_component(value: Any, fallback: str) -> str:
text = re.sub(r"[\x00-\x1f\x7f]+", " ", str(value or "")).strip()
return re.sub(r"\s+", " ", text) or fallback
def _next_title(
records: Iterable[Dict[str, Any]],
profile: str,
model: str,
title_template: str = "",
) -> str:
raw_profile = str(profile or "").strip()
raw_model = str(model or "").strip()
profile = _clean_title_component(profile, "profile").replace("《", "〈").replace("》", "〉")
model = _clean_title_component(model, "model")
template = str(title_template or "").strip() or "《{profile}》-{model}-{index:03d}"
matching_count = 0
for item in records:
if (
str(item.get("browser_profile_name") or "").strip() != raw_profile
or str(item.get("model_name") or "").strip() != raw_model
):
continue
matching_count += 1
index = matching_count + 1
try:
title = template.format(profile=profile, model=model, index=index)
except (KeyError, ValueError, IndexError):
title = f"《{profile}》-{model}-{index:03d}"
return _clean_title_component(title, f"《{profile}》-{model}-{index:03d}")
def _controlled_browser_metadata(values: Dict[str, Any], identity: Dict[str, Any]) -> Dict[str, Any]:
profile = {
key: str(identity.get(key) or "").strip()
for key in (
"name",
"profile_directory",
"profile_path",
"user_data_dir",
"browser_context_id",
"source_tab_id",
)
if str(identity.get(key) or "").strip()
}
return {
"version": 1,
"apiUrl": str(
values.get("controlled_browser_api_url")
or "http://127.0.0.1:8199/api/browser/open-profile-url"
).strip(),
"profile": profile,
}
def record_arena_rule_candidates(
command_id: str,
values: Dict[str, Any],
info: Dict[str, Any],
prompt: str = "",
source: str = "",
profile_resolver: Optional[Callable[[], Any]] = None,
) -> Dict[str, Any]:
"""
匹配 Arena 规则并保存候选结果到通用持久化存储与 Link Drawer。
"""
rules = values.get("rules") if isinstance(values.get("rules"), list) else []
response_sides = info.get("response_sides") if isinstance(info, dict) else []
if not isinstance(response_sides, (list, tuple)) or not response_sides:
response_sides = [str((info or {}).get("visible_text") or "")]
url = str((info or {}).get("url") or "").strip()
if not url:
return {"recorded": [], "matched": 0}
candidates: List[tuple[str, Dict[str, Any], int, int, str, Dict[str, Any]]] = []
profile_identity: Optional[Dict[str, Any]] = None
identity_unresolved = False
def _resolve_profile() -> Dict[str, Any]:
nonlocal profile_identity
if profile_identity is not None:
return profile_identity
resolved: Any = {}
if callable(profile_resolver):
try:
resolved = profile_resolver() or {}
except Exception:
resolved = {}
if isinstance(resolved, str):
resolved = {"name": resolved}
profile_identity = resolved if isinstance(resolved, dict) else {}
return profile_identity
store = _get_store()
existing_results = store.list_command_results(command_id, limit=2000)
existing_hits = {
str(item.get("rule_id") or "")
for item in existing_results
if isinstance(item, dict) and item.get("url") == url
}
# Dispatch detector using command_result_store._detector_accepts so test patches take effect
detector_fn = getattr(store, "_detector_accepts", _detector_accepts)
matches_rule_fn = getattr(store, "_matches_rule", _matches_rule)
for rule_index, rule in enumerate(rules):
if not isinstance(rule, dict) or rule.get("enabled", True) is False:
continue
rule_id = str(rule.get("id") or f"rule-{rule_index + 1}")
if rule_id in existing_hits:
continue
for side_index, response_text in enumerate(response_sides):
text = str(response_text or "").strip()
if not text:
continue
is_reasoning_side = False
has_reasoning_sides = info.get("has_reasoning_sides") if isinstance(info, dict) else None
if isinstance(has_reasoning_sides, list) and side_index < len(has_reasoning_sides):
is_reasoning_side = bool(has_reasoning_sides[side_index])
if rule.get("exclude_reasoning") and is_reasoning_side:
continue
matched, _ = matches_rule_fn(text, rule)
if not matched:
continue
accepted, detector = detector_fn(prompt, text, rule, values)
if accepted:
candidates.append((rule_id, rule, rule_index, side_index, text, detector))
break
if candidates:
identity = _resolve_profile()
if not str(identity.get("name") or "").strip():
identity_unresolved = True
logger.warning(
"[CMD_RESULT] browser profile name 为空,%d 条候选命中将被跳过。"
"请确认 profile_resolver 已正确返回 identity。",
len(candidates),
)
new_records: List[Dict[str, Any]] = []
if not identity_unresolved and candidates:
identity = _resolve_profile()
current_records = store.get_all_records()
for rule_id, rule, rule_index, side_index, text, detector in candidates:
if any(
item.get("command_id") == command_id
and item.get("rule_id") == rule_id
and item.get("url") == url
for item in current_records
if isinstance(item, dict)
):
continue
model = str(rule.get("model_name") or rule.get("name") or f"model-{rule_index + 1}").strip()
profile = str(identity.get("name") or "").strip()
title = _next_title(current_records, profile, model, str(rule.get("title_template") or ""))
drawer = _write_link_drawer(
_resolve_drawer_file(values.get("link_drawer_path")),
url,
title,
str(rule.get("drawer_group") or model),
_controlled_browser_metadata(values, identity),
)
record = {
"id": uuid.uuid4().hex,
"command_id": command_id,
"rule_id": rule_id,
"rule_name": str(rule.get("name") or model),
"model_name": model,
"browser_profile_name": profile,
"browser_profile": identity,
"title": title,
"url": url,
"side": "A" if side_index == 0 else "B",
"source": str(source or ""),
"response_preview": text[:500],
"detector": detector,
"drawer": drawer,
"created_at": time.time(),
}
current_records.append(record)
new_records.append(record)
if new_records:
# The final duplicate check and write must share one storage lock;
# otherwise two concurrent bridge callbacks can persist the same hit.
add_unique = getattr(store, "add_records_unique", None)
if callable(add_unique):
persisted_records = add_unique(
new_records,
("command_id", "rule_id", "url"),
)
else: # pragma: no cover - compatibility for lightweight test doubles
persisted_records = store.add_records(new_records)
new_records = persisted_records
return {
"recorded": new_records,
"matched": len(new_records),
"identity_unresolved": identity_unresolved,
}
__all__ = [
"_clean_title_component",
"_controlled_browser_metadata",
"_detector_accepts",
"_matches_rule",
"_next_title",
"_resolve_drawer_file",
"_split_terms",
"_write_link_drawer",
"record_arena_rule_candidates",
]
+358
View File
@@ -0,0 +1,358 @@
"""
app/services/arena_tab_listener.py - Arena 翻牌探测与结果桥接扩展监听器
从通用全局网络拦截层解耦的独立扩展模块。
通过 register_response_listener 挂载到 _GlobalNetworkInterceptionManager,
实现对 Arena 候选响应的事件桥接与前端 React Fiber store 的翻牌轮询。
"""
import threading
import time
from typing import Any, Callable, Dict, Optional, Set
from app.core.config import logger
from app.core.page_lifecycle import is_page_refresh_error
from app.core.tab_pool_parts.session import TabSession
from app.core.driver import driver_for_tab
def is_explicit_arena_direct_url(url: Any) -> bool:
"""Check if URL explicitly points to Arena direct chat page."""
try:
raw = str(url or "").strip()
if not raw:
return False
from urllib.parse import parse_qs, urlparse
parsed = urlparse(raw)
path = (parsed.path or "").rstrip("/").lower()
if path in {"/direct", "/text/direct", "/image/direct", "/code/direct", "/search/direct"}:
return True
if path.startswith(("/direct/", "/text/direct/", "/image/direct/", "/code/direct/", "/search/direct/")):
return True
if path.endswith("/direct"):
return True
query = parse_qs(parsed.query)
if query.get("mode") == ["direct"]:
return True
return False
except Exception:
return False
from app.core.tab_pool_parts._arena_snapshot import (
_ARENA_STORE_SNAPSHOT_JS,
_ARENA_STORE_SNAPSHOT_JSON_JS,
)
from app.core.cdp_hygiene import decode_js_json
from app.core.tab_pool_parts.idle_maintenance import note_network_activity
class ArenaTabListener:
"""Arena 翻牌探测与结果桥接监听器。"""
ARENA_REVEAL_POLL_INTERVAL_SEC = 3.0
ARENA_REVEAL_POLL_TIMEOUT_SEC = 120.0
RESULT_BRIDGE_MAX_ACTIVE_PER_SESSION = 2
def __init__(
self,
get_session_fn: Optional[Callable[[str], Optional[TabSession]]] = None,
is_shutdown_fn: Optional[Callable[[], bool]] = None,
):
self._get_session = get_session_fn
self._is_shutdown = is_shutdown_fn or (lambda: False)
self._result_event_handler = self._create_result_event_handler()
self._result_bridge_lock = threading.RLock()
self._result_bridge_active_by_session: Dict[str, int] = {}
self._arena_reveal_pollers: Dict[str, threading.Thread] = {}
self._arena_reveal_lock = threading.RLock()
self._arena_reveal_logged_signatures: Set[str] = set()
@staticmethod
def _create_result_event_handler():
try:
from app.services.result_event_bridge import create_result_event_handler
handler = create_result_event_handler()
if handler:
logger.info("[GlobalNet] Arena 结果事件桥接已启用(支持手动网页测试)")
return handler
except Exception as e:
logger.debug(f"[GlobalNet] Arena 结果事件桥接初始化失败(忽略): {e}")
return None
@staticmethod
def is_arena_candidate(event: Dict[str, Any]) -> bool:
"""快速判断事件是否可能属于 Arena。"""
url = str((event or {}).get("url") or "").strip().lower()
if not url:
return False
return any(host in url for host in ("lmarena.ai", "arena.ai", "lmsys.org"))
@classmethod
def _is_result_bridge_candidate(cls, event: Dict[str, Any]) -> bool:
if not cls.is_arena_candidate(event):
return False
url = str((event or {}).get("url") or "").strip().lower()
return any(
token in url
for token in (
"/nextjs-api/stream/",
"nextjs-api/stream",
"create-evaluation",
"post-to-evaluation",
"stream/create",
"stream/post",
)
)
@classmethod
def _is_reveal_snapshot_candidate(cls, event: Dict[str, Any]) -> bool:
if not cls.is_arena_candidate(event):
return False
url = str((event or {}).get("url") or "").strip().lower()
return any(
token in url
for token in (
"/nextjs-api/stream/",
"/rpc/i/",
"/api/history/",
"/c/",
"_rsc=",
)
)
def _claim_result_bridge_slot(self, session_id: str) -> bool:
key = str(session_id or "unknown")
with self._result_bridge_lock:
active = int(self._result_bridge_active_by_session.get(key, 0) or 0)
if active >= self.RESULT_BRIDGE_MAX_ACTIVE_PER_SESSION:
logger.debug_throttled(
f"global_net.result_bridge_busy.{key}",
f"[GlobalNet] Arena 结果桥接忙,跳过候选响应: {key}, active={active}",
interval_sec=10.0,
)
return False
self._result_bridge_active_by_session[key] = active + 1
return True
def _release_result_bridge_slot(self, session_id: str) -> None:
key = str(session_id or "unknown")
with self._result_bridge_lock:
active = int(self._result_bridge_active_by_session.get(key, 0) or 0)
if active <= 1:
self._result_bridge_active_by_session.pop(key, None)
else:
self._result_bridge_active_by_session[key] = active - 1
def _dispatch_result_bridge_async(
self,
session: TabSession,
response: Any,
event: Dict[str, Any],
stop_event: threading.Event,
) -> None:
if not self._result_event_handler:
return
if not self._is_result_bridge_candidate(event):
return
session_id = str(getattr(session, "id", "") or "unknown")
if not self._claim_result_bridge_slot(session_id):
return
try:
thread = threading.Thread(
target=self._dispatch_result_bridge,
args=(session, response, dict(event or {}), stop_event),
daemon=True,
name=f"global-net-arena-{session_id}",
)
thread.start()
except Exception as e:
self._release_result_bridge_slot(session_id)
logger.debug(f"[GlobalNet] Arena 结果桥接线程启动失败(忽略): {e}")
def _dispatch_result_bridge(
self,
session: TabSession,
response: Any,
event: Dict[str, Any],
stop_event: threading.Event,
) -> None:
session_id = str(getattr(session, "id", "") or "unknown")
try:
from app.core.tab_pool_parts.network import _GlobalNetworkInterceptionManager
raw_body, raw_body_source = _GlobalNetworkInterceptionManager.read_response_body(response, stop_event)
if not raw_body:
return
post_data = _GlobalNetworkInterceptionManager.extract_request_post_data(response)
self._result_event_handler(
{
"event": event,
"raw_body": raw_body,
"raw_body_source": raw_body_source,
"request_post_data": post_data,
"parse_result": {"done": True},
"parser_id": "lmarena_global",
"session_id": getattr(session, "id", ""),
"session": session,
}
)
except Exception as e:
logger.debug(f"[GlobalNet] Arena 结果事件桥接失败(忽略): {e}")
finally:
self._release_result_bridge_slot(session_id)
def _start_arena_reveal_poll(
self,
session: TabSession,
event: Dict[str, Any],
stop_event: threading.Event,
reason: str,
) -> None:
if not session or not self._is_reveal_snapshot_candidate(event):
return
session_id = str(getattr(session, "id", "") or "")
if not session_id:
return
try:
from app.core.workflow.arena_direct_guard import is_arena_direct_preset
tab = getattr(session, "tab", None)
current_url = getattr(tab, "url", "") if tab else ""
if current_url and is_explicit_arena_direct_url(current_url):
return
preset_name = str(getattr(session, "preset_name", "") or getattr(session, "preset", "") or "")
if preset_name and is_arena_direct_preset("arena.ai", preset_name):
return
except Exception:
pass
with self._arena_reveal_lock:
current = self._arena_reveal_pollers.get(session_id)
if current and current.is_alive():
return
thread = threading.Thread(
target=self._arena_reveal_poll_loop,
args=(session_id, stop_event, reason),
daemon=True,
name=f"global-net-reveal-{session_id}",
)
self._arena_reveal_pollers[session_id] = thread
thread.start()
def _arena_reveal_poll_loop(
self,
session_id: str,
stop_event: threading.Event,
reason: str,
) -> None:
try:
from app.services.result_event_bridge import emit_arena_snapshot_event
except Exception as e:
logger.debug(f"[GlobalNet] Arena 翻牌快照桥接不可用(忽略): {e}")
return
deadline = time.time() + self.ARENA_REVEAL_POLL_TIMEOUT_SEC
last_signature = ""
try:
while time.time() < deadline and not stop_event.is_set() and not self._is_shutdown():
session = self._get_session(session_id) if callable(self._get_session) else None
tab = getattr(session, "tab", None) if session is not None else None
if tab is None:
return
# P0-6:翻牌轮询期间视为活跃,避免被空闲冻结
note_network_activity(session)
try:
snapshot = decode_js_json(driver_for_tab(tab).run_js(_ARENA_STORE_SNAPSHOT_JSON_JS))
except Exception as e:
if not is_page_refresh_error(e):
logger.debug_throttled(
f"global_net.arena_reveal_snapshot.{session_id}",
f"[GlobalNet] 读取 Arena 翻牌快照失败(忽略): {e}",
interval_sec=10.0,
)
time.sleep(self.ARENA_REVEAL_POLL_INTERVAL_SEC)
continue
if not isinstance(snapshot, dict):
time.sleep(self.ARENA_REVEAL_POLL_INTERVAL_SEC)
continue
is_direct = bool(
snapshot.get("is_direct")
or str(snapshot.get("mode") or "").strip().lower() == "direct"
or is_explicit_arena_direct_url(snapshot.get("url"))
)
if is_direct:
return
snapshot["session_id"] = session_id
model_a = str(snapshot.get("model_a") or "").strip()
model_b = str(snapshot.get("model_b") or "").strip()
response_a = str(snapshot.get("response_a") or "")
response_b = str(snapshot.get("response_b") or "")
message_id_a = str(snapshot.get("message_id_a") or "").strip()
message_id_b = str(snapshot.get("message_id_b") or "").strip()
if not (model_a and model_b and message_id_a and message_id_b and message_id_a != message_id_b):
time.sleep(self.ARENA_REVEAL_POLL_INTERVAL_SEC)
continue
signature = (
f"{snapshot.get('conversation_id')}|{message_id_a}|"
f"{message_id_b}|{model_a}|{model_b}"
)
if signature != last_signature:
last_signature = signature
log_signature = f"{session_id}|{signature}"
with self._arena_reveal_lock:
should_log = log_signature not in self._arena_reveal_logged_signatures
if should_log:
self._arena_reveal_logged_signatures.add(log_signature)
if len(self._arena_reveal_logged_signatures) > 500:
self._arena_reveal_logged_signatures.clear()
if should_log:
logger.debug(
"[GlobalNet] Arena 翻牌快照更新: "
f"reason={reason}, model_a={model_a}, model_b={model_b}, "
f"a={len(response_a)}, b={len(response_b)}"
)
if response_a and response_b:
emit_arena_snapshot_event(snapshot)
return
time.sleep(self.ARENA_REVEAL_POLL_INTERVAL_SEC)
finally:
with self._arena_reveal_lock:
current = self._arena_reveal_pollers.get(session_id)
if current is threading.current_thread():
self._arena_reveal_pollers.pop(session_id, None)
def __call__(
self,
session: TabSession,
response: Any,
event: Dict[str, Any],
stop_event: threading.Event,
) -> None:
"""作为全局 response listener 的回调入口。"""
self._dispatch_result_bridge_async(session, response, event, stop_event)
self._start_arena_reveal_poll(session, event, stop_event, "network-event")
def register_arena_tab_listener(manager: Any) -> Optional[Callable[[], None]]:
"""向全局网络管理器挂载 Arena 翻牌与结果监听器。"""
if manager is None or not hasattr(manager, "register_response_listener"):
return None
get_session_fn = getattr(manager, "_get_session", None)
is_shutdown_fn = getattr(manager, "_is_shutdown", None)
listener = ArenaTabListener(get_session_fn=get_session_fn, is_shutdown_fn=is_shutdown_fn)
return manager.register_response_listener(
ArenaTabListener.is_arena_candidate,
listener,
)
+425
View File
@@ -0,0 +1,425 @@
"""
app/services/catalog_routing.py - 通用动态模型目录与路由派发器
职责:
- 统一管理各站点的动态模型目录提供者(CatalogProvider)
- 抽象统一的 CatalogRouter,支持动态目录模型匹配、Tab 挑选及模型列表发现
- 彻底解耦 API 层(chat.py)与具体业务模型目录(如 Arena Direct)的实现细节
"""
from __future__ import annotations
import random
import sys
from typing import Any, Callable, Dict, List, Optional, Protocol, Sequence, Set, Tuple
from app.core.config import get_logger
from app.utils.site_url import route_domain_matches
logger = get_logger("SERVICES.CATALOG_ROUTING")
def _get_chat_or_service_attr(name: str, fallback_module: Optional[Any] = None) -> Any:
chat_mod = sys.modules.get("app.api.chat")
if chat_mod and hasattr(chat_mod, name):
return getattr(chat_mod, name)
if fallback_module and hasattr(fallback_module, name):
return getattr(fallback_module, name)
return None
def select_catalog_tab(
browser: Any,
candidates: List[Dict[str, Any]],
preset_name: Any = None,
cursor_prefix: str = "arena_catalog",
) -> Optional[Dict[str, Any]]:
"""Select a tab from candidates respecting the tab pool's allocation policy."""
if not candidates:
return None
idle_candidates = [
item
for item in candidates
if str(item.get("status") or "").strip().lower() == "idle"
]
pool = idle_candidates or candidates
try:
allocation_mode = str(
getattr(browser.tab_pool, "allocation_mode", "") or ""
).strip().lower()
except Exception:
allocation_mode = ""
if allocation_mode not in {"first_idle", "round_robin", "random"}:
allocation_mode = "first_idle"
if allocation_mode == "round_robin":
from app.api import tab_routes as tab_routes_api
cursor_key = f"{cursor_prefix}::{str(preset_name or '').strip().casefold()}"
return tab_routes_api._select_round_robin_tab(pool, cursor_key)
if allocation_mode == "random":
return random.choice(pool)
return min(pool, key=lambda item: int(item.get("persistent_index") or 0))
class CatalogProvider(Protocol):
"""Protocol defining a dynamic catalog provider for model discovery and routing."""
@property
def name(self) -> str: ...
@property
def route_domain(self) -> str: ...
def match_tab_model(
self,
browser: Any,
config_engine: Any,
tab: Dict[str, Any],
requested_model: Any,
) -> Optional[Dict[str, Any]]: ...
def collect_catalog_configs(
self,
config_engine: Any,
tabs: List[Dict[str, Any]],
) -> List[Dict[str, Any]]: ...
def is_alias_suppressed(
self,
item: Dict[str, Any],
catalog_configs: List[Dict[str, Any]],
) -> bool: ...
def list_catalog_model_entries(
self,
browser: Any,
catalog_configs: List[Dict[str, Any]],
created: int,
) -> List[Dict[str, Any]]: ...
class ArenaCatalogProvider:
"""Dynamic catalog provider for Arena Direct models."""
@property
def name(self) -> str:
return "arena_direct"
@property
def route_domain(self) -> str:
return "arena.ai"
def match_tab_model(
self,
browser: Any,
config_engine: Any,
tab: Dict[str, Any],
requested_model: Any,
) -> Optional[Dict[str, Any]]:
import app.services.arena_direct_models as adm
get_catalog_fn = _get_chat_or_service_attr("get_arena_direct_catalog_for_tab", adm)
list_models_fn = _get_chat_or_service_attr("list_arena_direct_models", adm)
match_model_fn = _get_chat_or_service_attr("match_arena_direct_model", adm)
get_public_id_fn = _get_chat_or_service_attr("get_arena_direct_model_public_id", adm)
catalog_info = get_catalog_fn(config_engine, tab) if callable(get_catalog_fn) else None
catalog = catalog_info.get("catalog") if catalog_info else None
if not isinstance(catalog, dict) or not catalog.get("enabled"):
return None
try:
models = list_models_fn(browser, catalog_config=catalog, tab=tab)
except TypeError:
models = list_models_fn(browser, catalog_config=catalog)
model = match_model_fn(models, requested_model)
if not model:
return None
preset_name = str(catalog_info.get("preset_name") or "")
public_model_id = get_public_id_fn(model) if callable(get_public_id_fn) else ""
return {
"tab": tab,
"model": model,
"preset_name": preset_name,
"public_model_id": public_model_id,
"route_domain": self.route_domain,
}
def collect_catalog_configs(
self,
config_engine: Any,
tabs: List[Dict[str, Any]],
) -> List[Dict[str, Any]]:
import app.services.arena_direct_models as adm
get_catalog_fn = _get_chat_or_service_attr("get_arena_direct_catalog_for_tab", adm)
normalize_config_fn = _get_chat_or_service_attr("normalize_model_catalog_config", adm)
catalog_presets: List[Dict[str, Any]] = []
seen_catalog_keys: Set[Tuple[Any, ...]] = set()
for tab in tabs or []:
candidate = get_catalog_fn(config_engine, tab) if callable(get_catalog_fn) else None
cat_obj = candidate.get("catalog") if isinstance(candidate, dict) else None
if isinstance(cat_obj, dict) and cat_obj.get("enabled"):
cat = cat_obj
else:
cat = None
if route_domain_matches(self.route_domain, tab.get("route_domain") or tab.get("current_domain") or ""):
effective_preset_name = str(
tab.get("preset_name") or config_engine.get_default_preset(self.route_domain) or ""
).strip()
try:
preset = config_engine._get_site_data_readonly(self.route_domain, effective_preset_name or None)
if isinstance(preset, dict) and callable(normalize_config_fn):
cat_candidate = normalize_config_fn(preset.get("model_catalog"))
if cat_candidate.get("enabled") and cat_candidate.get("source") == "arena_direct":
cat = cat_candidate
except Exception:
pass
if cat and cat.get("enabled"):
cat_key = (
cat.get("source", ""),
cat.get("modality", ""),
tuple(cat.get("include_keywords") or []),
tuple(cat.get("exclude_keywords") or []),
bool(cat.get("enable_dark_pool", False)),
str(cat.get("dark_pool_since") or "").strip(),
tuple(cat.get("dark_pool_whitelist_keywords") or []),
tuple(cat.get("dark_pool_blacklist_keywords") or []),
)
if cat_key not in seen_catalog_keys:
seen_catalog_keys.add(cat_key)
catalog_presets.append(cat)
return catalog_presets
def is_alias_suppressed(
self,
item: Dict[str, Any],
catalog_configs: List[Dict[str, Any]],
) -> bool:
if not catalog_configs or not item.get("is_route_alias"):
return False
item_route_domains = list(item.get("route_domains") or [])
if item.get("route_domain"):
item_route_domains.append(item.get("route_domain"))
return any(route_domain_matches(self.route_domain, domain) for domain in item_route_domains)
def list_catalog_model_entries(
self,
browser: Any,
catalog_configs: List[Dict[str, Any]],
created: int,
) -> List[Dict[str, Any]]:
import app.services.arena_direct_models as adm
list_models_fn = _get_chat_or_service_attr("list_arena_direct_models", adm)
build_entries_fn = _get_chat_or_service_attr("build_openai_model_entries", adm)
entries: List[Dict[str, Any]] = []
for cat in catalog_configs:
cat_models = list_models_fn(browser, catalog_config=cat) if callable(list_models_fn) else []
model_entries = build_entries_fn(cat_models, created=created) if callable(build_entries_fn) else []
for item in model_entries:
entries.append({
"id": item.get("id"),
"owned_by": item.get("owned_by") or self.route_domain,
"display_name": item.get("display_name") or item.get("id"),
})
return entries
class CatalogRouter:
"""Universal dispatcher for dynamic catalog matching and model enumeration."""
def __init__(self, providers: Optional[Sequence[CatalogProvider]] = None):
self._providers: List[CatalogProvider] = list(providers or [ArenaCatalogProvider()])
def register_provider(self, provider: CatalogProvider) -> None:
self._providers.append(provider)
def match_route(
self,
browser: Any,
config_engine: Any,
tabs: List[Dict[str, Any]],
requested_model: Any,
preset_name: Any = None,
) -> Optional[Dict[str, Any]]:
"""Match a requested model against all registered catalog providers across open tabs."""
if not browser or not tabs:
return None
# Check if chat API has monkeypatched match/select function
chat_match_fn = _get_chat_or_service_attr("_match_arena_catalog_tab")
if callable(chat_match_fn) and not getattr(chat_match_fn, "_is_default_wrapper", False):
matched_tab_info = chat_match_fn(browser, config_engine, tabs, requested_model)
if matched_tab_info:
matched_tab = matched_tab_info.get("tab") or {}
matched_model = matched_tab_info.get("model") or {}
matched_preset = matched_tab_info.get("preset_name") or preset_name
import app.services.arena_direct_models as adm
get_public_id_fn = _get_chat_or_service_attr("get_arena_direct_model_public_id", adm)
public_model_id = get_public_id_fn(matched_model) if callable(get_public_id_fn) else str(requested_model or "")
catalog_tab_index = int(matched_tab.get("persistent_index") or 0) or None
return {
"route_domain": "arena.ai",
"route_type": "model_catalog",
"model_name": public_model_id or str(requested_model or ""),
"matched_id": public_model_id or str(requested_model or ""),
"match_type": "catalog",
"preset_name": matched_preset,
"available_model_ids": [public_model_id] if public_model_id else [],
"catalog_tab": matched_tab,
"catalog_tab_index": catalog_tab_index,
"matched_model": matched_model,
}
return None
is_url_excluded = getattr(getattr(browser, "tab_pool", None), "is_url_excluded", None)
for provider in self._providers:
matches: List[Dict[str, Any]] = []
for tab in tabs:
tab_url = str(tab.get("current_url") or tab.get("url") or "").strip()
if callable(is_url_excluded) and is_url_excluded(tab_url):
continue
matched = provider.match_tab_model(browser, config_engine, tab, requested_model)
if matched:
matches.append(matched)
if not matches:
continue
select_tab_fn = _get_chat_or_service_attr("_select_arena_catalog_tab")
if callable(select_tab_fn) and select_tab_fn is not select_catalog_tab:
selected_tab = select_tab_fn(
browser,
[entry["tab"] for entry in matches],
preset_name=preset_name,
)
else:
selected_tab = select_catalog_tab(
browser,
[entry["tab"] for entry in matches],
preset_name=preset_name,
)
if not selected_tab:
continue
selected_index = int(selected_tab.get("persistent_index") or 0)
selected_tab_id = str(selected_tab.get("id") or selected_tab.get("tab_id") or "").strip()
target_entry = next(
(
entry
for entry in matches
if entry["tab"] is selected_tab
or (selected_tab_id and str(entry["tab"].get("id") or entry["tab"].get("tab_id") or "").strip() == selected_tab_id)
or int(entry["tab"].get("persistent_index") or 0) == selected_index
),
None,
)
if not target_entry:
continue
catalog_tab = target_entry["tab"]
catalog_match = target_entry["model"]
matched_preset_name = target_entry["preset_name"]
public_model_id = target_entry.get("public_model_id") or ""
catalog_tab_index = int(catalog_tab.get("persistent_index") or 0) or None
matched_uuid = str(
catalog_match.get("id")
or catalog_match.get("uuid")
or catalog_match.get("arena_model_id")
or ""
)
matched_public_name = str(
catalog_match.get("public_name") or catalog_match.get("publicName") or ""
)
matched_display_name = str(
catalog_match.get("display_name")
or catalog_match.get("displayName")
or catalog_match.get("name")
or ""
)
logger.info(
f"[CATALOG_ROUTER:MATCH] 成功匹配动态目录模型: provider={provider.name}, "
f"requested_model={requested_model!r}, preset={matched_preset_name!r}, "
f"uuid={matched_uuid!r}, public_name={matched_public_name!r}, "
f"display_name={matched_display_name!r}, public_model_id={public_model_id!r}, "
f"target_tab_index={catalog_tab_index}, "
f"target_tab_url={catalog_tab.get('current_url') or catalog_tab.get('url')}"
)
return {
"route_domain": provider.route_domain,
"route_type": "model_catalog",
"model_name": public_model_id or str(requested_model or ""),
"matched_id": public_model_id or str(requested_model or ""),
"match_type": "catalog",
"preset_name": matched_preset_name,
"available_model_ids": [public_model_id] if public_model_id else [],
"catalog_tab": catalog_tab,
"catalog_tab_index": catalog_tab_index,
"matched_model": catalog_match,
}
return None
def collect_models_for_catalog(
self,
browser: Any,
config_engine: Any,
tabs: List[Dict[str, Any]],
append_entry: Callable[[str, Optional[str], Optional[str]], None],
created: int,
) -> None:
"""Collect models across tabs and catalog providers for the /v1/models endpoint."""
from app.utils.model_routing import collect_route_domain_models
# 1. 收集所有 Provider 的 Catalog 预设配置
provider_configs: Dict[str, List[Dict[str, Any]]] = {}
for provider in self._providers:
configs = provider.collect_catalog_configs(config_engine, tabs)
provider_configs[provider.name] = configs
# 2. 输出通用 Route Domain 模型,对被 Catalog 接管的 alias 模型进行抑制
for item in collect_route_domain_models(tabs):
suppressed = any(
provider.is_alias_suppressed(item, provider_configs.get(provider.name, []))
for provider in self._providers
)
if suppressed:
continue
append_entry(
item.get("id"),
item.get("route_domain") or "universal-web-api",
item.get("display_name") or item.get("id"),
)
# 3. 输出各 Catalog Provider 发现的动态模型列表
for provider in self._providers:
configs = provider_configs.get(provider.name, [])
if not configs:
continue
for model_entry in provider.list_catalog_model_entries(browser, configs, created):
append_entry(
model_entry.get("id"),
model_entry.get("owned_by") or provider.route_domain,
model_entry.get("display_name") or model_entry.get("id"),
)
catalog_router = CatalogRouter()
+460
View File
@@ -0,0 +1,460 @@
"""
app/services/cf_turnstile_solver.py - Cloudflare Turnstile / 5s 盾通用自动过盾服务
职责:
1. 探测并等待可点击人机验证按钮出现(支持动态选择器、自定义探测脚本或保底坐标);
2. 复用项目的低熵模式移动轨迹(smooth_move_mouse + cdp_precise_click)平滑移动并精确点击;
3. 确认验证通过(最多尝试指定次数,连续失败抛出异常);
4. 通过后比对当前 URL 与本次请求实际占用的目标 URL,支持参数化校验与重定向。
"""
from __future__ import annotations
import math
import random
import time
from typing import Any, Callable, Dict, List, Optional, Tuple
from urllib.parse import urlsplit
from app.core.config import logger as default_logger
from app.utils.human_mouse import cdp_precise_click as default_cdp_precise_click, smooth_move_mouse as default_smooth_move_mouse
from app.core.driver import driver_for_tab
_DEFAULT_MAX_ATTEMPTS = 3
DEFAULT_FIND_CLICK_POINT_JS = r"""
return (() => {
try {
const visibleRect = (el) => {
if (!el) return null;
const style = window.getComputedStyle(el);
const rect = el.getBoundingClientRect();
if (
style.display === 'none' ||
style.visibility === 'hidden' ||
Number(style.opacity || 1) <= 0.02 ||
rect.width < 8 ||
rect.height < 8
) {
return null;
}
const vw = window.innerWidth || document.documentElement.clientWidth || 0;
const vh = window.innerHeight || document.documentElement.clientHeight || 0;
if (rect.right < 0 || rect.bottom < 0 || rect.left > vw || rect.top > vh) {
return null;
}
const left = Math.max(0, rect.left);
const top = Math.max(0, rect.top);
const right = Math.min(vw, rect.right);
const bottom = Math.min(vh, rect.bottom);
return {
left,
top,
right,
bottom,
width: right - left,
height: bottom - top,
rawWidth: rect.width,
rawHeight: rect.height
};
};
const describe = (el) => {
const attrs = [
el.getAttribute('src'),
el.getAttribute('title'),
el.getAttribute('aria-label'),
el.getAttribute('name'),
el.id,
el.className
];
return attrs.map((v) => String(v || '')).join(' ').toLowerCase();
};
const candidates = [];
const pushCandidate = (el, kind, score) => {
const rect = visibleRect(el);
if (!rect) return;
const text = describe(el);
const checkboxLike = /checkbox|anchor|recaptcha|hcaptcha|turnstile|challenge|cloudflare/.test(text);
let clickX = rect.left + rect.width / 2;
let clickY = rect.top + rect.height / 2;
if (checkboxLike || rect.width >= 120) {
clickX = rect.left + Math.min(Math.max(rect.width * 0.16, 24), 46);
}
const insetX = Math.min(4, Math.max(0, (rect.width - 1) / 2));
const insetY = Math.min(4, Math.max(0, (rect.height - 1) / 2));
clickX = Math.max(rect.left + insetX, Math.min(rect.left + rect.width - 1 - insetX, clickX));
clickY = Math.max(rect.top + insetY, Math.min(rect.top + rect.height - 1 - insetY, clickY));
candidates.push({
kind,
score: score + (checkboxLike ? 20 : 0) + Math.min(rect.width, 320) / 100,
x: Math.round(clickX),
y: Math.round(clickY),
rect,
hint: text.slice(0, 160)
});
};
for (const el of document.querySelectorAll('iframe')) {
const text = describe(el);
const rect = visibleRect(el);
const isExpandedImage = !!rect && rect.width >= 360 && rect.height >= 300;
if (isExpandedImage) continue;
if (/recaptcha|google\.com\/recaptcha/.test(text)) pushCandidate(el, 'recaptcha_iframe', 100);
else if (/hcaptcha|hcaptcha\.com/.test(text)) pushCandidate(el, 'hcaptcha_iframe', 95);
else if (/turnstile|challenges\.cloudflare\.com|cloudflare|challenge/.test(text)) pushCandidate(el, 'cloudflare_iframe', 90);
}
for (const selector of [
'.cf-turnstile',
'#challenge-stage',
'#turnstile-wrapper',
'.g-recaptcha',
'.h-captcha',
'[data-sitekey]',
'[role="checkbox"]',
'input[type="checkbox"]'
]) {
for (const el of document.querySelectorAll(selector)) {
pushCandidate(el, selector, 50);
}
}
candidates.sort((a, b) => b.score - a.score);
const vw = window.innerWidth || document.documentElement.clientWidth || 800;
const vh = window.innerHeight || document.documentElement.clientHeight || 600;
if (candidates.length > 0) {
return { ok: true, ...candidates[0], viewport: { width: vw, height: vh } };
}
return {
ok: false,
reason: 'no_candidate_element',
viewport: { width: vw, height: vh },
fallback: {
x: Math.round(vw / 2 - 90),
y: Math.round(vh / 2 + 10)
}
};
} catch (e) {
return { ok: false, error: String(e) };
}
})();
"""
DEFAULT_PAGE_READY_PROBE_JS = r"""
return (() => {
try {
const isVisible = (el) => {
if (!el || !el.isConnected) return false;
const r = el.getBoundingClientRect();
const s = window.getComputedStyle(el);
return r.width >= 8 && r.height >= 8 && r.bottom > 0 && r.right > 0 &&
r.top < window.innerHeight && r.left < window.innerWidth &&
s.display !== 'none' && s.visibility !== 'hidden' &&
Number(s.opacity || 1) > 0.02;
};
const text = String(document.body && document.body.innerText ? document.body.innerText : '').toLowerCase();
const hasVerifyText = text.includes('security verification') ||
text.includes('protected by cloudflare') ||
text.includes('verify you are human') ||
text.includes('checking your browser') ||
text.includes('confirm you are human') ||
text.includes('确认您是真人') ||
text.includes('人机身份验证') ||
text.includes('请验证您是真人') ||
text.includes('正在进行安全验证');
const challengeIframes = Array.from(document.querySelectorAll(
'iframe[src*="challenges.cloudflare.com"], iframe[src*="challenge-platform"], iframe[title*="challenge" i], iframe[title*="cloudflare" i], iframe[title*="turnstile" i]'
));
const hasVisibleChallengeIframe = challengeIframes.some(isVisible);
if (hasVerifyText || hasVisibleChallengeIframe) {
return { resolved: false, reason: 'challenge_still_present' };
}
const hasInput = !!document.querySelector('textarea, [contenteditable="true"], input[placeholder*="ask" i], input[placeholder*="Type" i]');
const hasArenaUI = text.includes('new chat') || text.includes('leaderboard') || text.includes('battle mode') || hasInput;
return { resolved: true, ready: hasArenaUI || hasInput };
} catch (e) {
return { resolved: false, error: String(e) };
}
})();
"""
def _interruptible_sleep(seconds: float, raise_if_cancelled=None, step: float = 0.1) -> None:
deadline = time.time() + max(0.0, float(seconds))
while time.time() < deadline:
if raise_if_cancelled:
raise_if_cancelled()
remaining = deadline - time.time()
time.sleep(min(step, max(0.01, remaining)))
def is_valid_http_url(url: Optional[str]) -> bool:
"""通用 HTTP/HTTPS URL 有效性校验(排除 Cloudflare Challenge 自身页面)。"""
if not url or not isinstance(url, str):
return False
u = url.lower().strip()
if not (u.startswith("http://") or u.startswith("https://")):
return False
if "challenges.cloudflare.com" in u or "challenge-platform" in u:
return False
return True
def _urls_match(url1: Optional[str], url2: Optional[str]) -> bool:
u1 = str(url1 or "").strip().rstrip("/")
u2 = str(url2 or "").strip().rstrip("/")
if not u1 and not u2:
return True
if not u1 or not u2:
return False
if u1 == u2:
return True
try:
p1 = urlsplit(u1)
p2 = urlsplit(u2)
if p1.netloc.lower() == p2.netloc.lower() and p1.path.rstrip("/") == p2.path.rstrip("/"):
if p1.query == p2.query:
return True
except Exception:
pass
return False
def _get_occupied_url(
tab: Any,
session: Any = None,
default_url: str = "https://arena.ai/code",
validator_fn: Optional[Callable[[Optional[str]], bool]] = None,
) -> str:
"""提取本次请求在过盾前绑定的实际占用 URL。"""
validator = validator_fn or is_valid_http_url
if session is not None:
occupied = getattr(session, "_request_occupied_url", None)
if occupied and validator(occupied):
return str(occupied).strip()
last_known = getattr(session, "last_known_url", None)
if last_known and validator(last_known):
return str(last_known).strip()
try:
tab_url = str(getattr(tab, "url", "") or "").strip()
if tab_url and validator(tab_url):
return tab_url
except Exception:
pass
return default_url
def _wait_for_clickable_point(
tab: Any,
timeout_sec: float = 6.0,
raise_if_cancelled: Optional[Callable[[], None]] = None,
fallback_coords: Optional[Tuple[int, int]] = None,
find_click_point_js: Optional[str] = None,
log: Any = default_logger,
) -> Dict[str, Any]:
"""等待并获取可点击按钮坐标,若无法准确定位元素则使用固定/自适应保底坐标。"""
deadline = time.time() + max(1.0, float(timeout_sec))
last_fallback = None
js_code = find_click_point_js or DEFAULT_FIND_CLICK_POINT_JS
while time.time() < deadline:
if raise_if_cancelled:
raise_if_cancelled()
try:
probe = driver_for_tab(tab).run_js(js_code)
if isinstance(probe, dict):
if probe.get("ok"):
x = max(0, int(probe["x"]) + random.randint(-2, 2))
y = max(0, int(probe["y"]) + random.randint(-2, 2))
return {
"ok": True,
"x": x,
"y": y,
"kind": probe.get("kind", "element"),
"rect": probe.get("rect"),
}
elif probe.get("fallback"):
last_fallback = probe["fallback"]
except Exception as e:
log.debug(f"[CF_SOLVER] 寻找点击坐标异常: {e}")
_interruptible_sleep(0.3, raise_if_cancelled=raise_if_cancelled)
# 无法准确定位元素时启用保底坐标
if fallback_coords is not None:
fx, fy = max(0, int(fallback_coords[0])), max(0, int(fallback_coords[1]))
elif last_fallback and isinstance(last_fallback, dict):
fx = max(0, int(last_fallback.get("x", 310)))
fy = max(0, int(last_fallback.get("y", 310)))
else:
try:
vw = int(driver_for_tab(tab).run_js("return window.innerWidth || document.documentElement.clientWidth || 800") or 800)
vh = int(driver_for_tab(tab).run_js("return window.innerHeight || document.documentElement.clientHeight || 600") or 600)
fx = max(0, int(vw / 2 - 90))
fy = max(0, int(vh / 2 + 10))
except Exception:
fx, fy = 310, 310
fx = max(0, fx + random.randint(-4, 4))
fy = max(0, fy + random.randint(-4, 4))
log.info(f"[CF_SOLVER] 未定位到具体元素,启用固定/保底坐标点击: ({fx}, {fy})")
return {"ok": True, "x": fx, "y": fy, "kind": "fixed_fallback"}
def _wait_for_challenge_resolved(
tab: Any,
timeout_sec: float = 8.0,
raise_if_cancelled: Optional[Callable[[], None]] = None,
page_ready_probe_js: Optional[str] = None,
log: Any = default_logger,
) -> bool:
"""轮询等待验证通过(特征文字与 iframe 消失且页面就绪)。"""
deadline = time.time() + max(1.0, float(timeout_sec))
js_code = page_ready_probe_js or DEFAULT_PAGE_READY_PROBE_JS
while time.time() < deadline:
if raise_if_cancelled:
raise_if_cancelled()
try:
probe = driver_for_tab(tab).run_js(js_code)
if isinstance(probe, dict) and probe.get("resolved"):
return True
except Exception as e:
log.debug(f"[CF_SOLVER] 探测验证状态异常: {e}")
_interruptible_sleep(0.5, raise_if_cancelled=raise_if_cancelled)
return False
def solve_turnstile_challenge(
tab: Any,
session: Any = None,
target_url: Optional[str] = None,
is_valid_url_fn: Optional[Callable[[Optional[str]], bool]] = None,
ready_probe_js: Optional[str] = None,
click_point_js: Optional[str] = None,
fallback_coords: Optional[Tuple[int, int]] = None,
max_attempts: int = _DEFAULT_MAX_ATTEMPTS,
click_timeout_sec: float = 6.0,
resolve_timeout_sec: float = 8.0,
raise_if_cancelled: Optional[Callable[[], None]] = None,
smooth_move_fn: Optional[Callable[..., Any]] = None,
cdp_click_fn: Optional[Callable[..., Any]] = None,
logger: Any = None,
) -> Dict[str, Any]:
"""
通用 Cloudflare Turnstile / 5s 盾自动过盾流程。
"""
log = logger or default_logger
smooth_move = smooth_move_fn or default_smooth_move_mouse
cdp_click = cdp_click_fn or default_cdp_precise_click
if raise_if_cancelled:
raise_if_cancelled()
occupied_url = target_url or _get_occupied_url(
tab,
session,
validator_fn=is_valid_url_fn or is_valid_http_url,
)
log.info(f"[CF_SOLVER] 启动 Cloudflare 5s 盾自动处理: 实际占用目标 URL={occupied_url}")
attempts = 0
passed = False
while attempts < max_attempts:
attempts += 1
if raise_if_cancelled:
raise_if_cancelled()
log.info(f"[CF_SOLVER] 第 {attempts}/{max_attempts} 次尝试通过 5s 盾...")
click_target = _wait_for_clickable_point(
tab,
timeout_sec=click_timeout_sec,
raise_if_cancelled=raise_if_cancelled,
fallback_coords=fallback_coords,
find_click_point_js=click_point_js,
log=log,
)
click_x = max(0, int(click_target["x"]))
click_y = max(0, int(click_target["y"]))
log.info(
f"[CF_SOLVER] 准备低熵点击验证目标 (第 {attempts} 次): "
f"({click_x}, {click_y}), kind={click_target.get('kind', 'fallback')}"
)
start_x = max(0, click_x - random.randint(40, 80))
start_y = max(0, click_y - random.randint(30, 60))
smooth_move(tab, (start_x, start_y), (click_x, click_y))
_interruptible_sleep(random.uniform(0.06, 0.14), raise_if_cancelled=raise_if_cancelled)
click_ok = cdp_click(tab, click_x, click_y)
if not click_ok:
log.warning(f"[CF_SOLVER] CDP 精确点击未确认: ({click_x}, {click_y})")
passed = _wait_for_challenge_resolved(
tab,
timeout_sec=resolve_timeout_sec,
raise_if_cancelled=raise_if_cancelled,
page_ready_probe_js=ready_probe_js,
log=log,
)
if passed:
log.info(f"[CF_SOLVER] ✅ 第 {attempts} 次尝试后 5s 盾验证通过!")
break
else:
log.warning(f"[CF_SOLVER] 第 {attempts} 次点击后 5s 盾仍未通过,准备重试...")
_interruptible_sleep(random.uniform(1.5, 2.5), raise_if_cancelled=raise_if_cancelled)
if not passed:
err_msg = f"连续尝试 {max_attempts} 次仍未通过 Cloudflare 5s 盾人机验证"
log.error(f"[CF_SOLVER] ❌ {err_msg}")
raise RuntimeError(f"cf_turnstile_verify_failed_after_{max_attempts}_attempts")
current_url = str(getattr(tab, "url", "") or "").strip()
url_matches = _urls_match(current_url, occupied_url)
validator = is_valid_url_fn or is_valid_http_url
if not url_matches and occupied_url and validator(occupied_url):
log.warning(
f"[CF_SOLVER] 过盾后 URL 不一致: 当前 URL ({current_url}) != 实际占用 URL ({occupied_url}),"
f"正在跳转回实际占用 URL..."
)
try:
tab.get(occupied_url)
_interruptible_sleep(random.uniform(1.0, 2.0), raise_if_cancelled=raise_if_cancelled)
current_url = str(getattr(tab, "url", "") or "").strip()
log.info(f"[CF_SOLVER] 已跳转回实际占用 URL: {current_url}")
except Exception as e:
log.error(f"[CF_SOLVER] 跳转回实际占用 URL 失败: {e}")
raise
else:
log.info(f"[CF_SOLVER] 过盾后 URL 一致 ({current_url}),无需重定向。")
return {
"ok": True,
"attempts": attempts,
"current_url": current_url,
"occupied_url": occupied_url,
"url_redirected": not url_matches,
}
# 兼容别名
solve_arena_turnstile_challenge = solve_turnstile_challenge
__all__ = [
"DEFAULT_FIND_CLICK_POINT_JS",
"DEFAULT_PAGE_READY_PROBE_JS",
"_get_occupied_url",
"_interruptible_sleep",
"_urls_match",
"_wait_for_challenge_resolved",
"_wait_for_clickable_point",
"is_valid_http_url",
"solve_arena_turnstile_challenge",
"solve_turnstile_challenge",
]
+110
View File
@@ -0,0 +1,110 @@
"""
app/services/command_defs.py - Shared command definitions
Responsibilities:
- Default command payload
- Trigger/action type metadata
- Internal command-flow control exception
"""
import uuid
from typing import Any, Dict
TRIGGER_TYPES = {
"request_count": "请求计数",
"error_count": "错误计数",
"idle_timeout": "空闲超时",
"page_check": "页面检查,适合 Cloudflare 场景",
"command_check": "命令检查",
"command_triggered": "命令已触发",
"command_result_match": "命令返回结果",
"command_result_event": "命令结果事件",
"network_request_error": "网络异常",
}
ACTION_TYPES = {
"clear_cookies": "清除 Cookie",
"refresh_page": "刷新页面",
"new_chat": "新建对话",
"run_js": "执行 JavaScript",
"run_js_file": "执行 JS 文件",
"wait": "等待",
"execute_preset": "切换预设",
"execute_workflow": "执行工作流",
"switch_preset": "切换预设",
"navigate": "跳转 URL",
"switch_proxy": "切换 Clash 代理",
"send_webhook": "发送 Webhook / 通知",
"send_napcat": "发送 NapCat QQ 消息",
"execute_command_group": "执行命令组",
"abort_task": "中断任务",
"release_tab_lock": "解除标签页占用",
"click_element": "点击元素",
"click_coordinates": "点击坐标",
"click_captcha_challenge": "点击人机验证",
"write_element": "写入元素",
"read_element": "读取元素",
"http_request": "页面内请求",
"append_file": "追加到文件",
}
class CommandFlowAbort(Exception):
"""用于在动作链中提前中断后续步骤。"""
pass
def _new_command_id() -> str:
return f"cmd_{uuid.uuid4().hex[:8]}"
def get_default_command() -> Dict[str, Any]:
"""返回默认命令结构。"""
return {
"id": _new_command_id(),
"name": "新命令",
"enabled": True,
"log_enabled": True,
"log_level": "GLOBAL",
"mode": "simple",
"trigger": {
"type": "request_count",
"value": 10,
"command_id": "",
"action_ref": "",
"match_rule": "equals",
"expected_value": "",
"match_mode": "keyword",
"status_codes": "403,429,500,502,503,504",
"abort_on_match": True,
"command_ids": [],
"listen_all_commands": False,
"informative_only": True,
"scope": "all",
"domain": "",
"tab_index": None,
"priority": 2,
"allow_during_workflow": True,
"interrupt_policy": "auto",
"interrupt_message": "",
},
"actions": [
{"type": "clear_cookies"},
{"type": "refresh_page"},
],
"group_name": "",
"script": "",
"script_lang": "javascript",
"advanced_ui": {},
"last_triggered": None,
"trigger_count": 0,
}
__all__ = [
'ACTION_TYPES',
'CommandFlowAbort',
'TRIGGER_TYPES',
'_new_command_id',
'get_default_command',
]
+649
View File
@@ -0,0 +1,649 @@
"""
app/services/command_engine.py - 命令引擎
职责:
- 命令的 CRUD 管理
- 触发条件检查(在标签页释放后调用)
- 动作执行调度
- 高级模式脚本执行(JavaScript / Python)
存储位置:config/commands.json
"""
import copy
import os
import threading
from concurrent.futures import ThreadPoolExecutor
from typing import Dict, List, Optional, Any, TYPE_CHECKING
from app.core.page_lifecycle import (
BACKGROUND_WAKE_JS_TIMEOUT,
)
from app.services.command_defs import ACTION_TYPES, TRIGGER_TYPES, get_default_command
from app.services.command_engine_actions import CommandEngineActionsMixin
from app.services.command_engine_results import CommandEngineResultsMixin
from app.services.command_engine_runtime import CommandEngineRuntimeMixin
from app.services.command_engine_pages import CommandEnginePagesMixin
from app.services.command_engine_scheduler import CommandEngineSchedulerMixin
from app.services.command_engine_store import CommandEngineStoreMixin
from app.services.command_engine_crud import CommandEngineCrudMixin
from app.services.command_engine_triggers import CommandEngineTriggersMixin
from app.services.command_engine_page_check import CommandEnginePageCheckMixin
if TYPE_CHECKING:
from app.core.tab_pool import TabSession
# R2-3:模块级常量、日志器与可选依赖迁到 command_engine_common,这里重新导出以保持兼容
from app.services.command_engine_common import ( # noqa: E402,F401
FOLLOW_DEFAULT_PRESET,
HAS_REQUESTS,
_NON_ASCII_RE,
_WHITESPACE_RE,
_WORD_LIKE_KEYWORD_RE,
logger,
)
# ================= 常量 =================
class CommandEngine(
CommandEnginePagesMixin,
CommandEngineSchedulerMixin,
CommandEngineStoreMixin,
CommandEngineCrudMixin,
CommandEngineTriggersMixin,
CommandEnginePageCheckMixin,
CommandEngineRuntimeMixin,
CommandEngineResultsMixin,
CommandEngineActionsMixin,
):
"""命令引擎"""
def __init__(self):
self._config_engine = None
self._browser = None
self._commands_file = None
self._commands_local_file = None
self._commands_mtime = 0.0
self._commands_local_mtime = 0.0
# B8:最近一次「读失败」的命令文件 mtime;同一 mtime 不重复读取/刷错误日志
self._commands_failed_mtime: Optional[float] = None
# (cache_key, compact_text):page_check 去空白页面文本的单槽缓存
self._compact_haystack_cache: Optional[tuple] = None
self._commands_loaded = False
self._commands_cache: List[Dict[str, Any]] = []
self._command_runtime_stats: Dict[str, Dict[str, Any]] = {}
self._lock = threading.RLock()
self._commands_lock = threading.RLock()
self._scheduler_lifecycle_lock = threading.RLock()
self._shutdown_requested = False
# 触发状态:{(command_id, tab_id): {"req": int, "err": int, ...}}
self._trigger_states: Dict[tuple, Dict[str, Any]] = {}
self._pending_async_trigger_meta: Dict[tuple, Dict[str, Any]] = {}
# 最近命令执行结果:{(source_command_id, tab_id): {...}}
self._command_results: Dict[tuple, Dict[str, Any]] = {}
# 命令结果事件:{tab_id: [event, ...]}
self._command_result_events: Dict[str, List[Dict[str, Any]]] = {}
# 最近网络事件:{tab_id: [event, ...]}
self._network_events: Dict[str, List[Dict[str, Any]]] = {}
# 正在执行的命令(防止重复触发)
self._executing: set = set()
self._periodic_next_run: Dict[tuple, float] = {}
self._periodic_stop_event = threading.Event()
self._periodic_thread: Optional[threading.Thread] = None
self._pending_high_by_session: Dict[str, int] = {}
self._running_high_by_session: Dict[str, int] = {}
self._pending_high_by_domain: Dict[str, int] = {}
self._running_high_by_domain: Dict[str, int] = {}
try:
_max_async_workers = int(os.getenv("CMD_ASYNC_MAX_WORKERS", "20"))
except Exception:
_max_async_workers = 20
self._command_executor = ThreadPoolExecutor(
max_workers=max(1, _max_async_workers),
thread_name_prefix="cmd-exec",
)
try:
_baseline = int(os.getenv("CMD_REQUEST_PRIORITY_BASELINE", "2"))
except Exception:
_baseline = 2
self._request_priority_baseline = _baseline
self._activate_tab_on_command = str(
os.getenv("CMD_ACTIVATE_TAB_ON_COMMAND", "false")
).strip().lower() in {"1", "true", "yes", "y", "on"}
self._use_focus_emulation_on_command = str(
os.getenv("CMD_USE_FOCUS_EMULATION_ON_COMMAND", "true")
).strip().lower() in {"1", "true", "yes", "y", "on"}
self._wake_tab_before_page_check = str(
os.getenv("CMD_WAKE_TAB_BEFORE_PAGE_CHECK", "true")
).strip().lower() in {"1", "true", "yes", "y", "on"}
# 唤醒标签页的 CDP 状态(focus / visibility / Page.setWebLifecycleState=active)
# 是粘性的,不需要每条 page_check 命令都重发一遍。
# 每次唤醒是 4~6 次 CDP 往返 + 一次整页可见性补丁 JS,
# 而一个会话上往往同时挂着多条 page_check 命令,实际调用频率是每秒数次。
# 默认 2s 限频:既能把唤醒风暴压下去,最坏也只比原来晚 2s 唤醒
# (page_check 自身的最小轮询间隔就是 1.5s,量级相当)。
# 设为 0 可恢复"每次都唤醒"的旧行为;若观察到后台页仍被冻结可调小。
try:
_wake_min_interval = float(os.getenv("CMD_WAKE_TAB_MIN_INTERVAL_SEC", "2"))
except Exception:
_wake_min_interval = 2.0
self._wake_tab_min_interval_sec = max(0.0, _wake_min_interval)
self._tab_pool_auto_refresh = str(
os.getenv("CMD_TAB_POOL_AUTO_REFRESH", "true")
).strip().lower() in {"1", "true", "yes", "y", "on"}
try:
_refresh_interval = float(os.getenv("CMD_TAB_POOL_REFRESH_INTERVAL_SEC", "5"))
except Exception:
_refresh_interval = 5.0
self._tab_pool_refresh_interval_sec = max(1.0, _refresh_interval)
self._last_tab_pool_refresh_at = 0.0
# In memory saver mode, do not wake every idle tab on a fixed cadence.
# A page_check or real workflow still wakes its target tab on demand.
memory_saver_enabled = str(
os.getenv("BROWSER_MEMORY_SAVER", "false")
).strip().lower() in {"1", "true", "yes", "y", "on"}
# P0-6:开启空闲冻结时,周期 keepalive 会不断唤醒标签页,默认关闭
# (显式设置 CMD_PERIODIC_KEEPALIVE_ENABLED 仍以用户为准)。
try:
from app.core.tab_pool_parts.idle_maintenance import is_idle_freeze_enabled
idle_freeze_enabled = is_idle_freeze_enabled()
except Exception:
idle_freeze_enabled = False
keepalive_default = "false" if (memory_saver_enabled or idle_freeze_enabled) else "true"
self._periodic_keepalive_enabled = str(
os.getenv("CMD_PERIODIC_KEEPALIVE_ENABLED", keepalive_default)
).strip().lower() in {"1", "true", "yes", "y", "on"}
try:
_keepalive_interval = float(os.getenv("CMD_PERIODIC_KEEPALIVE_INTERVAL_SEC", "20"))
except Exception:
_keepalive_interval = 20.0
self._periodic_keepalive_interval_sec = max(5.0, _keepalive_interval)
try:
_page_check_js_timeout = float(os.getenv("CMD_PAGE_CHECK_JS_TIMEOUT_SEC", str(BACKGROUND_WAKE_JS_TIMEOUT)))
except Exception:
_page_check_js_timeout = BACKGROUND_WAKE_JS_TIMEOUT
self._page_check_js_timeout_sec = max(0.1, min(2.0, _page_check_js_timeout))
try:
_page_check_failure_backoff = float(os.getenv("CMD_PAGE_CHECK_FAILURE_BACKOFF_SEC", "30"))
except Exception:
_page_check_failure_backoff = 30.0
self._page_check_failure_backoff_sec = max(1.0, _page_check_failure_backoff)
try:
_page_check_refresh_grace = float(
os.getenv("CMD_PAGE_CHECK_REFRESH_GRACE_SEC", "2")
)
except Exception:
_page_check_refresh_grace = 2.0
self._page_check_refresh_grace_sec = max(0.2, min(10.0, _page_check_refresh_grace))
self._last_keepalive_by_session: Dict[str, float] = {}
self._last_tab_pool_wait_log_at = 0.0
self._last_periodic_summary_log_at = 0.0
self._periodic_active_log_interval_sec = 8.0
self._periodic_idle_log_interval_sec = 30.0
self._periodic_summary_log_enabled = str(
os.getenv("CMD_PERIODIC_SUMMARY_LOG_ENABLED", "false")
).strip().lower() in {"1", "true", "yes", "y", "on"}
# page_check observer: {session_id: set_of_keywords_installed}
self._observer_keywords_by_session: Dict[str, set] = {}
logger.debug("命令引擎已初始化")
if self._should_auto_start_scheduler():
self._start_periodic_scheduler()
def evict_session(self, session_id: str):
"""Evict runtime caches and pending state associated with a removed tab session."""
session_key = str(session_id or "").strip()
if not session_key:
return
logger.info(f"逐出会话缓存: {session_key}")
with self._lock:
keys_to_remove = [
key
for key in self._trigger_states
if isinstance(key, tuple) and len(key) > 1 and key[1] == session_key
]
for key in keys_to_remove:
self._trigger_states.pop(key, None)
meta_keys_to_remove = [
key
for key in self._pending_async_trigger_meta
if isinstance(key, tuple) and len(key) > 1 and key[1] == session_key
]
for key in meta_keys_to_remove:
self._pending_async_trigger_meta.pop(key, None)
result_keys_to_remove = [
key
for key in self._command_results
if isinstance(key, tuple) and len(key) > 1 and key[1] == session_key
]
for key in result_keys_to_remove:
self._command_results.pop(key, None)
exec_keys_to_remove = [
key
for key in self._executing
if isinstance(key, tuple) and len(key) > 1 and key[1] == session_key
]
for key in exec_keys_to_remove:
self._executing.discard(key)
run_keys_to_remove = [
key
for key in self._periodic_next_run
if isinstance(key, tuple) and len(key) > 1 and key[1] == session_key
]
for key in run_keys_to_remove:
self._periodic_next_run.pop(key, None)
self._command_result_events.pop(session_key, None)
self._network_events.pop(session_key, None)
self._pending_high_by_session.pop(session_key, None)
self._running_high_by_session.pop(session_key, None)
self._last_keepalive_by_session.pop(session_key, None)
self._observer_keywords_by_session.pop(session_key, None)
def _get_config_engine(self):
if self._config_engine is None:
from app.services.config_engine import config_engine
self._config_engine = config_engine
return self._config_engine
def _get_browser(self):
if self._browser is None:
from app.core.browser import get_browser
self._browser = get_browser(auto_connect=False)
return self._browser
@staticmethod
def _format_scope_label(scope: str) -> str:
mapping = {
"all": "全部标签页",
"domain": "同域",
"tab": "当前标签页",
}
return mapping.get(str(scope or "").strip().lower(), str(scope or "未指定"))
def _set_pending_async_trigger_meta(
self,
command: Dict[str, Any],
session: 'TabSession',
meta: Optional[Dict[str, Any]],
):
key = (str(command.get("id", "")).strip(), str(getattr(session, "id", "") or ""))
if not all(key):
return
with self._lock:
if meta:
self._pending_async_trigger_meta[key] = copy.deepcopy(meta)
else:
self._pending_async_trigger_meta.pop(key, None)
def _take_pending_async_trigger_meta(
self,
command: Dict[str, Any],
session: 'TabSession',
) -> Optional[Dict[str, Any]]:
key = (str(command.get("id", "")).strip(), str(getattr(session, "id", "") or ""))
if not all(key):
return None
with self._lock:
meta = self._pending_async_trigger_meta.pop(key, None)
return copy.deepcopy(meta) if meta else None
_PAGE_CHECK_OBSERVER_JS = r"""
(function() {
var kws = %KEYWORDS%;
function newSeen() {
// Set 判重是 O(1);旧实现用数组 + indexOf 是 O(n),整页遍历退化成
// O(n^2),长对话页面每次快照要跑上千万次比较,是浏览器端 CPU 的主要来源。
try {
if (typeof Set === 'function') {
var s = new Set();
return {
has: function(n) { return s.has(n); },
add: function(n) { s.add(n); }
};
}
} catch (e) {}
var arr = [];
return {
has: function(n) { return arr.indexOf(n) !== -1; },
add: function(n) { arr.push(n); }
};
}
function appendText(parts, value) {
if (typeof value === 'string' && value) parts.push(value);
}
function collectText(node, parts, seen) {
if (!node || seen.has(node)) return;
seen.add(node);
if (node.nodeType === 1) {
var tag = String(node.tagName || '').toUpperCase();
if (
tag === 'SCRIPT' ||
tag === 'STYLE' ||
tag === 'NOSCRIPT' ||
tag === 'TEMPLATE' ||
tag === 'META' ||
tag === 'HEAD'
) {
return;
}
try {
if (node.shadowRoot) collectText(node.shadowRoot, parts, seen);
} catch (e) {}
if (tag === 'IFRAME') {
try {
var doc = node.contentDocument || (node.contentWindow && node.contentWindow.document);
if (doc) collectText(doc.documentElement || doc.body, parts, seen);
} catch (e) {}
}
}
if (node.nodeType === 3) {
appendText(parts, String(node.nodeValue || '').trim());
}
var child = null;
try {
child = node.firstChild;
} catch (e) {}
while (child) {
collectText(child, parts, seen);
try {
child = child.nextSibling;
} catch (e) {
child = null;
}
}
}
function collectRootText(root, parts, seen) {
collectText(root, parts, seen);
}
function isElementVisible(el) {
if (!el) return false;
try {
var style = window.getComputedStyle ? window.getComputedStyle(el) : null;
var rect = el.getBoundingClientRect ? el.getBoundingClientRect() : null;
if (style) {
if (style.display === 'none' || style.visibility === 'hidden') return false;
if (Number(style.opacity || 1) <= 0.02) return false;
}
return !!rect && rect.width > 5 && rect.height > 5;
} catch (e) {
return false;
}
}
function hasVisibleSelector(selector) {
try {
var els = document.querySelectorAll(selector);
for (var i = 0; i < els.length; i++) {
if (isElementVisible(els[i])) return true;
}
return false;
} catch (e) {
return false;
}
}
function buildSnapshot() {
var parts = [];
var seen = newSeen();
appendText(parts, document.title || '');
collectRootText(document.documentElement || document.body, parts, seen);
var text = parts.join('\n').toLowerCase();
var cfIndicators = [];
if (text.indexOf('security verification') !== -1) cfIndicators.push('security verification');
if (text.indexOf('protected by cloudflare') !== -1) cfIndicators.push('protected by cloudflare');
if (text.indexOf('verify you are human') !== -1) cfIndicators.push('verify you are human');
if (text.indexOf('checking your browser') !== -1) cfIndicators.push('checking your browser');
if (text.indexOf('确认您是真人') !== -1) cfIndicators.push('确认您是真人');
if (
hasVisibleSelector('iframe[src*="challenges.cloudflare.com"]') ||
hasVisibleSelector('iframe[src*="turnstile"]') ||
hasVisibleSelector('.cf-turnstile') ||
hasVisibleSelector('[name="cf-turnstile-response"]') ||
hasVisibleSelector('[data-testid*="cf" i]')
) {
cfIndicators.push('cloudflare');
} else if (text.indexOf('cloudflare') !== -1) {
cfIndicators.push('cloudflare');
}
if (cfIndicators.length) {
text += '\n' + cfIndicators.join('\n');
}
var recaptchaIndicators = [];
if (text.indexOf('protected by recaptcha') !== -1) recaptchaIndicators.push('protected by recaptcha');
if (
hasVisibleSelector('iframe[src*="google.com/recaptcha"]') ||
hasVisibleSelector('iframe[src*="recaptcha"]') ||
hasVisibleSelector('.g-recaptcha') ||
hasVisibleSelector('[name="g-recaptcha-response"]') ||
hasVisibleSelector('[title*="recaptcha" i]') ||
hasVisibleSelector('[aria-label*="recaptcha" i]')
) {
recaptchaIndicators.push('recaptcha');
recaptchaIndicators.push('\u4eba\u673a\u8eab\u4efd\u9a8c\u8bc1');
} else if (text.indexOf('recaptcha') !== -1) {
recaptchaIndicators.push('recaptcha');
}
if (recaptchaIndicators.length) {
text += '\n' + recaptchaIndicators.join('\n');
}
return text;
}
if (window.__pcObserver && window.__pcKeywords) {
var same = kws.length === window.__pcKeywords.length &&
kws.every(function(k){ return window.__pcKeywords.indexOf(k) !== -1; });
if (same) return 'already_installed';
window.__pcObserver.disconnect();
window.__pcObserver = null;
}
window.__pcKeywords = kws;
window.__pcHits = {};
var pending = null;
var MIN_DELAY_MS = 200;
var MAX_DELAY_MS = 800;
// 自适应节流:下一次扫描的间隔取"上次扫描耗时 × 4",钳在 200~800ms。
// 换成 Set 判重后,即使 6 万节点的页面单次扫描也只要 30ms 左右,
// 算出来仍是下限 200ms —— 正常页面行为完全不变。
// 只有扫描耗时超过 50ms 的病态页面才会退避,且最多退到 800ms,
// 不会拖慢 page_check 对 Cloudflare 盾 / 报错文案的识别。
var nextDelayMs = MIN_DELAY_MS;
function doCheck() {
pending = null;
var startedAt = 0;
try {
startedAt = (window.performance && performance.now) ? performance.now() : Date.now();
} catch (e) {}
try {
var text = buildSnapshot();
window.__pcSnapshot = text;
for (var i = 0; i < window.__pcKeywords.length; i++) {
var k = window.__pcKeywords[i];
window.__pcHits[k] = text.indexOf(k) !== -1;
}
} catch(e) {}
try {
var endedAt = (window.performance && performance.now) ? performance.now() : Date.now();
var cost = Math.max(0, endedAt - startedAt);
nextDelayMs = Math.min(MAX_DELAY_MS, Math.max(MIN_DELAY_MS, Math.round(cost * 4)));
} catch (e) {
nextDelayMs = MIN_DELAY_MS;
}
}
doCheck();
window.__pcObserver = new MutationObserver(function() {
if (!pending) pending = setTimeout(doCheck, nextDelayMs);
});
var target = document.body || document.documentElement;
if (target) {
window.__pcObserver.observe(target, {
childList: true, subtree: true, characterData: true
});
}
return 'installed';
})();
"""
_PAGE_CHECK_SNAPSHOT_JS = r"""
return (function() {
function newSeen() {
// 同 _PAGE_CHECK_OBSERVER_JS:数组 indexOf 判重会让整页遍历退化成 O(n^2)。
try {
if (typeof Set === 'function') {
var s = new Set();
return {
has: function(n) { return s.has(n); },
add: function(n) { s.add(n); }
};
}
} catch (e) {}
var arr = [];
return {
has: function(n) { return arr.indexOf(n) !== -1; },
add: function(n) { arr.push(n); }
};
}
function appendText(parts, value) {
if (typeof value === 'string' && value) parts.push(value);
}
function collectText(node, parts, seen) {
if (!node || seen.has(node)) return;
seen.add(node);
if (node.nodeType === 1) {
var tag = String(node.tagName || '').toUpperCase();
if (
tag === 'SCRIPT' ||
tag === 'STYLE' ||
tag === 'NOSCRIPT' ||
tag === 'TEMPLATE' ||
tag === 'META' ||
tag === 'HEAD'
) {
return;
}
try {
if (node.shadowRoot) collectText(node.shadowRoot, parts, seen);
} catch (e) {}
if (tag === 'IFRAME') {
try {
var doc = node.contentDocument || (node.contentWindow && node.contentWindow.document);
if (doc) collectText(doc.documentElement || doc.body, parts, seen);
} catch (e) {}
}
}
if (node.nodeType === 3) {
appendText(parts, String(node.nodeValue || '').trim());
}
var child = null;
try {
child = node.firstChild;
} catch (e) {}
while (child) {
collectText(child, parts, seen);
try {
child = child.nextSibling;
} catch (e) {
child = null;
}
}
}
function collectRootText(root, parts, seen) {
collectText(root, parts, seen);
}
function isElementVisible(el) {
if (!el) return false;
try {
var style = window.getComputedStyle ? window.getComputedStyle(el) : null;
var rect = el.getBoundingClientRect ? el.getBoundingClientRect() : null;
if (style) {
if (style.display === 'none' || style.visibility === 'hidden') return false;
if (Number(style.opacity || 1) <= 0.02) return false;
}
return !!rect && rect.width > 5 && rect.height > 5;
} catch (e) {
return false;
}
}
function hasVisibleSelector(selector) {
try {
var els = document.querySelectorAll(selector);
for (var i = 0; i < els.length; i++) {
if (isElementVisible(els[i])) return true;
}
return false;
} catch (e) {
return false;
}
}
var parts = [];
appendText(parts, document.title || '');
collectRootText(document.documentElement || document.body, parts, newSeen());
var text = parts.join('\n').toLowerCase();
var cfIndicators = [];
if (text.indexOf('security verification') !== -1) cfIndicators.push('security verification');
if (text.indexOf('protected by cloudflare') !== -1) cfIndicators.push('protected by cloudflare');
if (text.indexOf('verify you are human') !== -1) cfIndicators.push('verify you are human');
if (text.indexOf('checking your browser') !== -1) cfIndicators.push('checking your browser');
if (text.indexOf('确认您是真人') !== -1) cfIndicators.push('确认您是真人');
if (
hasVisibleSelector('iframe[src*="challenges.cloudflare.com"]') ||
hasVisibleSelector('iframe[src*="turnstile"]') ||
hasVisibleSelector('.cf-turnstile') ||
hasVisibleSelector('[name="cf-turnstile-response"]') ||
hasVisibleSelector('[data-testid*="cf" i]')
) {
cfIndicators.push('cloudflare');
} else if (text.indexOf('cloudflare') !== -1) {
cfIndicators.push('cloudflare');
}
if (cfIndicators.length) {
text += '\n' + cfIndicators.join('\n');
}
var recaptchaIndicators = [];
if (text.indexOf('protected by recaptcha') !== -1) recaptchaIndicators.push('protected by recaptcha');
if (
hasVisibleSelector('iframe[src*="google.com/recaptcha"]') ||
hasVisibleSelector('iframe[src*="recaptcha"]') ||
hasVisibleSelector('.g-recaptcha') ||
hasVisibleSelector('[name="g-recaptcha-response"]') ||
hasVisibleSelector('[title*="recaptcha" i]') ||
hasVisibleSelector('[aria-label*="recaptcha" i]')
) {
recaptchaIndicators.push('recaptcha');
recaptchaIndicators.push('\u4eba\u673a\u8eab\u4efd\u9a8c\u8bc1');
} else if (text.indexOf('recaptcha') !== -1) {
recaptchaIndicators.push('recaptcha');
}
if (recaptchaIndicators.length) {
text += '\n' + recaptchaIndicators.join('\n');
}
return text;
})();
"""
def shutdown(self):
with self._scheduler_lifecycle_lock:
self._shutdown_requested = True
self._periodic_stop_event.set()
thread = self._periodic_thread
if thread and thread.is_alive():
thread.join(timeout=1.0)
try:
self._command_executor.shutdown(wait=False, cancel_futures=True)
except TypeError:
self._command_executor.shutdown(wait=False)
except Exception as e:
logger.debug(f"[CMD] 命令线程池关闭失败(忽略): {e}")
# ================= 单例 =================
command_engine = CommandEngine()
__all__ = [
'CommandEngine',
'command_engine',
'TRIGGER_TYPES',
'ACTION_TYPES',
'get_default_command',
]
File diff suppressed because it is too large. Load diff
+21
View File
@@ -0,0 +1,21 @@
"""CommandEngine 及其 mixin 共用的模块级对象(R2-3 拆分时从 command_engine.py 迁出,原处仍然重新导出)。"""
import re
from app.core.config import get_logger
try:
import requests # noqa: F401
HAS_REQUESTS = True
except ImportError: # pragma: no cover - requests 为必装依赖
requests = None # type: ignore[assignment]
HAS_REQUESTS = False
logger = get_logger("CMD_ENG")
FOLLOW_DEFAULT_PRESET = "__DEFAULT__"
# 页面文本匹配相关的正则预编译:这些会在 page_check 高频路径上
# 对整页快照(可能几十上百 KB)反复求值,逐次 re.* 的模式缓存查找是不必要的开销。
_WHITESPACE_RE = re.compile(r"\s+")
_NON_ASCII_RE = re.compile(r"[^\x00-\x7F]")
_WORD_LIKE_KEYWORD_RE = re.compile(r"[a-z0-9 _-]+")
+553
View File
@@ -0,0 +1,553 @@
"""CommandEngine 的一部分(R2-3 拆分):命令与分组的增删改查、分组执行与预览。
方法原样从 command_engine.py 迁出,行为不变;通过 mixin 组合回 CommandEngine,
类属性与实例状态都在 CommandEngine(command_engine.py)里定义。
"""
import copy
import uuid
from typing import Dict, List, Optional, Any, TYPE_CHECKING
from app.services.command_defs import _new_command_id, get_default_command
from app.services.command_engine_common import logger
if TYPE_CHECKING:
from app.core.tab_pool import TabSession # noqa: F401
class CommandEngineCrudMixin:
"""命令与分组的增删改查、分组执行与预览"""
def list_commands(self) -> List[Dict]:
"""获取所有命令"""
return self._merge_runtime_stats_into_commands(self._load_commands())
def get_command(self, command_id: str) -> Optional[Dict]:
command_key = str(command_id or "").strip()
if not command_key:
return None
command = None
with self._commands_lock:
self._get_config_engine()
self._refresh_commands_if_changed()
for cmd in self._commands_cache:
if not isinstance(cmd, dict):
continue
if str(cmd.get("id", "")).strip() == command_key:
command = copy.deepcopy(cmd)
break
if command is None:
return None
return self._merge_runtime_stats_into_command(command)
def get_command_config(self, command_id: str) -> Optional[Dict]:
"""获取单条命令原始配置,不合并展示用运行态统计。"""
command_key = str(command_id or "").strip()
if not command_key:
return None
with self._commands_lock:
self._get_config_engine()
self._refresh_commands_if_changed()
for cmd in self._commands_cache:
if not isinstance(cmd, dict):
continue
if str(cmd.get("id", "")).strip() == command_key:
return copy.deepcopy(cmd)
return None
def add_command(self, command: Dict = None) -> Optional[Dict]:
if command is None:
command = get_default_command()
else:
if not command.get("id"):
command["id"] = _new_command_id()
with self._commands_lock:
commands = self._load_commands()
command["name"] = self._ensure_unique_command_name(command.get("name"), commands)
command["group_name"] = self._normalize_group_name(command.get("group_name"))
command["advanced_ui"] = self._normalize_advanced_ui(command.get("advanced_ui"))
self._normalize_command_logging(command)
commands.append(command)
if not self._save_commands(commands):
return None
logger.info(f"[OK] 命令已添加: {command.get('name')} ({command['id']})")
return copy.deepcopy(command)
def duplicate_command(self, command_id: str) -> Optional[Dict]:
"""复制单条命令,并将副本紧跟在原命令之后。"""
command_key = str(command_id or "").strip()
if not command_key:
return None
with self._commands_lock:
commands = self._load_commands()
source_index = next(
(index for index, item in enumerate(commands) if item.get("id") == command_key),
-1,
)
if source_index < 0:
return None
duplicated = copy.deepcopy(commands[source_index])
duplicated["id"] = _new_command_id()
duplicated["name"] = self._ensure_unique_copy_name(
duplicated.get("name"),
{str(item.get("name") or "").strip() for item in commands},
)
self._normalize_command_logging(duplicated)
commands.insert(source_index + 1, duplicated)
if not self._save_commands(commands):
return None
logger.info(f"[OK] 命令已复制: {command_key} -> {duplicated['id']}")
return copy.deepcopy(duplicated)
def duplicate_group(self, group_name: str) -> Optional[Dict[str, Any]]:
"""复制整个命令组,并重映射副本组内的命令引用。"""
source_group = self._normalize_group_name(group_name)
if not source_group:
return None
with self._commands_lock:
commands = self._load_commands()
source_commands = [
item
for item in commands
if self._normalize_group_name(item.get("group_name")) == source_group
]
if not source_commands:
return None
existing_group_names = {
self._normalize_group_name(item.get("group_name"))
for item in commands
if self._normalize_group_name(item.get("group_name"))
}
target_group = self._ensure_unique_copy_name(source_group, existing_group_names)
command_id_map = {
str(item.get("id") or "").strip(): _new_command_id()
for item in source_commands
if str(item.get("id") or "").strip()
}
existing_command_names = {
str(item.get("name") or "").strip()
for item in commands
if str(item.get("name") or "").strip()
}
duplicated_commands: List[Dict[str, Any]] = []
for source in source_commands:
duplicated = copy.deepcopy(source)
source_id = str(source.get("id") or "").strip()
duplicated["id"] = command_id_map.get(source_id) or _new_command_id()
duplicated["name"] = self._ensure_unique_copy_name(
source.get("name"),
existing_command_names,
)
existing_command_names.add(duplicated["name"])
duplicated["group_name"] = target_group
self._remap_duplicated_command_references(
duplicated,
command_id_map,
source_group_name=source_group,
target_group_name=target_group,
)
self._normalize_command_logging(duplicated)
duplicated_commands.append(duplicated)
commands.extend(duplicated_commands)
if not self._save_commands(commands):
return None
logger.info(
f"[OK] 命令组已复制: {source_group} -> {target_group} "
f"({len(duplicated_commands)} 条命令)"
)
return {
"group_name": target_group,
"commands": copy.deepcopy(duplicated_commands),
"count": len(duplicated_commands),
}
def update_command(self, command_id: str, updates: Dict) -> Optional[Dict]:
updates = dict(updates or {})
with self._commands_lock:
commands = self._load_commands()
for i, cmd in enumerate(commands):
if cmd.get("id") == command_id:
updates.pop("id", None)
if "name" in updates:
updates["name"] = self._ensure_unique_command_name(
updates.get("name"),
commands,
exclude_id=command_id,
)
if "group_name" in updates:
updates["group_name"] = self._normalize_group_name(updates.get("group_name"))
if "advanced_ui" in updates:
updates["advanced_ui"] = self._normalize_advanced_ui(updates.get("advanced_ui"))
cmd.update(updates)
self._normalize_command_logging(cmd)
commands[i] = cmd
if not self._save_commands(commands):
return None
logger.debug(f"[OK] 命令已更新: {cmd.get('name')} ({command_id})")
return copy.deepcopy(cmd)
return None
def delete_command(self, command_id: str) -> bool:
with self._commands_lock:
commands = self._load_commands()
new_commands = [c for c in commands if c.get("id") != command_id]
if len(new_commands) == len(commands):
return False
if not self._save_commands(new_commands):
return False
# 清理触发状态
with self._lock:
keys_to_remove = [k for k in self._trigger_states if k[0] == command_id]
for k in keys_to_remove:
del self._trigger_states[k]
result_keys = [k for k in self._command_results if k[0] == command_id]
for k in result_keys:
del self._command_results[k]
self._command_runtime_stats.pop(command_id, None)
logger.info(f"[OK] 命令已删除: {command_id}")
return True
def reorder_commands(self, command_ids: List[str]) -> bool:
with self._commands_lock:
commands = self._load_commands()
cmd_map = {c["id"]: c for c in commands}
new_commands = []
for cid in command_ids:
if cid in cmd_map:
new_commands.append(cmd_map.pop(cid))
for remaining in cmd_map.values():
new_commands.append(remaining)
if len(new_commands) == len(commands) and all(
current is updated for current, updated in zip(commands, new_commands)
):
return True
return self._save_commands(new_commands)
return True
def set_commands_group(self, command_ids: List[str], group_name: str) -> int:
"""批量设置命令分组。group_name 为空时表示解散选中的命令。"""
target_ids = {str(cid).strip() for cid in (command_ids or []) if str(cid).strip()}
if not target_ids:
return 0
normalized_group = self._normalize_group_name(group_name)
updated = 0
with self._commands_lock:
commands = self._load_commands()
for cmd in commands:
if cmd.get("id") not in target_ids:
continue
if self._normalize_group_name(cmd.get("group_name")) == normalized_group:
continue
cmd["group_name"] = normalized_group
updated += 1
if updated > 0 and not self._save_commands(commands):
return -1
return updated
def rename_group(self, old_group_name: str, new_group_name: str) -> int:
"""重命名命令组。"""
source_name = self._normalize_group_name(old_group_name)
target_name = self._normalize_group_name(new_group_name)
if not source_name or not target_name or source_name == target_name:
return 0
updated = 0
with self._commands_lock:
commands = self._load_commands()
for cmd in commands:
if self._normalize_group_name(cmd.get("group_name")) != source_name:
continue
cmd["group_name"] = target_name
updated += 1
if updated > 0 and not self._save_commands(commands):
return -1
return updated
def set_commands_enabled(self, command_ids: List[str], enabled: bool) -> int:
"""批量更新命令启用状态。"""
target_ids = {str(cid).strip() for cid in (command_ids or []) if str(cid).strip()}
if not target_ids:
return 0
desired_enabled = bool(enabled)
updated = 0
with self._commands_lock:
commands = self._load_commands()
for cmd in commands:
if cmd.get("id") not in target_ids:
continue
current_enabled = bool(cmd.get("enabled", True))
if current_enabled == desired_enabled:
continue
cmd["enabled"] = desired_enabled
updated += 1
if updated > 0 and not self._save_commands(commands):
return -1
return updated
def disband_group(self, group_name: str) -> int:
"""解散整个命令组。"""
normalized_group = self._normalize_group_name(group_name)
if not normalized_group:
return 0
updated = 0
with self._commands_lock:
commands = self._load_commands()
for cmd in commands:
if self._normalize_group_name(cmd.get("group_name")) != normalized_group:
continue
cmd["group_name"] = ""
updated += 1
if updated > 0 and not self._save_commands(commands):
return -1
return updated
def set_group_enabled(self, group_name: str, enabled: bool) -> int:
"""直接更新整个命令组的启用状态。"""
normalized_group = self._normalize_group_name(group_name)
if not normalized_group:
return 0
desired_enabled = bool(enabled)
updated = 0
with self._commands_lock:
commands = self._load_commands()
for cmd in commands:
if self._normalize_group_name(cmd.get("group_name")) != normalized_group:
continue
current_enabled = bool(cmd.get("enabled", True))
if current_enabled == desired_enabled:
continue
cmd["enabled"] = desired_enabled
updated += 1
if updated > 0 and not self._save_commands(commands):
return -1
return updated
def list_command_groups(self) -> List[Dict[str, Any]]:
groups: Dict[str, Dict[str, Any]] = {}
for cmd in self._load_commands_for_checks():
group_name = self._normalize_group_name(cmd.get("group_name"))
if not group_name:
continue
bucket = groups.setdefault(group_name, {
"name": group_name,
"count": 0,
"enabled_count": 0,
"command_ids": [],
})
bucket["count"] += 1
bucket["enabled_count"] += 1 if cmd.get("enabled", True) else 0
bucket["command_ids"].append(cmd.get("id"))
return [groups[name] for name in sorted(groups.keys())]
def execute_command_group(
self,
group_name: str,
session: 'TabSession',
include_disabled: bool = False,
source_command_id: Optional[str] = None,
ancestry_chain: Optional[List[str]] = None,
acquire_policy: Optional[str] = None,
prepared_plan: Optional[Dict[str, Any]] = None,
) -> Dict[str, Any]:
"""在当前会话中顺序执行命令组内的命令。"""
normalized_group = self._normalize_group_name(group_name)
if not normalized_group:
return {"ok": False, "error": "empty_group_name"}
effective_policy = self._normalize_group_acquire_policy(acquire_policy)
plan = dict(prepared_plan or {}) if isinstance(prepared_plan, dict) else {}
if not plan:
plan = self.preview_command_group(
group_name=normalized_group,
session=session,
include_disabled=include_disabled,
source_command_id=source_command_id,
ancestry_chain=ancestry_chain,
)
candidates: List[Dict[str, Any]] = list(plan.pop("_candidate_commands", []))
results: List[Dict[str, Any]] = []
initial_scope_skipped = int(plan.get("scope_skipped", 0) or 0)
if not candidates:
return {
"ok": False,
"error": "group_empty_or_no_runnable_commands",
"group_name": normalized_group,
"scope_skipped": initial_scope_skipped,
"runnable_count": plan.get("runnable_count", 0),
}
chain_seed = list(ancestry_chain or [])
if not chain_seed and source_command_id:
chain_seed = [source_command_id]
group_acquired = False
group_task_id = f"group_{normalized_group}_{uuid.uuid4().hex[:12]}"
can_acquire = hasattr(session, "acquire_for_command")
if effective_policy != "inherit_session":
if not can_acquire:
if effective_policy == "require_acquire":
return {
"ok": False,
"error": "group_acquire_unavailable",
"group_name": normalized_group,
"acquire_policy": effective_policy,
}
else:
group_acquired = bool(session.acquire_for_command(group_task_id))
if not group_acquired and effective_policy == "require_acquire":
return {
"ok": False,
"error": "group_acquire_failed",
"group_name": normalized_group,
"acquire_policy": effective_policy,
}
try:
for cmd in candidates:
command_id = cmd.get("id")
if not command_id:
continue
if not self._matches_scope(cmd, session):
results.append({
"id": command_id,
"name": cmd.get("name", command_id),
"ok": False,
"error": "scope_mismatch",
})
continue
exec_key = (command_id, session.id)
with self._lock:
if exec_key in self._executing:
results.append({
"id": command_id,
"name": cmd.get("name", command_id),
"ok": False,
"error": "already_executing",
})
continue
self._executing.add(exec_key)
with self._command_logging_context(cmd):
try:
execution_result = self._execute_command(cmd, session, chain=chain_seed)
command_ok = not self._execution_needs_page_check_retry(execution_result)
results.append({
"id": command_id,
"name": cmd.get("name", command_id),
"ok": command_ok,
"result": execution_result,
**({"error": "execution_not_ok"} if not command_ok else {}),
})
except Exception as e:
logger.error(f"trigger check failed [{cmd.get('name')}]: {e}")
results.append({
"id": command_id,
"name": cmd.get("name", command_id),
"ok": False,
"error": str(e),
})
finally:
with self._lock:
self._executing.discard(exec_key)
finally:
if group_acquired:
try:
browser = self._get_browser()
pool = getattr(browser, "_tab_pool", None)
if pool is not None and hasattr(pool, "release"):
pool.release(
session.id,
check_triggers=False,
expected_task_id=group_task_id,
)
else:
session.release(
clear_page=False,
check_triggers=False,
expected_task_id=group_task_id,
)
except Exception as e:
logger.debug(f"[CMD] 命令组释放标签页失败(忽略): {e}")
success_count = sum(1 for item in results if item.get("ok"))
failure_count = sum(1 for item in results if not item.get("ok"))
return {
"ok": len(results) > 0 and failure_count == 0,
"partial_ok": success_count > 0,
"group_name": normalized_group,
"executed": success_count,
"total": len(results),
"failures": failure_count,
"results": results,
"acquire_policy": effective_policy,
"acquired": group_acquired,
"scope_skipped": sum(1 for item in results if item.get("error") == "scope_mismatch"),
"runnable_count": plan.get("runnable_count", 0),
}
def preview_command_group(
self,
group_name: str,
session: 'TabSession',
include_disabled: bool = False,
source_command_id: Optional[str] = None,
ancestry_chain: Optional[List[str]] = None,
) -> Dict[str, Any]:
normalized_group = self._normalize_group_name(group_name)
if not normalized_group:
return {
"ok": False,
"error": "empty_group_name",
"group_name": normalized_group,
"_candidate_commands": [],
"_scope_skipped_results": [],
}
with self._commands_lock:
commands = self._load_commands()
candidates: List[Dict[str, Any]] = []
scope_skipped_results: List[Dict[str, Any]] = []
total_candidates = 0
ancestors = {str(item or "").strip() for item in (ancestry_chain or []) if str(item or "").strip()}
if source_command_id:
ancestors.add(str(source_command_id).strip())
for cmd in commands:
if self._normalize_group_name(cmd.get("group_name")) != normalized_group:
continue
if not include_disabled and not cmd.get("enabled", True):
continue
if str(cmd.get("id", "")).strip() in ancestors:
continue
total_candidates += 1
candidates.append(cmd)
if not self._matches_scope(cmd, session):
scope_skipped_results.append({
"id": cmd.get("id"),
"name": cmd.get("name", cmd.get("id", "")),
"ok": False,
"error": "scope_mismatch",
})
continue
return {
"ok": bool(candidates) and not scope_skipped_results,
"fully_runnable": bool(candidates) and not scope_skipped_results,
"group_name": normalized_group,
"total_candidates": total_candidates,
"runnable_count": total_candidates - len(scope_skipped_results),
"scope_skipped": len(scope_skipped_results),
"_candidate_commands": candidates,
"_scope_skipped_results": scope_skipped_results,
}
+346
View File
@@ -0,0 +1,346 @@
"""CommandEngine 的一部分(R2-3 拆分):页面内容检查:表达式解析、文本匹配、快照求值与锁存。
方法原样从 command_engine.py 迁出,行为不变;通过 mixin 组合回 CommandEngine,
类属性与实例状态都在 CommandEngine(command_engine.py)里定义。
"""
import re
import time
from typing import Dict, List, Optional, Any, TYPE_CHECKING
from app.services.command_engine_common import _NON_ASCII_RE, _WHITESPACE_RE, _WORD_LIKE_KEYWORD_RE, logger
from app.core.driver import driver_for_tab
if TYPE_CHECKING:
from app.core.tab_pool import TabSession # noqa: F401
class CommandEnginePageCheckMixin:
"""页面内容检查:表达式解析、文本匹配、快照求值与锁存"""
@staticmethod
def _parse_page_check_expression(value: str):
"""解析 page_check 的 value 字段,支持 || (OR) 和 && (AND) 语法。
返回 (operator, keywords_list):
- ("or", [kw1, kw2, ...]) — 任意一个关键词命中即触发
- ("and", [kw1, kw2, ...]) — 所有关键词都命中才触发
- ("single", [value]) — 单关键词(向后兼容)
当 || 和 && 同时出现时,|| 优先拆分。
"""
raw = str(value or "").strip()
if not raw:
return ("single", [])
if "||" in raw:
parts = [p.strip() for p in raw.split("||") if p.strip()]
return ("or", parts) if parts else ("single", [])
if "&&" in raw:
parts = [p.strip() for p in raw.split("&&") if p.strip()]
return ("and", parts) if parts else ("single", [])
return ("single", [raw])
@staticmethod
def _normalize_match_text(value: Any) -> str:
text = str(value or "").strip().lower()
if not text:
return ""
return _WHITESPACE_RE.sub(" ", text)
def _get_compact_haystack(self, hay: str) -> str:
"""去空白版页面文本(带缓存)。
page_check 每个非 ASCII 关键词都要一份去空白文本,
而页面快照往往有几十上百 KB;原实现按关键词重复做全文 re.sub,
一次检查里同一段文本会被重复扫描 N 次。这里按内容缓存一份即可。
"""
if not hay:
return ""
cached = self._compact_haystack_cache
# 用对象身份比较:同一轮检查里各个关键词拿到的 hay 是同一个字符串对象,
# 比 (len, hash) 更简单,也不存在哈希碰撞返回错误结果的可能。
if cached is not None and cached[0] is hay:
return cached[1]
compact = _WHITESPACE_RE.sub("", hay)
self._compact_haystack_cache = (hay, compact)
return compact
def _text_contains_needle(
self,
haystack: str,
needle: str,
pre_normalized_haystack: Optional[str] = None,
) -> bool:
hay = pre_normalized_haystack if pre_normalized_haystack is not None else self._normalize_match_text(haystack)
ned = self._normalize_match_text(needle)
if not hay or not ned:
return False
if _NON_ASCII_RE.search(ned):
compact_hay = self._get_compact_haystack(hay)
compact_ned = _WHITESPACE_RE.sub("", ned)
if compact_hay and compact_ned and compact_ned in compact_hay:
return True
# For plain word-like keywords (for example "battle"), prefer whole-word
# matching to reduce accidental substring hits.
if _WORD_LIKE_KEYWORD_RE.fullmatch(ned):
pattern = rf"(?<![a-z0-9]){re.escape(ned)}(?![a-z0-9])"
return re.search(pattern, hay) is not None
return ned in hay
def _get_page_check_snapshot_text(self, session: 'TabSession') -> str:
if self._is_session_closed(session):
return ""
if self._is_page_check_refreshing(session):
return ""
now = time.time()
cached = getattr(session, "_pc_snapshot_cached", None)
if isinstance(cached, tuple) and len(cached) == 2:
cached_ts, cached_text = cached
try:
cached_ts = float(cached_ts or 0.0)
except Exception:
cached_ts = 0.0
if cached_ts > 0.0 and now - cached_ts < 0.5:
return str(cached_text or "")
if self._is_page_check_backing_off(session):
return ""
self._try_wake_tab(session, reason="page_check")
try:
snapshot = self._run_page_check_js(
session,
"return window.__pcObserver ? String(window.__pcSnapshot || '') : null"
)
self._record_page_check_js_success(session)
if snapshot is not None:
snapshot_text = str(snapshot or "")
if snapshot_text.strip():
setattr(session, "_pc_snapshot_cached", (now, snapshot_text))
return snapshot_text
except Exception as e:
if self._is_session_closed(session):
return ""
self._record_page_check_js_failure(session, e, "snapshot_observer")
if self._mark_session_closed_if_disconnected(session, e, "page_check_snapshot_observer"):
return ""
if self._is_page_check_refreshing(session):
return ""
pass
try:
page_text = str(self._run_page_check_js(session, self._PAGE_CHECK_SNAPSHOT_JS) or "")
self._record_page_check_js_success(session)
if page_text.strip():
setattr(session, "_pc_snapshot_cached", (now, page_text))
return page_text
except Exception as e:
if self._is_session_closed(session):
return ""
self._record_page_check_js_failure(session, e, "snapshot_body")
if self._mark_session_closed_if_disconnected(session, e, "page_check_snapshot_body"):
return ""
if self._is_page_check_refreshing(session):
return ""
pass
try:
title_text = str(self._run_page_check_js(session, "return document.title || '';") or "")
self._record_page_check_js_success(session)
setattr(session, "_pc_snapshot_cached", (now, title_text))
return title_text
except Exception as e:
if self._is_session_closed(session):
return ""
self._record_page_check_js_failure(session, e, "snapshot_title")
self._mark_session_closed_if_disconnected(session, e, "page_check_snapshot_title")
return ""
def _build_page_check_snapshot_preview(
self,
snapshot: str,
matched_keywords: Optional[List[str]] = None,
limit: int = 180,
) -> str:
normalized_snapshot = self._normalize_match_text(snapshot)
if not normalized_snapshot:
return ""
start = 0
for keyword in matched_keywords or []:
normalized_keyword = self._normalize_match_text(keyword)
if not normalized_keyword:
continue
index = normalized_snapshot.find(normalized_keyword)
if index != -1:
start = max(0, index - 60)
break
end = min(len(normalized_snapshot), start + max(40, limit))
preview = normalized_snapshot[start:end]
if start > 0:
preview = "..." + preview
if end < len(normalized_snapshot):
preview = preview + "..."
return preview.replace("'", '"')
def _evaluate_page_check_snapshot(
self,
snapshot: str,
op: str,
keywords: List[str],
) -> Dict[str, Any]:
if not keywords:
return {
"hit": False,
"matched_keywords": [],
"snapshot_preview": "",
}
normalized_snapshot = self._normalize_match_text(snapshot)
matched_keywords = [
keyword for keyword in keywords
if self._text_contains_needle(
snapshot,
keyword,
pre_normalized_haystack=normalized_snapshot,
)
]
if op == "or":
hit = bool(matched_keywords)
else:
hit = len(matched_keywords) == len(keywords)
return {
"hit": hit,
"matched_keywords": matched_keywords,
"snapshot_preview": self._build_page_check_snapshot_preview(snapshot, matched_keywords),
}
def _evaluate_page_check_expr(
self,
session: 'TabSession',
op: str,
keywords: List[str],
) -> Dict[str, Any]:
snapshot = self._get_page_check_snapshot_text(session)
result = self._evaluate_page_check_snapshot(snapshot, op, keywords)
result["snapshot_text"] = snapshot
return result
def _evaluate_page_check_probe(
self,
session: 'TabSession',
code: str,
) -> Dict[str, Any]:
self._try_wake_tab(session, reason="page_check_probe")
context = {
"parser_id": str(getattr(session, "_workflow_parser_id", "") or "").strip().lower(),
"target_side": str(getattr(session, "_workflow_target_side", "") or "").strip().lower(),
"runtime_id": str(getattr(session, "_workflow_runtime_id", "") or "").strip(),
}
try:
try:
driver_for_tab(session.tab).run_js(
"return (() => { window.__codexWorkflowContext = arguments[0] || {}; return true; })()",
context,
)
except Exception as context_error:
logger.debug(f"[CMD] 页面检查上下文注入失败(继续探测): {context_error}")
result = self._run_command_js(session.tab, code)
except Exception as e:
message = f"probe_js_failed: {e}"
return {
"hit": False,
"result": message,
"summary": message,
}
if isinstance(result, dict):
hit = bool(result.get("hit"))
summary = str(
result.get("summary", result.get("result", result))
or ""
).strip()
else:
hit = bool(result)
summary = "" if result in (None, False, "", 0) else str(result).strip()
return {
"hit": hit,
"result": result,
"summary": summary[:180],
}
def _log_page_check_hit_details(
self,
command: Dict,
session: 'TabSession',
match_info: Dict[str, Any],
):
matched_keywords = match_info.get("matched_keywords") or []
preview = str(match_info.get("snapshot_preview") or "").strip()
probe_summary = str(match_info.get("probe_summary") or "").strip()
probe_suffix = f", JS探测='{probe_summary}'" if probe_summary else ""
logger.info(
f"[CMD] 页面检查命中详情: {command.get('name')} "
f"(标签页={session.id}, 命中关键词={matched_keywords}, 快照预览='{preview}'{probe_suffix})"
)
def _check_page_content_expr(
self, session: 'TabSession', op: str, keywords: list
) -> bool:
"""根据解析后的表达式 (op, keywords) 检查页面内容。
- op="or" 任意一个关键词命中即返回 True
- op="and" 所有关键词都命中才返回 True
- op="single" 等同于 and(只有一个关键词)
"""
return bool(self._evaluate_page_check_expr(session, op, keywords).get("hit"))
def _check_page_content(self, session: 'TabSession', text: str) -> bool:
needle = str(text or "").strip()
if not needle:
return False
snapshot = self._get_page_check_snapshot_text(session)
return self._text_contains_needle(snapshot, needle)
def _reset_page_check_latch(self, command: Dict, session: 'TabSession', reason: str = ""):
"""Allow page_check commands to retrigger when previous execution did not complete successfully."""
trigger = command.get("trigger", {}) or {}
if str(trigger.get("type", "")).strip().lower() != "page_check":
return
if not trigger.get("reset_latch_on_failure", True):
# Only block latch reset if the command actually started executing (not blocked by session acquire timeout, scheduling failures, or page navigation)
if reason not in {"acquire_timeout", "workflow_schedule_failed"} and not (
reason.startswith("navigated") or reason.startswith("tab_navigated")
):
return
key = (command.get("id"), getattr(session, "id", ""))
if not key[0] or not key[1]:
return
normalized_text = str(trigger.get("value", "") or "").strip().lower()
with self._lock:
state = self._trigger_states.get(key)
if not state:
return
state["page_key"] = normalized_text
state["page_hit"] = False
state["page_stable"] = False
state["page_hit_since"] = 0.0
if bool(trigger.get("once_per_request", False)):
current_request_id = ""
try:
for attr_name in ("_command_request_id", "_bound_request_id"):
req_id = str(getattr(session, attr_name, "") or "").strip()
if req_id:
current_request_id = req_id
break
if not current_request_id:
task_id = str(getattr(session, "current_task_id", "") or "").strip()
if task_id.startswith("req-"):
current_request_id = task_id
except Exception:
pass
if current_request_id and "triggered_requests" in state:
state["triggered_requests"].discard(current_request_id)
if reason:
logger.debug(
f"[CMD] 页面检查锁存已重置: {command.get('name')} "
f"(标签页={session.id}, 原因={reason})"
)
+439
View File
@@ -0,0 +1,439 @@
"""CommandEngine 的一部分(R2-3 拆分):标签页交互:网络监听挂起/恢复、会话状态判断、焦点模拟、唤醒节流、页面检查脚本与观察者。
方法原样从 command_engine.py 迁出,行为不变;通过 mixin 组合回 CommandEngine,
类属性与实例状态都在 CommandEngine(command_engine.py)里定义。
"""
import copy
import json
import os
import time
from typing import Dict, List, Any, TYPE_CHECKING
from app.core.page_lifecycle import (
BACKGROUND_WAKE_CDP_TIMEOUT,
BACKGROUND_WAKE_JS_TIMEOUT,
install_visibility_emulation,
is_page_refresh_error,
restore_visibility_emulation,
)
from app.services.command_engine_common import logger
from app.core.driver import driver_for_tab
if TYPE_CHECKING:
from app.core.tab_pool import TabSession # noqa: F401
class CommandEnginePagesMixin:
"""标签页交互:网络监听挂起/恢复、会话状态判断、焦点模拟、唤醒节流、页面检查脚本与观察者"""
def _suspend_tab_global_network(self, session: 'TabSession', reason: str = "command"):
"""命令执行期间暂停标签页全局网络监听,避免和工作流监听冲突。"""
try:
browser = self._get_browser()
pool = getattr(browser, "_tab_pool", None)
if pool is not None and hasattr(pool, "suspend_global_network_monitor"):
pool.suspend_global_network_monitor(session.id, reason=reason)
except Exception as e:
logger.debug(f"[CMD] 暂停全局网络监听失败(忽略): {e}")
def _resume_tab_global_network(self, session: 'TabSession', reason: str = "command"):
"""命令执行结束后恢复标签页全局网络监听。"""
try:
browser = self._get_browser()
pool = getattr(browser, "_tab_pool", None)
if pool is not None and hasattr(pool, "resume_global_network_monitor"):
pool.resume_global_network_monitor(session.id, reason=reason)
except Exception as e:
logger.debug(f"[CMD] 恢复全局网络监听失败(忽略): {e}")
@staticmethod
def _session_status_value(session: 'TabSession') -> str:
return str(getattr(getattr(session, "status", None), "value", "")).lower()
def _is_session_closed(self, session: 'TabSession') -> bool:
return self._session_status_value(session) == "closed"
@staticmethod
def _looks_like_page_disconnected_error(error: Any) -> bool:
text = str(error or "").lower()
return (
"连接已断开" in text
or "connection disconnected" in text
or "target closed" in text
or "session closed" in text
or "websocket" in text and "closed" in text
)
@staticmethod
def _looks_like_page_refresh_error(error: Any) -> bool:
return is_page_refresh_error(error)
def _mark_session_closed_if_disconnected(self, session: 'TabSession', error: Any, reason: str) -> bool:
if not self._looks_like_page_disconnected_error(error):
return False
if getattr(session, "_cdp_recycle_in_progress", False):
# P0-3:空闲 CDP 会话回收期间的短暂断开不是标签页关闭
return True
try:
if hasattr(session, "mark_closed"):
session.mark_closed(reason)
except Exception:
pass
return True
def _set_focus_emulation(self, session: 'TabSession', enabled: bool):
"""Best-effort focus emulation without stealing OS/browser foreground focus."""
try:
driver_for_tab(session.tab).run_cdp(
"Emulation.setFocusEmulationEnabled",
enabled=bool(enabled),
_timeout=BACKGROUND_WAKE_CDP_TIMEOUT,
)
if enabled:
install_visibility_emulation(
session.tab,
owner=session,
reason="command_focus_emulation",
)
else:
restore_visibility_emulation(
session.tab,
owner=session,
reason="command_focus_emulation_end",
)
except Exception as e:
logger.debug(f"[CMD] 焦点模拟设置失败(忽略): enabled={enabled}, 错误={e}")
def _should_wake_tab_now(self, session: 'TabSession') -> bool:
"""按会话限频唤醒动作:唤醒后的 CDP 状态是粘性的,不必每轮都重发。"""
interval = float(self._wake_tab_min_interval_sec or 0.0)
if interval <= 0.0:
return True
try:
# 上一轮 page_check JS 已经出错,说明页面可能真的被冻结/丢弃了。
# 这种情况下不限频,保持原来"每轮都唤醒"的行为,
# 避免连续两轮失败把会话推进 30s 退避。
if int(getattr(session, "_pc_js_failures", 0) or 0) > 0:
setattr(session, "_pc_last_wake_at", time.time())
return True
except Exception:
return True
try:
now = time.time()
last_at = float(getattr(session, "_pc_last_wake_at", 0.0) or 0.0)
if last_at > 0.0 and (now - last_at) < interval:
return False
setattr(session, "_pc_last_wake_at", now)
except Exception:
return True
return True
def _invalidate_wake_throttle(self, session: 'TabSession') -> None:
"""让下一次 page_check 立即重新唤醒(JS 失败/页面可能已被冻结时调用)。"""
try:
setattr(session, "_pc_last_wake_at", 0.0)
except Exception:
pass
def _try_wake_tab(self, session: 'TabSession', reason: str = ""):
"""
Best-effort wake-up for background/discard-prone tabs.
Uses lifecycle and lightweight JS ping, without forcing browser focus.
"""
if self._is_session_closed(session):
return
# P0-6:page_check / keepalive 之前先解除空闲冻结(与唤醒节流无关,必须执行)
try:
from app.core.tab_pool_parts.idle_maintenance import resume_if_frozen
resume_if_frozen(session, reason=reason or "wake_tab")
except Exception:
pass
if not self._wake_tab_before_page_check:
return
if self._is_page_check_backing_off(session):
return
if not self._should_wake_tab_now(session):
return
focus_emulation_set = False
try:
driver_for_tab(session.tab).run_cdp(
"Emulation.setFocusEmulationEnabled",
enabled=True,
_timeout=BACKGROUND_WAKE_CDP_TIMEOUT,
)
focus_emulation_set = True
except Exception as e:
if self._mark_session_closed_if_disconnected(session, e, "wake_focus_emulation"):
return
try:
install_visibility_emulation(session.tab, owner=session, reason=reason or "wake_tab")
except Exception as e:
if self._mark_session_closed_if_disconnected(session, e, "wake_visibility"):
return
try:
driver_for_tab(session.tab).run_cdp(
"Page.setWebLifecycleState",
state="active",
_timeout=BACKGROUND_WAKE_CDP_TIMEOUT,
)
except Exception as e:
if self._mark_session_closed_if_disconnected(session, e, "wake_lifecycle"):
return
try:
driver_for_tab(session.tab).run_js("return document.readyState || '';", timeout=BACKGROUND_WAKE_JS_TIMEOUT)
except Exception as e:
if self._mark_session_closed_if_disconnected(session, e, "wake_ready_state"):
return
finally:
if focus_emulation_set:
try:
driver_for_tab(session.tab).run_cdp(
"Emulation.setFocusEmulationEnabled",
enabled=False,
_timeout=BACKGROUND_WAKE_CDP_TIMEOUT,
)
except Exception:
pass
def _run_page_check_js(self, session: 'TabSession', script: str) -> Any:
if self._is_session_closed(session):
raise RuntimeError("page_check session is closed")
return driver_for_tab(session.tab).run_js(script, timeout=self._page_check_js_timeout_sec)
def _is_page_check_backing_off(self, session: 'TabSession') -> bool:
until = float(getattr(session, "_pc_js_backoff_until", 0.0) or 0.0)
return until > time.time()
def _is_page_check_refreshing(self, session: 'TabSession') -> bool:
until = float(getattr(session, "_pc_refresh_grace_until", 0.0) or 0.0)
return until > time.time()
def _mark_page_check_refreshing(self, session: 'TabSession') -> None:
"""Pause page checks briefly while navigation replaces the document."""
self._invalidate_wake_throttle(session)
sid = str(getattr(session, "id", "") or "")
if sid:
with self._lock:
self._observer_keywords_by_session.pop(sid, None)
try:
setattr(
session,
"_pc_refresh_grace_until",
time.time() + self._page_check_refresh_grace_sec,
)
setattr(session, "_pc_js_failures", 0)
setattr(session, "_pc_js_backoff_until", 0.0)
setattr(session, "_pc_snapshot_cached", None)
except Exception:
pass
def _record_page_check_js_success(self, session: 'TabSession') -> None:
try:
setattr(session, "_pc_js_failures", 0)
setattr(session, "_pc_js_backoff_until", 0.0)
setattr(session, "_pc_refresh_grace_until", 0.0)
except Exception:
pass
def _record_page_check_js_failure(self, session: 'TabSession', error: Any, context: str) -> None:
if self._looks_like_page_refresh_error(error):
self._mark_page_check_refreshing(session)
return
# JS 执行失败通常意味着页面被冻结/丢弃或已导航,
# 清掉唤醒限频,让下一轮立刻重新唤醒,不受最小间隔限制。
self._invalidate_wake_throttle(session)
try:
failures = int(getattr(session, "_pc_js_failures", 0) or 0) + 1
setattr(session, "_pc_js_failures", failures)
if failures >= 2:
until = time.time() + self._page_check_failure_backoff_sec
setattr(session, "_pc_js_backoff_until", until)
logger.debug(
f"[CMD] 页面检查 JS 连续失败,暂停后台检查: "
f"标签页={getattr(session, 'id', '-')}, failures={failures}, "
f"backoff={self._page_check_failure_backoff_sec:.0f}s, context={context}, error={error}"
)
except Exception:
pass
def _collect_page_check_keywords(
self,
commands: List[Dict],
session: 'TabSession',
) -> set:
"""Collect all page_check keywords that apply to this session."""
keywords: set = set()
for cmd in commands:
if not cmd.get("enabled", True):
continue
trigger = cmd.get("trigger", {}) or {}
if str(trigger.get("type", "")).strip().lower() != "page_check":
continue
if not bool(trigger.get("periodic_enabled", True)):
continue
if not self._matches_scope(cmd, session):
continue
value = str(trigger.get("value", "") or "").strip()
if value:
_, parts = self._parse_page_check_expression(value)
for part in parts:
kw = part.lower().strip()
if kw:
keywords.add(kw)
return keywords
def _sync_session_bootstrap_js_files(
self,
commands: List[Dict],
session: 'TabSession',
) -> None:
"""Install explicitly opted-in JS files when a managed tab becomes idle.
`run_js_file` actions normally run only after their trigger fires. Some
page-resilience scripts must also be registered before a later reload,
so they opt in with ``bootstrap_on_session_ready``. The per-session
mtime cache keeps this out of the hot periodic path while still
replacing a registered script when its source changes.
"""
state = getattr(session, "_command_bootstrap_js_files", None)
if not isinstance(state, dict):
state = {}
setattr(session, "_command_bootstrap_js_files", state)
for command in commands:
if not isinstance(command, dict) or not command.get("enabled", True):
continue
if not self._matches_scope(command, session):
continue
command_id = str(command.get("id", "") or "").strip()
for action in command.get("actions", []) or []:
if not isinstance(action, dict):
continue
if str(action.get("type", "")).strip().lower() != "run_js_file":
continue
if not self._coerce_action_bool(action.get("bootstrap_on_session_ready"), False):
continue
resolved_path = self._resolve_action_file_path(action.get("file_path", ""))
if not resolved_path or not os.path.isfile(resolved_path):
logger.warning(
f"[CMD] 启动 JS 文件不存在: command={command_id or '-'}, path={resolved_path or action.get('file_path', '')!r}"
)
continue
try:
source_mtime = os.path.getmtime(resolved_path)
except OSError as error:
logger.warning(
f"[CMD] 读取启动 JS 文件时间失败: command={command_id or '-'}, path={resolved_path!r}, error={error}"
)
continue
action_id = str(action.get("action_id", "") or "").strip()
cache_key = (command_id, action_id, resolved_path.lower())
if state.get(cache_key) == source_mtime:
continue
try:
result = self._execute_action(copy.deepcopy(action), session)
except Exception as error:
logger.warning(
f"[CMD] 启动 JS 文件同步失败: command={command_id or '-'}, path={resolved_path!r}, error={error}"
)
continue
if self._is_action_soft_failure(result):
logger.warning(
f"[CMD] 启动 JS 文件同步未完成: command={command_id or '-'}, path={resolved_path!r}, result={result!r}"
)
continue
state[cache_key] = source_mtime
logger.debug(
f"[CMD] 已同步启动 JS 文件: command={command_id or '-'}, path={resolved_path!r}"
)
def _ensure_page_check_observer(
self,
session: 'TabSession',
keywords: set,
):
"""Inject or update MutationObserver for real-time page_check text detection."""
if not keywords:
return
if self._is_session_closed(session):
return
sid = str(getattr(session, "id", "") or "")
if not sid:
return
if self._is_page_check_backing_off(session):
return
if self._is_page_check_refreshing(session):
return
setattr(session, "_pc_observer_empty_cleanup_done", False)
# Skip if already installed with the same keywords AND observer is still alive
with self._lock:
installed = self._observer_keywords_by_session.get(sid)
if installed == keywords:
# Quick liveness check with throttling (outside lock — run_js is I/O).
now = time.time()
last_check = float(getattr(session, "_last_pc_observer_check_at", 0.0) or 0.0)
if now - last_check < 5.0:
return
setattr(session, "_last_pc_observer_check_at", now)
try:
alive = self._run_page_check_js(session, "return !!window.__pcObserver")
if alive:
self._record_page_check_js_success(session)
return
except Exception as e:
if self._is_session_closed(session):
return
self._record_page_check_js_failure(session, e, "observer_alive")
if self._mark_session_closed_if_disconnected(session, e, "page_check_observer_alive"):
return
if self._is_page_check_refreshing(session):
return
pass
# Observer lost — clear cache, re-inject below
with self._lock:
if self._observer_keywords_by_session.get(sid) == installed:
self._observer_keywords_by_session.pop(sid, None)
# Inject observer
sorted_kws = sorted(keywords)
js = self._PAGE_CHECK_OBSERVER_JS.replace(
"%KEYWORDS%", json.dumps(sorted_kws)
)
try:
result = self._run_page_check_js(session, js)
self._record_page_check_js_success(session)
with self._lock:
self._observer_keywords_by_session[sid] = set(keywords)
if result != "already_installed":
logger.debug(
f"[CMD] 页面检查观察器已注入: "
f"标签页={sid}, 关键词={sorted_kws}"
)
except Exception as e:
if self._is_session_closed(session):
return
self._record_page_check_js_failure(session, e, "observer_install")
self._mark_session_closed_if_disconnected(session, e, "page_check_observer_install")
def _clear_page_check_observer(self, session: 'TabSession') -> None:
if self._is_session_closed(session):
return
sid = str(getattr(session, "id", "") or "")
if not sid:
return
with self._lock:
installed = self._observer_keywords_by_session.pop(sid, None)
if installed is None and bool(getattr(session, "_pc_observer_empty_cleanup_done", False)):
return
if self._is_page_check_backing_off(session):
return
cleanup_js = """
return (() => {
if (window.__pcObserver && typeof window.__pcObserver.disconnect === 'function') {
window.__pcObserver.disconnect();
}
window.__pcObserver = null;
window.__pcKeywords = [];
window.__pcHits = {};
window.__pcSnapshot = '';
return true;
})();
"""
try:
self._run_page_check_js(session, cleanup_js)
self._record_page_check_js_success(session)
setattr(session, "_pc_observer_empty_cleanup_done", True)
except Exception as e:
if self._is_session_closed(session):
return
self._record_page_check_js_failure(session, e, "observer_cleanup")
self._mark_session_closed_if_disconnected(session, e, "page_check_observer_cleanup")
logger.debug(f"[CMD] 页面检查观察器注入失败: {e}")
+962
View File
@@ -0,0 +1,962 @@
import copy
import json
import re
import time
import uuid
from typing import Any, Dict, List, Optional, TYPE_CHECKING
from urllib.parse import urlsplit
from app.core.config import get_logger
if TYPE_CHECKING:
from app.core.tab_pool import TabSession
logger = get_logger("CMD_ENG")
class CommandEngineResultsMixin:
MAX_EVENT_QUEUE_ITEMS = 50
MAX_COMMAND_RESULT_EVENTS = 50
@staticmethod
def _normalize_domain_host(value: Any) -> str:
text = str(value or "").strip().lower()
if not text:
return ""
try:
if "://" in text:
parsed = urlsplit(text)
text = parsed.hostname or text
else:
text = text.split("/", 1)[0]
except Exception:
pass
if text.count(":") == 1:
host, port = text.rsplit(":", 1)
if port.isdigit():
text = host
return text.strip().strip(".")
def _domain_matches(self, target_domain: Any, actual_domain: Any) -> bool:
target = self._normalize_domain_host(target_domain)
actual = self._normalize_domain_host(actual_domain)
if not target or not actual:
return False
return actual == target or actual.endswith(f".{target}")
def _stringify_result(self, value: Any) -> str:
if value is None:
return ""
if isinstance(value, str):
return value
if isinstance(value, (dict, list, tuple)):
try:
return json.dumps(value, ensure_ascii=False, sort_keys=True)
except Exception:
return str(value)
return str(value)
def _match_value_rule(self, actual: str, expected: str, rule: str) -> bool:
normalized_rule = self._normalize_match_rule(rule)
if normalized_rule == "contains":
if not expected:
return False
return expected in actual
if normalized_rule == "not_equals":
return actual != expected
return actual == expected
def _get_command_result(self, command_id: str, tab_id: str) -> Optional[Dict[str, Any]]:
with self._lock:
return copy.deepcopy(self._command_results.get((command_id, tab_id)))
def _match_command_result_trigger(
self,
command: Dict[str, Any],
session: 'TabSession',
consume: bool = False,
) -> bool:
trigger = command.get("trigger", {})
source_id = str(trigger.get("command_id", "")).strip()
if not source_id:
return False
result_entry = self._get_command_result(source_id, session.id)
if not result_entry:
return False
action_ref = str(trigger.get("action_ref", "")).strip()
actual = result_entry.get("result", "")
if action_ref:
actual = result_entry.get("step_results", {}).get(action_ref, actual)
expected = self._stringify_result(trigger.get("expected_value", ""))
rule = trigger.get("match_rule", "equals")
if not self._match_value_rule(self._stringify_result(actual), expected, rule):
return False
if consume:
state, _ = self._ensure_trigger_state(command["id"], session)
token = str(result_entry.get("token", ""))
if state.get("result_token") == token:
return False
with self._lock:
state["result_token"] = token
return True
def _prepare_command_result_trigger_dispatch(
self,
command: Dict[str, Any],
session: 'TabSession',
) -> Optional[Dict[str, Any]]:
trigger = command.get("trigger", {})
source_id = str(trigger.get("command_id", "")).strip()
if not source_id:
return None
result_entry = self._get_command_result(source_id, session.id)
if not result_entry:
return None
action_ref = str(trigger.get("action_ref", "")).strip()
actual = result_entry.get("result", "")
if action_ref:
actual = result_entry.get("step_results", {}).get(action_ref, actual)
expected = self._stringify_result(trigger.get("expected_value", ""))
rule = trigger.get("match_rule", "equals")
if not self._match_value_rule(self._stringify_result(actual), expected, rule):
return None
token = str(result_entry.get("token", ""))
state, _ = self._ensure_trigger_state(command["id"], session)
with self._lock:
if state.get("result_token") == token:
return None
state["result_token"] = token
return {
"rollback": {"kind": "result_token", "token": token},
}
@staticmethod
def _extract_command_check_actual(
trigger: Dict[str, Any],
execution_result: Dict[str, Any],
) -> str:
action_ref = str(trigger.get("action_ref", "")).strip()
actual: Any = execution_result.get("result", "")
if action_ref:
for step in execution_result.get("steps", []) or []:
if str(step.get("action_ref", "")).strip() == action_ref:
actual = step.get("result", "")
break
return actual
def _run_command_check_source(
self,
command: Dict[str, Any],
source_command: Dict[str, Any],
session: 'TabSession',
) -> Optional[Dict[str, Any]]:
source_id = str((source_command or {}).get("id", "")).strip()
session_id = str(getattr(session, "id", "") or "").strip()
if not source_id or not session_id:
return None
exec_key = (source_id, session_id)
with self._lock:
if exec_key in self._executing:
return None
self._executing.add(exec_key)
with self._command_logging_context(source_command):
try:
return self._execute_command(
source_command,
session,
chain=[str((command or {}).get("id", "")).strip()],
record_result=False,
emit_followups=False,
update_trigger_stats=False,
)
except Exception as e:
logger.warning(
f"[CMD] 命令检查执行失败: {source_command.get('name', source_id)} "
f"(标签页={session_id}, 错误={e})"
)
return {
"mode": str(source_command.get("mode", "") or ""),
"result": f"command_check_failed: {e}",
"steps": [],
"error": str(e),
}
finally:
with self._lock:
self._executing.discard(exec_key)
def _prepare_command_check_dispatch(
self,
command: Dict[str, Any],
session: 'TabSession',
) -> Optional[Dict[str, Any]]:
trigger = command.get("trigger", {}) or {}
source_id = str(trigger.get("command_id", "")).strip()
target_id = str(command.get("id", "")).strip()
if not source_id or not target_id or source_id == target_id:
return None
source_command = self.get_command(source_id)
if not source_command:
return None
execution_result = self._run_command_check_source(command, source_command, session)
if not execution_result:
return None
actual_raw = self._extract_command_check_actual(trigger, execution_result)
actual = self._stringify_result(actual_raw)
expected = self._stringify_result(trigger.get("expected_value", ""))
rule = trigger.get("match_rule", "equals")
matched = self._match_value_rule(actual, expected, rule)
state, _ = self._ensure_trigger_state(command["id"], session)
signature = f"{source_id}|{str(trigger.get('action_ref', '')).strip()}|{actual}"
with self._lock:
if matched:
if str(state.get("command_check_sig", "") or "") == signature:
return None
state["command_check_sig"] = signature
state["command_check_actual"] = actual[:500]
else:
state["command_check_sig"] = ""
state["command_check_actual"] = actual[:500]
return None
return {
"interrupt_context": {
"command_check": {
"source_command_id": source_id,
"source_command_name": str(source_command.get("name", "") or source_id),
"actual_result": actual[:500],
"expected_value": expected[:200],
"match_rule": str(rule or "equals"),
}
}
}
def _record_command_result(
self,
command: Dict[str, Any],
session: 'TabSession',
execution_result: Dict[str, Any],
):
if not command.get("id"):
return
token = f"{int(time.time() * 1000)}_{uuid.uuid4().hex[:6]}"
step_results = {}
for step in execution_result.get("steps", []) or []:
ref = str(step.get("action_ref", "")).strip()
if not ref:
continue
step_results[ref] = self._stringify_result(step.get("result", ""))
entry = {
"token": token,
"timestamp": time.time(),
"result": self._stringify_result(execution_result.get("result", "")),
"step_results": step_results,
"mode": execution_result.get("mode", ""),
}
logger.debug(f"[CMD] 执行结果已记录: {command.get('name')} (标签页={session.id})")
event = self._build_command_result_event(command, execution_result, entry, session)
with self._lock:
self._command_results[(command["id"], session.id)] = entry
self._append_bounded_event(self._command_result_events, session.id, event)
def _build_command_result_event(
self,
command: Dict[str, Any],
execution_result: Dict[str, Any],
entry: Dict[str, Any],
session: 'TabSession',
) -> Dict[str, Any]:
summary, informative = self._summarize_command_result(command, execution_result, entry)
return {
"token": entry.get("token", ""),
"timestamp": float(entry.get("timestamp", time.time()) or time.time()),
"source_command_id": str(command.get("id", "")).strip(),
"source_command_name": str(command.get("name", "")).strip(),
"source_group_name": self._normalize_group_name(command.get("group_name")),
"result": str(entry.get("result", "") or ""),
"summary": summary,
"informative": bool(informative),
"mode": str(entry.get("mode", "") or ""),
"task_id": str(getattr(session, "current_task_id", "") or ""),
}
def _summarize_command_result(
self,
command: Dict[str, Any],
execution_result: Dict[str, Any],
entry: Dict[str, Any],
) -> tuple[str, bool]:
error_text = str(execution_result.get("error", "") or "").strip()
if error_text:
return (error_text, True)
result_text = str(entry.get("result", "") or "").strip()
if result_text:
generic_prefixes = (
"waited:",
"page_refreshed",
"cookies_cleared",
"new_chat_clicked",
"element_clicked:",
"element_stealth_clicked:",
"coordinates_clicked:",
"navigated:",
"preset_switched:",
)
informative_values = {"VERIFY", "BLACK", "READY", "UNKNOWN"}
if result_text in informative_values:
return (result_text, True)
lowered = result_text.lower()
if "failed" in lowered or "error" in lowered or "timeout" in lowered:
return (result_text, True)
if not lowered.startswith(generic_prefixes):
return (result_text, True)
for step in execution_result.get("steps", []) or []:
if not bool(step.get("ok", True)):
text = self._stringify_result(step.get("result", "")).strip()
return (text or "步骤执行失败", True)
cmd_name = str(command.get("name", "")).strip() or "命令"
return (f"{cmd_name} 执行完成", False)
def _get_latest_command_result_event(self, tab_id: str) -> Optional[Dict[str, Any]]:
with self._lock:
items = self._command_result_events.get(tab_id) or []
if not items:
return None
return copy.deepcopy(items[-1])
def _append_bounded_event(
self,
bucket: Dict[str, List[Dict[str, Any]]],
tab_id: str,
event: Dict[str, Any],
*,
max_events: Optional[int] = None,
) -> None:
queue = bucket.setdefault(tab_id, [])
queue.append(event)
limit = max(1, int(max_events or self.MAX_EVENT_QUEUE_ITEMS))
overflow = len(queue) - limit
if overflow > 0:
del queue[:overflow]
def _get_command_result_events(self, tab_id: str) -> List[Dict[str, Any]]:
with self._lock:
return copy.deepcopy(list(self._command_result_events.get(tab_id) or []))
def emit_external_command_result_event(
self,
session: 'TabSession',
*,
source_command_id: str,
source_command_name: str,
summary: Any,
result: Any = "",
informative: bool = True,
mode: str = "external",
group_name: Any = "",
trigger_commands: bool = True,
extra_fields: Optional[Dict[str, Any]] = None,
) -> Optional[Dict[str, Any]]:
if session is None:
return None
normalized_command_id = str(source_command_id or "").strip()
normalized_name = str(source_command_name or "").strip()
normalized_summary = str(summary or "").strip()
if not normalized_command_id or not normalized_name or not normalized_summary:
return None
event = {
"token": f"{int(time.time() * 1000)}_{uuid.uuid4().hex[:6]}",
"timestamp": time.time(),
"source_command_id": normalized_command_id,
"source_command_name": normalized_name,
"source_group_name": self._normalize_group_name(group_name),
"result": self._stringify_result(result),
"summary": normalized_summary,
"informative": bool(informative),
"mode": str(mode or "external").strip() or "external",
"task_id": str(getattr(session, "current_task_id", "") or ""),
}
if isinstance(extra_fields, dict):
for key, value in extra_fields.items():
normalized_key = str(key or "").strip()
if not normalized_key or normalized_key in event:
continue
event[normalized_key] = copy.deepcopy(value)
with self._lock:
self._append_bounded_event(
self._command_result_events,
session.id,
copy.deepcopy(event),
)
logger.info(
f"[CMD] 外部结果事件已记录: {normalized_name} "
f"(标签页={session.id}, 摘要={normalized_summary})"
)
if trigger_commands:
pseudo_command = {
"id": normalized_command_id,
"name": normalized_name,
"group_name": self._normalize_group_name(group_name),
}
self._trigger_result_event_commands(pseudo_command, session)
return copy.deepcopy(event)
def _parse_result_event_source_ids(self, trigger: Dict[str, Any]) -> tuple[bool, set[str]]:
listen_all = bool(trigger.get("listen_all_commands", False))
raw = trigger.get("command_ids", [])
values = raw if isinstance(raw, list) else str(raw or "").split(",")
normalized = {str(item or "").strip() for item in values if str(item or "").strip()}
if "*" in normalized:
listen_all = True
normalized.discard("*")
return listen_all, normalized
@staticmethod
def _get_consumed_history(state: Dict[str, Any], field: str) -> List[str]:
history_field = f"{field}s"
raw_history = state.get(history_field, [])
if isinstance(raw_history, list):
history = [str(item or "").strip() for item in raw_history if str(item or "").strip()]
else:
history = []
latest = str(state.get(field, "") or "").strip()
if latest and latest not in history:
history.append(latest)
return history[-50:]
def _result_event_matches_trigger(self, trigger: Dict[str, Any], event: Dict[str, Any]) -> bool:
listen_all = bool(trigger.get("listen_all_commands", False))
raw = trigger.get("command_ids", [])
values = raw if isinstance(raw, list) else str(raw or "").split(",")
command_ids = {str(item or "").strip() for item in values if str(item or "").strip()}
if "*" in command_ids:
listen_all = True
command_ids.discard("*")
source_id = str(event.get("source_command_id", "")).strip()
if not listen_all and command_ids and source_id not in command_ids:
return False
if not listen_all and not command_ids:
return False
if bool(trigger.get("informative_only", True)) and not bool(event.get("informative", False)):
return False
expected = self._stringify_result(
trigger.get("expected_value")
if trigger.get("expected_value") not in (None, "")
else trigger.get("value", "")
)
if expected:
actual_parts = []
for field in (
"summary",
"result",
"default_response",
"response_a",
"response_b",
"source_command_name",
"source_command_id",
):
value = self._stringify_result(event.get(field, ""))
if value:
actual_parts.append(value)
actual = "\n".join(actual_parts)
if not self._match_value_rule(actual, expected, trigger.get("match_rule", "contains")):
return False
return True
@staticmethod
def _normalize_result_event_dedupe_fields(trigger: Dict[str, Any]) -> List[str]:
raw = trigger.get("dedupe_key_fields", trigger.get("result_event_dedupe_fields", []))
if isinstance(raw, str):
values = str(raw or "").replace(",", ",").split(",")
elif isinstance(raw, (list, tuple, set)):
values = list(raw)
else:
values = []
fields = [str(item or "").strip() for item in values if str(item or "").strip()]
return fields
def _build_result_event_dedupe_signature(
self,
trigger: Dict[str, Any],
event: Dict[str, Any],
) -> str:
fields = self._normalize_result_event_dedupe_fields(trigger)
if not fields:
return ""
parts: List[str] = []
for field in fields:
value = event.get(field, "")
parts.append(f"{field}={str(value or '').strip()}")
return "|".join(parts)
def _remember_consumed_token(
self,
state: Dict[str, Any],
field: str,
token: str,
) -> None:
token = str(token or "").strip()
if not token:
return
history = self._get_consumed_history(state, field)
if token in history:
history.remove(token)
history.append(token)
state[field] = token
overflow = len(history) - 50
if overflow > 0:
del history[:overflow]
state[f"{field}s"] = history
def _get_next_command_result_event(
self,
command: Dict[str, Any],
session: 'TabSession',
consume: bool = False,
copy_event: bool = True,
) -> Optional[Dict[str, Any]]:
trigger = command.get("trigger", {}) or {}
state, _ = self._ensure_trigger_state(command["id"], session)
with self._lock:
items = list(self._command_result_events.get(session.id) or [])
consumed = set(self._get_consumed_history(state, "result_event_token"))
latched_signature = str(state.get("result_event_latch_sig", "") or "").strip()
for event in items:
token = str(event.get("token", "")).strip()
if not token or token in consumed:
continue
if not self._result_event_matches_trigger(trigger, event):
continue
signature = self._build_result_event_dedupe_signature(trigger, event)
if signature and signature == latched_signature:
if consume:
self._remember_consumed_token(state, "result_event_token", token)
continue
if consume:
self._remember_consumed_token(state, "result_event_token", token)
if signature:
state["result_event_latch_sig"] = signature
return copy.deepcopy(event) if copy_event else event
return None
def _match_command_result_event_trigger(
self,
command: Dict[str, Any],
session: 'TabSession',
consume: bool = False,
) -> bool:
return self._get_next_command_result_event(
command,
session,
consume=consume,
copy_event=False,
) is not None
def _prepare_command_result_event_dispatch(
self,
command: Dict[str, Any],
session: 'TabSession',
) -> Optional[Dict[str, Any]]:
event = self._get_next_command_result_event(command, session, consume=True)
if not event:
return None
token = str(event.get("token", "")).strip()
return {
"rollback": {"kind": "result_event_token", "token": token},
"interrupt_context": {"command_result_event": event},
}
def _normalize_status_codes(self, raw_codes: Any) -> set[int]:
if isinstance(raw_codes, (list, tuple, set)):
values = raw_codes
else:
values = str(raw_codes or "").replace(",", ",").split(",")
result: set[int] = set()
for item in values:
text = str(item).strip()
if not text:
continue
try:
result.add(int(text))
except Exception:
continue
return result
def _pattern_to_listen_hint(self, pattern: str, mode: str) -> str:
raw = str(pattern or "").strip()
if not raw:
return ""
if str(mode or "").strip().lower() == "regex":
if "|" in raw:
return "http"
tokens = re.split(r"[\^\$\.\*\+\?\(\)\[\]\{\}\|\\]+", raw)
tokens = [t.strip() for t in tokens if t and len(t.strip()) >= 3]
if not tokens:
wildcard_fallback = raw.replace(".*", "").replace("\\/", "/")
wildcard_fallback = wildcard_fallback.strip("* ").strip()
return wildcard_fallback[:120]
tokens.sort(key=len, reverse=True)
return tokens[0][:120]
simplified = raw.strip("* ").strip()
return simplified[:120]
# H12:网络事件 URL 正则的求值预算(与工作流 matches 运算符一致:25ms)
URL_REGEX_MAX_PATTERN = 512
URL_REGEX_MAX_INPUT = 8192
URL_REGEX_TIMEOUT_SEC = 0.025
@classmethod
def _bounded_regex_search(cls, pattern: str, url: str) -> Optional[bool]:
"""带超时的正则匹配:返回 True/False;模式无效返回 None;超时视为不匹配。"""
import regex as _regex
try:
return bool(_regex.search(
pattern,
url[: cls.URL_REGEX_MAX_INPUT],
flags=_regex.IGNORECASE,
timeout=cls.URL_REGEX_TIMEOUT_SEC,
))
except TimeoutError:
logger.warning(f"[CMD] URL 正则超过 25ms 求值预算,按不匹配处理: {pattern[:120]}")
return False
except _regex.error:
return None
def _matches_url_rule(self, url: str, pattern: str, mode: str) -> bool:
if not pattern:
return True
if mode == "regex":
if len(pattern) > self.URL_REGEX_MAX_PATTERN:
logger.warning(f"[CMD] URL 正则超过 {self.URL_REGEX_MAX_PATTERN} 字符,按关键词匹配")
return pattern.lower() in url.lower()
matched = self._bounded_regex_search(pattern, url)
if matched is not None:
return matched
logger.warning(f"[CMD] 无效正则,回退通配/关键词匹配: {pattern}")
wildcard = str(pattern).replace(".", r"\.").replace("*", ".*")
matched = self._bounded_regex_search(wildcard, url)
if matched is not None:
return matched
simplified = str(pattern).replace("*", "").strip()
if simplified:
return simplified.lower() in url.lower()
return pattern.lower() in url.lower()
def _matches_network_trigger(self, trigger: Dict[str, Any], event: Dict[str, Any]) -> bool:
url = str(event.get("url", "") or "")
status = event.get("status")
pattern = str(trigger.get("url_pattern") or trigger.get("value") or "").strip()
match_mode = str(trigger.get("match_mode", "keyword")).strip().lower()
codes = self._normalize_status_codes(trigger.get("status_codes", ""))
if not self._matches_url_rule(url, pattern, match_mode):
return False
if codes and int(status or 0) not in codes:
return False
return True
def _build_network_signature(self, event: Dict[str, Any]) -> str:
event_id = str(event.get("event_id", "")).strip()
if event_id:
return event_id
ts = str(event.get("timestamp", ""))
return f"{ts}:{event.get('status')}:{event.get('url', '')}"
def _consume_network_signature(self, command_id: str, session: 'TabSession', signature: str) -> bool:
if not command_id:
return False
state, _ = self._ensure_trigger_state(command_id, session)
with self._lock:
if signature in self._get_consumed_history(state, "net_sig"):
return False
self._remember_consumed_token(state, "net_sig", signature)
return True
def _prepare_network_trigger_dispatch(
self,
command: Dict[str, Any],
session: 'TabSession',
event: Optional[Dict[str, Any]] = None,
) -> Optional[Dict[str, Any]]:
trigger = command.get("trigger", {}) or {}
if event is None:
for candidate in self._snapshot_network_events(session.id):
if not self._matches_network_trigger(trigger, candidate):
continue
signature = self._build_network_signature(candidate)
if not self._consume_network_signature(command.get("id"), session, signature):
continue
event = candidate
break
else:
if not self._matches_network_trigger(trigger, event):
return None
signature = self._build_network_signature(event)
if not self._consume_network_signature(command.get("id"), session, signature):
return None
if event is None:
return None
signature = self._build_network_signature(event)
return {
"rollback": {"kind": "net_sig", "token": signature},
"interrupt_context": {"network_event": copy.deepcopy(event)},
}
def _get_latest_network_event(self, tab_id: str) -> Optional[Dict[str, Any]]:
with self._lock:
items = self._network_events.get(tab_id) or []
if not items:
return None
return copy.deepcopy(items[-1])
def _snapshot_network_events(self, tab_id: str) -> List[Dict[str, Any]]:
with self._lock:
return list(self._network_events.get(tab_id) or [])
def _get_network_events(self, tab_id: str) -> List[Dict[str, Any]]:
with self._lock:
return copy.deepcopy(list(self._network_events.get(tab_id) or []))
def _matches_scope(self, command: Dict, session: 'TabSession') -> bool:
trigger = command.get("trigger", {})
scope = trigger.get("scope", "all")
if scope == "domain":
target_domain = str(trigger.get("domain", "") or "").strip().lower()
session_domain = self._get_session_domain(session)
if target_domain and session_domain:
return self._domain_matches(target_domain, session_domain)
return not target_domain
if scope == "tab":
target_index = trigger.get("tab_index")
return target_index is None or session.persistent_index == target_index
return True
def _rollback_trigger_consumption(
self,
command: Dict[str, Any],
session: 'TabSession',
rollback: Optional[Dict[str, Any]],
):
if not rollback:
return
state, _ = self._ensure_trigger_state(command["id"], session)
field_map = {
"result_token": "result_token",
"result_event_token": "result_event_token",
"net_sig": "net_sig",
}
field = field_map.get(str(rollback.get("kind", "")).strip())
token = str(rollback.get("token", "") or "")
if not field or not token:
return
with self._lock:
history = [item for item in self._get_consumed_history(state, field) if item != token]
state[f"{field}s"] = history
state[field] = history[-1] if history else ""
def _trigger_chained_commands(
self,
source_command: Dict,
session: 'TabSession',
chain: Optional[List[str]] = None,
interrupt_context: Optional[Dict[str, Any]] = None,
):
source_id = source_command.get("id")
if not source_id:
return
chain = list(chain or [])
next_chain = chain + [source_id]
try:
commands = self._load_commands()
except Exception as e:
logger.debug(f"链式命令加载失败,跳过: {e}")
return
for cmd in commands:
if not cmd.get("enabled", True):
continue
target_id = cmd.get("id")
trigger = cmd.get("trigger", {})
if trigger.get("type") != "command_triggered":
continue
if trigger.get("command_id") != source_id:
continue
if not target_id or target_id in next_chain:
continue
if not self._matches_scope(cmd, session):
continue
exec_key = (target_id, session.id)
with self._lock:
if exec_key in self._executing:
continue
logger.info(
f"[CMD] 链式触发: {source_command.get('name')} -> {cmd.get('name')} "
f"(标签页={session.id})"
)
if interrupt_context is not None:
self._dispatch_interrupt_followup_command(
cmd, session, next_chain, interrupt_context
)
else:
self._execute_command_async(cmd, session, chain=next_chain)
def _trigger_result_match_commands(
self,
source_command: Dict[str, Any],
session: 'TabSession',
chain: Optional[List[str]] = None,
interrupt_context: Optional[Dict[str, Any]] = None,
):
source_id = source_command.get("id")
if not source_id:
return
chain = list(chain or [])
next_chain = chain + [source_id]
try:
commands = self._load_commands()
except Exception as e:
logger.debug(f"条件分支命令加载失败,跳过: {e}")
return
for cmd in commands:
if not cmd.get("enabled", True):
continue
target_id = cmd.get("id")
trigger = cmd.get("trigger", {})
if trigger.get("type") != "command_result_match":
continue
if str(trigger.get("command_id", "")).strip() != source_id:
continue
if not target_id or target_id in next_chain:
continue
if not self._matches_scope(cmd, session):
continue
dispatch = self._prepare_command_result_trigger_dispatch(cmd, session)
if not dispatch:
continue
logger.info(
f"[CMD] 条件分支触发: {source_command.get('name')} -> {cmd.get('name')} "
f"(标签页={session.id})"
)
combined_context = copy.deepcopy(interrupt_context) if interrupt_context is not None else None
extra_context = dispatch.get("interrupt_context")
if extra_context:
combined_context = combined_context or {}
combined_context.update(copy.deepcopy(extra_context))
if interrupt_context is not None:
self._dispatch_interrupt_followup_command(
cmd, session, next_chain, combined_context or interrupt_context
)
else:
self._execute_command_async(
cmd,
session,
chain=next_chain,
interrupt_context=combined_context,
trigger_rollback=dispatch.get("rollback"),
)
def _trigger_result_event_commands(
self,
source_command: Dict[str, Any],
session: 'TabSession',
chain: Optional[List[str]] = None,
interrupt_context: Optional[Dict[str, Any]] = None,
):
source_id = str(source_command.get("id", "")).strip()
if not source_id:
return
chain = list(chain or [])
next_chain = chain + [source_id]
try:
commands = self._load_commands()
except Exception as e:
logger.debug(f"结果事件命令加载失败,跳过: {e}")
return
for cmd in commands:
if not cmd.get("enabled", True):
continue
target_id = str(cmd.get("id", "")).strip()
if not target_id or target_id in next_chain:
continue
trigger = cmd.get("trigger", {}) or {}
if str(trigger.get("type", "")).strip() != "command_result_event":
continue
if not self._matches_scope(cmd, session):
continue
dispatch = self._prepare_command_result_event_dispatch(cmd, session)
if not dispatch:
continue
logger.info(
f"[CMD] 结果事件触发: {source_command.get('name')} -> {cmd.get('name')} "
f"(标签页={session.id})"
)
combined_context = copy.deepcopy(interrupt_context) if interrupt_context is not None else None
extra_context = dispatch.get("interrupt_context")
if extra_context:
combined_context = combined_context or {}
combined_context.update(copy.deepcopy(extra_context))
if interrupt_context is not None:
self._dispatch_interrupt_followup_command(
cmd, session, next_chain, combined_context or interrupt_context
)
else:
self._execute_command_async(
cmd,
session,
chain=next_chain,
interrupt_context=combined_context,
trigger_rollback=dispatch.get("rollback"),
)
+581
View File
@@ -0,0 +1,581 @@
import copy
import threading
import time
import uuid
from typing import Any, Dict, List, Optional, TYPE_CHECKING
from app.core.config import get_logger
if TYPE_CHECKING:
from app.core.tab_pool import TabSession
logger = get_logger("CMD_ENG")
class CommandEngineRuntimeMixin:
def _workflow_queue_key(
self,
command: Dict[str, Any],
trigger_rollback: Optional[Dict[str, Any]] = None,
) -> str:
cmd_id = str((command or {}).get("id", "")).strip()
rollback = trigger_rollback or {}
kind = str(rollback.get("kind", "") or "").strip()
token = str(rollback.get("token", "") or "").strip()
if cmd_id and kind and token:
return f"{cmd_id}:{kind}:{token}"
return cmd_id
def _dedupe_deferred_items(self, items: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
deduped: List[Dict[str, Any]] = []
seen: set[str] = set()
for item in items or []:
if not isinstance(item, dict):
continue
queue_key = str(item.get("queue_key", "") or "").strip()
if not queue_key:
queue_key = self._workflow_queue_key(
item.get("command") or {},
item.get("trigger_rollback") if isinstance(item.get("trigger_rollback"), dict) else None,
)
if queue_key:
item = dict(item)
item["queue_key"] = queue_key
if queue_key and queue_key in seen:
continue
if queue_key:
seen.add(queue_key)
deduped.append(item)
return deduped
def _has_active_workflow(self, session: 'TabSession') -> bool:
runtime = self._get_active_workflow_runtime(session)
return bool(runtime and runtime.get("active"))
def begin_workflow_runtime(
self,
session: 'TabSession',
*,
task_id: str = "",
preset_name: str = "",
priority: Optional[int] = None,
) -> Dict[str, Any]:
runtime = {
"runtime_id": uuid.uuid4().hex,
"task_id": str(task_id or "").strip(),
"preset_name": str(preset_name or "").strip(),
"parser_id": str(getattr(session, "_workflow_parser_id", "") or "").strip().lower(),
"target_side": str(getattr(session, "_workflow_target_side", "") or "").strip().lower(),
"priority": self._normalize_priority(priority, self._get_request_priority_baseline()),
"current_step_index": -1,
"current_step_action": "",
"pending_interrupts": [],
"pending_interrupt_ids": set(),
"deferred_commands": [],
"deferred_command_ids": set(),
"active": True,
"interrupting": False,
}
with self._lock:
stack = list(getattr(session, "_workflow_runtime_stack", None) or [])
stack.append(runtime)
setattr(session, "_workflow_runtime_stack", stack)
setattr(session, "_workflow_runtime_id", runtime["runtime_id"])
setattr(session, "_workflow_target_side", runtime["target_side"])
return runtime
def update_workflow_runtime_step(
self,
session: 'TabSession',
step_index: int,
step: Optional[Dict[str, Any]] = None,
):
runtime = self._get_active_workflow_runtime(session)
if not runtime:
return
runtime["current_step_index"] = int(step_index)
runtime["current_step_action"] = str((step or {}).get("action", "") or "")
def update_workflow_target_side(self, session: 'TabSession', side: str) -> bool:
normalized = str(side or "").strip().lower()
if normalized not in {"left", "right"}:
return False
runtime = self._get_active_workflow_runtime(session)
if not runtime:
return False
runtime["target_side"] = normalized
setattr(session, "_workflow_target_side", normalized)
return True
def finish_workflow_runtime(
self,
session: 'TabSession',
*,
aborted: bool = False,
) -> Optional[Dict[str, Any]]:
with self._lock:
stack = list(getattr(session, "_workflow_runtime_stack", None) or [])
if not stack:
return None
runtime = stack.pop()
setattr(session, "_workflow_runtime_stack", stack)
pending_interrupts = list(runtime.get("pending_interrupts") or [])
deferred_commands = list(runtime.get("deferred_commands") or [])
stop_reason = str(getattr(session, "_workflow_stop_reason", "") or "").strip()
preserve_pending_interrupts = bool(pending_interrupts) and (
not aborted or stop_reason in {"command_interrupt", "command_interrupt_abort"}
)
if preserve_pending_interrupts:
deferred_ids = {
str(item.get("queue_key", "") or "").strip()
or self._workflow_queue_key(
item.get("command") or {},
item.get("trigger_rollback") if isinstance(item.get("trigger_rollback"), dict) else None,
)
for item in deferred_commands
if isinstance(item, dict)
}
for item in pending_interrupts:
if not isinstance(item, dict):
continue
command = copy.deepcopy(item.get("command") or {})
queue_key = str(item.get("queue_key", "") or "").strip() or self._workflow_queue_key(
command,
item.get("trigger_rollback") if isinstance(item.get("trigger_rollback"), dict) else None,
)
if queue_key and queue_key in deferred_ids:
continue
deferred_commands.append({
"command": command,
"chain": list(item.get("chain") or []),
"interrupt_context": copy.deepcopy(item.get("interrupt_context") or {}),
"trigger_rollback": copy.deepcopy(item.get("trigger_rollback") or {}),
"queue_key": queue_key,
"queued_at": float(item.get("requested_at", time.time()) or time.time()),
})
if queue_key:
deferred_ids.add(queue_key)
runtime["active"] = False
runtime["interrupting"] = False
runtime["pending_interrupts"] = []
runtime["pending_interrupt_ids"] = set()
preserve_deferred = bool(deferred_commands) and (
not aborted or stop_reason == "command_interrupt_abort"
)
if stack:
parent = stack[-1]
if preserve_deferred:
parent_deferred = parent.setdefault("deferred_commands", [])
parent_ids = parent.setdefault("deferred_command_ids", set())
for item in deferred_commands:
queue_key = str(item.get("queue_key", "") or "").strip() or self._workflow_queue_key(
item.get("command") or {},
item.get("trigger_rollback") if isinstance(item.get("trigger_rollback"), dict) else None,
)
if queue_key and queue_key in parent_ids:
continue
parent_deferred.append(item)
if queue_key:
parent_ids.add(queue_key)
elif preserve_deferred:
existing = list(getattr(session, "_pending_post_workflow_commands", None) or [])
existing_ids = {
str((item.get("command") or {}).get("id", "")).strip()
for item in existing
if isinstance(item, dict)
}
merged = self._dedupe_deferred_items(existing + deferred_commands)
setattr(session, "_pending_post_workflow_commands", merged)
else:
setattr(session, "_pending_post_workflow_commands", [])
parent = stack[-1] if stack else None
if parent:
setattr(session, "_workflow_runtime_id", str(parent.get("runtime_id") or ""))
setattr(session, "_workflow_parser_id", str(parent.get("parser_id") or ""))
setattr(session, "_workflow_target_side", str(parent.get("target_side") or ""))
else:
setattr(session, "_workflow_runtime_id", "")
setattr(session, "_workflow_parser_id", "")
setattr(session, "_workflow_target_side", "")
return runtime
def flush_deferred_workflow_commands(self, session: 'TabSession'):
if bool(getattr(self, "_shutdown_requested", False)):
return
with self._lock:
items = list(getattr(session, "_pending_post_workflow_commands", None) or [])
setattr(session, "_pending_post_workflow_commands", [])
if not items:
return
requeued = 0
for item in items:
command = copy.deepcopy((item or {}).get("command") or {})
if not command or not command.get("enabled", True):
continue
dispatched = self._execute_command_async(
command,
session,
chain=list((item or {}).get("chain") or []),
interrupt_context=copy.deepcopy((item or {}).get("interrupt_context") or {}) or None,
trigger_rollback=copy.deepcopy((item or {}).get("trigger_rollback") or {}) or None,
)
if dispatched:
requeued += 1
logger.info(
f"[CMD] 已将延后命令交回命令调度器执行 "
f"(标签页={getattr(session, 'id', '')}, count={requeued})"
)
def schedule_deferred_workflow_commands(
self,
session: 'TabSession',
*,
delay_sec: float = 0.25,
):
if bool(getattr(self, "_shutdown_requested", False)):
return False
delay = max(0.0, float(delay_sec or 0.0))
thread = threading.Thread(
target=self.flush_deferred_workflow_commands,
args=(session,),
daemon=True,
name=f"cmd-resume-delay-{getattr(session, 'id', 'tab')}",
)
if delay > 0:
# Timer waits before dispatching, while command execution itself stays in
# the command scheduler path instead of touching the tab from this helper.
thread = threading.Timer(delay, self.flush_deferred_workflow_commands, args=(session,))
thread.daemon = True
thread.name = f"cmd-resume-delay-{getattr(session, 'id', 'tab')}"
thread.start()
return True
def _get_workflow_interrupt_policy(self, command: Dict[str, Any]) -> str:
trigger = command.get("trigger", {}) or {}
policy = str(
trigger.get("interrupt_policy", command.get("interrupt_policy", "auto"))
or "auto"
).strip().lower()
return policy if policy in {"auto", "resume", "abort"} else "auto"
def _build_workflow_interrupt_message(self, command: Dict[str, Any]) -> str:
trigger = command.get("trigger", {}) or {}
custom = str(
trigger.get("interrupt_message", command.get("interrupt_message", ""))
or ""
).strip()
if custom:
return custom
return f"触发 {command.get('name', '命令')},后续工作流已打断,请重试"
def _allow_command_during_workflow(self, command: Dict[str, Any]) -> bool:
trigger = command.get("trigger", {}) or {}
raw = trigger.get("allow_during_workflow", None)
if raw is None:
trigger_type = str(trigger.get("type", "")).strip().lower()
if trigger_type != "page_check":
return False
policy = self._get_workflow_interrupt_policy(command)
if policy in {"resume", "abort"}:
return True
return self._get_command_priority(command) > self._get_request_priority_baseline()
return bool(raw)
def _queue_workflow_interrupt(
self,
command: Dict[str, Any],
session: 'TabSession',
chain: Optional[List[str]] = None,
interrupt_context: Optional[Dict[str, Any]] = None,
trigger_rollback: Optional[Dict[str, Any]] = None,
) -> bool:
runtime = self._get_active_workflow_runtime(session)
if not runtime or not runtime.get("active"):
return False
if not self._allow_command_during_workflow(command):
return False
queue_key = self._workflow_queue_key(command, trigger_rollback)
if not queue_key:
return False
with self._lock:
pending_ids = runtime.setdefault("pending_interrupt_ids", set())
if queue_key in pending_ids:
return False
runtime.setdefault("pending_interrupts", []).append({
"command": copy.deepcopy(command),
"chain": list(chain or []),
"interrupt_context": copy.deepcopy(interrupt_context or {}),
"trigger_rollback": copy.deepcopy(trigger_rollback or {}),
"queue_key": queue_key,
"requested_at": time.time(),
"priority": self._get_command_priority(command),
})
pending_ids.add(queue_key)
setattr(session, "_workflow_stop_reason", "command_interrupt")
logger.info(
f"[CMD] 请求暂停工作流: {command.get('name')} "
f"(标签页={session.id}, 工作流优先级={runtime.get('priority')}, "
f"命令优先级={self._get_command_priority(command)})"
)
return True
def _schedule_command_for_active_workflow(
self,
command: Dict[str, Any],
session: 'TabSession',
chain: Optional[List[str]] = None,
interrupt_context: Optional[Dict[str, Any]] = None,
trigger_rollback: Optional[Dict[str, Any]] = None,
) -> bool:
runtime = self._get_active_workflow_runtime(session)
if not runtime or not runtime.get("active"):
return False
if self._allow_command_during_workflow(command):
return self._queue_workflow_interrupt(
command,
session,
chain=chain,
interrupt_context=interrupt_context,
trigger_rollback=trigger_rollback,
)
self._defer_command_until_workflow_resume(
runtime,
command,
chain=chain,
interrupt_context=interrupt_context,
trigger_rollback=trigger_rollback,
)
logger.info(
f"[CMD] 工作流忙碌,延后执行命令: {command.get('name')} "
f"(标签页={session.id}, 工作流优先级={runtime.get('priority')}, "
f"命令优先级={self._get_command_priority(command)})"
)
return True
def workflow_interrupt_requested(self, session: 'TabSession') -> bool:
runtime = self._get_active_workflow_runtime(session)
if not runtime:
return False
with self._lock:
return bool(runtime.get("pending_interrupts"))
def _defer_command_until_workflow_resume(
self,
runtime: Dict[str, Any],
command: Dict[str, Any],
chain: Optional[List[str]] = None,
interrupt_context: Optional[Dict[str, Any]] = None,
trigger_rollback: Optional[Dict[str, Any]] = None,
):
queue_key = self._workflow_queue_key(command, trigger_rollback)
if not queue_key:
return
with self._lock:
deferred_ids = runtime.setdefault("deferred_command_ids", set())
if queue_key in deferred_ids:
return
runtime.setdefault("deferred_commands", []).append({
"command": copy.deepcopy(command),
"chain": list(chain or []),
"interrupt_context": copy.deepcopy(interrupt_context or {}),
"trigger_rollback": copy.deepcopy(trigger_rollback or {}),
"queue_key": queue_key,
"queued_at": time.time(),
})
deferred_ids.add(queue_key)
def _mark_interrupt_abort(
self,
command: Dict[str, Any],
interrupt_context: Dict[str, Any],
):
if interrupt_context.get("abort"):
return
policy = self._get_workflow_interrupt_policy(command)
priority = self._get_command_priority(command)
workflow_priority = self._normalize_priority(
interrupt_context.get("workflow_priority", self._get_request_priority_baseline()),
self._get_request_priority_baseline(),
)
if policy == "abort" or (policy == "auto" and priority > workflow_priority):
interrupt_context["abort"] = True
interrupt_context["message"] = self._build_workflow_interrupt_message(command)
interrupt_context["abort_by"] = command.get("name", "")
def _dispatch_interrupt_followup_command(
self,
command: Dict[str, Any],
session: 'TabSession',
chain: Optional[List[str]],
interrupt_context: Dict[str, Any],
):
runtime = interrupt_context.get("runtime") or {}
workflow_priority = self._normalize_priority(
interrupt_context.get("workflow_priority", self._get_request_priority_baseline()),
self._get_request_priority_baseline(),
)
priority = self._get_command_priority(command)
policy = self._get_workflow_interrupt_policy(command)
if priority > workflow_priority or policy == "abort":
logger.info(
f"[CMD] 工作流暂停期间继续执行高优先级命令: {command.get('name')} "
f"(标签页={session.id}, 命令优先级={priority}, 工作流优先级={workflow_priority})"
)
self._execute_command(command, session, chain=chain, interrupt_context=interrupt_context)
self._mark_interrupt_abort(command, interrupt_context)
return
logger.info(
f"[CMD] 延后命令至工作流恢复后执行: {command.get('name')} "
f"(标签页={session.id}, 命令优先级={priority}, 工作流优先级={workflow_priority})"
)
self._defer_command_until_workflow_resume(
runtime,
command,
chain=chain,
interrupt_context=interrupt_context,
)
@staticmethod
def _interrupt_item_sort_key(item: Dict[str, Any]) -> tuple[int, float]:
try:
priority = int(item.get("priority", 0) or 0)
except (TypeError, ValueError):
priority = 0
try:
requested_at = float(item.get("requested_at", 0.0) or 0.0)
except (TypeError, ValueError):
requested_at = 0.0
return -priority, requested_at
def _take_next_pending_interrupt(
self,
runtime: Dict[str, Any],
) -> Optional[Dict[str, Any]]:
with self._lock:
raw_pending = runtime.get("pending_interrupts") or []
pending = raw_pending if isinstance(raw_pending, list) else list(raw_pending)
pending = [item for item in pending if isinstance(item, dict)]
if not pending:
runtime["pending_interrupts"] = []
runtime["pending_interrupt_ids"] = set()
return None
best_index = 0
best_key = self._interrupt_item_sort_key(pending[0])
for index in range(1, len(pending)):
key = self._interrupt_item_sort_key(pending[index])
if key < best_key:
best_index = index
best_key = key
item = pending.pop(best_index)
runtime["pending_interrupts"] = pending
return item
def _drop_pending_interrupts_by_key(
self,
runtime: Dict[str, Any],
queue_key: str,
) -> None:
if not queue_key:
return
with self._lock:
runtime["pending_interrupts"] = [
existing for existing in runtime.get("pending_interrupts", [])
if isinstance(existing, dict)
and (
str(existing.get("queue_key", "") or "").strip()
or self._workflow_queue_key(
(existing.get("command") or {}),
existing.get("trigger_rollback") if isinstance(existing.get("trigger_rollback"), dict) else None,
)
) != queue_key
]
def handle_pending_workflow_interrupts(self, session: 'TabSession') -> Dict[str, Any]:
runtime = self._get_active_workflow_runtime(session)
if not runtime or runtime.get("interrupting"):
return {"handled": False, "abort": False, "message": ""}
with self._lock:
pending = list(runtime.get("pending_interrupts") or [])
if not pending:
return {"handled": False, "abort": False, "message": ""}
with self._lock:
if runtime.get("interrupting"):
return {"handled": False, "abort": False, "message": ""}
runtime["interrupting"] = True
interrupt_context = {
"workflow_priority": runtime.get("priority", self._get_request_priority_baseline()),
"runtime": runtime,
"abort": False,
"message": "",
"abort_by": "",
}
try:
while True:
item = self._take_next_pending_interrupt(runtime)
if not item:
break
cmd = copy.deepcopy(item.get("command") or {})
chain = list(item.get("chain") or [])
queue_key = str(item.get("queue_key", "") or "").strip() or self._workflow_queue_key(
cmd,
item.get("trigger_rollback") if isinstance(item.get("trigger_rollback"), dict) else None,
)
interrupt_payload = copy.deepcopy(item.get("interrupt_context") or {})
self._drop_pending_interrupts_by_key(runtime, queue_key)
with self._lock:
runtime.setdefault("pending_interrupt_ids", set()).discard(queue_key)
if not cmd or not cmd.get("enabled", True):
continue
logger.info(
f"[CMD] 工作流已暂停,执行插队命令: {cmd.get('name')} "
f"(标签页={session.id}, 步骤={runtime.get('current_step_index')})"
)
merged_context = dict(interrupt_context)
if interrupt_payload:
merged_context.update(interrupt_payload)
self._execute_command(cmd, session, chain=chain, interrupt_context=merged_context)
self._mark_interrupt_abort(cmd, interrupt_context)
if interrupt_context.get("abort"):
with self._lock:
runtime["pending_interrupts"] = []
runtime["pending_interrupt_ids"] = set()
break
finally:
with self._lock:
runtime["interrupting"] = False
if interrupt_context.get("abort"):
setattr(session, "_workflow_stop_reason", "command_interrupt_abort")
else:
setattr(session, "_workflow_stop_reason", None)
return {
"handled": True,
"abort": bool(interrupt_context.get("abort")),
"message": str(interrupt_context.get("message", "") or ""),
"abort_by": str(interrupt_context.get("abort_by", "") or ""),
}
+266
View File
@@ -0,0 +1,266 @@
"""CommandEngine 的一部分(R2-3 拆分):周期调度:保活、标签页池刷新、周期检查循环。
方法原样从 command_engine.py 迁出,行为不变;通过 mixin 组合回 CommandEngine,
类属性与实例状态都在 CommandEngine(command_engine.py)里定义。
"""
import os
import random
import threading
import time
from typing import Dict, List, Any, TYPE_CHECKING
from app.services.command_engine_common import logger
if TYPE_CHECKING:
from app.core.tab_pool import TabSession # noqa: F401
class CommandEngineSchedulerMixin:
"""周期调度:保活、标签页池刷新、周期检查循环"""
@staticmethod
def _should_auto_start_scheduler() -> bool:
# The application lifespan explicitly owns scheduler startup/shutdown.
# Starting here makes a plain module import (including pytest test
# collection) leak a background thread with no matching shutdown.
return str(os.getenv("CMD_ENGINE_AUTO_START", "false")).strip().lower() in {
"1",
"true",
"yes",
"y",
"on",
}
def _should_log_periodic_summary(self, now_ts: float, due_total: int) -> bool:
if not self._periodic_summary_log_enabled:
return False
interval = (
self._periodic_active_log_interval_sec
if due_total > 0
else self._periodic_idle_log_interval_sec
)
if (now_ts - self._last_periodic_summary_log_at) < interval:
return False
self._last_periodic_summary_log_at = now_ts
return True
def _refresh_tab_pool_if_due(self, pool: Any):
if not self._tab_pool_auto_refresh:
return
now = time.time()
if (now - self._last_tab_pool_refresh_at) < self._tab_pool_refresh_interval_sec:
return
self._last_tab_pool_refresh_at = now
try:
if hasattr(pool, "refresh_tabs"):
pool.refresh_tabs()
except Exception as e:
logger.debug(f"[CMD] 刷新标签页池失败(忽略): {e}")
def _maybe_periodic_keepalive(self, session: 'TabSession', now_ts: float):
if not self._periodic_keepalive_enabled:
return
sid = str(getattr(session, "id", "") or "")
if not sid:
return
last_at = float(self._last_keepalive_by_session.get(sid, 0.0) or 0.0)
if (now_ts - last_at) < self._periodic_keepalive_interval_sec:
return
self._last_keepalive_by_session[sid] = now_ts
self._try_wake_tab(session, reason="periodic_keepalive")
def _start_periodic_scheduler(self):
with self._scheduler_lifecycle_lock:
if self._shutdown_requested:
return False
if self._periodic_thread and self._periodic_thread.is_alive():
return True
self._periodic_stop_event.clear()
self._periodic_thread = threading.Thread(
target=self._periodic_loop,
daemon=True,
name="cmd-periodic-checker",
)
self._periodic_thread.start()
logger.debug("[CMD] 周期调度器已启动")
return True
def is_scheduler_running(self) -> bool:
with self._scheduler_lifecycle_lock:
thread = self._periodic_thread
return bool(
not self._shutdown_requested
and thread
and thread.is_alive()
)
def ensure_scheduler_running(self):
"""Best-effort watchdog: start periodic checker if it is not running."""
return self._start_periodic_scheduler()
def _periodic_loop(self):
while not self._periodic_stop_event.wait(1.0):
try:
self._run_periodic_checks()
except Exception as e:
logger.debug(f"[CMD] 周期调度循环异常(忽略): {e}")
def _get_periodic_trigger_timing(self, trigger: Dict, session_id: str) -> tuple[float, float]:
interval = max(1.0, self._coerce_float(trigger.get("periodic_interval_sec", 8), 8.0))
jitter = max(0.0, self._coerce_float(trigger.get("periodic_jitter_sec", 2), 2.0))
trigger_type = str(trigger.get("type", "")).strip().lower()
has_keyword_expression = bool(str(trigger.get("value", "") or "").strip())
if trigger_type == "page_check" and has_keyword_expression:
with self._lock:
has_observer = session_id in self._observer_keywords_by_session
if has_observer:
interval = min(interval, 1.5)
jitter = 0.0
return interval, jitter
def _run_periodic_checks(self):
try:
commands = self._load_commands_for_checks()
except Exception:
return
if not commands:
return
try:
browser = self._get_browser()
except Exception:
return
pool = getattr(browser, "_tab_pool", None)
if pool is None:
try:
pool = browser.tab_pool
logger.debug("[CMD] 周期调度器已初始化标签页池")
except Exception as e:
now = time.time()
if (now - self._last_tab_pool_wait_log_at) >= 10:
self._last_tab_pool_wait_log_at = now
logger.debug(f"[CMD] 周期调度器等待标签页池初始化: {e}")
return
if not hasattr(pool, "get_idle_sessions_snapshot"):
return
self._refresh_tab_pool_if_due(pool)
if hasattr(pool, "get_sessions_snapshot"):
sessions = pool.get_sessions_snapshot()
else:
sessions = pool.get_idle_sessions_snapshot()
if not sessions:
return
current_session_ids = {str(getattr(s, "id", "") or "") for s in sessions}
now = time.time()
active_keys = set()
due_total = 0
enabled_commands = [
(idx, cmd) for idx, cmd in enumerate(commands)
if cmd.get("enabled", True)
]
for session in sessions:
session_id = str(getattr(session, "id", "") or "")
if not session_id or session_id not in current_session_ids:
continue
session_status = self._session_status_value(session)
if session_status not in {"idle", "busy"}:
continue
is_busy_workflow = session_status == "busy" and self._has_active_workflow(session)
if session_status == "busy" and not is_busy_workflow:
continue
if session_status == "idle":
self._maybe_periodic_keepalive(session, now)
try:
self._sync_session_bootstrap_js_files(commands, session)
except Exception as error:
logger.debug(f"[CMD] 启动 JS 文件同步异常(忽略): {error}")
# Inject/update MutationObserver for page_check keywords on this session
try:
pc_keywords = self._collect_page_check_keywords(commands, session)
if pc_keywords:
self._ensure_page_check_observer(session, pc_keywords)
else:
self._clear_page_check_observer(session)
except Exception:
pass
due_commands: List[tuple[int, int, Dict[str, Any]]] = []
for idx, cmd in enabled_commands:
cmd_id = str(cmd.get("id", "")).strip()
if not cmd_id:
continue
trigger = cmd.get("trigger", {}) or {}
if not bool(trigger.get("periodic_enabled", True)):
continue
trigger_type = str(trigger.get("type", "")).strip().lower()
if (
is_busy_workflow
and trigger_type == "page_check"
and not self._should_evaluate_page_check_while_busy_workflow(cmd)
):
# While a workflow is actively running, low-priority page_check
# commands should not consume transient intermediate states.
continue
key = (cmd_id, session.id)
active_keys.add(key)
interval, jitter = self._get_periodic_trigger_timing(trigger, session_id)
with self._lock:
next_at = float(self._periodic_next_run.get(key, 0.0))
if now < next_at:
continue
delay = interval + (random.uniform(0.0, jitter) if jitter > 0 else 0.0)
with self._lock:
self._periodic_next_run[key] = now + delay
due_commands.append((self._get_command_priority(cmd), idx, cmd))
due_total += len(due_commands)
due_commands.sort(key=lambda item: (-item[0], item[1]))
for _, _, cmd in due_commands:
with self._command_logging_context(cmd):
current_status = str(getattr(getattr(session, "status", None), "value", "")).lower()
if current_status == "busy" and not self._has_active_workflow(session):
break
if current_status not in {"idle", "busy"}:
break
if self._should_trigger(cmd, session):
meta = self._take_pending_async_trigger_meta(cmd, session) or {}
if current_status == "busy":
scheduled = self._schedule_command_for_active_workflow(
cmd,
session,
interrupt_context=meta.get("interrupt_context"),
trigger_rollback=meta.get("rollback"),
)
if not scheduled:
if meta.get("rollback"):
self._rollback_trigger_consumption(cmd, session, meta.get("rollback"))
self._finalize_request_count_trigger_state(cmd, session, rollback=True)
self._reset_page_check_latch(cmd, session, reason="workflow_schedule_failed")
else:
self._execute_command_async(
cmd,
session,
interrupt_context=meta.get("interrupt_context"),
trigger_rollback=meta.get("rollback"),
)
if self._should_log_periodic_summary(now, due_total):
logger.debug(
f"[CMD] 周期检查: 会话数={len(sessions)}, "
f"已启用周期命令={len(active_keys)}, 本轮到点={due_total}"
)
with self._lock:
stale_keys = [k for k in self._periodic_next_run if k not in active_keys]
for key in stale_keys:
self._periodic_next_run.pop(key, None)
stale_keepalive_keys = [k for k in self._last_keepalive_by_session if k not in {s.id for s in sessions}]
for key in stale_keepalive_keys:
self._last_keepalive_by_session.pop(key, None)
+680
View File
@@ -0,0 +1,680 @@
"""CommandEngine 的一部分(R2-3 拆分):命令的读写与规范化:commands.json / 本地状态、运行统计合并、命名去重、run_js_file 清理。
方法原样从 command_engine.py 迁出,行为不变;通过 mixin 组合回 CommandEngine,
类属性与实例状态都在 CommandEngine(command_engine.py)里定义。
"""
import copy
import json
import os
import re
from typing import Dict, List, Optional, Any, TYPE_CHECKING
from app.core.config import command_log_context
from app.services.command_engine_common import FOLLOW_DEFAULT_PRESET, logger
if TYPE_CHECKING:
from app.core.tab_pool import TabSession # noqa: F401
class CommandEngineStoreMixin:
"""命令的读写与规范化:commands.json / 本地状态、运行统计合并、命名去重、run_js_file 清理"""
def _get_commands_file(self) -> str:
if self._commands_file is None:
from app.services.config_engine import ConfigConstants
self._commands_file = ConfigConstants.COMMANDS_FILE
return self._commands_file
def _get_commands_local_file(self) -> str:
if self._commands_local_file is None:
from app.services.config_engine import ConfigConstants
self._commands_local_file = ConfigConstants.COMMANDS_LOCAL_FILE
return self._commands_local_file
def _read_commands_file(self) -> Optional[List[Dict]]:
"""读取命令配置。
B8:返回 ``None`` 表示「读取失败」(JSON 损坏、结构不对、IO 错误),调用方必须保留
last-known-good;只有文件不存在或合法的空列表才返回 ``[]``。旧实现把两者混为 ``[]``,
编辑器半写入等短暂损坏会清空全部命令并触发 run_js_file 清理。
"""
commands_file = self._get_commands_file()
commands = []
if os.path.exists(commands_file):
try:
with open(commands_file, "r", encoding="utf-8-sig") as f:
data = json.load(f)
if isinstance(data, dict):
data = data.get("commands", [])
if isinstance(data, list):
commands = [entry for entry in data if isinstance(entry, dict)]
for entry in commands:
self._normalize_command_logging(entry)
entry["advanced_ui"] = self._normalize_advanced_ui(entry.get("advanced_ui"))
else:
logger.warning(f"命令配置文件格式无效: {commands_file}")
return None
except json.JSONDecodeError as e:
logger.error(f"命令配置文件格式错误: {e}")
return None
except Exception as e:
logger.error(f"加载命令配置失败: {e}")
return None
return self._apply_local_command_state(commands)
def _load_command_state_entries(self) -> Optional[List[Dict[str, Any]]]:
local_file = self._get_commands_local_file()
if not os.path.exists(local_file):
self._commands_local_mtime = 0.0
return []
try:
with open(local_file, "r", encoding="utf-8-sig") as f:
data = json.load(f)
self._commands_local_mtime = os.path.getmtime(local_file)
if not isinstance(data, dict):
logger.error(f"本地命令状态文件格式无效: {local_file}")
return None
entries = data.get("commands", [])
if not isinstance(entries, list):
logger.error(f"本地命令状态 commands 字段格式无效: {local_file}")
return None
return [entry for entry in entries if isinstance(entry, dict)]
except json.JSONDecodeError as e:
logger.error(f"本地命令状态文件格式错误: {e}")
return None
except Exception as e:
logger.error(f"加载本地命令状态失败: {e}")
return None
def _fallback_local_command_state_entries(self) -> List[Dict[str, Any]]:
entries: List[Dict[str, Any]] = []
for cmd in self._commands_cache or []:
if not isinstance(cmd, dict):
continue
entries.append({
"id": str(cmd.get("id", "")).strip(),
"name": str(cmd.get("name", "")).strip(),
"enabled": bool(cmd.get("enabled", True)),
"group_name": self._normalize_group_name(cmd.get("group_name")),
})
if entries:
logger.warning("本地命令状态读取失败,沿用内存中的上一轮状态覆盖")
return entries
def _apply_local_command_state(self, commands: List[Dict]) -> List[Dict]:
entries = self._load_command_state_entries()
if entries is None:
entries = self._fallback_local_command_state_entries()
if not entries or not commands:
return commands
by_id = {}
by_name = {}
for entry in entries:
command_id = str(entry.get("id", "")).strip()
command_name = str(entry.get("name", "")).strip()
if command_id:
by_id[command_id] = entry
if command_name:
by_name[command_name] = entry
applied = 0
for cmd in commands:
if not isinstance(cmd, dict):
continue
command_id = str(cmd.get("id", "")).strip()
command_name = str(cmd.get("name", "")).strip()
entry = by_id.get(command_id) or by_name.get(command_name)
if not entry:
continue
if "enabled" in entry:
cmd["enabled"] = bool(entry.get("enabled"))
if "group_name" in entry:
cmd["group_name"] = self._normalize_group_name(entry.get("group_name"))
applied += 1
if applied > 0:
logger.debug(f"已应用 {applied} 条本地命令状态覆盖")
return commands
def _save_local_command_state(self, commands: List[Dict[str, Any]]) -> bool:
local_file = self._get_commands_local_file()
tmp_file = local_file + ".tmp"
entries = []
for cmd in commands:
if not isinstance(cmd, dict):
continue
entries.append({
"id": str(cmd.get("id", "")).strip(),
"name": str(cmd.get("name", "")).strip(),
"enabled": bool(cmd.get("enabled", True)),
"group_name": self._normalize_group_name(cmd.get("group_name")),
})
try:
os.makedirs(os.path.dirname(local_file), exist_ok=True)
with open(tmp_file, "w", encoding="utf-8") as f:
json.dump({"commands": entries}, f, indent=2, ensure_ascii=False)
f.flush()
os.fsync(f.fileno())
os.replace(tmp_file, local_file)
try:
self._commands_local_mtime = os.path.getmtime(local_file) if os.path.exists(local_file) else 0.0
except Exception as mtime_error:
logger.warning(f"本地命令状态已保存但更新时间戳失败: {mtime_error}")
return True
except Exception as e:
logger.error(f"保存本地命令状态失败: {e}")
try:
if os.path.exists(tmp_file):
os.remove(tmp_file)
except Exception:
pass
return False
def _snapshot_local_command_state(self) -> Optional[tuple[bool, bytes]]:
local_file = self._get_commands_local_file()
try:
if not os.path.exists(local_file):
return (False, b"")
with open(local_file, "rb") as f:
return (True, f.read())
except Exception as e:
logger.error(f"读取本地命令状态快照失败: {e}")
return None
def _restore_local_command_state(self, snapshot: Optional[tuple[bool, bytes]]) -> None:
if snapshot is None:
return
local_file = self._get_commands_local_file()
tmp_file = local_file + ".restore.tmp"
existed, payload = snapshot
try:
if existed:
os.makedirs(os.path.dirname(local_file), exist_ok=True)
with open(tmp_file, "wb") as f:
f.write(payload)
f.flush()
os.fsync(f.fileno())
os.replace(tmp_file, local_file)
elif os.path.exists(local_file):
os.remove(local_file)
try:
self._commands_local_mtime = os.path.getmtime(local_file) if os.path.exists(local_file) else 0.0
except Exception as mtime_error:
logger.warning(f"本地命令状态已恢复但更新时间戳失败: {mtime_error}")
except Exception as e:
logger.error(f"恢复本地命令状态失败: {e}")
try:
if os.path.exists(tmp_file):
os.remove(tmp_file)
except Exception:
pass
@staticmethod
def _iter_run_js_file_actions(command: Optional[Dict[str, Any]]):
for action in list((command or {}).get("actions") or []):
if not isinstance(action, dict):
continue
if str(action.get("type", "")).strip().lower() != "run_js_file":
continue
yield action
def _collect_enabled_run_js_file_keys(
self,
commands: List[Dict[str, Any]],
*,
preinject_only: bool = False,
) -> set[str]:
keys: set[str] = set()
for command in commands or []:
if not isinstance(command, dict) or not bool(command.get("enabled", True)):
continue
for action in self._iter_run_js_file_actions(command):
if preinject_only and not self._coerce_action_bool(
action.get("inject_on_new_document", True),
True,
):
continue
resolved_path = self._resolve_action_file_path(action.get("file_path", ""))
if resolved_path:
keys.add(resolved_path.lower())
return keys
def _collect_run_js_file_cleanup_actions(
self,
previous_commands: List[Dict[str, Any]],
next_commands: List[Dict[str, Any]],
) -> List[Dict[str, Any]]:
active_keys = self._collect_enabled_run_js_file_keys(next_commands)
active_preinject_keys = self._collect_enabled_run_js_file_keys(
next_commands,
preinject_only=True,
)
next_by_id: Dict[str, Dict[str, Any]] = {}
next_by_name: Dict[str, Dict[str, Any]] = {}
for command in next_commands or []:
if not isinstance(command, dict):
continue
command_id = str(command.get("id", "")).strip()
command_name = str(command.get("name", "")).strip()
if command_id:
next_by_id[command_id] = command
if command_name:
next_by_name[command_name] = command
cleanup_actions: List[Dict[str, Any]] = []
seen_keys: set[str] = set()
for command in previous_commands or []:
if not isinstance(command, dict) or not bool(command.get("enabled", True)):
continue
command_id = str(command.get("id", "")).strip()
command_name = str(command.get("name", "")).strip()
next_command = next_by_id.get(command_id) or next_by_name.get(command_name)
command_still_enabled = next_command is not None and bool(next_command.get("enabled", True))
for action in self._iter_run_js_file_actions(command):
resolved_path = self._resolve_action_file_path(action.get("file_path", ""))
action_key = resolved_path.lower() if resolved_path else f"{command_id}:{action.get('action_id', '')}"
if command_still_enabled:
was_preinjected = self._coerce_action_bool(
action.get("inject_on_new_document", True),
True,
)
if not was_preinjected:
continue
if resolved_path and resolved_path.lower() in active_preinject_keys:
continue
elif resolved_path and resolved_path.lower() in active_keys:
continue
if action_key in seen_keys:
continue
seen_keys.add(action_key)
cleanup_actions.append(copy.deepcopy(action))
return cleanup_actions
def _cleanup_disabled_run_js_file_actions(self, actions: List[Dict[str, Any]]) -> None:
if not actions:
return
try:
browser = self._get_browser()
pool = getattr(browser, "_tab_pool", None)
if pool is None or not hasattr(pool, "get_sessions_snapshot"):
return
sessions = list(pool.get_sessions_snapshot() or [])
except Exception as e:
logger.debug(f"[CMD] run_js_file 清理前获取会话失败(忽略): {e}")
return
cleaned = 0
for session in sessions:
for action in actions:
try:
result = self._cleanup_run_js_file_action(
session,
action,
reason="command_disabled",
)
if result.get("removed_init_script") or result.get("ran_teardown"):
cleaned += 1
if not result.get("ok", True):
logger.debug(
f"[CMD] run_js_file 清理存在告警: "
f"session={getattr(session, 'id', '')}, path={result.get('path', '')}, "
f"errors={result.get('errors', [])}"
)
except Exception as e:
logger.debug(
f"[CMD] run_js_file 清理失败(忽略): "
f"session={getattr(session, 'id', '')}, error={e}"
)
if cleaned > 0:
logger.info(f"[CMD] 已清理 {cleaned} 个 run_js_file 注入实例")
def _refresh_commands_if_changed(self, force: bool = False):
with self._commands_lock:
commands_file = self._get_commands_file()
current_mtime = os.path.getmtime(commands_file) if os.path.exists(commands_file) else 0.0
commands_local_file = self._get_commands_local_file()
current_local_mtime = os.path.getmtime(commands_local_file) if os.path.exists(commands_local_file) else 0.0
if (
force
or not self._commands_loaded
or current_mtime != self._commands_mtime
or current_local_mtime != self._commands_local_mtime
):
if (
not force
and self._commands_loaded
and self._commands_failed_mtime is not None
and current_mtime == self._commands_failed_mtime
and current_local_mtime == self._commands_local_mtime
):
# 同一份损坏文件已判定过失败:继续使用 last-known-good,等待文件被修好
return
previous_snapshot = copy.deepcopy(self._commands_cache) if self._commands_loaded else []
next_snapshot = self._read_commands_file()
if next_snapshot is None:
# B8:读失败 ≠ 合法空配置。保留上一版命令与预注入脚本,不推进 mtime、不做清理;
# 记录失败 mtime 以免每次轮询重复读取/刷日志,文件修复(mtime 变化)后自动重载。
self._commands_failed_mtime = current_mtime
if self._commands_loaded:
logger.warning(
f"[CMD] 命令配置读取失败,继续使用上一版 {len(self._commands_cache)} 条命令"
"(修复 commands.json 后将自动重新加载)"
)
else:
# 首次加载就失败:没有 last-known-good,只能以空配置启动,但不标记为已加载成功的 mtime
self._commands_cache = []
self._commands_loaded = True
return
self._commands_failed_mtime = None
cleanup_actions = self._collect_run_js_file_cleanup_actions(previous_snapshot, next_snapshot)
self._commands_cache = next_snapshot
self._commands_mtime = current_mtime
self._commands_loaded = True
self._cleanup_disabled_run_js_file_actions(cleanup_actions)
def _save_commands(self, commands: List[Dict]) -> bool:
commands_file = self._get_commands_file()
tmp_file = commands_file + ".tmp"
local_snapshot: Optional[tuple[bool, bytes]] = None
local_state_written = False
try:
with self._commands_lock:
commands_snapshot = copy.deepcopy(commands)
previous_snapshot = copy.deepcopy(self._commands_cache) if self._commands_loaded else []
os.makedirs(os.path.dirname(commands_file), exist_ok=True)
local_snapshot = self._snapshot_local_command_state()
if local_snapshot is None:
return False
if not self._save_local_command_state(commands_snapshot):
self._restore_local_command_state(local_snapshot)
return False
local_state_written = True
with open(tmp_file, "w", encoding="utf-8") as f:
json.dump({"commands": commands_snapshot}, f, indent=2, ensure_ascii=False)
f.flush()
os.fsync(f.fileno())
os.replace(tmp_file, commands_file)
try:
self._commands_mtime = os.path.getmtime(commands_file) if os.path.exists(commands_file) else 0.0
except Exception as mtime_error:
logger.warning(f"命令配置已保存但更新时间戳失败: {mtime_error}")
self._commands_loaded = True
self._commands_failed_mtime = None
self._commands_cache = commands_snapshot
cleanup_actions = self._collect_run_js_file_cleanup_actions(previous_snapshot, commands_snapshot)
self._cleanup_disabled_run_js_file_actions(cleanup_actions)
return True
except Exception as e:
logger.error(f"保存命令配置失败: {e}")
try:
if os.path.exists(tmp_file):
os.remove(tmp_file)
except Exception:
pass
if local_state_written:
self._restore_local_command_state(local_snapshot)
return False
def _runtime_stats_for_command_ids(self, command_ids: set[str]) -> Dict[str, Dict[str, Any]]:
if not command_ids:
return {}
with self._lock:
return {
command_id: copy.deepcopy(stats)
for command_id, stats in self._command_runtime_stats.items()
if command_id in command_ids and isinstance(stats, dict)
}
def _merge_runtime_stats_into_commands(self, commands: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
if not commands:
return commands
command_ids = {
str(cmd.get("id", "")).strip()
for cmd in commands
if isinstance(cmd, dict) and str(cmd.get("id", "")).strip()
}
runtime_stats = self._runtime_stats_for_command_ids(command_ids)
if not runtime_stats:
return commands
for cmd in commands:
cmd_id = str(cmd.get("id", "")).strip()
if not cmd_id:
continue
stats = runtime_stats.get(cmd_id)
if not stats:
continue
cmd.update(stats)
return commands
def _merge_runtime_stats_into_command(self, command: Dict[str, Any]) -> Dict[str, Any]:
command_id = str((command or {}).get("id", "")).strip()
if not command_id:
return command
runtime_stats = self._runtime_stats_for_command_ids({command_id})
stats = runtime_stats.get(command_id)
if stats:
command.update(stats)
return command
def _normalize_group_name(self, group_name: Any) -> str:
return str(group_name or "").strip()
@staticmethod
def _normalize_advanced_ui(ui: Any) -> Dict[str, Any]:
if not isinstance(ui, dict):
return {}
kind = str(ui.get("kind") or "none").strip().lower()
if kind not in {"none", "form"}:
kind = "none"
title = str(ui.get("title") or "").strip()
description = str(ui.get("description") or "").strip()
fields = ui.get("fields")
if not isinstance(fields, list):
fields = []
normalized_fields: List[Dict[str, Any]] = []
for field in fields:
if not isinstance(field, dict):
continue
field_type = str(field.get("type") or "text").strip().lower()
if field_type not in {"text", "textarea", "number", "boolean", "select", "password", "rule_list", "image"}:
continue
options = field.get("options")
if not isinstance(options, list):
options = []
rows = field.get("rows")
try:
rows = int(rows)
except (TypeError, ValueError, OverflowError):
rows = rows
normalized_fields.append({
"key": str(field.get("key") or "").strip(),
"label": str(field.get("label") or "").strip(),
"type": field_type,
"default": field.get("default"),
"help": str(field.get("help") or "").strip(),
"placeholder": str(field.get("placeholder") or "").strip(),
"required": bool(field.get("required", False)),
"options": options,
"rows": rows,
"item_defaults": field.get("item_defaults") if isinstance(field.get("item_defaults"), dict) else {},
})
values = ui.get("values")
if not isinstance(values, dict):
values = {}
return {
"kind": kind,
"title": title,
"description": description,
"results_enabled": bool(ui.get("results_enabled", False)),
"fields": normalized_fields,
"values": values,
}
@staticmethod
def _coerce_bool_flag(value: Any, default: bool = True) -> bool:
if value is None:
return default
if isinstance(value, bool):
return value
if isinstance(value, (int, float)):
return bool(value)
text = str(value).strip().lower()
if text in {"1", "true", "yes", "y", "on"}:
return True
if text in {"0", "false", "no", "n", "off"}:
return False
return default
@staticmethod
def _normalize_command_log_level(value: Any) -> str:
level = str(value or "GLOBAL").strip().upper()
return level if level in {"GLOBAL", "DEBUG", "INFO", "WARNING", "ERROR"} else "GLOBAL"
def _normalize_command_logging(self, command: Optional[Dict[str, Any]]) -> Optional[Dict[str, Any]]:
if not isinstance(command, dict):
return command
command["log_enabled"] = self._coerce_bool_flag(command.get("log_enabled", True), True)
command["log_level"] = self._normalize_command_log_level(command.get("log_level"))
return command
def _get_command_log_context(self, command: Optional[Dict[str, Any]]) -> Dict[str, Any]:
normalized = self._normalize_command_logging(command if isinstance(command, dict) else {}) or {}
return {
"enabled": bool(normalized.get("log_enabled", True)),
"level": self._normalize_command_log_level(normalized.get("log_level")),
}
def _command_logging_context(self, command: Optional[Dict[str, Any]]):
return command_log_context(self._get_command_log_context(command))
@staticmethod
def _normalize_group_acquire_policy(value: Any) -> str:
policy = str(value or "inherit_session").strip().lower()
return policy if policy in {"inherit_session", "try_acquire", "require_acquire"} else "inherit_session"
def _repair_mojibake_text(self, text: Any) -> str:
value = str(text or "").strip()
if not value:
return ""
candidates = [value]
for source_encoding in ("latin-1", "cp1252"):
try:
repaired = value.encode(source_encoding).decode("utf-8")
except (UnicodeEncodeError, UnicodeDecodeError):
continue
if repaired and repaired not in candidates:
candidates.append(repaired)
return candidates[-1]
def _should_follow_default_preset(self, preset_name: Any) -> bool:
value = str(preset_name or "").strip()
return value in {"", FOLLOW_DEFAULT_PRESET}
def _resolve_preset_name(self, preset_name: Any, session: Optional['TabSession'] = None) -> str:
if self._should_follow_default_preset(preset_name):
return ""
raw_name = str(preset_name or "").strip()
if not raw_name:
return ""
repaired_name = self._repair_mojibake_text(raw_name)
if repaired_name == raw_name:
return raw_name
domain = str(getattr(session, "current_domain", "") or "").strip() if session else ""
if not domain:
logger.warning(f"[CMD] 检测到乱码预设名,已自动修正: {raw_name} -> {repaired_name}")
return repaired_name
try:
config_engine = self._get_config_engine()
site = getattr(config_engine, "sites", {}).get(domain, {}) or {}
presets = site.get("presets", {}) or {}
if not presets or repaired_name in presets:
logger.warning(f"[CMD] 检测到乱码预设名,已自动修正: {raw_name} -> {repaired_name}")
return repaired_name
except Exception as e:
logger.debug(f"[CMD] 预设名乱码修正校验失败(忽略): {e}")
return raw_name
def _ensure_unique_command_name(
self,
raw_name: Any,
commands: List[Dict[str, Any]],
exclude_id: Optional[str] = None,
) -> str:
existing = {
str(cmd.get("name", "")).strip()
for cmd in commands
if cmd.get("id") != exclude_id and str(cmd.get("name", "")).strip()
}
base_name = str(raw_name or "").strip() or "新命令"
if base_name != "新命令" and base_name not in existing:
return base_name
root = re.sub(r"\d+$", "", base_name).rstrip() or "新命令"
pattern = re.compile(rf"^{re.escape(root)}(\d+)$")
next_num = 1
for name in existing:
match = pattern.match(name)
if match:
next_num = max(next_num, int(match.group(1)) + 1)
candidate = f"{root}{next_num}"
while candidate in existing:
next_num += 1
candidate = f"{root}{next_num}"
return candidate
@staticmethod
def _ensure_unique_copy_name(raw_name: Any, existing_names: set[str], max_length: int = 100) -> str:
source_name = str(raw_name or "").strip() or "未命名"
root = re.sub(r"\s*-\s*副本(?:\s+\d+)?$", "", source_name).strip() or source_name
def build_candidate(copy_number: int) -> str:
suffix = " - 副本" if copy_number == 1 else f" - 副本 {copy_number}"
available = max(1, max_length - len(suffix))
return f"{root[:available].rstrip()}{suffix}"
copy_number = 1
candidate = build_candidate(copy_number)
while candidate in existing_names:
copy_number += 1
candidate = build_candidate(copy_number)
return candidate
@staticmethod
def _remap_duplicated_command_references(
command: Dict[str, Any],
command_id_map: Dict[str, str],
source_group_name: str = "",
target_group_name: str = "",
) -> None:
trigger = command.get("trigger")
if isinstance(trigger, dict):
source_id = str(trigger.get("command_id") or "").strip()
if source_id in command_id_map:
trigger["command_id"] = command_id_map[source_id]
source_ids = trigger.get("command_ids")
if isinstance(source_ids, list):
trigger["command_ids"] = [
command_id_map.get(str(command_id or "").strip(), command_id)
for command_id in source_ids
]
if not source_group_name or not target_group_name:
return
for action in command.get("actions") or []:
if not isinstance(action, dict) or action.get("type") != "execute_command_group":
continue
if str(action.get("group_name") or "").strip() == source_group_name:
action["group_name"] = target_group_name
def _load_commands(self) -> List[Dict]:
"""从配置引擎加载命令列表原始快照,避免共享可变引用。"""
with self._commands_lock:
self._get_config_engine()
self._refresh_commands_if_changed()
snapshot = self._commands_cache
return copy.deepcopy(snapshot)
def _load_commands_for_checks(self) -> List[Dict]:
"""Load a lightweight command snapshot for read-mostly trigger checks."""
with self._commands_lock:
self._get_config_engine()
self._refresh_commands_if_changed()
snapshot = self._commands_cache
commands = [dict(cmd) for cmd in snapshot if isinstance(cmd, dict)]
return commands
+918
View File
@@ -0,0 +1,918 @@
"""CommandEngine 的一部分(R2-3 拆分):触发判定:请求计数、网络事件、优先级与忙碌判断、导航事件分发。
方法原样从 command_engine.py 迁出,行为不变;通过 mixin 组合回 CommandEngine,
类属性与实例状态都在 CommandEngine(command_engine.py)里定义。
"""
import time
import uuid
from typing import Dict, List, Optional, Any, TYPE_CHECKING
from app.utils.site_url import extract_remote_site_domain
from app.services.command_engine_common import logger
if TYPE_CHECKING:
from app.core.tab_pool import TabSession # noqa: F401
class CommandEngineTriggersMixin:
"""触发判定:请求计数、网络事件、优先级与忙碌判断、导航事件分发"""
def check_triggers(self, session: 'TabSession'):
"""
检查所有命令的触发条件
在 TabSession.release() 后调用(锁外、后台,不阻塞主流程)
"""
self.ensure_scheduler_running()
try:
commands = self._load_commands_for_checks()
except Exception as e:
logger.debug(f"命令加载失败,跳过触发检查: {e}")
return
if not commands:
return
ordered_commands = [
(idx, cmd) for idx, cmd in enumerate(commands)
if cmd.get("enabled", True)
]
ordered_commands.sort(key=lambda item: (-self._get_command_priority(item[1]), item[0]))
for _, cmd in ordered_commands:
with self._command_logging_context(cmd):
try:
if self._should_trigger(cmd, session):
meta = self._take_pending_async_trigger_meta(cmd, session) or {}
self._execute_command_async(
cmd,
session,
interrupt_context=meta.get("interrupt_context"),
trigger_rollback=meta.get("rollback"),
)
except Exception as e:
logger.error(f"trigger check failed [{cmd.get('name')}]: {e}")
def check_workflow_triggers_now(self, session: 'TabSession') -> bool:
"""Check commands for an active workflow and enqueue matching interrupts immediately."""
if not self._has_active_workflow(session):
return False
self.ensure_scheduler_running()
try:
commands = self._load_commands_for_checks()
except Exception as e:
logger.debug(f"命令加载失败,跳过工作流触发检查: {e}")
return False
ordered_commands = [
(idx, cmd) for idx, cmd in enumerate(commands)
if cmd.get("enabled", True)
]
ordered_commands.sort(key=lambda item: (-self._get_command_priority(item[1]), item[0]))
scheduled_any = False
for _, cmd in ordered_commands:
with self._command_logging_context(cmd):
try:
trigger = cmd.get("trigger", {}) or {}
trigger_type = str(trigger.get("type", "")).strip().lower()
if (
trigger_type == "page_check"
and not self._should_evaluate_page_check_while_busy_workflow(cmd)
):
continue
if not self._should_trigger(cmd, session):
continue
meta = self._take_pending_async_trigger_meta(cmd, session) or {}
scheduled = self._schedule_command_for_active_workflow(
cmd,
session,
interrupt_context=meta.get("interrupt_context"),
trigger_rollback=meta.get("rollback"),
)
if scheduled:
scheduled_any = True
elif meta.get("rollback"):
self._rollback_trigger_consumption(cmd, session, meta.get("rollback"))
except Exception as e:
logger.error(f"workflow trigger check failed [{cmd.get('name')}]: {e}")
return scheduled_any
def submit_background_task(self, fn, *args, **kwargs):
"""Submit non-trigger command work to the bounded command executor."""
return self._command_executor.submit(fn, *args, **kwargs)
def handle_network_event(self, session: 'TabSession', event: Dict[str, Any]) -> bool:
"""
处理实时网络事件。
返回值:
- True: 命中了网络拦截,当前监听应暂停并交回工作流处理
- False: 不需要暂停当前监听
"""
self.ensure_scheduler_running()
if not event:
return False
event_copy = dict(event)
if not event_copy.get("event_id"):
event_copy["event_id"] = f"net_{uuid.uuid4().hex[:10]}"
event_copy.setdefault("timestamp", time.time())
with self._lock:
self._append_bounded_event(self._network_events, session.id, event_copy)
try:
commands = self._load_commands_for_checks()
except Exception as e:
logger.debug(f"命令加载失败,跳过网络事件触发: {e}")
return False
should_interrupt_listener = False
for cmd in commands:
if not cmd.get("enabled", True):
continue
trigger = cmd.get("trigger", {})
if trigger.get("type") != "network_request_error":
continue
if not self._matches_scope(cmd, session):
continue
with self._command_logging_context(cmd):
dispatch = self._prepare_network_trigger_dispatch(cmd, session, event_copy)
if not dispatch:
continue
if self._has_active_workflow(session):
scheduled = self._schedule_command_for_active_workflow(
cmd,
session,
interrupt_context=dispatch.get("interrupt_context"),
trigger_rollback=dispatch.get("rollback"),
)
if scheduled:
should_interrupt_listener = (
should_interrupt_listener
or self.workflow_interrupt_requested(session)
)
else:
scheduled = self._execute_command_async(
cmd,
session,
interrupt_context=dispatch.get("interrupt_context"),
trigger_rollback=dispatch.get("rollback"),
)
if scheduled:
should_interrupt_listener = (
should_interrupt_listener
or bool(trigger.get("abort_on_match", True))
)
if scheduled:
continue
if dispatch.get("rollback"):
self._rollback_trigger_consumption(cmd, session, dispatch.get("rollback"))
return should_interrupt_listener
def has_network_interception_for_session(self, session: 'TabSession') -> bool:
"""当前会话是否存在可生效的网络异常拦截触发器。"""
try:
commands = self._load_commands_for_checks()
except Exception:
return False
for cmd in commands:
if not cmd.get("enabled", True):
continue
trigger = cmd.get("trigger", {})
if trigger.get("type") != "network_request_error":
continue
if self._matches_scope(cmd, session):
return True
return False
def get_network_listen_pattern(self, session: 'TabSession') -> str:
"""
依据网络异常拦截命令推断一个 listen_pattern。
仅用于事件监听,不要求完全精准。
"""
try:
commands = self._load_commands_for_checks()
except Exception:
return "http"
hints: List[str] = []
for cmd in commands:
if not cmd.get("enabled", True):
continue
trigger = cmd.get("trigger", {})
if trigger.get("type") != "network_request_error":
continue
if not self._matches_scope(cmd, session):
continue
pattern = str(trigger.get("url_pattern") or trigger.get("value") or "").strip()
if not pattern:
continue
hint = self._pattern_to_listen_hint(pattern, str(trigger.get("match_mode", "keyword")))
if hint:
hints.append(hint)
if not hints:
return "http"
hints.sort(key=len, reverse=True)
return hints[0]
def _ensure_trigger_state(
self,
command_id: str,
session: 'TabSession',
state_key: Optional[tuple] = None,
initial_req: Optional[int] = None,
initial_err: Optional[int] = None,
) -> tuple[Dict[str, Any], bool]:
state_key = state_key or (command_id, session.id)
req_baseline = session.request_count if initial_req is None else int(initial_req)
err_baseline = session.error_count if initial_err is None else int(initial_err)
with self._lock:
state = self._trigger_states.get(state_key)
if state is None:
state = {
"req": req_baseline,
"err": err_baseline,
"result_token": "",
"net_sig": "",
"page_key": "",
"page_hit": False,
"page_last_fire_at": 0.0,
}
self._trigger_states[state_key] = state
return state, True
return state, False
def _coerce_int(self, value: Any, default: int) -> int:
try:
return int(value)
except (TypeError, ValueError, OverflowError):
return default
def _coerce_float(self, value: Any, default: float) -> float:
try:
return float(value)
except (TypeError, ValueError, OverflowError):
return default
@staticmethod
def _counter_inc(counter: Dict[str, int], key: str):
if not key:
return
counter[key] = int(counter.get(key, 0)) + 1
@staticmethod
def _counter_dec(counter: Dict[str, int], key: str):
if not key:
return
next_value = int(counter.get(key, 0)) - 1
if next_value > 0:
counter[key] = next_value
else:
counter.pop(key, None)
def _normalize_priority(self, value: Any, default: int = 2) -> int:
try:
p = int(value)
except (TypeError, ValueError, OverflowError):
p = int(default)
return p
def _get_request_priority_baseline(self) -> int:
return self._normalize_priority(getattr(self, "_request_priority_baseline", 2), 2)
def _get_command_priority(self, command: Dict) -> int:
trigger = command.get("trigger", {}) or {}
raw = trigger.get("priority", trigger.get("command_priority", 2))
return self._normalize_priority(raw, 2)
def _should_evaluate_page_check_while_busy_workflow(self, command: Dict) -> bool:
trigger = command.get("trigger", {}) or {}
raw = trigger.get("check_while_busy_workflow", None)
if raw is None:
return True
return bool(raw)
def _command_affects_domain(self, command: Dict) -> bool:
actions = command.get("actions", [])
for action in actions or []:
if str((action or {}).get("type", "")).strip() == "clear_cookies":
return True
return False
def _has_busy_peer_on_domain(self, domain: str, exclude_session_id: str = "") -> bool:
normalized = str(domain or "").strip().lower()
if not normalized:
return False
try:
browser = self._get_browser()
pool = getattr(browser, "_tab_pool", None)
if pool is None:
return False
status = pool.get_status() if hasattr(pool, "get_status") else {}
for tab in status.get("tabs", []) or []:
sid = str(tab.get("id", "") or "")
if exclude_session_id and sid == exclude_session_id:
continue
if str(tab.get("status", "")).lower() != "busy":
continue
tab_domain = str(tab.get("current_domain", "") or "").strip().lower()
if self._domain_matches(normalized, tab_domain):
return True
except Exception:
return False
return False
def should_block_request_for_session(self, session: "TabSession", task_id: str = "") -> bool:
if session is None:
return False
task = str(task_id or "").strip().lower()
if task.startswith("cmd_") or task.startswith("group_") or task.startswith("cmd_test_") or task.startswith("group_test_"):
return False
session_id = str(getattr(session, "id", "") or "")
domain = self._get_session_domain(session)
with self._lock:
if int(self._pending_high_by_session.get(session_id, 0)) > 0:
return True
if int(self._running_high_by_session.get(session_id, 0)) > 0:
return True
if domain and int(self._pending_high_by_domain.get(domain, 0)) > 0:
return True
if domain and int(self._running_high_by_domain.get(domain, 0)) > 0:
return True
return False
def _get_session_domain(self, session: 'TabSession') -> str:
try:
url = str(getattr(session.tab, "url", "") or "")
domain = str(extract_remote_site_domain(url) or "").strip().lower()
if domain:
session.current_domain = domain
return domain
except Exception:
pass
domain = str(getattr(session, "current_domain", "") or "").strip().lower()
if domain:
return domain
return ""
def _get_active_workflow_runtime(self, session: 'TabSession') -> Optional[Dict[str, Any]]:
stack = getattr(session, "_workflow_runtime_stack", None) or []
if not stack:
return None
runtime = stack[-1]
return runtime if isinstance(runtime, dict) else None
def _build_request_count_state_key(self, command: Dict, session: 'TabSession') -> tuple:
trigger = command.get("trigger", {}) or {}
scope = str(trigger.get("scope", "all") or "all").strip().lower()
if scope == "all":
return (command["id"], "__scope:all")
if scope == "domain":
domain_key = str(trigger.get("domain", "") or "").strip().lower()
if not domain_key:
domain_key = self._get_session_domain(session)
return (command["id"], f"__scope:domain:{domain_key or '_'}")
return (command["id"], session.id)
def _get_scope_request_count(self, command: Dict, session: 'TabSession') -> int:
trigger = command.get("trigger", {}) or {}
scope = str(trigger.get("scope", "all") or "all").strip().lower()
if scope == "tab":
try:
return int(getattr(session, "request_count", 0) or 0)
except Exception:
return 0
try:
browser = self._get_browser()
pool = getattr(browser, "_tab_pool", None)
if pool is None:
return int(getattr(session, "request_count", 0) or 0)
status = pool.get_status() if hasattr(pool, "get_status") else {}
tabs = status.get("tabs", []) or []
except Exception:
return int(getattr(session, "request_count", 0) or 0)
if scope == "all":
total = 0
for tab in tabs:
try:
total += int(tab.get("request_count", 0) or 0)
except Exception:
continue
return total
if scope == "domain":
target_domain = str(trigger.get("domain", "") or "").strip().lower()
if not target_domain:
target_domain = self._get_session_domain(session)
if not target_domain:
return int(getattr(session, "request_count", 0) or 0)
total = 0
for tab in tabs:
try:
tab_domain = str(tab.get("current_domain", "") or "").strip().lower()
if not tab_domain:
url = str(tab.get("url", "") or "")
if "://" in url:
tab_domain = url.split("//", 1)[1].split("/", 1)[0].strip().lower()
if self._domain_matches(target_domain, tab_domain):
total += int(tab.get("request_count", 0) or 0)
except Exception:
continue
return total
try:
return int(getattr(session, "request_count", 0) or 0)
except Exception:
return 0
def _should_trigger(self, command: Dict, session: 'TabSession') -> bool:
trigger = command.get("trigger", {})
trigger_type = trigger.get("type", "")
scope = trigger.get("scope", "all")
# Scope pre-check
if scope == "domain":
target_domain = str(trigger.get("domain", "") or "").strip().lower()
session_domain = self._get_session_domain(session)
if target_domain and session_domain:
if not self._domain_matches(target_domain, session_domain):
return False
elif target_domain:
return False
elif scope == "tab":
target_index = trigger.get("tab_index")
if target_index is not None and session.persistent_index != target_index:
return False
# Skip if same command already executing on this tab
exec_key = (command["id"], session.id)
with self._lock:
if exec_key in self._executing:
return False
request_state_key = None
scope_request_count = None
initial_req = None
if trigger_type == "request_count":
request_state_key = self._build_request_count_state_key(command, session)
scope_request_count = self._get_scope_request_count(command, session)
try:
# Count the current completed request as the first hit so
# a threshold of N fires on the Nth completed request rather
# than the (N+1)th after state initialization.
initial_req = max(0, int(scope_request_count) - 1)
except Exception:
initial_req = scope_request_count
# Initialize or load trigger state
state, is_new = self._ensure_trigger_state(
command["id"],
session,
state_key=request_state_key,
initial_req=initial_req if trigger_type == "request_count" else scope_request_count,
)
if is_new and trigger_type not in {"page_check", "command_check"}:
return False # Newly initialized: wait for next check cycle
# Evaluate trigger condition by trigger type
if trigger_type == "request_count":
threshold = max(1, self._coerce_int(trigger.get("value", 10), 10))
current_count = (
int(scope_request_count)
if scope_request_count is not None
else self._get_scope_request_count(command, session)
)
with self._lock:
baseline = int(state.get("req", 0))
delta = current_count - baseline
should_fire = delta >= threshold
if should_fire:
# 先记录旧基线;若后续因标签页忙碌/超时未实际执行,可回滚以便重试。
state["req_prev"] = baseline
state["req_pending"] = True
state["req"] = current_count
logger.debug_throttled(
f"cmd.request_count.{command.get('id')}:{session.id}:{self._format_scope_label(scope)}",
f"[CMD] 请求计数检查: {command.get('name')} "
f"(当前={current_count}, 基线={baseline}, 增量={delta}, 阈值={threshold}, "
f"标签页={session.id}, 范围={self._format_scope_label(scope)})",
interval_sec=10.0,
)
if should_fire:
logger.info(
f"[CMD] 触发命令: {command.get('name')} "
f"(请求增量={delta}, 阈值={threshold}, 标签页={session.id}, "
f"范围={self._format_scope_label(scope)})"
)
return True
elif trigger_type == "error_count":
threshold = max(1, self._coerce_int(trigger.get("value", 3), 3))
delta = session.error_count - state["err"]
if delta >= threshold:
logger.info(
f"[CMD] 触发命令: {command.get('name')} "
f"(错误增量={delta}, 阈值={threshold})"
)
with self._lock:
state["err"] = session.error_count
return True
elif trigger_type == "idle_timeout":
threshold_sec = max(1.0, self._coerce_float(trigger.get("value", 300), 300.0))
idle = time.time() - session.last_used_at
if idle >= threshold_sec:
logger.info(
f"[CMD] 触发命令: {command.get('name')} "
f"(空闲时长={idle:.0f}秒, 阈值={threshold_sec}秒)"
)
return True
elif trigger_type == "page_check":
check_text = str(trigger.get("value", ""))
normalized_text = check_text.lower().strip()
op, keywords = self._parse_page_check_expression(check_text)
probe_js = str(trigger.get("probe_js", "") or "").strip()
match_info = (
self._evaluate_page_check_expr(session, op, keywords)
if keywords else {
"hit": True if probe_js else False,
"matched_keywords": [],
"snapshot_preview": "",
}
)
current_hit = bool(match_info.get("hit"))
probe_always = bool(trigger.get("probe_always", False))
if probe_js and (current_hit or probe_always):
probe_info = self._evaluate_page_check_probe(session, probe_js)
match_info["probe_hit"] = bool(probe_info.get("hit"))
match_info["probe_result"] = probe_info.get("result")
match_info["probe_summary"] = str(probe_info.get("summary") or "").strip()
current_hit = bool(probe_info.get("hit"))
match_info["hit"] = current_hit
fire_mode = str(trigger.get("fire_mode", "edge") or "edge").strip().lower()
cooldown_sec = max(0.0, self._coerce_float(trigger.get("cooldown_sec", 0), 0.0))
stable_for_sec = max(0.0, self._coerce_float(trigger.get("stable_for_sec", 0), 0.0))
now_ts = time.time()
once_per_request = bool(trigger.get("once_per_request", False))
current_request_id = ""
if once_per_request:
try:
for attr_name in ("_command_request_id", "_bound_request_id"):
req_id = str(getattr(session, attr_name, "") or "").strip()
if req_id:
current_request_id = req_id
break
if not current_request_id:
task_id = str(getattr(session, "current_task_id", "") or "").strip()
if task_id.startswith("req-"):
current_request_id = task_id
except Exception:
pass
with self._lock:
if once_per_request and current_request_id:
triggered_reqs = state.setdefault("triggered_requests", set())
if current_request_id in triggered_reqs:
return False
prev_key = str(state.get("page_key", ""))
prev_hit = bool(state.get("page_hit", False)) if prev_key == normalized_text else False
prev_stable = bool(state.get("page_stable", False)) if prev_key == normalized_text else False
hit_since = float(state.get("page_hit_since", 0.0) or 0.0) if prev_key == normalized_text else 0.0
if current_hit:
if not prev_hit or hit_since <= 0:
hit_since = now_ts
else:
hit_since = 0.0
state["page_key"] = normalized_text
state["page_hit"] = current_hit
state["page_hit_since"] = hit_since
last_fire_at = float(state.get("page_last_fire_at", 0.0) or 0.0)
stable_hit = bool(current_hit and ((now_ts - hit_since) >= stable_for_sec))
state["page_stable"] = stable_hit
if fire_mode == "level":
if stable_hit and (cooldown_sec <= 0 or (now_ts - last_fire_at) >= cooldown_sec):
if once_per_request and current_request_id:
triggered_reqs = state.setdefault("triggered_requests", set())
if current_request_id in triggered_reqs:
return False
triggered_reqs.add(current_request_id)
state["page_last_fire_at"] = now_ts
logger.info(
f"[CMD] 触发命令: {command.get('name')} "
f"(页面检查命中, 模式=持续触发, 文本='{check_text[:30]}', "
f"冷却={cooldown_sec}秒)"
)
self._log_page_check_hit_details(command, session, match_info)
return True
else:
if stable_hit and not prev_stable:
if once_per_request and current_request_id:
triggered_reqs = state.setdefault("triggered_requests", set())
if current_request_id in triggered_reqs:
return False
triggered_reqs.add(current_request_id)
state["page_last_fire_at"] = now_ts
logger.info(
f"[CMD] 触发命令: {command.get('name')} "
f"(页面检查命中, 模式=边沿触发, 文本='{check_text[:30]}')"
)
self._log_page_check_hit_details(command, session, match_info)
return True
elif trigger_type == "command_result_match":
dispatch = self._prepare_command_result_trigger_dispatch(command, session)
if dispatch:
self._set_pending_async_trigger_meta(command, session, dispatch)
logger.info(
f"[CMD] 触发命令: {command.get('name')} "
f"(命中命令结果匹配条件)"
)
return True
elif trigger_type == "command_check":
dispatch = self._prepare_command_check_dispatch(command, session)
if dispatch:
self._set_pending_async_trigger_meta(command, session, dispatch)
check_info = (dispatch.get("interrupt_context") or {}).get("command_check", {})
logger.info(
f"[CMD] 触发命令: {command.get('name')} "
f"(命令检查命中: 来源={check_info.get('source_command_name', '')}, "
f"结果={str(check_info.get('actual_result', '') or '')[:80]})"
)
return True
elif trigger_type == "command_result_event":
dispatch = self._prepare_command_result_event_dispatch(command, session)
if dispatch:
self._set_pending_async_trigger_meta(command, session, dispatch)
logger.info(
f"[CMD] 触发命令: {command.get('name')} "
f"(命中命令结果事件条件)"
)
return True
elif trigger_type == "network_request_error":
dispatch = self._prepare_network_trigger_dispatch(command, session)
if dispatch:
event = (dispatch.get("interrupt_context") or {}).get("network_event", {})
self._set_pending_async_trigger_meta(command, session, dispatch)
if (dispatch.get("rollback") or {}).get("token"):
logger.info(
f"[CMD] 触发命令: {command.get('name')} "
f"(网络异常状态码={event.get('status')}, 地址={event.get('url', '')[:80]})"
)
return True
return False
def _is_command_applicable_on_navigation(self, cmd: Dict[str, Any]) -> bool:
"""判断命令是否适用于页面导航/刷新事件触发。"""
if not isinstance(cmd, dict) or not bool(cmd.get("enabled", True)):
return False
trigger = cmd.get("trigger", {}) or {}
# 1. 显式配置 check_on_navigation / check_on_page_load
if bool(trigger.get("check_on_navigation", False)) or bool(cmd.get("check_on_navigation", False)):
return True
if bool(trigger.get("check_on_page_load", False)) or bool(cmd.get("check_on_page_load", False)):
return True
# 2. page_check 触发器天然依赖页面 DOM/内容,页面跳转刷新后必须立即重新检查
trigger_type = str(trigger.get("type", "")).strip().lower()
if trigger_type == "page_check":
return True
# 3. 包含 bootstrap_on_session_ready 的 run_js_file 命令需要在页面就绪后注入/执行
for action in list(cmd.get("actions") or []):
if isinstance(action, dict) and bool(action.get("bootstrap_on_session_ready", False)):
return True
return False
def notify_tab_navigated(
self,
session: 'TabSession',
reason: str = "page_reload",
async_dispatch: bool = True,
) -> None:
"""
通用的页面导航/刷新通知接口。
当标签页发生刷新(F5、location.reload、断流恢复刷新等)或导航(URL 跳转、新标签页打开等)时调用。
执行操作:
1. 即时重置 JS 执行失败退避锁(_pc_js_backoff_until = 0.0)和失败计数(_pc_js_failures = 0)
2. 清空页面快照缓存与刷新宽限期,重置 observer 探测节流与缓存
3. 清除 observer 缓存与 haystack 缓存,重置适用命令的周期倒计时与 edge 锁存
4. 异步或同步派发页面导航后评估(_dispatch_tab_navigated_evaluation):
- 同步 bootstrap 启动 JS 文件注入(Page.addScriptToEvaluateOnNewDocument + 立即执行)
- 重新注入/同步 MutationObserver
- 毫秒级触发并评估适用命令
"""
if session is None:
return
session_id = getattr(session, "id", "")
if not session_id:
return
# 1. 同步清理 session 上的快照与退避状态
try:
setattr(session, "_pc_js_failures", 0)
setattr(session, "_pc_js_backoff_until", 0.0)
setattr(session, "_pc_refresh_grace_until", 0.0)
setattr(session, "_pc_snapshot_cached", None)
setattr(session, "_last_pc_observer_check_at", 0.0)
setattr(session, "_pc_observer_empty_cleanup_done", False)
if hasattr(self, "_invalidate_wake_throttle"):
self._invalidate_wake_throttle(session)
except Exception as e:
logger.debug(f"[CMD] 重置会话退避状态失败(忽略): {e}")
# 2. 清理引擎层全局缓存与适用命令的状态
try:
commands = self._load_commands_for_checks()
except Exception as e:
logger.debug(f"[CMD] 导航通知加载命令失败: {e}")
commands = []
applicable_cmds = []
with self._lock:
self._observer_keywords_by_session.pop(session_id, None)
self._compact_haystack_cache = None
for cmd in commands:
cmd_id = cmd.get("id")
if not cmd_id:
continue
if self._is_command_applicable_on_navigation(cmd):
self._periodic_next_run[(cmd_id, session_id)] = 0.0
applicable_cmds.append(cmd)
for cmd in applicable_cmds:
self._reset_page_check_latch(cmd, session, reason=f"navigated:{reason}")
logger.debug(
f"[CMD] 标签页导航/刷新通知: session={session_id}, reason={reason}, async={async_dispatch}"
)
# 3. 派发评估
if async_dispatch:
self.submit_background_task(self._dispatch_tab_navigated_evaluation, session, reason)
else:
self._dispatch_tab_navigated_evaluation(session, reason)
def _dispatch_tab_navigated_evaluation(
self,
session: 'TabSession',
reason: str = "page_reload",
) -> None:
"""在页面导航/刷新后执行 bootstrap 脚本注入、observer 同步及命令评估调度。"""
if self._is_session_closed(session):
return
self.ensure_scheduler_running()
try:
commands = self._load_commands_for_checks()
except Exception as e:
logger.debug(f"[CMD] 导航后评估加载命令失败: {e}")
return
if not commands:
return
# 1. 优先执行/注入 bootstrap 启动脚本(例如 Page.addScriptToEvaluateOnNewDocument 及即时注入)
try:
self._sync_session_bootstrap_js_files(commands, session)
except Exception as e:
logger.warning(f"[CMD] 导航后同步 bootstrap 脚本失败: {e}")
# 2. 同步 Page Check Observer(如果有 page_check 关键词)
try:
pc_keywords = self._collect_page_check_keywords(commands, session)
if pc_keywords:
self._ensure_page_check_observer(session, pc_keywords)
else:
self._clear_page_check_observer(session)
except Exception as e:
logger.debug(f"[CMD] 导航后同步 observer 失败: {e}")
# 3. 评估并调度适用的命令
applicable_commands = [
(idx, cmd) for idx, cmd in enumerate(commands)
if self._is_command_applicable_on_navigation(cmd) and self._matches_scope(cmd, session)
]
applicable_commands.sort(key=lambda item: (-self._get_command_priority(item[1]), item[0]))
for _, cmd in applicable_commands:
with self._command_logging_context(cmd):
try:
current_status = str(getattr(getattr(session, "status", None), "value", "")).lower()
if current_status == "busy" and not self._has_active_workflow(session):
continue
if current_status not in {"idle", "busy"}:
continue
if self._should_trigger(cmd, session):
meta = self._take_pending_async_trigger_meta(cmd, session) or {}
if current_status == "busy":
scheduled = self._schedule_command_for_active_workflow(
cmd,
session,
interrupt_context=meta.get("interrupt_context"),
trigger_rollback=meta.get("rollback"),
)
if not scheduled:
if meta.get("rollback"):
self._rollback_trigger_consumption(cmd, session, meta.get("rollback"))
self._finalize_request_count_trigger_state(cmd, session, rollback=True)
self._reset_page_check_latch(cmd, session, reason="workflow_schedule_failed")
else:
self._execute_command_async(
cmd,
session,
interrupt_context=meta.get("interrupt_context"),
trigger_rollback=meta.get("rollback"),
)
except Exception as e:
logger.error(f"[CMD] 导航后触发检查失败 [{cmd.get('name')}]: {e}")
def _finalize_request_count_trigger_state(
self,
command: Dict,
session: 'TabSession',
*,
rollback: bool = False
):
"""
收尾 request_count 触发状态:
- rollback=True: 未实际执行时回滚 req 到触发前基线,避免触发被“吃掉”
- rollback=False: 实际开始执行后清理 pending 标记
"""
trigger = command.get("trigger", {}) or {}
if str(trigger.get("type", "")).strip().lower() != "request_count":
return
key = self._build_request_count_state_key(command, session)
with self._lock:
state = self._trigger_states.get(key)
if not state:
return
pending = bool(state.pop("req_pending", False))
prev_req = state.pop("req_prev", None)
if rollback and pending and prev_req is not None:
try:
state["req"] = int(prev_req)
except Exception:
pass
@staticmethod
def _execution_needs_page_check_retry(execution_result: Any) -> bool:
"""
Determine whether a page_check-triggered command should be retried.
Retry signal is inferred from:
- Step-level ok flag being False (exception during execution)
- Action results shaped like {"ok": False, ...}
- String results containing known failure prefixes (e.g. "js_failed:")
"""
if not isinstance(execution_result, dict):
return False
_FAIL_PREFIXES = (
"js_failed:",
"python_failed:",
"unsupported_lang:",
"ERROR:",
"refresh_failed:",
"navigate_failed:",
)
direct_result = execution_result.get("result")
if isinstance(direct_result, dict) and direct_result.get("ok") is False:
return True
if isinstance(direct_result, str) and direct_result.startswith(_FAIL_PREFIXES):
return True
steps = execution_result.get("steps")
if isinstance(steps, list):
for step in steps:
if not isinstance(step, dict):
continue
# Check step-level ok flag (set by _execute_simple on exceptions)
if step.get("ok") is False:
return True
step_result = step.get("result")
if isinstance(step_result, dict) and step_result.get("ok") is False:
return True
if isinstance(step_result, str) and step_result.startswith(_FAIL_PREFIXES):
return True
return False
def _normalize_match_rule(self, rule: Any) -> str:
rule_value = str(rule or "").strip().lower()
mapping = {
"eq": "equals",
"equal": "equals",
"equals": "equals",
"is": "equals",
"contains": "contains",
"include": "contains",
"includes": "contains",
"ne": "not_equals",
"not_equal": "not_equals",
"not_equals": "not_equals",
}
return mapping.get(rule_value, "equals")
+225
View File
@@ -0,0 +1,225 @@
"""
app/services/command_result_store.py - 通用命令执行结果持久化存储层 (CRUD)
职责:
- 提供命令结果记录的底层 JSON 文件读写与增删改查;
- 维护原子写入与最大记录数上限控制;
- 存储层保持纯粹 CRUD,不耦合特定厂商或业务规则。
"""
from __future__ import annotations
import json
import logging
import os
import tempfile
import threading
import uuid
from pathlib import Path
from typing import Any, Callable, Dict, Iterable, List, Optional
try:
import requests
except ImportError: # pragma: no cover
requests = None
logger = logging.getLogger(__name__)
_PROJECT_ROOT = Path(__file__).resolve().parents[2]
RESULTS_FILE = _PROJECT_ROOT / "config" / "command_results.local.json"
_LOCK = threading.RLock()
_MAX_RECORDS = 5000
def _atomic_write_json(path: Path, payload: Dict[str, Any]) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
fd, temp_name = tempfile.mkstemp(prefix=f"{path.stem}_", suffix=".tmp", dir=str(path.parent))
try:
with os.fdopen(fd, "w", encoding="utf-8", newline="\n") as handle:
json.dump(payload, handle, ensure_ascii=False, indent=2)
handle.write("\n")
handle.flush()
os.fsync(handle.fileno())
os.replace(temp_name, path)
finally:
if os.path.exists(temp_name):
os.remove(temp_name)
def _load_results() -> Dict[str, Any]:
if not RESULTS_FILE.exists():
return {"version": 1, "records": []}
try:
with RESULTS_FILE.open("r", encoding="utf-8") as handle:
payload = json.load(handle)
except (OSError, ValueError):
return {"version": 1, "records": []}
records = payload.get("records") if isinstance(payload, dict) else []
return {"version": 1, "records": records if isinstance(records, list) else []}
def get_all_records() -> List[Dict[str, Any]]:
"""获取所有记录副本。"""
with _LOCK:
return list(_load_results()["records"])
def get_record(record_id: str) -> Optional[Dict[str, Any]]:
"""根据 ID 获取单条记录。"""
rid = str(record_id or "").strip()
if not rid:
return None
with _LOCK:
for item in _load_results()["records"]:
if isinstance(item, dict) and str(item.get("id") or "") == rid:
return dict(item)
return None
def list_command_results(command_id: str, limit: int = 500) -> List[Dict[str, Any]]:
"""根据 command_id 检索结果列表,按创建时间倒序。"""
command_id = str(command_id or "").strip()
with _LOCK:
records = _load_results()["records"]
filtered = [item for item in records if isinstance(item, dict) and item.get("command_id") == command_id]
filtered.sort(key=lambda item: float(item.get("created_at") or 0), reverse=True)
return filtered[: max(1, min(int(limit or 500), 2000))]
def add_record(record: Dict[str, Any]) -> Dict[str, Any]:
"""新增单条命令执行结果。"""
if not isinstance(record, dict):
raise ValueError("record must be a dict")
rec = dict(record)
if not rec.get("id"):
rec["id"] = uuid.uuid4().hex
with _LOCK:
payload = _load_results()
records = payload["records"]
records.append(rec)
payload["records"] = records[-_MAX_RECORDS:]
_atomic_write_json(RESULTS_FILE, payload)
return rec
def add_records(records: Iterable[Dict[str, Any]]) -> List[Dict[str, Any]]:
"""批量新增命令执行结果。"""
items: List[Dict[str, Any]] = []
for raw in records:
if isinstance(raw, dict):
item = dict(raw)
if not item.get("id"):
item["id"] = uuid.uuid4().hex
items.append(item)
if not items:
return []
with _LOCK:
payload = _load_results()
existing = payload["records"]
existing.extend(items)
payload["records"] = existing[-_MAX_RECORDS:]
_atomic_write_json(RESULTS_FILE, payload)
return items
def add_records_unique(
records: Iterable[Dict[str, Any]],
unique_fields: Iterable[str],
) -> List[Dict[str, Any]]:
"""Add records while atomically suppressing duplicates by business key."""
fields = tuple(str(field) for field in unique_fields if str(field))
if not fields:
return add_records(records)
items: List[Dict[str, Any]] = []
for raw in records:
if isinstance(raw, dict):
item = dict(raw)
if not item.get("id"):
item["id"] = uuid.uuid4().hex
items.append(item)
if not items:
return []
def key(item: Dict[str, Any]) -> tuple[Any, ...]:
return tuple(str(item.get(field) or "") for field in fields)
with _LOCK:
payload = _load_results()
existing = payload["records"]
seen = {key(item) for item in existing if isinstance(item, dict)}
accepted = []
for item in items:
item_key = key(item)
if item_key in seen:
continue
seen.add(item_key)
accepted.append(item)
if accepted:
existing.extend(accepted)
payload["records"] = existing[-_MAX_RECORDS:]
_atomic_write_json(RESULTS_FILE, payload)
return accepted
def delete_record(record_id: str) -> bool:
"""根据 ID 删除单条记录。"""
rid = str(record_id or "").strip()
if not rid:
return False
with _LOCK:
payload = _load_results()
before = len(payload["records"])
payload["records"] = [
item for item in payload["records"]
if not (isinstance(item, dict) and str(item.get("id") or "") == rid)
]
removed = before != len(payload["records"])
if removed:
_atomic_write_json(RESULTS_FILE, payload)
return removed
def clear_command_results(command_id: str, rule_id: str = "") -> int:
"""清空指定 command_id(及可选 rule_id)的结果。"""
command_id = str(command_id or "").strip()
rule_id = str(rule_id or "").strip()
with _LOCK:
payload = _load_results()
before = len(payload["records"])
payload["records"] = [
item
for item in payload["records"]
if item.get("command_id") != command_id
or (rule_id and str(item.get("rule_id") or "") != rule_id)
]
removed = before - len(payload["records"])
if removed:
_atomic_write_json(RESULTS_FILE, payload)
return removed
# 业务转发兼容层(保持既有测试与导入不中断)
from app.services.arena_rule_service import ( # noqa: E402
_detector_accepts,
_matches_rule,
record_arena_rule_candidates,
)
__all__ = [
"RESULTS_FILE",
"_atomic_write_json",
"_detector_accepts",
"_load_results",
"_matches_rule",
"add_record",
"add_records",
"add_records_unique",
"clear_command_results",
"delete_record",
"get_all_records",
"get_record",
"list_command_results",
"record_arena_rule_candidates",
"requests",
]
+13
View File
@@ -0,0 +1,13 @@
"""
app/services/config - 配置管理模块
职责:
- 导出配置引擎单例(保持向后兼容)
"""
from .engine import ConfigEngine, ConfigConstants, DEFAULT_WORKFLOW
# 创建单例
config_engine = ConfigEngine()
__all__ = ['config_engine', 'ConfigEngine', 'ConfigConstants', 'DEFAULT_WORKFLOW']
+48
View File
@@ -0,0 +1,48 @@
"""
app/services/config/cache.py - lightweight in-memory cache for config reads.
"""
import threading
import time
from typing import Any, Dict, Tuple
_CACHE_MISS = object()
class ConfigCache:
"""Small TTL cache used only for derived config read results."""
def __init__(self, ttl: float = 5.0):
try:
ttl_value = float(ttl)
except (TypeError, ValueError):
ttl_value = 5.0
self._ttl = max(0.0, ttl_value)
self._cache: Dict[str, Tuple[Any, float]] = {}
self._lock = threading.RLock()
def get(self, key: str, default: Any = _CACHE_MISS) -> Any:
now = time.monotonic()
with self._lock:
entry = self._cache.get(key)
if entry is None:
return default
value, created_at = entry
if self._ttl and now - created_at > self._ttl:
self._cache.pop(key, None)
return default
return value
def set(self, key: str, value: Any) -> None:
with self._lock:
self._cache[key] = (value, time.monotonic())
def invalidate(self, key: str = None) -> None:
with self._lock:
if key is None:
self._cache.clear()
else:
self._cache.pop(key, None)
+298
View File
@@ -0,0 +1,298 @@
"""
app/services/config/engine.py - 配置引擎主类
职责:
- 配置文件读写
- 站点配置管理
- 配置缓存与热重载
- 图片配置、提取器管理
"""
import json
import os
import copy
import re
import threading
import time
from app.core.config import get_logger
from typing import Dict, Optional, List, Any, Set, Union, Callable
from urllib.parse import urljoin
from app.core.parsers import ParserRegistry
from app.models.schemas import (
SiteConfig,
WorkflowStep,
SelectorDefinition,
ADVANCED_FIELDS,
PRESET_ADVANCED_FIELDS,
SITE_ADVANCED_FIELDS,
get_default_image_extraction_config,
get_default_file_paste_config,
get_default_prompt_padding_config,
get_default_site_advanced_config,
get_default_attachment_monitor_config,
get_default_send_confirmation_config,
get_modality_policy,
get_enabled_modalities,
is_modality_enabled,
normalize_modalities_config,
normalize_modality_policy,
)
from app.services.extractor_manager import extractor_manager
from app.services.parser_manager import parser_manager
from app.utils.site_rules import derive_site_card_id, get_site_rule
from app.utils.site_discovery import automatic_discovery_allowed, specific_chat_selectors, selectors_match_chat_html
from app.core.request_transport import (
get_default_request_transport_config,
normalize_request_transport_config,
)
from .cache import ConfigCache
from .site_store import SiteStore
from .managers import GlobalConfigManager, ImagePresetsManager
from .processors import HTMLCleaner, SelectorValidator, AIAnalyzer
# R2-3:模块级定义迁到 engine_common,这里重新导出以保持兼容(外部与测试都从 engine 导入这些名字)
from .engine_common import ( # noqa: E402,F401
ConfigConstants,
DEFAULT_PRESET_NAME,
DEFAULT_WORKFLOW,
PRESET_FIELDS,
_MISSING,
_SITE_STARTUP_JS_PATTERNS,
_coerce_bool,
_merge_config_patch,
get_default_network_config,
get_default_stream_config,
logger,
)
from .engine_local_overrides import ConfigLocalOverridesMixin # noqa: E402
from .engine_migrations import ConfigMigrationsMixin # noqa: E402
from .engine_presets import ConfigPresetsMixin # noqa: E402
from .engine_sites import ConfigSitesMixin # noqa: E402
from .engine_media import ConfigMediaMixin # noqa: E402
from .engine_stream import ConfigStreamMixin # noqa: E402
# ================= 配置引擎主类 =================
class ConfigEngine(
ConfigLocalOverridesMixin,
ConfigMigrationsMixin,
ConfigPresetsMixin,
ConfigSitesMixin,
ConfigMediaMixin,
ConfigStreamMixin,
):
"""配置引擎主类"""
def __init__(self):
self.config_file = ConfigConstants.CONFIG_FILE # 旧版单文件,仅用于自动迁移
self.sites_dir = ConfigConstants.SITES_DIR
self.site_store = SiteStore(self.sites_dir, self.config_file)
self.local_sites_file = ConfigConstants.SITES_LOCAL_FILE
self._io_lock = threading.RLock()
self.last_mtime = 0.0
self.last_signature: tuple = ()
self.last_local_mtime = 0.0
self.sites: Dict[str, SiteConfig] = {}
self._global_default_presets: Dict[str, str] = {}
self._local_default_presets: Dict[str, str] = {}
self._local_sites_payload: Dict[str, Any] = {}
self._cache = ConfigCache(ttl=5.0)
# 子管理器
self.global_config = GlobalConfigManager()
self.image_presets = ImagePresetsManager(ConfigConstants.IMAGE_PRESETS_FILE)
# 加载配置
self._load_config()
self._migrate_global_commands()
# 处理器
self.html_cleaner = HTMLCleaner()
self.validator = SelectorValidator(self.global_config.get_fallback_selectors())
self.ai_analyzer = AIAnalyzer(self.global_config)
# 迁移旧配置(顺序重要:先转预设格式,再补缺失字段,最后清理残留)
self._migrate_loaded_config()
self._apply_local_site_overrides()
logger.debug(f"配置引擎已初始化,已加载 {len(self.sites)} 个站点配置")
def _load_config(self):
with self._io_lock:
return self._load_config_locked()
def _load_config_locked(self):
"""初始化加载站点配置(config/sites/ 目录;发现旧版 sites.json 时自动迁移)"""
try:
data = self.site_store.load()
self.last_signature = self.site_store.signature()
if data is None:
logger.info(f"站点配置目录 {self.sites_dir} 为空,将在首次保存时创建")
self._apply_local_site_overrides()
return
self.last_mtime = time.time()
# 提取并加载 _global;缺失时重置为默认全局配置。
global_section = data.pop("_global", {})
self.global_config.load(global_section if isinstance(global_section, dict) else {})
# 过滤内部键
self.sites = {
k: v for k, v in data.items()
if not k.startswith('_')
}
self._refresh_global_default_presets_from_sites()
self._apply_local_site_overrides()
logger.debug(f"已加载站点配置: {self.sites_dir}({len(self.sites)} 个站点)")
except Exception as e:
logger.error(f"加载配置失败: {e}")
def refresh_if_changed(self):
"""检查文件是否变化,如果变化则重载"""
if not self.site_store.exists() and not os.path.exists(self.local_sites_file):
return
try:
current_signature = self.site_store.signature()
current_local_mtime = os.path.getmtime(self.local_sites_file) if os.path.exists(self.local_sites_file) else 0.0
if current_signature != self.last_signature or current_local_mtime != self.last_local_mtime:
logger.debug("⚡ 检测到站点配置变化,热重载")
self.reload_config()
except Exception as e:
logger.error(f"检查文件变化失败: {e}")
def reload_config(self):
with self._io_lock:
return self._reload_config_locked()
def _reload_config_locked(self):
"""重新加载配置(Hot Reload)"""
self._cache.invalidate()
if not self.site_store.exists():
logger.warning("重载失败:站点配置目录为空")
return
try:
previous = {"_global": self.global_config.to_dict(), **self.sites}
data = self.site_store.load(previous=previous)
if data is None:
logger.warning("重载失败:站点配置目录为空")
return
signature = self.site_store.signature()
# 提取并加载 _global;缺失时重置为默认全局配置。
global_section = data.pop("_global", {})
self.global_config.load(global_section if isinstance(global_section, dict) else {})
self.validator.fallback_selectors = self.global_config.get_fallback_selectors()
# 过滤内部键
self.sites = {
k: v for k, v in data.items()
if not k.startswith('_')
}
self.last_signature = signature
self.last_mtime = time.time()
self._refresh_global_default_presets_from_sites()
self._migrate_loaded_config()
self._apply_local_site_overrides()
logger.debug(f"✅ 配置已热重载 (Sites: {len(self.sites)})")
except json.JSONDecodeError as e:
logger.error(f"❌ 重载配置失败(JSON格式错误),保留旧配置: {e}")
except Exception as e:
logger.error(f"❌ 重载配置失败: {e}")
def save_config(self):
"""公开的保存方法(供 API 调用)"""
return self._save_config()
def _save_config(self) -> bool:
with self._io_lock:
return self._save_config_locked()
def _save_config_locked(self) -> bool:
"""保存站点配置(只重写有变化的站点文件,每个文件原子替换)"""
local_snapshot: Optional[tuple[bool, bytes, Dict[str, Any], Dict[str, str]]] = None
local_overrides_written = False
default_maps_snapshot = (
copy.deepcopy(self._global_default_presets),
copy.deepcopy(self._local_default_presets),
)
try:
self._prune_default_preset_maps()
persisted_sites = {}
for domain, site in self.sites.items():
if domain.startswith('_') or not isinstance(site, dict):
continue
site_copy = copy.deepcopy(site)
from app.utils.site_url import route_domain_matches
if route_domain_matches("arena.ai", domain):
try:
from app.services.arena_model_catalog import load_arena_model_catalog_data
cat_data = load_arena_model_catalog_data()
d_key = str(domain).strip().lower().strip(".")
domain_cat = cat_data.get(d_key, {})
except Exception:
domain_cat = None
presets_dict = site_copy.get("presets")
if isinstance(presets_dict, dict):
for p_key, p_val in presets_dict.items():
if isinstance(p_val, dict) and "model_catalog" in p_val:
if domain_cat is not None and str(p_key).strip() in domain_cat:
p_val.pop("model_catalog", None)
# 同步清理内存中的残留
mem_presets = site.get("presets")
if isinstance(mem_presets, dict):
for p_key, p_val in mem_presets.items():
if isinstance(p_val, dict) and "model_catalog" in p_val:
if domain_cat is not None and str(p_key).strip() in domain_cat:
p_val.pop("model_catalog", None)
persisted_default = self._get_persisted_default_preset(domain, site_copy)
if persisted_default:
site_copy["default_preset"] = persisted_default
else:
site_copy.pop("default_preset", None)
persisted_sites[domain] = site_copy
# 构建完整配置(包含 _global)
full_config = {
"_global": self.global_config.to_dict(),
**persisted_sites
}
local_snapshot = self._snapshot_local_site_overrides_locked()
if local_snapshot is None:
self._global_default_presets, self._local_default_presets = default_maps_snapshot
return False
if not self._save_local_site_overrides():
self._restore_local_site_overrides_locked(local_snapshot)
self._global_default_presets, self._local_default_presets = default_maps_snapshot
return False
local_overrides_written = True
written = self.site_store.save(full_config)
self.last_signature = self.site_store.signature()
self.last_mtime = time.time()
self._cache.invalidate()
logger.info(f"配置已保存: {self.sites_dir}(重写 {len(written)} 个站点文件)")
return True
except Exception as e:
logger.error(f"保存配置失败: {e}")
if local_overrides_written:
self._restore_local_site_overrides_locked(local_snapshot)
self._global_default_presets, self._local_default_presets = default_maps_snapshot
return False
def _migrate_loaded_config(self):
"""Run migrations that apply to freshly loaded site config data."""
self._migrate_to_presets()
self.migrate_site_configs()
self._migrate_site_advanced_to_presets()
self._cleanup_preset_residuals()
try:
from app.services.arena_model_catalog import migrate_and_cleanup_sites_model_catalog
migrate_and_cleanup_sites_model_catalog(self)
except Exception as exc:
logger.warning(f"Arena 模型目录自动迁移失败: {exc}")
__all__ = ['ConfigEngine', 'ConfigConstants', 'DEFAULT_WORKFLOW', 'DEFAULT_PRESET_NAME']
+128
View File
@@ -0,0 +1,128 @@
"""ConfigEngine 及其 mixin 共用的模块级定义(R2-3 拆分时从 engine.py 原样迁出,engine.py 仍重新导出)。"""
import os
import copy
import re
from app.core.config import get_logger
from typing import Dict, List, Any
from app.models.schemas import (
WorkflowStep,
get_default_attachment_monitor_config,
get_default_send_confirmation_config,
)
from app.core.request_transport import get_default_request_transport_config
logger = get_logger("CFG_ENG")
# ================= 常量配置 =================
class ConfigConstants:
"""配置引擎常量"""
_PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
# R1-2:站点配置的事实来源是 config/sites/ 目录(每站点一个文件);CONFIG_FILE 只作为旧版单文件的迁移来源
CONFIG_FILE = os.getenv("SITES_CONFIG_FILE", os.path.join(_PROJECT_ROOT, "config", "sites.json"))
SITES_DIR = os.getenv("SITES_CONFIG_DIR", os.path.join(_PROJECT_ROOT, "config", "sites"))
SITES_LOCAL_FILE = os.getenv("SITES_LOCAL_FILE", os.path.join(_PROJECT_ROOT, "config", "sites.local.json"))
COMMANDS_FILE = os.getenv("COMMANDS_CONFIG_FILE", os.path.join(_PROJECT_ROOT, "config", "commands.json"))
COMMANDS_LOCAL_FILE = os.getenv("COMMANDS_LOCAL_FILE", os.path.join(_PROJECT_ROOT, "config", "commands.local.json"))
IMAGE_PRESETS_FILE = os.path.join(_PROJECT_ROOT, "config", "image_presets.json")
MAX_HTML_CHARS = int(os.getenv("MAX_HTML_CHARS", "120000"))
TEXT_TRUNCATE_LENGTH = 80
AI_MAX_RETRIES = 3
AI_RETRY_BASE_DELAY = 1.0
AI_RETRY_MAX_DELAY = 10.0
AI_REQUEST_TIMEOUT = 120
_MISSING = object()
def _coerce_bool(value: Any, default: bool = False) -> bool:
"""Parse bool-like config values without treating every non-empty string as true."""
if isinstance(value, bool):
return value
if value is None:
return default
if isinstance(value, str):
lowered = value.strip().lower()
if lowered in {"1", "true", "yes", "y", "on"}:
return True
if lowered in {"0", "false", "no", "n", "off"}:
return False
return default
if isinstance(value, (int, float)):
return value != 0
return bool(value)
# ================= 预设常量 =================
DEFAULT_PRESET_NAME = "主预设"
# 预设内包含的配置字段(用于迁移和校验)
PRESET_FIELDS = [
"selectors", "workflow", "stream_config",
"image_extraction", "file_paste", "prompt_padding", "stealth",
"extractor_id", "extractor_verified"
]
# 默认工作流
DEFAULT_WORKFLOW: List[WorkflowStep] = [
{"action": "CLICK", "target": "new_chat_btn", "optional": True, "value": None},
{"action": "WAIT", "target": "", "optional": False, "value": "0.5"},
{"action": "FILL_INPUT", "target": "input_box", "optional": False, "value": None},
{"action": "CLICK", "target": "send_btn", "optional": True, "value": None},
{"action": "KEY_PRESS", "target": "Enter", "optional": True, "value": None},
{"action": "STREAM_WAIT", "target": "result_container", "optional": False, "value": None}
]
def get_default_stream_config() -> Dict[str, Any]:
"""获取默认流式配置"""
return {
"mode": "dom", # dom / network
"request_transport": get_default_request_transport_config(),
"hard_timeout": 300, # 全局硬超时(秒)
"send_confirmation": get_default_send_confirmation_config(),
"attachment_monitor": get_default_attachment_monitor_config(),
# 网络监听配置(可选)
"network": None
}
def get_default_network_config() -> Dict[str, Any]:
"""获取默认网络监听配置"""
return {
"listen_pattern": "", # URL 匹配模式(必填)
"stream_match_mode": "keyword", # 流目标匹配模式(keyword / regex)
"stream_match_pattern": "", # 流目标匹配表达式(为空时回退到 listen_pattern)
"parser": "", # 解析器 ID(必填)
"first_response_timeout": 300.0, # 等待首个匹配目标流量的最大时间
"silence_threshold": 3.0, # 静默超时(秒)
"response_interval": 0.5 # 轮询间隔(秒)
}
def _merge_config_patch(base: Any, patch: Any) -> Any:
"""Deep-merge object patches while letting scalars/lists replace existing values."""
if not isinstance(base, dict) or not isinstance(patch, dict):
return copy.deepcopy(patch)
merged = dict(base)
for key, value in patch.items():
existing = base.get(key)
if isinstance(value, dict) and isinstance(existing, dict):
merged[key] = _merge_config_patch(existing, value)
else:
merged[key] = copy.deepcopy(value)
return merged
_SITE_STARTUP_JS_PATTERNS = (
re.compile(r"""location\.(?:assign|replace)\(\s*['"]([^'"]+)['"]\s*\)""", re.IGNORECASE),
re.compile(r"""location\.href\s*=\s*['"]([^'"]+)['"]""", re.IGNORECASE),
)
@@ -0,0 +1,226 @@
"""ConfigEngine 的一部分(R2-3 拆分):本地站点覆盖(sites.local.json)与默认预设状态。
方法原样从 engine.py 迁出,行为不变;通过 mixin 组合回 ConfigEngine,
实例状态都在 ConfigEngine.__init__(engine.py)里初始化。
"""
import json
import os
import copy
from typing import Dict, Optional, Any
from .engine_common import logger
class ConfigLocalOverridesMixin:
"""本地站点覆盖(sites.local.json)与默认预设状态"""
def _load_local_site_overrides(self) -> Dict[str, str]:
with self._io_lock:
return self._load_local_site_overrides_locked()
def _load_local_site_overrides_locked(self) -> Dict[str, str]:
"""加载本地站点覆盖配置,并保留未识别字段。"""
if not os.path.exists(self.local_sites_file):
self.last_local_mtime = 0.0
self._local_sites_payload = {}
return {}
try:
with open(self.local_sites_file, "r", encoding="utf-8-sig") as f:
data = json.load(f)
self.last_local_mtime = os.path.getmtime(self.local_sites_file)
self._local_sites_payload = data if isinstance(data, dict) else {}
defaults = self._local_sites_payload.get("default_presets", {})
if not isinstance(defaults, dict):
return {}
return {
str(domain).strip(): str(preset).strip()
for domain, preset in defaults.items()
if str(domain).strip() and str(preset).strip()
}
except json.JSONDecodeError as e:
logger.error(f"本地站点覆盖配置格式错误: {e}")
return copy.deepcopy(self._local_default_presets or {})
except Exception as e:
logger.error(f"加载本地站点覆盖配置失败: {e}")
return copy.deepcopy(self._local_default_presets or {})
def _prune_default_preset_maps(self) -> None:
active_domains = {
domain for domain, site in self.sites.items()
if not domain.startswith('_') and isinstance(site, dict)
}
self._global_default_presets = {
domain: preset
for domain, preset in self._global_default_presets.items()
if domain in active_domains and str(preset or "").strip()
}
self._local_default_presets = {
domain: preset
for domain, preset in self._local_default_presets.items()
if domain in active_domains and str(preset or "").strip()
}
def _refresh_global_default_presets_from_sites(self) -> None:
next_defaults: Dict[str, str] = {}
for domain, site in self.sites.items():
if domain.startswith('_') or not isinstance(site, dict):
continue
resolved = self._resolve_default_preset_name(site)
if resolved:
next_defaults[domain] = resolved
self._global_default_presets = next_defaults
def _get_persisted_default_preset(self, domain: str, site: Dict[str, Any]) -> Optional[str]:
presets = site.get("presets", {})
if not isinstance(presets, dict) or not presets:
self._global_default_presets.pop(domain, None)
return None
candidate = str(self._global_default_presets.get(domain, "") or "").strip()
if candidate and candidate in presets:
return candidate
resolved = self._resolve_default_preset_name(site)
if resolved:
self._global_default_presets[domain] = resolved
return resolved
self._global_default_presets.pop(domain, None)
return None
def _sync_site_default_preset_state(self, domain: str, site: Dict[str, Any]) -> bool:
presets = site.get("presets", {})
if not isinstance(presets, dict) or not presets:
self._global_default_presets.pop(domain, None)
self._local_default_presets.pop(domain, None)
if "default_preset" in site:
del site["default_preset"]
return True
return False
persisted_default = self._get_persisted_default_preset(domain, site)
local_default = str(self._local_default_presets.get(domain, "") or "").strip()
if local_default and local_default not in presets:
self._local_default_presets.pop(domain, None)
local_default = ""
if local_default and persisted_default and local_default == persisted_default:
self._local_default_presets.pop(domain, None)
local_default = ""
effective_default = local_default or persisted_default
if not effective_default:
effective_default = self._resolve_default_preset_name(site)
if effective_default and site.get("default_preset") != effective_default:
site["default_preset"] = effective_default
return True
if not effective_default and "default_preset" in site:
del site["default_preset"]
return True
return False
def _apply_local_site_overrides(self):
"""将本地默认预设选择覆盖到当前站点配置。"""
self._local_default_presets = self._load_local_site_overrides()
applied = 0
for domain, site in self.sites.items():
if not isinstance(site, dict):
continue
self._sync_site_default_preset_state(domain, site)
local_default = self._local_default_presets.get(domain)
if local_default and site.get("default_preset") == local_default:
applied += 1
if applied > 0:
logger.debug(f"已应用 {applied} 个本地默认预设覆盖")
def _save_local_site_overrides(self) -> bool:
with self._io_lock:
return self._save_local_site_overrides_locked()
def _save_local_site_overrides_locked(self) -> bool:
"""保存本地站点覆盖配置。"""
tmp_file = self.local_sites_file + ".tmp"
self._prune_default_preset_maps()
defaults = {}
for domain, preset_name in self._local_default_presets.items():
site = self.sites.get(domain)
if not isinstance(site, dict):
continue
presets = site.get("presets", {})
preset_value = str(preset_name or "").strip()
if isinstance(presets, dict) and preset_value and preset_value in presets:
defaults[domain] = preset_value
payload = dict(self._local_sites_payload or {})
if os.path.exists(self.local_sites_file):
try:
with open(self.local_sites_file, "r", encoding="utf-8-sig") as f:
latest_payload = json.load(f)
if isinstance(latest_payload, dict):
payload.update(latest_payload)
except Exception:
pass
payload["default_presets"] = defaults
try:
os.makedirs(os.path.dirname(self.local_sites_file), exist_ok=True)
with open(tmp_file, "w", encoding="utf-8", newline="\n") as f:
json.dump(payload, f, indent=2, ensure_ascii=False)
f.flush()
os.fsync(f.fileno())
os.replace(tmp_file, self.local_sites_file)
try:
self.last_local_mtime = os.path.getmtime(self.local_sites_file)
except Exception as e:
logger.warning(f"本地站点覆盖已保存但更新时间戳失败: {e}")
self._local_sites_payload = payload
return True
except Exception as e:
logger.error(f"保存本地站点覆盖配置失败: {e}")
try:
if os.path.exists(tmp_file):
os.remove(tmp_file)
except Exception:
pass
return False
def _snapshot_local_site_overrides_locked(self) -> Optional[tuple[bool, bytes, Dict[str, Any], Dict[str, str]]]:
try:
payload_snapshot = copy.deepcopy(self._local_sites_payload or {})
defaults_snapshot = copy.deepcopy(self._local_default_presets or {})
if not os.path.exists(self.local_sites_file):
return (False, b"", payload_snapshot, defaults_snapshot)
with open(self.local_sites_file, "rb") as f:
return (True, f.read(), payload_snapshot, defaults_snapshot)
except Exception as e:
logger.error(f"读取本地站点覆盖快照失败: {e}")
return None
def _restore_local_site_overrides_locked(self, snapshot: Optional[tuple[bool, bytes, Dict[str, Any], Dict[str, str]]]) -> None:
if snapshot is None:
return
tmp_file = self.local_sites_file + ".restore.tmp"
existed, payload, payload_snapshot, defaults_snapshot = snapshot
try:
if existed:
os.makedirs(os.path.dirname(self.local_sites_file), exist_ok=True)
with open(tmp_file, "wb") as f:
f.write(payload)
f.flush()
os.fsync(f.fileno())
os.replace(tmp_file, self.local_sites_file)
elif os.path.exists(self.local_sites_file):
os.remove(self.local_sites_file)
try:
self.last_local_mtime = os.path.getmtime(self.local_sites_file) if os.path.exists(self.local_sites_file) else 0.0
except Exception as e:
logger.warning(f"本地站点覆盖已恢复但更新时间戳失败: {e}")
self._local_sites_payload = copy.deepcopy(payload_snapshot)
self._local_default_presets = copy.deepcopy(defaults_snapshot)
except Exception as e:
logger.error(f"恢复本地站点覆盖失败: {e}")
try:
if os.path.exists(tmp_file):
os.remove(tmp_file)
except Exception:
pass
+685
View File
@@ -0,0 +1,685 @@
"""ConfigEngine 的一部分(R2-3 拆分):图片提取、文件粘贴、提示词填充配置与图片预设。
方法原样从 engine.py 迁出,行为不变;通过 mixin 组合回 ConfigEngine,
实例状态都在 ConfigEngine.__init__(engine.py)里初始化。
"""
import copy
from typing import Dict, Optional, Any
from app.models.schemas import (
get_default_image_extraction_config,
get_default_file_paste_config,
get_default_prompt_padding_config,
get_default_attachment_monitor_config,
get_default_send_confirmation_config,
get_enabled_modalities,
is_modality_enabled,
normalize_modalities_config,
normalize_modality_policy,
)
from .engine_common import DEFAULT_PRESET_NAME, _coerce_bool, _merge_config_patch, logger
class ConfigMediaMixin:
"""图片提取、文件粘贴、提示词填充配置与图片预设"""
def get_site_image_config(self, domain: str, preset_name: str = None) -> Dict:
"""获取站点的图片提取配置"""
self.refresh_if_changed()
default_config = get_default_image_extraction_config()
data = self._get_site_data(domain, preset_name)
if data is None:
return copy.deepcopy(default_config)
image_config = data.get("image_extraction", {})
return self._validate_image_config(image_config)
def set_site_image_config(self, domain: str, config: Dict,
preset_name: str = None) -> bool:
"""设置站点的图片提取配置"""
self.refresh_if_changed()
data = self._get_site_data(domain, preset_name)
if data is None:
logger.warning(f"站点或预设不存在: {domain}/{preset_name}")
return False
current_config = self._validate_image_config(data.get("image_extraction", {}))
merged_config = _merge_config_patch(current_config, config if isinstance(config, dict) else {})
validated = self._validate_image_config(merged_config)
if not self._assign_preset_field_and_save(data, "image_extraction", validated):
return False
logger.info(f"站点 {domain} [{preset_name or DEFAULT_PRESET_NAME}] 多模态提取配置已更新")
return True
def _validate_image_config(self, config: Dict) -> Dict:
"""验证并规范化多模态提取配置"""
default = get_default_image_extraction_config()
result = copy.deepcopy(default)
if not config:
return result
raw_modalities = config.get("modalities")
if isinstance(raw_modalities, dict):
result["modalities"] = normalize_modalities_config(raw_modalities)
legacy_enabled = None
if "enabled" in config:
legacy_enabled = _coerce_bool(config.get("enabled"), False)
result["enabled"] = legacy_enabled
if (
legacy_enabled is not None
and not isinstance(raw_modalities, dict)
):
result["modalities"]["image"] = normalize_modality_policy("image", legacy_enabled)
elif legacy_enabled is not None and isinstance(raw_modalities, dict):
if not legacy_enabled:
for key in ("image", "audio", "video"):
result["modalities"][key] = normalize_modality_policy(key, False)
elif not get_enabled_modalities(result.get("modalities")):
result["modalities"]["image"] = normalize_modality_policy("image", True)
if "selector" in config and config["selector"]:
result["selector"] = str(config["selector"]).strip()
if not result["selector"]:
result["selector"] = "img"
if "audio_selector" in config and config["audio_selector"]:
result["audio_selector"] = str(config["audio_selector"]).strip()
if not result["audio_selector"]:
result["audio_selector"] = default["audio_selector"]
if "video_selector" in config and config["video_selector"]:
result["video_selector"] = str(config["video_selector"]).strip()
if not result["video_selector"]:
result["video_selector"] = default["video_selector"]
if "container_selector" in config:
val = config["container_selector"]
result["container_selector"] = str(val).strip() if val else None
if "final_target_strategy" in config:
val = str(config["final_target_strategy"] or "").strip().lower()
if val in ("container", "latest_reply", "latest_visual_reply"):
result["final_target_strategy"] = val
if "latest_visual_column" in config:
val = str(config["latest_visual_column"] or "").strip().lower()
if val in ("left", "right"):
result["latest_visual_column"] = val
if "allow_container_fallback" in config:
result["allow_container_fallback"] = _coerce_bool(config.get("allow_container_fallback"), False)
if "force_postprocess" in config:
result["force_postprocess"] = _coerce_bool(config.get("force_postprocess"), False)
if "arena_image_generation" in config:
result["arena_image_generation"] = _coerce_bool(
config.get("arena_image_generation"), False
)
if "direct_postprocess_modalities" in config:
raw_direct_modalities = config.get("direct_postprocess_modalities")
if isinstance(raw_direct_modalities, (list, tuple, set)):
allowed_direct_modalities = []
for item in raw_direct_modalities:
media_type = str(item or "").strip().lower()
if (
media_type in {"image", "audio", "video"}
and is_modality_enabled(result.get("modalities"), media_type)
and media_type not in allowed_direct_modalities
):
allowed_direct_modalities.append(media_type)
if allowed_direct_modalities:
result["direct_postprocess_modalities"] = allowed_direct_modalities
if "debounce_seconds" in config:
try:
val = float(config["debounce_seconds"])
result["debounce_seconds"] = max(0, min(val, 30))
except (ValueError, TypeError):
pass
if "wait_for_load" in config:
result["wait_for_load"] = _coerce_bool(config.get("wait_for_load"), True)
if "load_timeout_seconds" in config:
try:
val = float(config["load_timeout_seconds"])
result["load_timeout_seconds"] = max(1, min(val, 60))
except (ValueError, TypeError):
pass
if "download_blobs" in config:
result["download_blobs"] = _coerce_bool(config.get("download_blobs"), False)
if "max_size_mb" in config:
try:
val = int(config["max_size_mb"])
result["max_size_mb"] = max(1, min(val, 100))
except (ValueError, TypeError):
pass
if "canvas_export_mime" in config:
val = str(config["canvas_export_mime"] or "").strip().lower()
if val in {"image/jpeg", "image/webp", "image/png"}:
result["canvas_export_mime"] = val
if "canvas_export_quality" in config:
try:
val = float(config["canvas_export_quality"])
result["canvas_export_quality"] = max(0.1, min(val, 1.0))
except (ValueError, TypeError):
pass
if "src_allow_patterns" in config:
raw_patterns = config.get("src_allow_patterns")
if isinstance(raw_patterns, str):
raw_patterns = raw_patterns.replace("\r\n", "\n").replace(";", "\n").split("\n")
if isinstance(raw_patterns, (list, tuple, set)):
patterns = []
seen = set()
for item in raw_patterns:
text = str(item or "").strip()
if not text or text in seen:
continue
seen.add(text)
patterns.append(text)
result["src_allow_patterns"] = patterns
if "mode" in config:
val = str(config["mode"]).lower()
if val in ("all", "first", "last"):
result["mode"] = val
if "audio_capture_enabled" in config:
result["audio_capture_enabled"] = _coerce_bool(config.get("audio_capture_enabled"), False)
if "audio_capture_mute_playback" in config:
result["audio_capture_mute_playback"] = _coerce_bool(config.get("audio_capture_mute_playback"), False)
if "audio_capture_preload_enabled" in config:
result["audio_capture_preload_enabled"] = _coerce_bool(config.get("audio_capture_preload_enabled"), False)
if "audio_capture_reload_before_workflow" in config:
result["audio_capture_reload_before_workflow"] = _coerce_bool(config.get("audio_capture_reload_before_workflow"), False)
if "audio_capture_preserve_graph" in config:
result["audio_capture_preserve_graph"] = _coerce_bool(config.get("audio_capture_preserve_graph"), False)
if "audio_capture_terminal_settle_seconds" in config:
try:
val = float(config["audio_capture_terminal_settle_seconds"])
result["audio_capture_terminal_settle_seconds"] = max(0.0, min(val, 5.0))
except (ValueError, TypeError):
pass
if "audio_trigger_selector" in config:
result["audio_trigger_selector"] = str(config["audio_trigger_selector"] or "").strip()
if "audio_trigger_labels" in config:
raw_labels = config.get("audio_trigger_labels")
if isinstance(raw_labels, (list, tuple)):
labels = [
str(item).strip()
for item in raw_labels
if str(item).strip()
]
if labels:
result["audio_trigger_labels"] = labels
if "audio_capture_max_wait_seconds" in config:
try:
val = float(config["audio_capture_max_wait_seconds"])
result["audio_capture_max_wait_seconds"] = max(1.0, min(val, 120.0))
except (ValueError, TypeError):
pass
if "audio_capture_min_wait_seconds" in config:
try:
val = float(config["audio_capture_min_wait_seconds"])
result["audio_capture_min_wait_seconds"] = max(0.2, min(val, 30.0))
except (ValueError, TypeError):
pass
if "audio_capture_hard_max_wait_seconds" in config:
try:
val = float(config["audio_capture_hard_max_wait_seconds"])
result["audio_capture_hard_max_wait_seconds"] = max(1.0, min(val, 180.0))
except (ValueError, TypeError):
pass
if "audio_capture_estimated_chars_per_second" in config:
try:
val = float(config["audio_capture_estimated_chars_per_second"])
result["audio_capture_estimated_chars_per_second"] = max(1.0, min(val, 20.0))
except (ValueError, TypeError):
pass
if "audio_capture_wait_padding_seconds" in config:
try:
val = float(config["audio_capture_wait_padding_seconds"])
result["audio_capture_wait_padding_seconds"] = max(0.0, min(val, 10.0))
except (ValueError, TypeError):
pass
network_capture = dict(result.get("audio_network_capture") or {})
raw_network_capture = config.get("audio_network_capture")
if isinstance(raw_network_capture, dict):
if "enabled" in raw_network_capture:
network_capture["enabled"] = _coerce_bool(raw_network_capture.get("enabled"), False)
if "timeout_seconds" in raw_network_capture:
try:
val = float(raw_network_capture["timeout_seconds"])
network_capture["timeout_seconds"] = max(0.1, min(val, 15.0))
except (ValueError, TypeError):
pass
if "transport" in raw_network_capture:
val = str(raw_network_capture["transport"] or "").strip()
if val in {"page_websocket_probe"}:
network_capture["transport"] = val
if "extractor" in raw_network_capture:
val = str(raw_network_capture["extractor"] or "").strip()
if val in {"voicegenie_ogg_pages", "voicegenie_binary_stream"}:
network_capture["extractor"] = val
if "settle_seconds" in raw_network_capture:
try:
val = float(raw_network_capture["settle_seconds"])
network_capture["settle_seconds"] = max(0.05, min(val, 5.0))
except (ValueError, TypeError):
pass
if "url_patterns" in raw_network_capture:
raw_patterns = raw_network_capture.get("url_patterns")
if isinstance(raw_patterns, (list, tuple)):
patterns = [
str(item).strip()
for item in raw_patterns
if str(item).strip()
]
if patterns:
network_capture["url_patterns"] = patterns
if "max_payload_bytes" in raw_network_capture:
# 修复:此前漏处理该键,导致前端设置的单条载荷上限被静默重置为默认 10MB
try:
val = int(raw_network_capture["max_payload_bytes"])
network_capture["max_payload_bytes"] = max(65536, min(val, 104857600))
except (ValueError, TypeError):
pass
# 兼容旧平铺字段,最终统一收口到新对象
if "audio_network_capture_enabled" in config:
network_capture["enabled"] = _coerce_bool(config.get("audio_network_capture_enabled"), False)
if "audio_network_capture_timeout_seconds" in config:
try:
val = float(config["audio_network_capture_timeout_seconds"])
network_capture["timeout_seconds"] = max(0.1, min(val, 15.0))
except (ValueError, TypeError):
pass
if "audio_network_url_patterns" in config:
raw_patterns = config.get("audio_network_url_patterns")
if isinstance(raw_patterns, (list, tuple)):
patterns = [
str(item).strip()
for item in raw_patterns
if str(item).strip()
]
if patterns:
network_capture["url_patterns"] = patterns
result["audio_network_capture"] = network_capture
browser_tts_fallback = dict(result.get("audio_browser_tts_fallback") or {})
raw_browser_tts_fallback = config.get("audio_browser_tts_fallback")
if isinstance(raw_browser_tts_fallback, dict):
if "enabled" in raw_browser_tts_fallback:
browser_tts_fallback["enabled"] = _coerce_bool(raw_browser_tts_fallback.get("enabled"), False)
if "provider" in raw_browser_tts_fallback:
val = str(raw_browser_tts_fallback["provider"] or "").strip()
if val in {"doubao_samantha"}:
browser_tts_fallback["provider"] = val
if "speaker" in raw_browser_tts_fallback:
val = str(raw_browser_tts_fallback["speaker"] or "").strip()
if val:
browser_tts_fallback["speaker"] = val
if "speech_rate" in raw_browser_tts_fallback:
try:
val = int(raw_browser_tts_fallback["speech_rate"])
browser_tts_fallback["speech_rate"] = max(-100, min(val, 100))
except (ValueError, TypeError):
pass
if "pitch" in raw_browser_tts_fallback:
try:
val = int(raw_browser_tts_fallback["pitch"])
browser_tts_fallback["pitch"] = max(-100, min(val, 100))
except (ValueError, TypeError):
pass
if "format" in raw_browser_tts_fallback:
val = str(raw_browser_tts_fallback["format"] or "").strip().lower()
if val in {"aac"}:
browser_tts_fallback["format"] = val
if "timeout_seconds" in raw_browser_tts_fallback:
try:
val = float(raw_browser_tts_fallback["timeout_seconds"])
browser_tts_fallback["timeout_seconds"] = max(3.0, min(val, 120.0))
except (ValueError, TypeError):
pass
for key in (
"pc_version",
"aid",
"real_aid",
"language",
"device_platform",
"pkg_type",
"region",
"sys_region",
"use_olympus_account",
"samantha_web",
):
if key in raw_browser_tts_fallback:
val = str(raw_browser_tts_fallback[key] or "").strip()
if val:
browser_tts_fallback[key] = val
result["audio_browser_tts_fallback"] = browser_tts_fallback
if "audio_capture_poll_seconds" in config:
try:
val = float(config["audio_capture_poll_seconds"])
result["audio_capture_poll_seconds"] = max(0.05, min(val, 5.0))
except (ValueError, TypeError):
pass
if "audio_capture_silence_seconds" in config:
try:
val = float(config["audio_capture_silence_seconds"])
result["audio_capture_silence_seconds"] = max(0.2, min(val, 30.0))
except (ValueError, TypeError):
pass
if "audio_capture_activity_threshold" in config:
try:
val = float(config["audio_capture_activity_threshold"])
result["audio_capture_activity_threshold"] = max(0.0001, min(val, 0.2))
except (ValueError, TypeError):
pass
if "audio_capture_activity_silence_seconds" in config:
try:
val = float(config["audio_capture_activity_silence_seconds"])
result["audio_capture_activity_silence_seconds"] = max(0.2, min(val, 10.0))
except (ValueError, TypeError):
pass
result["enabled"] = bool(get_enabled_modalities(result.get("modalities")))
return result
def get_site_file_paste_config(self, domain: str, preset_name: str = None) -> dict:
"""获取站点的文件粘贴配置"""
self.refresh_if_changed()
default_config = get_default_file_paste_config()
data = self._get_site_data(domain, preset_name)
if data is None:
result = copy.deepcopy(default_config)
result["send_confirmation"] = get_default_send_confirmation_config()
result["attachment_monitor"] = get_default_attachment_monitor_config()
return result
return self._validate_file_paste_config(
data.get("file_paste", {}),
legacy_stream_config=data.get("stream_config"),
include_attachment_defaults=True,
)
def set_site_file_paste_config(self, domain: str, config: dict,
preset_name: str = None) -> bool:
"""设置站点的文件粘贴配置"""
self.refresh_if_changed()
data = self._get_site_data(domain, preset_name)
if data is None:
logger.warning(f"站点或预设不存在: {domain}/{preset_name}")
return False
current_config = self._validate_file_paste_config(
data.get("file_paste", {}),
legacy_stream_config=data.get("stream_config"),
)
merged_config = _merge_config_patch(current_config, config if isinstance(config, dict) else {})
validated = self._validate_file_paste_config(
merged_config,
legacy_stream_config=data.get("stream_config"),
)
if not self._assign_preset_field_and_save(data, "file_paste", validated):
return False
logger.info(f"站点 {domain} [{preset_name or DEFAULT_PRESET_NAME}] 文件粘贴配置已更新")
return True
def get_all_file_paste_configs(self) -> dict:
"""获取所有站点的文件粘贴配置(使用各站点当前默认预设)"""
self.refresh_if_changed()
result = {}
for domain in self.sites:
if domain.startswith('_'):
continue
data = self._get_site_data(domain)
if data is None:
continue
result[domain] = self._validate_file_paste_config(
data.get("file_paste", {}),
legacy_stream_config=data.get("stream_config"),
include_attachment_defaults=True,
)
return result
def get_site_prompt_padding_config(self, domain: str, preset_name: str = None) -> Dict[str, Any]:
"""获取站点的提示词开头注入配置"""
self.refresh_if_changed()
data = self._get_site_data(domain, preset_name)
if data is None:
return copy.deepcopy(get_default_prompt_padding_config())
return self._validate_prompt_padding_config(data.get("prompt_padding", {}))
def set_site_prompt_padding_config(
self,
domain: str,
config: Dict[str, Any],
preset_name: str = None,
) -> bool:
"""设置站点的提示词开头注入配置"""
self.refresh_if_changed()
data = self._get_site_data(domain, preset_name)
if data is None:
logger.warning(f"站点或预设不存在: {domain}/{preset_name}")
return False
current_config = self._validate_prompt_padding_config(data.get("prompt_padding", {}))
merged_config = _merge_config_patch(current_config, config if isinstance(config, dict) else {})
validated = self._validate_prompt_padding_config(merged_config)
if not self._assign_preset_field_and_save(data, "prompt_padding", validated):
return False
logger.info(f"站点 {domain} [{preset_name or DEFAULT_PRESET_NAME}] 提示词开头注入配置已更新")
return True
def _validate_file_paste_config(
self,
config: dict,
*,
legacy_stream_config: Optional[Dict[str, Any]] = None,
include_attachment_defaults: bool = False,
) -> dict:
"""验证并规范化文件粘贴配置"""
default = get_default_file_paste_config()
result = copy.deepcopy(default)
if not isinstance(config, dict):
config = {}
if "enabled" in config:
result["enabled"] = _coerce_bool(config.get("enabled"), False)
if "threshold" in config:
try:
val = int(config["threshold"])
result["threshold"] = max(1000, min(val, 10000000))
except (ValueError, TypeError):
pass
if "temp_file_type" in config:
val = str(config.get("temp_file_type") or "").strip().lower().lstrip(".")
if val in {"txt", "pdf", "chunk", "error"}:
result["temp_file_type"] = val
if "hint_text" in config:
val = str(config["hint_text"]).strip()
# 限制长度,避免过长的引导文本
hint_val = val[:500] if val else ""
result["hint_text"] = hint_val
# 智能向后兼容:结合老配置的策略类型,防止新字段被无关的旧数据污染
old_temp_type = config.get("temp_file_type", "txt")
if "txt_hint_text" not in config:
result["txt_hint_text"] = hint_val if old_temp_type == "txt" else "完全专注于文件内容"
if "pdf_hint_text" not in config:
result["pdf_hint_text"] = hint_val if old_temp_type == "pdf" else "完全专注于文件内容"
if "error_hint_text" not in config:
result["error_hint_text"] = hint_val if old_temp_type == "error" else "输入文本长度超过限制,已中止发送"
if "txt_hint_text" in config:
val = str(config["txt_hint_text"]).strip()
result["txt_hint_text"] = val[:500] if val else ""
if "pdf_hint_text" in config:
val = str(config["pdf_hint_text"]).strip()
result["pdf_hint_text"] = val[:500] if val else ""
if "error_hint_text" in config:
val = str(config["error_hint_text"]).strip()
result["error_hint_text"] = val[:500] if val else ""
if "reacquire_input_after_upload" in config:
result["reacquire_input_after_upload"] = _coerce_bool(config.get("reacquire_input_after_upload"), False)
if "post_upload_input_selector" in config:
val = str(config["post_upload_input_selector"] or "").strip()
result["post_upload_input_selector"] = val[:500] if val else ""
if "post_upload_settle" in config:
try:
val = float(config["post_upload_settle"])
result["post_upload_settle"] = max(0.0, min(val, 30.0))
except (ValueError, TypeError):
pass
if "upload_signal_timeout" in config:
try:
val = float(config["upload_signal_timeout"])
result["upload_signal_timeout"] = max(0.5, min(val, 120.0))
except (ValueError, TypeError):
pass
if "upload_signal_grace" in config:
try:
val = float(config["upload_signal_grace"])
result["upload_signal_grace"] = max(0.0, min(val, 120.0))
except (ValueError, TypeError):
pass
default_state_probe = copy.deepcopy(default.get("state_probe") or {})
raw_state_probe = config.get("state_probe")
state_probe = copy.deepcopy(default_state_probe)
if isinstance(raw_state_probe, dict):
if "enabled" in raw_state_probe:
state_probe["enabled"] = _coerce_bool(raw_state_probe.get("enabled"), False)
if "code" in raw_state_probe:
code = str(raw_state_probe["code"] or "").strip()
state_probe["code"] = code[:20000] if code else ""
if state_probe:
result["state_probe"] = state_probe
legacy_send_confirmation = {}
if isinstance(legacy_stream_config, dict) and isinstance(legacy_stream_config.get("send_confirmation"), dict):
legacy_send_confirmation.update(legacy_stream_config.get("send_confirmation") or {})
raw_send_confirmation = config.get("send_confirmation")
if isinstance(raw_send_confirmation, dict):
legacy_send_confirmation.update(raw_send_confirmation)
if legacy_send_confirmation or include_attachment_defaults:
result["send_confirmation"] = self._validate_send_confirmation_config(legacy_send_confirmation)
legacy_attachment_monitor = {}
if isinstance(legacy_stream_config, dict) and isinstance(legacy_stream_config.get("attachment_monitor"), dict):
legacy_attachment_monitor.update(legacy_stream_config.get("attachment_monitor") or {})
raw_attachment_monitor = config.get("attachment_monitor")
if isinstance(raw_attachment_monitor, dict):
legacy_attachment_monitor.update(raw_attachment_monitor)
if legacy_attachment_monitor or include_attachment_defaults:
result["attachment_monitor"] = self._validate_attachment_monitor_config(legacy_attachment_monitor)
from app.utils.attachments import attachment_config
if "attachments" in config or include_attachment_defaults:
result["attachments"] = attachment_config(config.get("attachments"))
return result
def _validate_prompt_padding_config(self, config: Dict[str, Any]) -> Dict[str, Any]:
"""验证并规范化提示词开头注入配置。"""
result = copy.deepcopy(get_default_prompt_padding_config())
if not isinstance(config, dict):
return result
if "enabled" in config:
result["enabled"] = _coerce_bool(config.get("enabled"), False)
if "marker_text" in config:
marker_text = str(config.get("marker_text") or "").strip()
result["marker_text"] = marker_text
if "segments_per_side" in config:
try:
segments_per_side = int(config.get("segments_per_side"))
except (TypeError, ValueError):
segments_per_side = int(result["segments_per_side"])
result["segments_per_side"] = max(0, min(segments_per_side, 64))
if "random_insert_enabled" in config:
result["random_insert_enabled"] = _coerce_bool(
config.get("random_insert_enabled"), False
)
if "random_insert_chars" in config:
result["random_insert_chars"] = str(
config.get("random_insert_chars") or ""
)
return result
def list_image_presets(self):
"""列出所有可用的图片配置预设"""
return self.image_presets.list_presets()
def get_image_preset(self, domain: str):
"""获取指定站点的预设信息"""
return self.image_presets.get_preset_for_display(domain)
def apply_image_preset(self, domain: str, preset_domain: str):
"""将预设配置应用到站点"""
preset_config = self.image_presets.get_preset(preset_domain)
if not preset_config:
raise ValueError(f"找不到预设: {preset_domain}")
return self.set_site_image_config(domain, preset_config)
def reload_presets(self):
"""重新加载图片预设"""
self.image_presets.reload()
+374
View File
@@ -0,0 +1,374 @@
"""ConfigEngine 的一部分(R2-3 拆分):历史配置迁移与命令文件读写。
方法原样从 engine.py 迁出,行为不变;通过 mixin 组合回 ConfigEngine,
实例状态都在 ConfigEngine.__init__(engine.py)里初始化。
"""
import json
import os
import copy
from typing import Dict, List, Any
from app.models.schemas import (
PRESET_ADVANCED_FIELDS,
SITE_ADVANCED_FIELDS,
get_default_image_extraction_config,
get_default_file_paste_config,
get_default_prompt_padding_config,
)
from app.utils.site_rules import get_site_rule
from .engine_common import ConfigConstants, DEFAULT_PRESET_NAME, PRESET_FIELDS, logger
class ConfigMigrationsMixin:
"""历史配置迁移与命令文件读写"""
def _load_commands_file(self) -> List[Dict[str, Any]]:
"""加载独立命令配置文件"""
commands_file = ConfigConstants.COMMANDS_FILE
if not os.path.exists(commands_file):
return []
try:
with open(commands_file, "r", encoding="utf-8") as f:
data = json.load(f)
if isinstance(data, dict):
data = data.get("commands", [])
if isinstance(data, list):
return data
logger.warning(f"命令配置文件格式无效: {commands_file}")
return []
except json.JSONDecodeError as e:
logger.error(f"命令配置文件格式错误: {e}")
return []
except Exception as e:
logger.error(f"加载命令配置失败: {e}")
return []
def _save_commands_file(self, commands: List[Dict[str, Any]]) -> bool:
"""保存独立命令配置文件"""
commands_file = ConfigConstants.COMMANDS_FILE
tmp_file = commands_file + ".tmp"
try:
os.makedirs(os.path.dirname(commands_file), exist_ok=True)
payload = {"commands": commands}
with open(tmp_file, "w", encoding="utf-8", newline="\n") as f:
json.dump(payload, f, indent=2, ensure_ascii=False)
f.flush()
os.fsync(f.fileno())
os.replace(tmp_file, commands_file)
logger.info(f"命令配置已保存: {commands_file}")
return True
except Exception as e:
logger.error(f"保存命令配置失败: {e}")
try:
if os.path.exists(tmp_file):
os.remove(tmp_file)
except Exception:
pass
return False
@staticmethod
def _merge_commands(existing: List[Dict[str, Any]], incoming: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
"""保留已有命令,仅追加不存在的命令"""
merged = list(existing or [])
seen_ids = {
str(cmd.get("id", "")).strip()
for cmd in merged
if isinstance(cmd, dict) and str(cmd.get("id", "")).strip()
}
seen_names = {
str(cmd.get("name", "")).strip()
for cmd in merged
if isinstance(cmd, dict) and str(cmd.get("name", "")).strip()
}
for cmd in incoming or []:
if not isinstance(cmd, dict):
continue
command_id = str(cmd.get("id", "")).strip()
command_name = str(cmd.get("name", "")).strip()
if command_id and command_id in seen_ids:
continue
if command_name and command_name in seen_names:
continue
merged.append(cmd)
if command_id:
seen_ids.add(command_id)
if command_name:
seen_names.add(command_name)
return merged
def _migrate_global_commands(self):
"""将旧版 _global.commands 迁移到独立文件"""
legacy_commands = self.global_config.get("commands", [])
existing_commands = self._load_commands_file()
if not isinstance(legacy_commands, list):
legacy_commands = []
merged_commands = self._merge_commands(existing_commands, legacy_commands)
if legacy_commands or not os.path.exists(ConfigConstants.COMMANDS_FILE):
self._save_commands_file(merged_commands)
if self.global_config.remove("commands"):
self._save_config()
def _migrate_to_presets(self):
"""
将旧格式(扁平)站点配置迁移为预设格式
旧格式: { "selectors": {...}, "workflow": [...], ... }
新格式: { "presets": { "主预设": { "selectors": {...}, ... } } }
"""
migrated_count = 0
for domain in list(self.sites.keys()):
if domain.startswith('_'):
continue
site_config = self.sites[domain]
# 已经是预设格式,跳过
if "presets" in site_config:
continue
# 将所有已知配置字段提取到主预设中
preset_data = {}
remaining = {}
for key, value in site_config.items():
if key in PRESET_FIELDS:
preset_data[key] = value
elif key == "advanced":
remaining[key] = value
else:
# 未知字段也放入预设(保留用户自定义数据)
preset_data[key] = value
# 构建新格式
self.sites[domain] = {
"default_preset": DEFAULT_PRESET_NAME,
"presets": {
DEFAULT_PRESET_NAME: preset_data
},
**remaining,
}
migrated_count += 1
logger.debug(f"迁移站点配置: {domain} → 预设格式")
if migrated_count > 0:
self._save_config()
logger.info(f"✅ 已迁移 {migrated_count} 个站点配置为预设格式")
def _cleanup_preset_residuals(self):
"""
清理站点配置中预设外的残留字段
当站点已有 presets 结构时,顶层不应再有 selectors/workflow/file_paste 等字段。
这些残留通常由旧版 bug 或手动编辑产生。
"""
cleaned_count = 0
default_fixed_count = 0
for domain in list(self.sites.keys()):
if domain.startswith('_'):
continue
site_config = self.sites[domain]
# 只处理已有 presets 结构的站点
if "presets" not in site_config:
continue
# 找出预设外的残留字段
residual_keys = []
for key in list(site_config.keys()):
if key == "presets":
continue
if key in PRESET_FIELDS:
residual_keys.append(key)
# 删除残留
for key in residual_keys:
del site_config[key]
cleaned_count += 1
logger.debug(f"清理残留: {domain}.{key}")
if self._normalize_site_default_preset(domain, site_config):
default_fixed_count += 1
logger.debug(f"修正默认预设: {domain} -> {site_config.get('default_preset')}")
if cleaned_count > 0 or default_fixed_count > 0:
self._save_config()
logger.info(
f"✅ 已清理 {cleaned_count} 个预设外残留字段,"
f"修正 {default_fixed_count} 个站点默认预设"
)
def _migrate_site_advanced_to_presets(self):
"""
规范化已有预设级 advanced,避免启动时改写站点级 advanced 语义。
站点根级 advanced 仍作为所有预设的共享基线;预设级 advanced 只保存
覆盖项。因此这里不再把站点级时序字段复制到所有预设,也不从站点级
删除这些字段,只清理明显放错位置的 Cookie 字段并规范化已有覆盖项。
"""
cleaned_count = 0
changed = False
for domain, site_config in self.sites.items():
if domain.startswith("_") or not isinstance(site_config, dict):
continue
presets = site_config.get("presets", {})
if not isinstance(presets, dict) or not presets:
continue
for preset_name, preset_data in presets.items():
if not isinstance(preset_data, dict):
continue
preset_advanced = preset_data.get("advanced")
if not isinstance(preset_advanced, dict):
continue
for key in list(SITE_ADVANCED_FIELDS):
if key in preset_advanced:
del preset_advanced[key]
cleaned_count += 1
changed = True
normalized_preset_advanced = self._normalize_site_advanced_config(
preset_advanced
)
for key in list(preset_advanced.keys()):
if key in PRESET_ADVANCED_FIELDS:
value = normalized_preset_advanced[key]
if preset_advanced.get(key) != value:
preset_advanced[key] = value
cleaned_count += 1
changed = True
if preset_advanced != preset_data.get("advanced"):
logger.debug(f"规范化预设高级配置: {domain}/{preset_name}")
if not preset_advanced:
del preset_data["advanced"]
cleaned_count += 1
changed = True
if changed:
self._save_config()
logger.info(
f"✅ 已规范化预设高级配置,清理 {cleaned_count} 个残留项"
)
def _guess_stealth(self, domain: str) -> bool:
"""Guess whether stealth mode should default to enabled."""
rule = get_site_rule(domain)
if "stealth_default" in rule:
enabled = bool(rule.get("stealth_default"))
if enabled:
logger.info(f"检测到默认启用低熵模式的域名: {domain}")
return enabled
return False
def migrate_site_configs(self):
"""迁移旧版站点配置,补充各预设中缺失的字段"""
migrated_count = 0
default_image_config = get_default_image_extraction_config()
default_file_paste = get_default_file_paste_config()
default_prompt_padding = get_default_prompt_padding_config()
obsolete_stream_keys = {
"silence_threshold",
"initial_wait",
"enable_wrapper_search",
"rerender_wait",
"content_shrink_tolerance",
}
obsolete_network_keys = set()
for domain, site_config in self.sites.items():
if domain.startswith("_"):
continue
presets = site_config.get("presets", {})
for preset_name, preset_data in presets.items():
if "image_extraction" not in preset_data:
preset_data["image_extraction"] = copy.deepcopy(default_image_config)
migrated_count += 1
logger.debug(f"迁移: {domain}/{preset_name} (添加 image_extraction)")
else:
normalized_image_config = self._validate_image_config(preset_data.get("image_extraction", {}))
if normalized_image_config != preset_data.get("image_extraction"):
preset_data["image_extraction"] = normalized_image_config
migrated_count += 1
logger.debug(f"迁移: {domain}/{preset_name} (规范化 image_extraction)")
if "file_paste" not in preset_data:
preset_data["file_paste"] = copy.deepcopy(default_file_paste)
migrated_count += 1
logger.debug(f"迁移: {domain}/{preset_name} (添加 file_paste)")
if "prompt_padding" not in preset_data:
preset_data["prompt_padding"] = copy.deepcopy(default_prompt_padding)
migrated_count += 1
logger.debug(f"迁移: {domain}/{preset_name} (添加 prompt_padding)")
else:
normalized_prompt_padding = self._validate_prompt_padding_config(
preset_data.get("prompt_padding", {})
)
if normalized_prompt_padding != preset_data.get("prompt_padding"):
preset_data["prompt_padding"] = normalized_prompt_padding
migrated_count += 1
logger.debug(f"迁移: {domain}/{preset_name} (规范化 prompt_padding)")
stream_config = preset_data.get("stream_config")
moved_attachment_rules = False
if not isinstance(stream_config, dict):
stream_config = {}
preset_data["stream_config"] = stream_config
normalized_file_paste = self._validate_file_paste_config(
preset_data.get("file_paste", {}),
legacy_stream_config=stream_config,
)
if normalized_file_paste != preset_data.get("file_paste"):
preset_data["file_paste"] = normalized_file_paste
migrated_count += 1
logger.debug(f"迁移: {domain}/{preset_name} (规范化 file_paste)")
for legacy_key in ("send_confirmation", "attachment_monitor"):
if legacy_key in stream_config:
del stream_config[legacy_key]
moved_attachment_rules = True
if isinstance(stream_config, dict):
removed_stream = False
for key in list(obsolete_stream_keys):
if key in stream_config:
del stream_config[key]
removed_stream = True
network_config = stream_config.get("network")
removed_network = False
if isinstance(network_config, dict):
for key in list(obsolete_network_keys):
if key in network_config:
del network_config[key]
removed_network = True
if removed_stream or removed_network or moved_attachment_rules:
migrated_count += 1
logger.debug(f"迁移: {domain}/{preset_name} (清理废弃的流式配置字段)")
if migrated_count > 0:
self._save_config()
logger.info(f"已迁移 {migrated_count} 个预设配置")
return migrated_count
+667
View File
@@ -0,0 +1,667 @@
"""ConfigEngine 的一部分(R2-3 拆分):预设的解析、创建、改名、删除及引用同步。
方法原样从 engine.py 迁出,行为不变;通过 mixin 组合回 ConfigEngine,
实例状态都在 ConfigEngine.__init__(engine.py)里初始化。
"""
import json
import copy
from typing import Dict, Optional, List, Any
from .engine_common import DEFAULT_PRESET_NAME, DEFAULT_WORKFLOW, _MISSING, logger
class ConfigPresetsMixin:
"""预设的解析、创建、改名、删除及引用同步"""
def _resolve_default_preset_name(self, site: Dict[str, Any]) -> Optional[str]:
"""解析站点有效默认预设名(不修改原对象)"""
presets = site.get("presets", {})
if not presets:
return None
configured_default = site.get("default_preset")
if isinstance(configured_default, str):
configured_default = configured_default.strip()
else:
configured_default = None
if configured_default and configured_default in presets:
return configured_default
if DEFAULT_PRESET_NAME in presets:
return DEFAULT_PRESET_NAME
return next(iter(presets))
def _normalize_site_default_preset(self, domain: str, site: Dict[str, Any]) -> bool:
"""
规范化站点 default_preset 字段
Returns:
是否发生修改
"""
return self._sync_site_default_preset_state(domain, site)
def _get_site_data(self, domain: str, preset_name: str = None) -> Optional[Dict]:
"""
获取指定站点的预设配置数据(可变引用)
查找顺序:
- preset_name 显式提供时:只匹配该预设(含别名)
- preset_name 为空时:按站点默认预设 / 主预设 / 第一个可用预设回退
Args:
domain: 站点域名
preset_name: 预设名称,None 则使用默认
Returns:
预设配置字典的引用(可直接修改),或 None
"""
if domain not in self.sites:
return None
site = self.sites[domain]
presets = site.get("presets", {})
if not presets:
return None
requested_preset = str(preset_name or "").strip()
if requested_preset:
resolved_target = self._resolve_preset_alias_key(requested_preset, presets)
if resolved_target != requested_preset:
logger.debug(f"预设别名命中: '{requested_preset}' -> '{resolved_target}'")
if resolved_target in presets:
return presets[resolved_target]
return None
resolved_default = self._resolve_default_preset_name(site)
if resolved_default and resolved_default in presets:
return presets[resolved_default]
if DEFAULT_PRESET_NAME in presets:
return presets[DEFAULT_PRESET_NAME]
first_key = next(iter(presets))
logger.warning(f"默认预设不存在,使用第一个预设: '{first_key}'")
return presets[first_key]
@staticmethod
def _resolve_preset_alias_key(target: Any, presets: Dict[str, Any]) -> str:
"""兼容历史命名:允许不带“预设_”前缀的名字命中真实预设键。"""
normalized = str(target or "").strip()
if not normalized or not isinstance(presets, dict):
return normalized
if normalized in presets:
return normalized
candidates = []
if normalized.startswith("预设_"):
stripped = normalized[len("预设_"):].strip()
if stripped:
candidates.append(stripped)
else:
candidates.append(f"预设_{normalized}")
for candidate in candidates:
if candidate in presets:
return candidate
return normalized
def _get_site_data_readonly(self, domain: str, preset_name: str = None) -> Optional[Dict]:
"""获取预设配置的深拷贝(只读用途)"""
with self._io_lock:
data = self._get_site_data(domain, preset_name)
if data is None:
return None
return copy.deepcopy(data)
def _get_preset_data_exact(self, domain: str, preset_name: str = None) -> Optional[Dict]:
"""获取预设配置引用;显式预设不存在时不回退到默认预设。"""
site = self.sites.get(domain)
if not isinstance(site, dict):
return None
presets = site.get("presets", {})
if not isinstance(presets, dict) or not presets:
return None
if preset_name is None or not str(preset_name).strip():
target = self._resolve_default_preset_name(site)
else:
target = str(preset_name).strip()
resolved_target = self._resolve_preset_alias_key(target, presets)
if resolved_target and resolved_target in presets:
return presets[resolved_target]
return None
def list_presets(self, domain: str) -> List[str]:
"""获取指定站点的所有预设名称"""
self.refresh_if_changed()
if domain not in self.sites:
return []
site = self.sites[domain]
presets = site.get("presets", {})
return list(presets.keys())
def get_default_preset(self, domain: str) -> Optional[str]:
"""获取指定站点的默认预设名称(已解析回退)"""
self.refresh_if_changed()
cache_key = f"default_preset:{domain}"
cached = self._cache.get(cache_key, _MISSING)
if cached is not _MISSING:
return cached
site = self.sites.get(domain)
if not site:
self._cache.set(cache_key, None)
return None
result = self._resolve_default_preset_name(site)
self._cache.set(cache_key, result)
return result
def _assign_preset_field_and_save(
self,
preset_data: Dict[str, Any],
field_name: str,
value: Any,
) -> bool:
previous_value = preset_data.get(field_name, _MISSING)
if previous_value is not _MISSING:
previous_value = copy.deepcopy(previous_value)
preset_data[field_name] = value
if self._save_config():
return True
if previous_value is _MISSING:
preset_data.pop(field_name, None)
else:
preset_data[field_name] = previous_value
return False
def set_default_preset(self, domain: str, preset_name: str) -> bool:
"""设置指定站点的默认预设"""
self.refresh_if_changed()
site = self.sites.get(domain)
if not site:
return False
presets = site.get("presets", {})
if preset_name not in presets:
logger.warning(f"默认预设设置失败,预设不存在: {domain}/{preset_name}")
return False
previous_default = site.get("default_preset", _MISSING)
if previous_default is not _MISSING:
previous_default = copy.deepcopy(previous_default)
previous_default_maps = (
copy.deepcopy(getattr(self, "_global_default_presets", {})),
copy.deepcopy(getattr(self, "_local_default_presets", {})),
)
persisted_default = self._get_persisted_default_preset(domain, site)
if persisted_default == preset_name:
self._local_default_presets.pop(domain, None)
else:
self._local_default_presets[domain] = preset_name
site["default_preset"] = preset_name
if not self._save_local_site_overrides():
if previous_default is _MISSING:
site.pop("default_preset", None)
else:
site["default_preset"] = previous_default
self._global_default_presets, self._local_default_presets = previous_default_maps
return False
logger.info(f"✅ 站点 {domain} 默认预设已设置为: '{preset_name}'(仅本地覆盖)")
return True
def create_preset(self, domain: str, new_name: str,
source_name: str = None) -> bool:
"""
创建新预设(克隆自现有预设)
Args:
domain: 站点域名
new_name: 新预设名称
source_name: 要克隆的源预设名称,None 则克隆主预设
Returns:
是否成功
"""
self.refresh_if_changed()
if domain not in self.sites:
logger.warning(f"站点不存在: {domain}")
return False
site = self.sites[domain]
presets = site.get("presets", {})
if new_name in presets:
logger.warning(f"预设已存在: {new_name}")
return False
# 获取源预设
if source_name is not None and str(source_name or "").strip():
requested_source = str(source_name or "").strip()
source = self._resolve_preset_alias_key(requested_source, presets)
source_data = presets.get(source)
if not source_data:
logger.warning(f"源预设不存在: {requested_source}")
return False
else:
source = DEFAULT_PRESET_NAME
source_data = presets.get(source)
if not source_data:
# 未显式指定源时才尝试第一个可用预设
if presets:
source = next(iter(presets))
source_data = presets[source]
else:
logger.warning(f"没有可克隆的源预设")
return False
previous_default = site.get("default_preset", _MISSING)
if previous_default is not _MISSING:
previous_default = copy.deepcopy(previous_default)
previous_presets = copy.deepcopy(presets)
previous_global_default = copy.deepcopy(self._global_default_presets)
previous_local_default = copy.deepcopy(self._local_default_presets)
# 深拷贝创建新预设
presets[new_name] = copy.deepcopy(source_data)
self._normalize_site_default_preset(domain, site)
if not self._save_config():
site["presets"] = previous_presets
if previous_default is _MISSING:
site.pop("default_preset", None)
else:
site["default_preset"] = previous_default
self._global_default_presets = previous_global_default
self._local_default_presets = previous_local_default
return False
logger.info(f"✅ 站点 {domain} 创建预设: '{new_name}' (克隆自 '{source}')")
return True
@staticmethod
def _build_preset_rename_map(old_name: Any, new_name: Any) -> Dict[str, str]:
old_raw = str(old_name or "").strip()
new_raw = str(new_name or "").strip()
if not old_raw or not new_raw:
return {}
def _strip_prefix(value: str) -> str:
if value.startswith("预设_"):
return value[len("预设_"):].strip()
return value
def _add_prefix(value: str) -> str:
return value if value.startswith("预设_") else f"预设_{value}"
mapping: Dict[str, str] = {}
old_plain = _strip_prefix(old_raw)
new_plain = _strip_prefix(new_raw)
old_prefixed = _add_prefix(old_raw)
new_prefixed = _add_prefix(new_raw)
for src, dst in (
(old_raw, new_raw),
(old_plain, new_plain),
(old_prefixed, new_prefixed),
):
src_text = str(src or "").strip()
dst_text = str(dst or "").strip()
if src_text and dst_text:
mapping[src_text] = dst_text
return mapping
@staticmethod
def _command_targets_domain(command: Dict[str, Any], domain: str) -> bool:
normalized_domain = str(domain or "").strip().lower()
if not normalized_domain or not isinstance(command, dict):
return False
trigger = command.get("trigger", {}) or {}
command_domain = str(trigger.get("domain", "") or "").strip().lower()
if not command_domain:
return False
return (
command_domain == normalized_domain
or command_domain.endswith(f".{normalized_domain}")
or normalized_domain.endswith(f".{command_domain}")
)
def _rename_preset_references_in_commands(
self,
domain: str,
rename_map: Dict[str, str],
) -> int:
if not rename_map:
return 0
commands = self._load_commands_file()
if not commands:
return 0
updated = 0
for command in commands:
if not self._command_targets_domain(command, domain):
continue
actions = command.get("actions", [])
if not isinstance(actions, list):
continue
for action in actions:
if not isinstance(action, dict):
continue
current_preset = str(action.get("preset_name", "") or "").strip()
replacement = rename_map.get(current_preset)
if not replacement or replacement == current_preset:
continue
action["preset_name"] = replacement
updated += 1
if updated > 0:
self._save_commands_file(commands)
return updated
def _rename_preset_references_in_active_tabs(
self,
domain: str,
rename_map: Dict[str, str],
) -> int:
if not rename_map:
return 0
try:
from app.core import browser as browser_module
instance = getattr(browser_module, "_browser_instance", None)
if instance is None:
return 0
pool = getattr(instance, "_tab_pool", None)
if pool is None or not hasattr(pool, "get_sessions_snapshot"):
return 0
updated = 0
for session in pool.get_sessions_snapshot():
session_domain = str(getattr(session, "current_domain", "") or "").strip().lower()
if session_domain != str(domain or "").strip().lower():
continue
current_preset = str(getattr(session, "preset_name", "") or "").strip()
replacement = rename_map.get(current_preset)
if not replacement or replacement == current_preset:
continue
if hasattr(session, "set_preset"):
session.set_preset(replacement, source=getattr(session, "preset_override_source", None))
else:
session.preset_name = replacement
updated += 1
return updated
except Exception as e:
logger.debug(f"同步活动标签页预设引用失败(忽略): {e}")
return 0
def _clear_preset_references_in_active_tabs(self, domain: str, preset_name: str) -> int:
"""预设被删除后,把在用标签页里指向它的 preset_name 置为 None(回落站点默认预设)。"""
target = str(preset_name or "").strip()
if not target:
return 0
# 兼容历史命名:带/不带“预设_”前缀都视为命中同一个预设
targets = {target}
if target.startswith("预设_"):
stripped = target[len("预设_"):].strip()
if stripped:
targets.add(stripped)
else:
targets.add(f"预设_{target}")
try:
from app.core import browser as browser_module
instance = getattr(browser_module, "_browser_instance", None)
if instance is None:
return 0
pool = getattr(instance, "_tab_pool", None)
if pool is None or not hasattr(pool, "get_sessions_snapshot"):
return 0
cleared = 0
for session in pool.get_sessions_snapshot():
session_domain = str(getattr(session, "current_domain", "") or "").strip().lower()
if session_domain != str(domain or "").strip().lower():
continue
current_preset = str(getattr(session, "preset_name", "") or "").strip()
if current_preset not in targets:
continue
if hasattr(session, "set_preset"):
session.set_preset(None, source=None)
else:
session.preset_name = None
cleared += 1
return cleared
except Exception as e:
logger.debug(f"清理活动标签页预设引用失败(忽略): {e}")
return 0
def _sync_preset_overrides_file(self, domain: str, old_preset: str, new_preset: Optional[str] = None) -> int:
"""当预设重命名或删除时,同步清理/更新 preset_overrides.local.json"""
target = str(old_preset or "").strip()
if not target:
return 0
from pathlib import Path
config_path = Path("config/preset_overrides.local.json")
if not config_path.exists():
return 0
from app.utils.site_url import extract_remote_site_domain
import json
try:
with open(config_path, "r", encoding="utf-8-sig") as f:
payload = json.load(f)
if not isinstance(payload, dict):
return 0
urls = payload.get("urls", {})
if not isinstance(urls, dict):
return 0
domain_norm = str(domain or "").strip().lower()
changed = 0
keys_to_delete = []
for url_key, p_name in list(urls.items()):
url_domain = str(extract_remote_site_domain(url_key) or "").strip().lower()
if domain_norm and url_domain != domain_norm:
continue
if p_name == target or p_name == f"预设_{target}" or (target.startswith("预设_") and p_name == target[3:]):
if new_preset:
urls[url_key] = new_preset
changed += 1
else:
keys_to_delete.append(url_key)
changed += 1
for k in keys_to_delete:
urls.pop(k, None)
if changed > 0:
from app.core.config import atomic_write_json
atomic_write_json(config_path, payload)
try:
from app.core import browser as browser_module
instance = getattr(browser_module, "_browser_instance", None)
pool = getattr(instance, "_tab_pool", None)
if pool is not None and hasattr(pool, "apply_runtime_config"):
pool.apply_runtime_config(preset_overrides=payload)
except Exception:
pass
return changed
except Exception as e:
logger.debug(f"同步预设本地配置失败: {e}")
return 0
def delete_preset(self, domain: str, preset_name: str) -> bool:
"""
删除预设(不允许删除最后一个预设)
Args:
domain: 站点域名
preset_name: 要删除的预设名称
Returns:
是否成功
"""
self.refresh_if_changed()
if domain not in self.sites:
return False
site = self.sites[domain]
presets = site.get("presets", {})
resolved_preset_name = self._resolve_preset_alias_key(preset_name, presets)
if resolved_preset_name not in presets:
logger.warning(f"预设不存在: {preset_name}")
return False
if len(presets) <= 1:
logger.warning(f"不能删除最后一个预设")
return False
previous_default = site.get("default_preset", _MISSING)
if previous_default is not _MISSING:
previous_default = copy.deepcopy(previous_default)
removed_preset = copy.deepcopy(presets[resolved_preset_name])
previous_global_default = copy.deepcopy(self._global_default_presets)
previous_local_default = copy.deepcopy(self._local_default_presets)
del presets[resolved_preset_name]
self._normalize_site_default_preset(domain, site)
if not self._save_config():
presets[resolved_preset_name] = removed_preset
if previous_default is _MISSING:
site.pop("default_preset", None)
else:
site["default_preset"] = previous_default
self._global_default_presets = previous_global_default
self._local_default_presets = previous_local_default
return False
# 预设已删除并落盘:清理在用标签页对它的残留引用,否则 session.preset_name 会指向不存在的预设
cleared_tab_refs = 0
try:
cleared_tab_refs = self._clear_preset_references_in_active_tabs(domain, resolved_preset_name)
except Exception as e:
logger.warning(f"清理活动标签页预设引用失败(忽略): {e}")
# 同步清理持久化 preset_overrides.local.json
try:
self._sync_preset_overrides_file(domain, resolved_preset_name, new_preset=None)
except Exception as e:
logger.warning(f"清理本地预设记忆文件失败(忽略): {e}")
logger.info(
f"✅ 站点 {domain} 删除预设: '{resolved_preset_name}' "
f"(活动标签页同步 {cleared_tab_refs} 处)"
)
return True
def rename_preset(self, domain: str, old_name: str, new_name: str) -> bool:
"""重命名预设"""
self.refresh_if_changed()
if domain not in self.sites:
return False
site = self.sites[domain]
presets = site.get("presets", {})
resolved_old_name = self._resolve_preset_alias_key(old_name, presets)
if resolved_old_name not in presets:
return False
if new_name in presets:
logger.warning(f"预设名已存在: {new_name}")
return False
rename_map = self._build_preset_rename_map(resolved_old_name, new_name)
default_preset = site.get("default_preset")
previous_default = site.get("default_preset", _MISSING)
if previous_default is not _MISSING:
previous_default = copy.deepcopy(previous_default)
previous_presets = copy.deepcopy(presets)
previous_global_default = copy.deepcopy(self._global_default_presets)
previous_local_default = copy.deepcopy(self._local_default_presets)
local_default = str(self._local_default_presets.get(domain, "") or "").strip()
# 保持顺序:创建有序副本
new_presets = {}
for key, value in presets.items():
if key == resolved_old_name:
new_presets[new_name] = value
else:
new_presets[key] = value
site["presets"] = new_presets
if default_preset == resolved_old_name:
site["default_preset"] = new_name
if local_default:
local_replacement = rename_map.get(local_default)
if local_replacement:
self._local_default_presets[domain] = local_replacement
self._normalize_site_default_preset(domain, site)
if not self._save_config():
site["presets"] = previous_presets
if previous_default is _MISSING:
site.pop("default_preset", None)
else:
site["default_preset"] = previous_default
self._global_default_presets = previous_global_default
self._local_default_presets = previous_local_default
return False
updated_command_refs = self._rename_preset_references_in_commands(domain, rename_map)
updated_tab_refs = self._rename_preset_references_in_active_tabs(domain, rename_map)
try:
self._sync_preset_overrides_file(domain, resolved_old_name, new_preset=new_name)
except Exception as e:
logger.warning(f"同步本地预设记忆文件失败(忽略): {e}")
logger.info(
f"站点 {domain} 重命名预设: '{resolved_old_name}' → '{new_name}' "
f"(命令引用同步 {updated_command_refs} 处, 活动标签页同步 {updated_tab_refs} 处)"
)
return True
def get_preset_selectors(self, domain: str, preset_name: str = None) -> Dict:
"""获取指定预设的选择器配置"""
data = self._get_site_data_readonly(domain, preset_name)
return data.get("selectors", {}) if data else {}
def set_preset_selectors(self, domain: str, selectors: Dict,
preset_name: str = None) -> bool:
"""设置指定预设的选择器配置"""
self.refresh_if_changed()
data = self._get_site_data(domain, preset_name)
if data is None:
return False
if not self._assign_preset_field_and_save(data, "selectors", selectors):
return False
logger.info(f"站点 {domain} [{preset_name or DEFAULT_PRESET_NAME}] 选择器已更新")
return True
def get_preset_workflow(self, domain: str, preset_name: str = None) -> List:
"""获取指定预设的工作流配置"""
data = self._get_site_data_readonly(domain, preset_name)
return data.get("workflow", DEFAULT_WORKFLOW) if data else DEFAULT_WORKFLOW
def set_preset_workflow(self, domain: str, workflow: List,
preset_name: str = None) -> bool:
"""设置指定预设的工作流配置"""
from app.core.workflow.flow_runtime import has_control_flow, validate_workflow
if has_control_flow(workflow):
validate_workflow(workflow)
self.refresh_if_changed()
data = self._get_site_data(domain, preset_name)
if data is None:
return False
if not self._assign_preset_field_and_save(data, "workflow", workflow):
return False
logger.info(f"站点 {domain} [{preset_name or DEFAULT_PRESET_NAME}] 工作流已更新")
return True
+535
View File
@@ -0,0 +1,535 @@
"""ConfigEngine 的一部分(R2-3 拆分):站点目录、起始页推断、站点高级配置与选择器定义。
方法原样从 engine.py 迁出,行为不变;通过 mixin 组合回 ConfigEngine,
实例状态都在 ConfigEngine.__init__(engine.py)里初始化。
"""
import copy
from typing import Dict, Optional, List, Any, Set, Union, Callable
from urllib.parse import urljoin
from app.models.schemas import (
SiteConfig,
SelectorDefinition,
ADVANCED_FIELDS,
PRESET_ADVANCED_FIELDS,
SITE_ADVANCED_FIELDS,
get_default_image_extraction_config,
get_default_file_paste_config,
get_default_prompt_padding_config,
get_default_site_advanced_config,
)
from app.utils.site_rules import derive_site_card_id, get_site_rule
from app.utils.site_discovery import (
automatic_discovery_allowed,
specific_chat_selectors,
selectors_match_chat_html,
)
from .engine_common import (
DEFAULT_PRESET_NAME,
DEFAULT_WORKFLOW,
_MISSING,
_SITE_STARTUP_JS_PATTERNS,
_coerce_bool,
get_default_stream_config,
logger,
)
class ConfigSitesMixin:
"""站点目录、起始页推断、站点高级配置与选择器定义"""
def _extract_startup_url_from_script(self, script: str) -> str:
text = str(script or "").strip()
if not text:
return ""
for pattern in _SITE_STARTUP_JS_PATTERNS:
match = pattern.search(text)
if match:
return str(match.group(1) or "").strip()
return ""
def _normalize_startup_url(self, domain: str, raw_url: str) -> str:
normalized_domain = str(domain or "").strip().lower().strip(".")
text = str(raw_url or "").strip()
if not normalized_domain:
return text
if not text:
return f"https://{normalized_domain}"
if text.startswith(("http://", "https://")):
return text
return urljoin(f"https://{normalized_domain}", text)
def _infer_site_startup_url(self, domain: str, site: Dict[str, Any]) -> str:
preset_name = self._resolve_default_preset_name(site)
preset_data = self._get_site_data_readonly(domain, preset_name)
workflow = preset_data.get("workflow", []) if isinstance(preset_data, dict) else []
if isinstance(workflow, list):
for step in workflow:
if str((step or {}).get("action") or "").strip().upper() != "JS_EXEC":
continue
startup_url = self._extract_startup_url_from_script((step or {}).get("value"))
if startup_url:
return self._normalize_startup_url(domain, startup_url)
return self._normalize_startup_url(domain, "")
def get_site_catalog_entry(self, domain: str, fallback_order: int = 0) -> Optional[Dict[str, Any]]:
self.refresh_if_changed()
site = self.sites.get(domain)
if not isinstance(site, dict) or str(domain or "").startswith("_"):
return None
rule = get_site_rule(domain)
startup_url = rule.get("startup_url") or self._infer_site_startup_url(domain, site)
display_name = str(rule.get("display_name") or domain).strip() or str(domain)
card_id = str(rule.get("card_id") or derive_site_card_id(domain)).strip() or derive_site_card_id(domain)
guide_priority = rule.get("guide_priority")
if not isinstance(guide_priority, int):
guide_priority = int(fallback_order)
entry = {
"domain": str(domain),
"display_name": display_name,
"url": self._normalize_startup_url(domain, startup_url),
"card_id": card_id,
"guide_priority": guide_priority,
}
if isinstance(rule.get("route_aliases"), list):
entry["route_aliases"] = [str(item) for item in rule.get("route_aliases", [])]
if "stealth_default" in rule:
entry["stealth_default"] = bool(rule.get("stealth_default"))
return entry
def list_site_catalog(self) -> List[Dict[str, Any]]:
self.refresh_if_changed()
ordered_domains = [
domain for domain, site in self.sites.items()
if not domain.startswith("_") and isinstance(site, dict)
]
entries: List[Dict[str, Any]] = []
for index, domain in enumerate(ordered_domains):
entry = self.get_site_catalog_entry(domain, fallback_order=index)
if entry:
entries.append(entry)
entries.sort(key=lambda item: (int(item.get("guide_priority", 0)), str(item.get("domain") or "")))
return entries
def _normalize_site_advanced_config(
self,
raw_config: Optional[Dict[str, Any]] = None,
*,
base_config: Optional[Dict[str, Any]] = None,
) -> Dict[str, Any]:
"""合并并规范化高级配置。"""
normalized = {
**get_default_site_advanced_config(),
}
if isinstance(base_config, dict):
normalized.update(copy.deepcopy(base_config))
if isinstance(raw_config, dict):
normalized.update(copy.deepcopy(raw_config))
normalized["independent_cookies"] = _coerce_bool(normalized.get("independent_cookies"), False)
normalized["independent_cookies_auto_takeover"] = _coerce_bool(
normalized.get("independent_cookies_auto_takeover"), False
)
normalized["input_box_stability_wait_enabled"] = _coerce_bool(
normalized.get("input_box_stability_wait_enabled"), False
)
normalized["input_box_stability_wait_after_new_chat_only"] = _coerce_bool(
normalized.get("input_box_stability_wait_after_new_chat_only"), True
)
try:
timeout_value = float(normalized.get("input_box_stability_wait_timeout", 1.5))
except Exception:
timeout_value = 1.5
normalized["input_box_stability_wait_timeout"] = max(0.1, min(timeout_value, 10.0))
normalized["url_transition_wait_on_new_chat"] = _coerce_bool(
normalized.get("url_transition_wait_on_new_chat"), False
)
raw_patterns = normalized.get("url_transition_wait_patterns") or []
if isinstance(raw_patterns, str):
raw_patterns = raw_patterns.replace("\n", ",").replace(";", ",").split(",")
if not isinstance(raw_patterns, (list, tuple, set)):
raw_patterns = []
normalized["url_transition_wait_patterns"] = [
str(pattern or "").strip()
for pattern in raw_patterns
if str(pattern or "").strip()
]
normalized["send_confirmation_check_enabled"] = _coerce_bool(
normalized.get("send_confirmation_check_enabled"), False
)
try:
send_timeout_value = float(normalized.get("send_confirmation_check_timeout", 1.5))
except Exception:
send_timeout_value = 1.5
normalized["send_confirmation_check_timeout"] = max(0.1, min(send_timeout_value, 10.0))
normalized["skip_new_chat_on_retry"] = _coerce_bool(
normalized.get("skip_new_chat_on_retry"), False
)
return normalized
def get_site_advanced_config(self, domain: str, preset_name: str = None) -> Dict[str, Any]:
"""获取站点高级配置;传入 preset_name 时叠加对应预设的覆盖项。"""
self.refresh_if_changed()
site = self.sites.get(domain)
if not site:
return self._normalize_site_advanced_config()
raw_config = site.get("advanced") if isinstance(site, dict) else None
normalized = self._normalize_site_advanced_config(
raw_config if isinstance(raw_config, dict) else {}
)
requested_preset = str(preset_name or "").strip()
if requested_preset:
preset_data = self._get_preset_data_exact(domain, requested_preset)
if preset_data is None:
logger.warning(f"站点高级配置预设不存在: {domain}/{requested_preset}")
return normalized
preset_advanced = (
preset_data.get("advanced")
if isinstance(preset_data, dict)
else None
)
if isinstance(preset_advanced, dict):
preset_advanced = {
key: value
for key, value in preset_advanced.items()
if key in PRESET_ADVANCED_FIELDS
}
normalized = self._normalize_site_advanced_config(
preset_advanced,
base_config=normalized,
)
return normalized
def set_site_advanced_config(self, domain: str, config: Dict[str, Any]) -> bool:
"""设置站点级高级配置。"""
with self._io_lock:
self.refresh_if_changed()
site = self.sites.get(domain)
if not site:
logger.warning(f"站点不存在: {domain}")
return False
previous_advanced = site.get("advanced", _MISSING)
if previous_advanced is not _MISSING:
previous_advanced = copy.deepcopy(previous_advanced)
existing = site.get("advanced") if isinstance(site.get("advanced"), dict) else {}
stored = copy.deepcopy(existing)
normalized = self._normalize_site_advanced_config(config or {}, base_config=existing)
provided_keys = {
key
for key in ((config or {}).keys() if isinstance(config, dict) else set())
if key in ADVANCED_FIELDS
}
if not provided_keys and previous_advanced is _MISSING:
return True
for key in ADVANCED_FIELDS:
if key in provided_keys:
stored[key] = normalized[key]
site["advanced"] = stored
if self._save_config_locked():
return True
if previous_advanced is _MISSING:
site.pop("advanced", None)
else:
site["advanced"] = previous_advanced
return False
def set_preset_advanced_config(
self,
domain: str,
config: Dict[str, Any],
preset_name: str = None,
*,
prune_inherited_fields: Optional[Set[str]] = None,
) -> bool:
"""设置当前预设的高级配置;拒绝混入站点级字段。"""
with self._io_lock:
self.refresh_if_changed()
site = self.sites.get(domain)
if not isinstance(site, dict):
logger.warning(f"站点不存在: {domain}")
return False
requested_preset = str(preset_name or "").strip()
if not requested_preset:
logger.warning(f"预设级高级配置缺少 preset_name: {domain}")
return False
data = self._get_preset_data_exact(domain, requested_preset)
if data is None:
logger.warning(f"站点或预设不存在: {domain}/{requested_preset}")
return False
raw_config = config if isinstance(config, dict) else {}
invalid_site_keys = {
key
for key in raw_config.keys()
if key in SITE_ADVANCED_FIELDS
}
if invalid_site_keys:
joined = ", ".join(sorted(invalid_site_keys))
logger.warning(f"预设级高级配置不能包含站点级字段: {domain}/{requested_preset} ({joined})")
return False
previous_advanced = data.get("advanced", _MISSING)
if previous_advanced is not _MISSING:
previous_advanced = copy.deepcopy(previous_advanced)
stored = copy.deepcopy(data.get("advanced") or {})
if not isinstance(stored, dict):
stored = {}
for key in list(SITE_ADVANCED_FIELDS):
stored.pop(key, None)
site_advanced = site.get("advanced") if isinstance(site.get("advanced"), dict) else {}
inherited = self._normalize_site_advanced_config(site_advanced)
normalized = self._normalize_site_advanced_config(
raw_config,
base_config=stored,
)
provided_keys = {
key
for key in raw_config.keys()
if key in PRESET_ADVANCED_FIELDS
}
if not provided_keys and previous_advanced is _MISSING:
return True
prune_inherited_keys = set(prune_inherited_fields or set())
for key in PRESET_ADVANCED_FIELDS:
if key in provided_keys:
if (
key in prune_inherited_keys
and key not in stored
and normalized[key] == inherited.get(key)
):
stored.pop(key, None)
else:
stored[key] = normalized[key]
if not stored and previous_advanced is _MISSING:
return True
data["advanced"] = stored
if self._save_config_locked():
return True
if previous_advanced is _MISSING:
data.pop("advanced", None)
else:
data["advanced"] = previous_advanced
return False
def list_sites(self) -> Dict[str, Any]:
"""获取所有站点配置(过滤内部键)"""
self.refresh_if_changed()
with self._io_lock:
return copy.deepcopy({
domain: config
for domain, config in self.sites.items()
if not domain.startswith('_')
})
def get_site_config(self, domain: str, html_content: Optional[Union[str, Callable[[], str]]] = None,
preset_name: str = None) -> Optional[SiteConfig]:
"""
获取站点配置(缓存 + AI 分析)
Args:
domain: 站点域名
html_content: 页面 HTML 或用于延迟获取 HTML 的可调用对象(用于 AI 分析未知站点)
preset_name: 预设名称,None 则使用默认预设
"""
self.refresh_if_changed()
if domain in self.sites:
cache_key = f"site_config:{domain}:{preset_name or ''}"
cached = self._cache.get(cache_key, _MISSING)
if cached is not _MISSING:
return copy.deepcopy(cached)
site = self.sites.get(domain, {})
config = self._get_site_data(domain, preset_name)
if config is None:
logger.warning(f"站点 {domain} 无可用预设")
return None
used_preset = preset_name or self._resolve_default_preset_name(site) or DEFAULT_PRESET_NAME
previous_config = None
# 补充缺失字段
changed = False
if "workflow" not in config:
if previous_config is None:
previous_config = copy.deepcopy(config)
config["workflow"] = DEFAULT_WORKFLOW
changed = True
if "image_extraction" not in config:
if previous_config is None:
previous_config = copy.deepcopy(config)
config["image_extraction"] = get_default_image_extraction_config()
changed = True
if "file_paste" not in config:
if previous_config is None:
previous_config = copy.deepcopy(config)
config["file_paste"] = get_default_file_paste_config()
changed = True
else:
normalized_file_paste = self._validate_file_paste_config(
config.get("file_paste", {}),
legacy_stream_config=config.get("stream_config"),
)
if normalized_file_paste != config.get("file_paste"):
if previous_config is None:
previous_config = copy.deepcopy(config)
config["file_paste"] = normalized_file_paste
changed = True
if "prompt_padding" not in config:
if previous_config is None:
previous_config = copy.deepcopy(config)
config["prompt_padding"] = get_default_prompt_padding_config()
changed = True
else:
normalized_prompt_padding = self._validate_prompt_padding_config(
config.get("prompt_padding", {})
)
if normalized_prompt_padding != config.get("prompt_padding"):
if previous_config is None:
previous_config = copy.deepcopy(config)
config["prompt_padding"] = normalized_prompt_padding
changed = True
if changed:
completed_config = copy.deepcopy(config)
if not self._save_config():
config.clear()
config.update(previous_config)
logger.warning(
f"站点 {domain} [{used_preset}] 配置自动补全保存失败,"
"已仅对当前请求返回补全配置"
)
return completed_config
logger.debug(f"使用缓存配置: {domain} [预设: {used_preset}]")
result = copy.deepcopy(config)
self._cache.set(cache_key, result)
return copy.deepcopy(result)
if not automatic_discovery_allowed(domain):
logger.info(f"跳过非聊天站点的自动收录: {domain}(已有手动配置不受影响)")
return None
resolved_html = ""
if callable(html_content):
try:
resolved_html = str(html_content() or "")
except Exception as e:
logger.warning(f"获取页面 HTML 失败: {e}")
resolved_html = ""
elif html_content is not None:
resolved_html = str(html_content or "")
if resolved_html.strip():
logger.info(f"🔍 未知域名 {domain},启动 AI 识别...")
clean_html = self.html_cleaner.clean(resolved_html)
selectors = self.ai_analyzer.analyze(clean_html)
if specific_chat_selectors(selectors):
selectors = self.validator.validate(selectors)
if selectors_match_chat_html(selectors, clean_html):
new_preset: SiteConfig = {
"selectors": selectors,
"workflow": DEFAULT_WORKFLOW,
"stealth": self._guess_stealth(domain),
"stream_config": copy.deepcopy(get_default_stream_config()),
"image_extraction": get_default_image_extraction_config(),
"file_paste": get_default_file_paste_config(),
"prompt_padding": get_default_prompt_padding_config(),
}
self.sites[domain] = {
"default_preset": DEFAULT_PRESET_NAME,
"presets": {
DEFAULT_PRESET_NAME: new_preset
}
}
if self._save_config():
logger.info(f"✅ 配置已生成并保存: {domain}")
else:
logger.warning(f"⚠️ 配置已生成但保存失败,仅保留在当前运行内存: {domain}")
return copy.deepcopy(new_preset)
logger.warning(f"⚠️ AI 分析失败,使用通用回退配置: {domain}")
else:
logger.warning(f"⚠️ 未知域名 {domain} 且未提供网页 HTML,使用通用回退配置")
fallback_selectors = self.global_config.get_fallback_selectors()
fallback_preset: SiteConfig = {
"selectors": fallback_selectors,
"workflow": DEFAULT_WORKFLOW,
"stealth": False,
"stream_config": copy.deepcopy(get_default_stream_config()),
"image_extraction": get_default_image_extraction_config(),
"file_paste": get_default_file_paste_config(),
"prompt_padding": get_default_prompt_padding_config(),
}
# A generic fallback is only for this caller. It is not proof that this
# domain hosts an AI chat product, and must never enter the site catalog.
logger.info(f"通用回退仅用于当前请求,未自动添加站点: {domain}")
return copy.deepcopy(fallback_preset)
def delete_site_config(self, domain: str) -> bool:
"""删除指定站点配置"""
self.refresh_if_changed()
if domain in self.sites:
removed_site = copy.deepcopy(self.sites[domain])
previous_global_default = copy.deepcopy(self._global_default_presets)
previous_local_default = copy.deepcopy(self._local_default_presets)
del self.sites[domain]
if not self._save_config():
self.sites[domain] = removed_site
self._global_default_presets = previous_global_default
self._local_default_presets = previous_local_default
return False
logger.info(f"已删除配置: {domain}")
return True
return False
def get_selector_definitions(self) -> List[SelectorDefinition]:
"""获取元素定义列表"""
return self.global_config.get_selector_definitions()
def set_selector_definitions(self, definitions: List[SelectorDefinition]) -> bool:
"""设置元素定义列表并保存"""
previous_definitions = self.global_config.get_selector_definitions()
previous_fallback_selectors = copy.deepcopy(self.validator.fallback_selectors)
self.global_config.set_selector_definitions(definitions)
# 更新验证器的回退选择器
self.validator.fallback_selectors = self.global_config.get_fallback_selectors()
# 保存配置
if not self._save_config():
self.global_config.set_selector_definitions(previous_definitions)
self.validator.fallback_selectors = previous_fallback_selectors
return False
logger.info(f"元素定义已更新: {len(definitions)} 个")
return True
+485
View File
@@ -0,0 +1,485 @@
"""ConfigEngine 的一部分(R2-3 拆分):提取器、流式/网络监听配置及其校验、解析器列表。
方法原样从 engine.py 迁出,行为不变;通过 mixin 组合回 ConfigEngine,
实例状态都在 ConfigEngine.__init__(engine.py)里初始化。
"""
import copy
from typing import Dict, Optional, List, Any
from app.core.parsers import ParserRegistry
from app.models.schemas import get_default_attachment_monitor_config, get_default_send_confirmation_config
from app.services.extractor_manager import extractor_manager
from app.services.parser_manager import parser_manager
from app.core.request_transport import normalize_request_transport_config
from .engine_common import (
DEFAULT_PRESET_NAME,
_MISSING,
_coerce_bool,
_merge_config_patch,
get_default_network_config,
get_default_stream_config,
logger,
)
class ConfigStreamMixin:
"""提取器、流式/网络监听配置及其校验、解析器列表"""
def get_site_extractor(self, domain: str, preset_name: str = None):
"""获取站点的提取器实例"""
self.refresh_if_changed()
data = self._get_site_data(domain, preset_name)
if data is not None:
return extractor_manager.get_extractor_for_site(data)
return extractor_manager.get_extractor()
def set_site_extractor(self, domain: str, extractor_id: str,
preset_name: str = None) -> bool:
"""为站点设置提取器"""
self.refresh_if_changed()
data = self._get_site_data(domain, preset_name)
if data is None:
logger.warning(f"站点或预设不存在: {domain}/{preset_name}")
return False
from app.core.extractors import ExtractorRegistry
if not ExtractorRegistry.exists(extractor_id):
logger.error(f"提取器不存在: {extractor_id}")
return False
previous_extractor_id = data.get("extractor_id", _MISSING)
if previous_extractor_id is not _MISSING:
previous_extractor_id = copy.deepcopy(previous_extractor_id)
previous_verified = data.get("extractor_verified", _MISSING)
if previous_verified is not _MISSING:
previous_verified = copy.deepcopy(previous_verified)
data["extractor_id"] = extractor_id
data["extractor_verified"] = False
if not self._save_config():
if previous_extractor_id is _MISSING:
data.pop("extractor_id", None)
else:
data["extractor_id"] = previous_extractor_id
if previous_verified is _MISSING:
data.pop("extractor_verified", None)
else:
data["extractor_verified"] = previous_verified
return False
logger.info(f"站点 {domain} [{preset_name or DEFAULT_PRESET_NAME}] 已绑定提取器: {extractor_id}")
return True
def set_site_extractor_verified(self, domain: str, verified: bool = True,
preset_name: str = None) -> bool:
"""设置站点提取器验证状态"""
self.refresh_if_changed()
data = self._get_site_data(domain, preset_name)
if data is None:
return False
if not self._assign_preset_field_and_save(data, "extractor_verified", verified):
return False
return True
def get_site_stream_config(self, domain: str, preset_name: str = None) -> Dict[str, Any]:
"""
获取站点的流式配置
Args:
domain: 站点域名
preset_name: 预设名称
Returns:
完整的流式配置(包含默认值)
"""
self.refresh_if_changed()
cache_key = f"stream_config:{domain}:{preset_name or ''}"
cached = self._cache.get(cache_key, _MISSING)
if cached is not _MISSING:
return copy.deepcopy(cached)
default_config = get_default_stream_config()
data = self._get_site_data(domain, preset_name)
if data is None:
result = copy.deepcopy(default_config)
self._cache.set(cache_key, result)
return copy.deepcopy(result)
stream_config = data.get("stream_config", {})
# 合并默认值
result = copy.deepcopy(default_config)
# 更新顶层字段
for key in ["mode", "hard_timeout"]:
if key in stream_config:
result[key] = stream_config[key]
if isinstance(stream_config.get("request_transport"), dict):
result["request_transport"] = normalize_request_transport_config(
stream_config.get("request_transport")
)
# 处理 send_confirmation 配置
if isinstance(stream_config.get("send_confirmation"), dict):
result["send_confirmation"].update(stream_config["send_confirmation"])
# 处理 attachment_monitor 配置
if isinstance(stream_config.get("attachment_monitor"), dict):
result["attachment_monitor"].update(stream_config["attachment_monitor"])
file_paste_config = data.get("file_paste", {})
if isinstance(file_paste_config, dict):
if isinstance(file_paste_config.get("send_confirmation"), dict):
result["send_confirmation"].update(file_paste_config["send_confirmation"])
if isinstance(file_paste_config.get("attachment_monitor"), dict):
result["attachment_monitor"].update(file_paste_config["attachment_monitor"])
# 处理 network 配置
if stream_config.get("network"):
network_default = get_default_network_config()
network_config = stream_config["network"]
result["network"] = network_default.copy()
for key in [
"listen_pattern",
"stream_match_mode",
"stream_match_pattern",
"parser",
"hard_timeout",
"first_response_timeout",
"first_content_timeout",
"initial_target_body_wait",
"silence_threshold",
"response_interval",
]:
if key in network_config:
result["network"][key] = network_config[key]
self._cache.set(cache_key, copy.deepcopy(result))
return result
def set_site_stream_config(self, domain: str, config: Dict[str, Any],
preset_name: str = None) -> bool:
"""
设置站点的流式配置
Args:
domain: 站点域名
config: 流式配置(部分或完整)
preset_name: 预设名称
Returns:
是否成功
"""
self.refresh_if_changed()
data = self._get_site_data(domain, preset_name)
if data is None:
logger.warning(f"站点或预设不存在: {domain}/{preset_name}")
return False
previous_file_paste = data.get("file_paste", _MISSING)
if previous_file_paste is not _MISSING:
previous_file_paste = copy.deepcopy(previous_file_paste)
previous_stream = data.get("stream_config", _MISSING)
if previous_stream is not _MISSING:
previous_stream = copy.deepcopy(previous_stream)
legacy_file_paste_updates: Dict[str, Any] = {}
if isinstance(config.get("send_confirmation"), dict):
legacy_file_paste_updates["send_confirmation"] = config.get("send_confirmation") or {}
if isinstance(config.get("attachment_monitor"), dict):
legacy_file_paste_updates["attachment_monitor"] = config.get("attachment_monitor") or {}
# 验证并规范化配置
current_config = self.get_site_stream_config(domain, preset_name)
merged_config = _merge_config_patch(current_config, config if isinstance(config, dict) else {})
validated = self._validate_stream_config(merged_config)
validated.pop("send_confirmation", None)
validated.pop("attachment_monitor", None)
if legacy_file_paste_updates:
existing_file_paste = self._validate_file_paste_config(
data.get("file_paste", {}),
legacy_stream_config=data.get("stream_config"),
)
merged_file_paste = _merge_config_patch(existing_file_paste, legacy_file_paste_updates)
data["file_paste"] = self._validate_file_paste_config(
merged_file_paste,
legacy_stream_config=data.get("stream_config"),
)
data["stream_config"] = validated
if not self._save_config():
if previous_stream is _MISSING:
data.pop("stream_config", None)
else:
data["stream_config"] = previous_stream
if previous_file_paste is _MISSING:
data.pop("file_paste", None)
else:
data["file_paste"] = previous_file_paste
return False
logger.info(f"站点 {domain} [{preset_name or DEFAULT_PRESET_NAME}] 流式配置已更新 (mode={validated.get('mode')})")
return True
def _validate_stream_config(self, config: Dict[str, Any]) -> Dict[str, Any]:
"""
验证并规范化流式配置
Args:
config: 原始配置
Returns:
规范化后的配置
"""
result = get_default_stream_config()
if not config:
return result
mode_explicitly_set = False
# 验证 mode
if "mode" in config:
mode = str(config["mode"]).lower()
if mode in ("dom", "network"):
result["mode"] = mode
mode_explicitly_set = True
# 验证数值字段
for key in ["hard_timeout"]:
if key in config:
try:
val = float(config[key])
if key == "hard_timeout":
result[key] = max(10, min(val, 600))
except (ValueError, TypeError):
pass
# 验证 send_confirmation 配置
if isinstance(config.get("send_confirmation"), dict):
result["send_confirmation"] = self._validate_send_confirmation_config(
config["send_confirmation"]
)
if isinstance(config.get("request_transport"), dict):
result["request_transport"] = normalize_request_transport_config(
config["request_transport"]
)
# 验证 attachment_monitor 配置
if isinstance(config.get("attachment_monitor"), dict):
result["attachment_monitor"] = self._validate_attachment_monitor_config(
config["attachment_monitor"]
)
# 验证 network 配置
if config.get("network"):
network_config = self._validate_network_config(config["network"])
if network_config:
result["network"] = network_config
# 仅在调用方没有显式指定 mode 时,才根据 network 配置自动切到 network。
# 这样切回 DOM 时可以保留 parser/listen_pattern,而不会被后端强制改回 network。
if (
not mode_explicitly_set
and network_config.get("parser")
and network_config.get("listen_pattern")
):
result["mode"] = "network"
return result
def _validate_send_confirmation_config(self, config: Dict[str, Any]) -> Dict[str, Any]:
"""验证发送成功判定配置。"""
result = get_default_send_confirmation_config()
if not isinstance(config, dict):
return result
numeric_ranges = {
"post_click_observe_window": (0.0, 15.0),
"pre_retry_probe_window": (0.0, 5.0),
"retry_observe_window": (0.0, 15.0),
"attachment_observe_window": (0.0, 30.0),
"retry_interval": (0.0, 30.0),
"retry_cooldown_window": (0.0, 30.0),
}
for key, (minimum, maximum) in numeric_ranges.items():
if key not in config:
continue
try:
value = float(config[key])
except (TypeError, ValueError):
continue
result[key] = max(minimum, min(value, maximum))
if "max_retry_count" in config:
try:
value = int(config["max_retry_count"])
except (TypeError, ValueError):
value = None
if value is not None:
result["max_retry_count"] = max(0, min(value, 10))
retry_action = str(config.get("retry_action") or "").strip().lower()
if retry_action in {"click_send_btn", "key_press"}:
result["retry_action"] = retry_action
if "retry_key_combo" in config:
retry_key_combo = str(config.get("retry_key_combo") or "").strip()
if retry_key_combo:
result["retry_key_combo"] = retry_key_combo[:64]
bool_fields = [
"retry_on_unconfirmed_send",
"accept_attachment_change",
"accept_attachment_disappear",
"accept_probe_confirmation",
"retry_block_on_stop_button",
"retry_block_if_generating",
"trust_network_activity",
"trust_generating_indicator",
"trust_send_disabled_with_input_shrink",
]
for key in bool_fields:
if key not in config:
continue
result[key] = _coerce_bool(config.get(key), bool(result.get(key, False)))
sensitivity = str(config.get("attachment_sensitivity") or "").strip().lower()
if sensitivity in {"low", "medium", "high"}:
result["attachment_sensitivity"] = sensitivity
return result
def _validate_attachment_monitor_config(self, config: Dict[str, Any]) -> Dict[str, Any]:
"""验证附件上传/发送前判定规则。"""
result = get_default_attachment_monitor_config()
if not isinstance(config, dict):
return result
list_fields = [
"root_selectors",
"attachment_selectors",
"pending_selectors",
"status_selectors",
"content_exclusion_selectors",
"busy_text_markers",
"ignored_busy_text_markers",
"send_button_disabled_markers",
]
for key in list_fields:
raw_value = config.get(key)
if raw_value is None:
continue
if not isinstance(raw_value, list):
continue
cleaned = []
for item in raw_value:
value = str(item or "").strip()
if value and value not in cleaned:
cleaned.append(value)
result[key] = cleaned
numeric_ranges = {
"idle_timeout": (0.5, 60.0),
"hard_max_wait": (1.0, 300.0),
}
for key, (minimum, maximum) in numeric_ranges.items():
if key not in config:
continue
try:
value = float(config[key])
except (TypeError, ValueError):
continue
result[key] = max(minimum, min(value, maximum))
bool_fields = [
"require_attachment_present",
"require_upload_signal_before_ready",
"continue_once_on_unconfirmed_send",
]
for key in bool_fields:
if key not in config:
continue
result[key] = _coerce_bool(config.get(key), bool(result.get(key, False)))
return result
def _validate_network_config(self, config: Dict[str, Any]) -> Optional[Dict[str, Any]]:
"""
验证网络监听配置
Args:
config: 原始网络配置
Returns:
规范化后的配置,无效则返回 None
"""
if not config:
return None
result = get_default_network_config()
# listen_pattern(必填)
if "listen_pattern" in config:
pattern = str(config["listen_pattern"]).strip()
if pattern:
result["listen_pattern"] = pattern
if "stream_match_mode" in config:
match_mode = str(config["stream_match_mode"]).strip().lower()
if match_mode in {"keyword", "regex"}:
result["stream_match_mode"] = match_mode
if "stream_match_pattern" in config:
result["stream_match_pattern"] = str(config["stream_match_pattern"]).strip()
# parser(必填,需验证存在性)
if "parser" in config:
parser_id = str(config["parser"]).strip()
if parser_id:
# 验证解析器是否存在
if ParserRegistry.exists(parser_id):
result["parser"] = parser_id
else:
logger.warning(f"解析器不存在: {parser_id}")
# 仍然保存,允许后续添加解析器
result["parser"] = parser_id
# 验证数值字段
for key in ["first_response_timeout", "silence_threshold", "response_interval"]:
if key in config:
try:
val = float(config[key])
if key == "first_response_timeout":
result[key] = max(0.5, min(val, 300))
elif key == "silence_threshold":
result[key] = max(0.5, min(val, 30))
elif key == "response_interval":
result[key] = max(0.1, min(val, 5))
except (ValueError, TypeError):
pass
# 检查是否有有效配置
if not result["listen_pattern"] or not result["parser"]:
return None
return result
def list_available_parsers(self) -> List[Dict[str, str]]:
"""
列出所有可用的响应解析器
Returns:
解析器信息列表
"""
return parser_manager.list_parsers()
def get_extractor_manager(self):
"""获取提取器管理器实例"""
return extractor_manager
def get_parser_manager(self):
"""获取解析器管理器实例"""
return parser_manager
+440
View File
@@ -0,0 +1,440 @@
"""
app/services/config/managers.py - 配置子管理器
职责:
- 全局配置管理(元素定义)
- 图片预设管理
"""
import json
import os
import copy
from app.core.config import get_logger
from typing import Dict, List, Any, Optional
from app.models.schemas import (
SelectorDefinition,
get_default_selector_definitions,
get_default_image_extraction_config
)
logger = get_logger("CFG_MGR")
# ================= 全局配置管理器 =================
class GlobalConfigManager:
"""
全局配置管理器
管理 _global 节点中的配置,包括:
- selector_definitions: 元素定义列表
- 以及任意扩展字段(通过 get/set 访问)
"""
def __init__(self):
self._selector_definitions: List[SelectorDefinition] = get_default_selector_definitions()
self._extra: Dict[str, Any] = {} # 通用扩展存储
def load(self, global_section: Dict[str, Any]):
"""从 _global 节点加载配置"""
self._extra = {}
self._selector_definitions = get_default_selector_definitions()
if not global_section:
return
if "selector_definitions" in global_section:
defs = global_section["selector_definitions"]
if isinstance(defs, list):
self._selector_definitions = self._merge_selector_definitions(defs)
logger.debug(f"已加载 {len(defs)} 个元素定义")
# 加载所有非 selector_definitions 的字段到通用存储
for key, value in global_section.items():
if key != "selector_definitions":
self._extra[key] = copy.deepcopy(value)
if "commands" in self._extra:
cmd_count = len(self._extra["commands"]) if isinstance(self._extra["commands"], list) else 0
logger.debug(f"已加载 {cmd_count} 个自动化命令")
def get_selector_definitions(self) -> List[SelectorDefinition]:
"""获取元素定义列表"""
return copy.deepcopy(self._selector_definitions)
def set_selector_definitions(self, definitions: List[SelectorDefinition]):
"""设置元素定义列表"""
self._selector_definitions = self._merge_selector_definitions(definitions)
@staticmethod
def _normalize_selector_definition(raw_item: Any) -> Optional[SelectorDefinition]:
if not isinstance(raw_item, dict):
return None
key = raw_item.get("key")
if not isinstance(key, str):
return None
key = key.strip()
if not key:
return None
description = raw_item.get("description", "")
if not isinstance(description, str):
description = str(description) if description is not None else ""
return {
"key": key,
"description": description,
"enabled": bool(raw_item.get("enabled", True)),
"required": bool(raw_item.get("required", False)),
}
def _merge_selector_definitions(self, definitions: List[SelectorDefinition]) -> List[SelectorDefinition]:
"""合并内置默认字段,确保升级后新增的选择器能自动出现。"""
merged = []
existing_keys = set()
for item in definitions or []:
normalized = self._normalize_selector_definition(item)
if not normalized:
continue
key = normalized["key"]
if key in existing_keys:
continue
merged.append(normalized)
existing_keys.add(key)
for default_def in get_default_selector_definitions():
normalized_default = self._normalize_selector_definition(default_def)
if not normalized_default:
continue
key = normalized_default.get("key")
if key not in existing_keys:
merged.append(normalized_default)
existing_keys.add(key)
return merged
def get_enabled_definitions(self) -> List[SelectorDefinition]:
"""获取启用的元素定义"""
return [
copy.deepcopy(d)
for d in self._selector_definitions
if isinstance(d, dict) and d.get("enabled", True)
]
def get_fallback_selectors(self) -> Dict[str, Optional[str]]:
"""
生成回退选择器字典
基于元素定义生成,用于 SelectorValidator
"""
fallback_map = {
"input_box": "textarea",
"send_btn": 'button[type="submit"]',
"result_container": "div",
"new_chat_btn": None,
"retry_send_btn": None,
"message_wrapper": None,
"generating_indicator": None,
"upload_btn": None,
"file_input": 'input[type="file"]',
"drop_zone": None,
}
result = {}
for d in self._selector_definitions:
if not isinstance(d, dict):
continue
key = d.get("key")
if not isinstance(key, str) or not key:
continue
result[key] = fallback_map.get(key, None)
return result
def build_prompt_selector_list(self) -> str:
"""
生成 AI 提示词中的元素查找列表
只包含 enabled=True 的元素
"""
lines = []
for d in self._selector_definitions:
if not isinstance(d, dict):
continue
if not d.get("enabled", True):
continue
key = str(d.get("key") or "").strip()
if not key:
continue
desc = str(d.get("description") or "")
required = d.get("required", False)
if required:
tag = "[REQUIRED]"
else:
tag = "[OPTIONAL, return null if not found]"
lines.append(f"- `{key}`: {desc} {tag}")
return "\n".join(lines)
def build_prompt_json_keys(self) -> str:
"""
生成 AI 提示词中的 JSON 输出格式说明
"""
lines = []
for d in self._selector_definitions:
if not isinstance(d, dict):
continue
if not d.get("enabled", True):
continue
key = str(d.get("key") or "").strip()
if not key:
continue
desc = str(d.get("description") or "")
lines.append(f'- `{key}`: {desc}')
return "\n".join(lines)
def get(self, key: str, default: Any = None) -> Any:
"""获取通用配置值"""
return copy.deepcopy(self._extra.get(key, default))
def set(self, key: str, value: Any):
"""设置通用配置值"""
self._extra[key] = copy.deepcopy(value)
def remove(self, key: str) -> bool:
"""删除通用配置值"""
if key not in self._extra:
return False
del self._extra[key]
return True
def to_dict(self) -> Dict[str, Any]:
"""导出为字典(用于保存)"""
result = {
"selector_definitions": self._selector_definitions
}
# 合并通用存储中的所有字段
result.update(self._extra)
return copy.deepcopy(result)
# ================= 图片预设管理器 =================
class ImagePresetsManager:
"""
图片提取预设管理器
职责:
- 加载预设配置文件
- 提供站点预设查询
- 应用预设到站点配置
"""
def __init__(self, presets_file: str):
self.presets_file = presets_file
self.presets: Dict[str, Any] = {}
self._load_presets()
@staticmethod
def _default_presets_payload() -> Dict[str, Any]:
return {
"_default": {
"name": "Default",
"description": "Built-in fallback image extraction preset",
"image_extraction": get_default_image_extraction_config(),
}
}
@staticmethod
def _get_preset_image_config(preset_data: Any) -> Optional[Dict[str, Any]]:
if not isinstance(preset_data, dict):
return None
config = preset_data.get("image_extraction")
return copy.deepcopy(config) if isinstance(config, dict) else None
@staticmethod
def _get_preset_text(preset_data: Dict[str, Any], key: str, default: str = "") -> str:
value = preset_data.get(key, default)
if value is None:
return default
if isinstance(value, (str, int, float, bool)):
return str(value)
return default
def _find_matching_preset(self, domain: str) -> Optional[tuple[str, Dict[str, Any]]]:
domain = str(domain or "").strip().lower().rstrip(".")
if not domain:
return None
normalized_presets = []
for preset_domain, preset_data in self.presets.items():
if preset_domain.startswith("_") or not isinstance(preset_data, dict):
continue
normalized_preset_domain = str(preset_domain).strip().lower().rstrip(".")
if not normalized_preset_domain:
continue
normalized_presets.append((preset_domain, preset_data, normalized_preset_domain))
for preset_domain, preset_data, normalized_preset_domain in normalized_presets:
if domain == normalized_preset_domain:
return preset_domain, preset_data
suffix_matches = [
item
for item in normalized_presets
if domain.endswith(f".{item[2]}")
]
if suffix_matches:
preset_domain, preset_data, _ = max(
suffix_matches,
key=lambda item: len(item[2]),
)
logger.debug(f"使用模糊匹配预设: {domain} -> {preset_domain}")
return preset_domain, preset_data
return None
def _use_default_presets_or_keep_existing(self, keep_existing: bool) -> None:
if keep_existing and self.presets:
logger.warning("图片预设加载失败,保留当前内存中的预设")
return
self.presets = self._default_presets_payload()
def _load_presets(self, keep_existing_on_error: bool = False):
"""加载预设配置文件"""
if not os.path.exists(self.presets_file):
logger.warning(f"图片预设文件不存在: {self.presets_file}")
self._use_default_presets_or_keep_existing(keep_existing_on_error)
return
try:
with open(self.presets_file, 'r', encoding='utf-8') as f:
data = json.load(f)
if not isinstance(data, dict):
logger.error("预设文件格式错误: 顶层必须是对象")
self._use_default_presets_or_keep_existing(keep_existing_on_error)
return
# 移除元数据
loaded_presets = {
str(k): v
for k, v in data.items()
if k != "_meta" and isinstance(v, dict)
}
skipped_count = len([k for k, v in data.items() if k != "_meta" and not isinstance(v, dict)])
if skipped_count:
logger.warning(f"已跳过 {skipped_count} 个格式无效的图片预设")
if not loaded_presets:
self._use_default_presets_or_keep_existing(keep_existing_on_error)
logger.debug(f"已加载 {len(self.presets)} 个图片预设")
return
self.presets = loaded_presets
logger.debug(f"已加载 {len(self.presets)} 个图片预设")
except json.JSONDecodeError as e:
logger.error(f"预设文件格式错误: {e}")
self._use_default_presets_or_keep_existing(keep_existing_on_error)
except Exception as e:
logger.error(f"加载预设失败: {e}")
self._use_default_presets_or_keep_existing(keep_existing_on_error)
def get_preset(self, domain: str) -> Optional[Dict]:
"""
获取站点的预设配置
Args:
domain: 站点域名
Returns:
预设配置字典,不存在返回 None
"""
match = self._find_matching_preset(domain)
if match is None:
return None
return self._get_preset_image_config(match[1])
def list_presets(self) -> List[Dict[str, Any]]:
"""
列出所有可用预设
Returns:
预设列表,每项包含 domain、name、description、enabled
"""
result = []
for domain, data in self.presets.items():
if domain == "_meta":
continue
if not isinstance(data, dict):
continue
config = self._get_preset_image_config(data) or {}
item = {
"domain": domain,
"name": self._get_preset_text(data, "name", domain),
"description": self._get_preset_text(data, "description", ""),
"enabled": config.get("enabled", False),
"notes": self._get_preset_text(data, "notes", ""),
"config": config
}
result.append(item)
result.sort(key=lambda x: x["domain"])
return result
def get_preset_for_display(self, domain: str) -> Dict[str, Any]:
"""
获取用于显示的预设信息
Args:
domain: 站点域名
Returns:
包含 available、preset_domain、config 的字典
"""
match = self._find_matching_preset(domain)
if match is None:
preset_config = None
else:
preset_config = self._get_preset_image_config(match[1])
if preset_config:
matched_domain, matched_data = match
return {
"available": True,
"preset_domain": matched_domain,
"name": self._get_preset_text(
matched_data,
"name",
"",
),
"config": preset_config
}
return {
"available": False,
"preset_domain": None,
"name": None,
"config": None
}
def reload(self):
"""重新加载预设文件"""
self._load_presets(keep_existing_on_error=True)
__all__ = ['GlobalConfigManager', 'ImagePresetsManager']
+661
View File
@@ -0,0 +1,661 @@
"""
app/services/config/processors.py - 配置处理器
职责:
- HTML 清理
- 选择器验证
- AI 页面分析
"""
import json
import importlib.util
import os
import re
import time
from app.core.config import get_logger
from app.utils.site_discovery import admitted_selectors
from typing import Dict, Optional
from urllib import request, error
import bs4
from bs4 import BeautifulSoup
from app.core.config import AppConfig
from .managers import GlobalConfigManager
logger = get_logger("CFG_PRC")
_BS4_PARSER = "lxml" if importlib.util.find_spec("lxml") is not None else "html.parser"
# ================= 常量 =================
# HTML 清理配置
MAX_HTML_CHARS = int(os.getenv("MAX_HTML_CHARS", "120000"))
TEXT_TRUNCATE_LENGTH = 80
# AI 请求配置
AI_MAX_RETRIES = 3
AI_RETRY_BASE_DELAY = 1.0
AI_RETRY_MAX_DELAY = 10.0
AI_REQUEST_TIMEOUT = 120
# 无效选择器语法模式
INVALID_SYNTAX_PATTERNS = [
(r'~\s*\.\.', '~ .. 无效语法'),
(r'\.\.\s*$', '结尾 .. 无效'),
(r'>>\s', '>> 无效语法'),
(r':has\(', ':has() 兼容性差'),
(r'\s~\s*$', '结尾 ~ 无效'),
]
# ================= HTML 清理器 =================
class HTMLCleaner:
"""HTML 清理器"""
TAGS_TO_REMOVE = [
'script', 'style', 'meta', 'link', 'noscript',
'img', 'video', 'audio', 'iframe', 'canvas',
'path', 'rect', 'circle', 'polygon', 'defs', 'clipPath',
'header', 'footer', 'nav', 'aside',
]
ALLOWED_ATTRS = [
'id', 'class', 'name', 'placeholder', 'aria-label', 'role',
'data-testid', 'type', 'disabled', 'value', 'title', 'tabindex',
'contenteditable', 'href'
]
INTERACTIVE_TAGS = ['input', 'textarea', 'button', 'form', 'a']
CORE_AREA_SELECTORS = [
'[role="main"]',
'main',
'#app',
'#root',
'.chat',
'.conversation',
'.message',
]
def __init__(self, max_chars: int = None, text_truncate: int = None):
self.max_chars = max_chars or MAX_HTML_CHARS
self.text_truncate = text_truncate or TEXT_TRUNCATE_LENGTH
def clean(self, html: str) -> str:
"""深度清理 HTML"""
logger.debug("开始 HTML 清理...")
original_length = len(html)
soup = BeautifulSoup(html, _BS4_PARSER)
try:
interactive_elements = self._extract_interactive_elements(soup)
for tag in soup(self.TAGS_TO_REMOVE):
tag.decompose()
for element in soup(text=lambda t: isinstance(t, bs4.element.Comment)):
element.extract()
for tag in soup.find_all(True):
if tag.string and len(tag.string) > self.text_truncate:
tag.string = tag.string[:self.text_truncate] + "..."
attrs = dict(tag.attrs)
for attr in attrs:
if attr not in self.ALLOWED_ATTRS:
del tag.attrs[attr]
if 'class' in tag.attrs and isinstance(tag.attrs['class'], list):
tag.attrs['class'] = " ".join(tag.attrs['class'])
clean_html = str(soup.body) if soup.body else str(soup)
clean_html = re.sub(r'\s+', ' ', clean_html).strip()
if len(clean_html) > self.max_chars:
logger.warning(f"HTML 过长 ({len(clean_html)}),执行智能截断...")
clean_html = self._smart_truncate(clean_html, interactive_elements)
final_length = len(clean_html)
reduction = 100 - (final_length / original_length * 100) if original_length > 0 else 0
logger.info(f"HTML 清理完成: {original_length} → {final_length} 字符 (减少 {reduction:.1f}%)")
return clean_html
finally:
soup.decompose()
def _extract_interactive_elements(self, soup: BeautifulSoup) -> str:
elements = []
for tag_name in self.INTERACTIVE_TAGS:
for element in soup.find_all(tag_name):
context = self._get_element_with_context(element, levels=2)
if context:
elements.append(context)
unique_elements = list(dict.fromkeys(elements))
return "\n".join(unique_elements)
def _get_element_with_context(self, element, levels: int = 2) -> str:
try:
current = element
for _ in range(levels):
if current.parent and current.parent.name not in ['body', 'html', '[document]']:
current = current.parent
else:
break
html_str = str(current)
if len(html_str) > 2000:
html_str = html_str[:2000] + "..."
return html_str
except Exception:
return str(element)
def _smart_truncate(self, html: str, interactive_html: str) -> str:
interactive_budget = int(self.max_chars * 0.3)
if len(interactive_html) > interactive_budget:
interactive_html = interactive_html[:interactive_budget]
remaining_budget = self.max_chars - len(interactive_html) - 100
if remaining_budget <= 0:
logger.warning("交互元素已占满预算")
return interactive_html
core_html = self._extract_core_area(html)
if core_html and len(core_html) <= remaining_budget:
result = core_html + "\n<!-- INTERACTIVE ELEMENTS -->\n" + interactive_html
return result[:self.max_chars]
head_budget = remaining_budget // 3
tail_budget = remaining_budget // 3
head_part = self._truncate_at_tag_boundary(html[:head_budget * 2], head_budget, from_end=False)
tail_part = self._truncate_at_tag_boundary(html[-tail_budget * 2:], tail_budget, from_end=True)
result = (
head_part +
"\n<!-- TRUNCATED: MIDDLE SECTION -->\n" +
"<!-- INTERACTIVE ELEMENTS START -->\n" +
interactive_html +
"\n<!-- INTERACTIVE ELEMENTS END -->\n" +
tail_part
)
if len(result) > self.max_chars:
result = result[:self.max_chars]
return result
def _extract_core_area(self, html: str) -> Optional[str]:
soup = None
try:
soup = BeautifulSoup(html, _BS4_PARSER)
for selector in self.CORE_AREA_SELECTORS:
try:
element = soup.select_one(selector)
if element:
core_html = str(element)
logger.debug(f"找到核心区域: {selector} ({len(core_html)} chars)")
return core_html
except Exception:
continue
return None
except Exception as e:
logger.debug(f"提取核心区域失败: {e}")
return None
finally:
if soup is not None:
soup.decompose()
def _truncate_at_tag_boundary(self, html: str, max_len: int, from_end: bool = False) -> str:
if len(html) <= max_len:
return html
if from_end:
start_pos = len(html) - max_len
tag_start = html.find('<', start_pos)
if tag_start != -1 and tag_start < len(html) - 100:
return html[tag_start:]
return html[-max_len:]
else:
tag_end = html.rfind('>', 0, max_len)
if tag_end != -1 and tag_end > 100:
return html[:tag_end + 1]
return html[:max_len]
# ================= 选择器验证器 =================
class SelectorValidator:
"""选择器验证器"""
def __init__(self, fallback_selectors: Dict[str, Optional[str]] = None):
self.fallback_selectors = fallback_selectors or {}
def validate(self, selectors: Dict[str, Optional[str]]) -> Dict[str, Optional[str]]:
"""验证并修复选择器"""
fixed = {}
for key, selector in selectors.items():
if selector is None:
fixed[key] = self.fallback_selectors.get(key)
continue
is_invalid = False
invalid_reason = ""
for pattern, reason in INVALID_SYNTAX_PATTERNS:
if re.search(pattern, selector):
is_invalid = True
invalid_reason = reason
break
if is_invalid:
logger.warning(f"❌ 无效选择器 [{key}]: {selector}")
logger.warning(f" 原因: {invalid_reason}")
repaired = self._try_repair(selector)
if repaired:
logger.info(f" ✅ 修复为: {repaired}")
fixed[key] = repaired
else:
fallback = self.fallback_selectors.get(key)
logger.info(f" 🔄 回退为: {fallback}")
fixed[key] = fallback
else:
if re.search(r'\._[a-f0-9]{5,}|^\.[a-f0-9]{6,}', selector):
logger.info(f"ℹ️ 哈希类名 [{key}]: {selector} (可能不稳定,但保留)")
fixed[key] = selector
return fixed
def _try_repair(self, selector: str) -> Optional[str]:
tag_match = re.match(r'^(\w+)', selector)
if not tag_match:
return None
tag = tag_match.group(1)
attr_patterns = [
r'(\[name=["\']?\w+["\']?\])',
r'(\[type=["\']?\w+["\']?\])',
r'(\[role=["\']?\w+["\']?\])',
r'(#[\w-]+)',
]
for pattern in attr_patterns:
match = re.search(pattern, selector)
if match:
return tag + match.group(1)
return tag
# ================= AI 分析器 =================
class AIAnalyzer:
"""AI 页面分析器(支持 OpenAI 兼容 / Gemini 官方 / Claude 官方)"""
# ===== Provider 注册表 =====
# 每个 provider 定义如何构建请求和解析响应
# 新增 provider 只需在此处注册,无需修改其他逻辑
PROVIDERS = {
"openai": {
"build_url": "_build_url_openai",
"build_headers": "_build_headers_openai",
"build_body": "_build_body_openai",
"parse_response": "_parse_response_openai",
},
"gemini": {
"build_url": "_build_url_gemini",
"build_headers": "_build_headers_gemini",
"build_body": "_build_body_gemini",
"parse_response": "_parse_response_gemini",
},
"claude": {
"build_url": "_build_url_claude",
"build_headers": "_build_headers_claude",
"build_body": "_build_body_claude",
"parse_response": "_parse_response_claude",
},
}
# URL 关键词 → provider 类型的映射(用于自动检测)
URL_PROVIDER_HINTS = [
("generativelanguage.googleapis.com", "gemini"),
("api.anthropic.com", "claude"),
("openai.azure.com", "azure"), # 未来可扩展
]
def __init__(self, global_config: GlobalConfigManager = None):
self.api_key = AppConfig.get_helper_api_key()
base_url = AppConfig.get_helper_base_url()
self.base_url = base_url.rstrip('/') if base_url else "http://127.0.0.1:5104/v1"
self.model = AppConfig.get_helper_model()
self.global_config = global_config
# 确定 provider 类型
configured_provider = AppConfig.get_helper_api_provider()
if configured_provider == "auto" or configured_provider not in self.PROVIDERS:
self.provider = self._detect_provider(self.base_url)
else:
self.provider = configured_provider
if not self.api_key:
logger.warning("⚠️ 未配置 HELPER_API_KEY,AI 分析功能将不可用")
else:
logger.debug(f"AI 分析器已配置: Provider={self.provider}, URL={self.base_url}, Model={self.model}")
# ================= 公共接口 =================
def analyze(self, html: str) -> Optional[Dict[str, str]]:
"""分析 HTML 并返回选择器"""
if not self.api_key:
logger.error("API Key 未配置")
return None
prompt = self._build_prompt(html)
for attempt in range(AI_MAX_RETRIES):
try:
logger.info(f"正在请求 AI 分析(尝试 {attempt + 1}/{AI_MAX_RETRIES},Provider: {self.provider})...")
response = self._request_ai(prompt)
if response:
analysis = self._extract_json(response)
if analysis is not None:
selectors = admitted_selectors(analysis)
if selectors:
logger.info("✅ 已确认 AI 聊天页并识别到专用定位器")
return selectors
logger.info("未确认 AI 聊天界面或缺少专用回复容器,不自动收录")
return None
logger.warning(f"第 {attempt + 1} 次分析失败")
except Exception as e:
logger.error(f"AI 请求异常: {e}")
if attempt < AI_MAX_RETRIES - 1:
delay = min(
AI_RETRY_BASE_DELAY * (2 ** attempt),
AI_RETRY_MAX_DELAY
)
jitter = delay * 0.1 * (0.5 - os.urandom(1)[0] / 255)
sleep_time = delay + jitter
logger.info(f"等待 {sleep_time:.2f}s 后重试...")
time.sleep(sleep_time)
logger.error("❌ AI 分析失败(已达最大重试次数)")
return None
# ================= Provider 自动检测 =================
def _detect_provider(self, base_url: str) -> str:
"""根据 URL 自动推断 API 类型"""
if not base_url:
return "openai"
url_lower = base_url.lower()
for keyword, provider in self.URL_PROVIDER_HINTS:
if keyword in url_lower:
logger.info(f"自动检测到 API 类型: {provider}(基于 URL 包含 '{keyword}')")
return provider
# 默认回退到 OpenAI 兼容格式
return "openai"
# ================= 统一请求调度 =================
def _request_ai(self, prompt: str) -> Optional[str]:
"""向 AI API 发送请求(根据 provider 自动适配格式)"""
provider_config = self.PROVIDERS.get(self.provider)
if not provider_config:
logger.error(f"未知的 Provider: {self.provider},回退到 openai")
provider_config = self.PROVIDERS["openai"]
# 通过注册表动态调用对应的构建/解析方法
url = getattr(self, provider_config["build_url"])(prompt)
headers = getattr(self, provider_config["build_headers"])()
body = getattr(self, provider_config["build_body"])(prompt)
parse_fn = getattr(self, provider_config["parse_response"])
try:
req = request.Request(
url,
data=json.dumps(body).encode('utf-8'),
headers=headers
)
with request.urlopen(req, timeout=AI_REQUEST_TIMEOUT) as response:
response_text = response.read().decode('utf-8')
try:
json_resp = json.loads(response_text)
return parse_fn(json_resp)
except json.JSONDecodeError:
logger.error(f"AI 响应 JSON 解析失败,原始响应前 500 字符: {response_text[:500]}")
return None
except error.HTTPError as e:
error_body = ""
try:
error_body = e.read().decode('utf-8')[:500]
except Exception:
pass
logger.error(f"HTTP 错误 {e.code}: {e.reason}" + (f",响应: {error_body}" if error_body else ""))
return None
except error.URLError as e:
logger.error(f"网络错误: {e.reason}")
return None
except Exception as e:
# 兜底:捕获 socket.timeout 等未被上层覆盖的异常
logger.error(f"请求异常: {type(e).__name__}: {e}")
return None
# ================= OpenAI 兼容格式 =================
def _build_url_openai(self, prompt: str) -> str:
return f"{self.base_url}/chat/completions"
def _build_headers_openai(self) -> Dict[str, str]:
return {
"Content-Type": "application/json",
"Authorization": f"Bearer {self.api_key}",
}
def _build_body_openai(self, prompt: str) -> Dict:
return {
"model": self.model,
"messages": [{"role": "user", "content": prompt}],
"temperature": 0.1,
}
def _parse_response_openai(self, json_resp: Dict) -> Optional[str]:
try:
if "choices" in json_resp and len(json_resp["choices"]) > 0:
return json_resp["choices"][0]["message"]["content"]
except (KeyError, IndexError, TypeError) as e:
logger.error(f"OpenAI 响应解析失败: {e}")
return None
# ================= Gemini 官方格式 =================
def _build_url_gemini(self, prompt: str) -> str:
# Gemini 官方 API: base_url 应为 https://generativelanguage.googleapis.com
# 完整端点: /v1beta/models/{model}:generateContent?key={api_key}
base = self.base_url.rstrip('/')
# 如果用户配置的 base_url 已经包含 /v1beta 或 /v1,不再追加
if "/v1beta" not in base and "/v1/" not in base:
base = f"{base}/v1beta"
return f"{base}/models/{self.model}:generateContent?key={self.api_key}"
def _build_headers_gemini(self) -> Dict[str, str]:
# Gemini 官方通过 URL 参数传递 key,不需要 Authorization 头
return {
"Content-Type": "application/json",
}
def _build_body_gemini(self, prompt: str) -> Dict:
return {
"contents": [
{
"parts": [
{"text": prompt}
]
}
],
"generationConfig": {
"temperature": 0.1,
},
}
def _parse_response_gemini(self, json_resp: Dict) -> Optional[str]:
try:
candidates = json_resp.get("candidates", [])
if candidates:
parts = candidates[0].get("content", {}).get("parts", [])
if parts:
return parts[0].get("text")
except (KeyError, IndexError, TypeError) as e:
logger.error(f"Gemini 响应解析失败: {e}")
# 检查是否有错误信息
if "error" in json_resp:
err = json_resp["error"]
logger.error(f"Gemini API 错误: [{err.get('code')}] {err.get('message', '')}")
return None
# ================= Claude 官方格式 =================
def _build_url_claude(self, prompt: str) -> str:
base = self.base_url.rstrip('/')
# Claude 官方端点: /v1/messages
if not base.endswith("/v1"):
base = f"{base}/v1"
return f"{base}/messages"
def _build_headers_claude(self) -> Dict[str, str]:
return {
"Content-Type": "application/json",
"x-api-key": self.api_key,
"anthropic-version": "2023-06-01",
}
def _build_body_claude(self, prompt: str) -> Dict:
return {
"model": self.model,
"max_tokens": 4096,
"messages": [{"role": "user", "content": prompt}],
"temperature": 0.1,
}
def _parse_response_claude(self, json_resp: Dict) -> Optional[str]:
try:
content = json_resp.get("content", [])
if content:
# Claude 响应的 content 是列表,每项有 type 和 text
text_parts = [block["text"] for block in content if block.get("type") == "text"]
if text_parts:
return "\n".join(text_parts)
except (KeyError, IndexError, TypeError) as e:
logger.error(f"Claude 响应解析失败: {e}")
# 检查是否有错误信息
if "error" in json_resp:
err = json_resp["error"]
logger.error(f"Claude API 错误: [{err.get('type')}] {err.get('message', '')}")
return None
# ================= Prompt 构建(不变)=================
def _build_prompt(self, clean_html: str) -> str:
"""构建 AI 提示词(动态版)"""
if self.global_config:
selector_list = self.global_config.build_prompt_selector_list()
json_keys = self.global_config.build_prompt_json_keys()
else:
selector_list = self._build_default_selector_list()
json_keys = self._build_default_json_keys()
lines = [
"You are a web interface classifier. First determine whether this is an AI CONVERSATION interface; do not assume every page is a chat app.",
"Return {\"is_chat_site\": false} for search engines, search results, login/consent pages, articles, ordinary forms, and pages with insufficient evidence.",
"A textarea, submit button, generic div, or the word AI alone is NOT evidence of an AI conversation interface.",
"Only for a genuine AI chat page with a composer and a specific assistant-reply container, return is_chat_site: true and the selector keys below. Do not invent selectors for missing elements.",
"",
"## CRITICAL RULES:",
"1. **Uniqueness is Key**: Ensure selectors matches ONLY the intended element.",
"2. **Distinguish AI vs User**: For `result_container`, specify the selector to target the **AI's response text** only. It MUST exclude user prompts, sidebars, or chat history.",
"3. **Use Hierarchy**: If a class like `.prose` or `.markdown` is used for both User and AI, you MUST find a unique parent class to differentiate (e.g., `.bot-msg .prose`).",
"4. **Syntax**: Use standard CSS selectors. Spaces for descendants (e.g., `div.bot p`) are encouraged for precision.",
"5. **No Invalid Syntax**: Do NOT use `xpath`, `~`, `:has()`, or `text()`.",
"",
"## PREFERENCE ORDER:",
"1. `id`, `name`, `data-testid` (Most preferred)",
"2. `button[type=\"submit\"]`",
"3. Unique parent class + target class (e.g., `.response-area .content`)",
"4. Hashed classes (only if no other option exists)",
"",
"## ELEMENTS TO FIND:",
selector_list,
"",
"## REQUIRED OUTPUT (JSON ONLY):",
"Return a JSON object with boolean is_chat_site plus these keys ONLY when is_chat_site is true:",
json_keys,
"",
"## HTML:",
clean_html
]
return "\n".join(lines)
def _build_default_selector_list(self) -> str:
"""默认的元素列表(回退用)"""
return """- `input_box`: The text input area (textarea/input). [REQUIRED]
- `send_btn`: The button that sends the message. [REQUIRED]
- `result_container`: The container for the AI's generated text response. [REQUIRED]
- `new_chat_btn`: Button to start a fresh conversation. [OPTIONAL, return null if not found]"""
def _build_default_json_keys(self) -> str:
"""默认的 JSON 键说明(回退用)"""
return """- `input_box`: The text input area
- `send_btn`: The send button
- `result_container`: AI response container
- `new_chat_btn`: New chat button (or null)"""
# ================= JSON 提取(不变)=================
def _extract_json(self, text: str) -> Optional[Dict]:
"""从 AI 响应中提取 JSON"""
try:
match = re.search(r'```(?:json)?\s*(\{[\s\S]*?\})\s*```', text)
if match:
return json.loads(match.group(1))
match = re.search(r'(\{[\s\S]*\})', text)
if match:
return json.loads(match.group(1))
return json.loads(text)
except json.JSONDecodeError as e:
logger.error(f"JSON 解析失败: {e}")
return None
__all__ = ['HTMLCleaner', 'SelectorValidator', 'AIAnalyzer']
+154
View File
@@ -0,0 +1,154 @@
"""R1-1:站点适配器 Schema v1。
只校验“形状”:类型、必填项、嵌套关系,以及 default_preset 必须指向已有预设。
语义校验(工作流 DSL、选择器能否命中等)仍由 ``flow_runtime.validate_workflow`` 和
ConfigEngine 的各个 ``_validate_*`` 负责。为了不误伤现有配置和用户自建预设,未知字段一律放行,
预设内部也不设必填项。
单站点文件(R1-2,``config/sites/<站点>.json``)的外层信封::
{"schema_version": 1, "site": "chat.deepseek.com", "adapter_version": "2026.09.26",
"min_app_version": "3.0.0", "last_verified": "2026-09-26", "config": {...}}
``config`` 与旧版 ``config/sites.json`` 中对应节点的内容完全相同。
"""
from __future__ import annotations
from typing import Any, Dict, Iterable, List
from jsonschema import Draft202012Validator
SITE_FILE_SCHEMA_VERSION = 1
GLOBAL_KEY = "_global"
_OBJECT = {"type": "object"}
WORKFLOW_STEP_SCHEMA: Dict[str, Any] = {
"type": "object",
"required": ["action"],
"properties": {
"action": {"type": "string", "minLength": 1},
"target": {"type": ["string", "null"]},
"optional": {"type": "boolean"},
"label": {"type": "string"},
"selector": {"type": "string"},
"execution": _OBJECT,
"flow_version": {"type": "integer"},
"retry_safe": {"type": "boolean"},
},
}
PRESET_SCHEMA: Dict[str, Any] = {
"type": "object",
"properties": {
"selectors": {"type": "object", "additionalProperties": {"type": ["string", "null"]}},
"workflow": {"type": "array", "items": WORKFLOW_STEP_SCHEMA},
"stealth": {"type": "boolean"},
"stream_config": _OBJECT,
"image_extraction": _OBJECT,
"file_paste": _OBJECT,
"prompt_padding": _OBJECT,
"advanced": _OBJECT,
},
}
SITE_CONFIG_SCHEMA: Dict[str, Any] = {
"$schema": "https://json-schema.org/draft/2020-12/schema",
"type": "object",
"required": ["presets"],
"properties": {
"presets": {"type": "object", "minProperties": 1, "additionalProperties": PRESET_SCHEMA},
"default_preset": {"type": "string", "minLength": 1},
"advanced": _OBJECT,
},
}
GLOBAL_CONFIG_SCHEMA: Dict[str, Any] = {
"$schema": "https://json-schema.org/draft/2020-12/schema",
"type": "object",
"properties": {
"selector_definitions": {
"type": "array",
"items": {
"type": "object",
"required": ["key"],
"properties": {
"key": {"type": "string", "minLength": 1},
"description": {"type": "string"},
"enabled": {"type": "boolean"},
"required": {"type": "boolean"},
},
},
},
},
}
SITE_FILE_SCHEMA: Dict[str, Any] = {
"$schema": "https://json-schema.org/draft/2020-12/schema",
"type": "object",
"required": ["schema_version", "site", "config"],
"properties": {
"schema_version": {"const": SITE_FILE_SCHEMA_VERSION},
"site": {"type": "string", "minLength": 1},
"adapter_version": {"type": "string"},
"min_app_version": {"type": "string", "pattern": r"^\d+(\.\d+){0,3}$"},
"last_verified": {"type": "string", "pattern": r"^\d{4}-\d{2}-\d{2}$"},
"config": _OBJECT,
},
}
_VALIDATORS = {
name: Draft202012Validator(schema)
for name, schema in (
("site", SITE_CONFIG_SCHEMA),
("global", GLOBAL_CONFIG_SCHEMA),
("file", SITE_FILE_SCHEMA),
)
}
def _format_path(prefix: str, parts: Iterable[Any]) -> str:
path = prefix
for part in parts:
path += f"[{part}]" if isinstance(part, int) else f".{part}"
return path
def _schema_errors(kind: str, payload: Any, prefix: str) -> List[str]:
errors = sorted(_VALIDATORS[kind].iter_errors(payload), key=lambda e: list(map(str, e.absolute_path)))
return [f"{_format_path(prefix, error.absolute_path)}: {error.message}" for error in errors]
def site_config_errors(site: str, config: Any) -> List[str]:
"""校验单个站点(或 ``_global``)的配置,返回可读的路径级错误列表;空列表表示通过。"""
prefix = f"sites[{site}]"
if site == GLOBAL_KEY:
return _schema_errors("global", config, prefix)
errors = _schema_errors("site", config, prefix)
if not errors and isinstance(config, dict):
default = config.get("default_preset")
presets = config.get("presets") or {}
if default and default not in presets:
errors.append(f"{prefix}.default_preset: '{default}' 不在 presets 中(可选:{', '.join(presets)})")
return errors
def site_file_errors(payload: Any) -> List[str]:
"""校验单站点文件(信封 + config)。"""
errors = _schema_errors("file", payload, "file")
if errors:
return errors
return site_config_errors(payload["site"], payload["config"])
def sites_errors(sites: Any) -> Dict[str, List[str]]:
"""校验整份站点配置字典(旧 sites.json 结构),返回 {站点: [错误...]},只包含有错误的站点。"""
if not isinstance(sites, dict):
return {"*": [f"sites: 应为对象,实际为 {type(sites).__name__}"]}
result: Dict[str, List[str]] = {}
for site, config in sites.items():
errors = site_config_errors(str(site), config)
if errors:
result[str(site)] = errors
return result
+312
View File
@@ -0,0 +1,312 @@
"""R1-2:站点配置目录存储(``config/sites/<站点>.json``)。
旧版把全部站点放在一个 271KB 的 ``config/sites.json`` 里;现在每个站点(以及 ``_global``)一个文件,
外层是 R1-1 定义的信封(见 ``site_schema.py``)。本模块负责:
- 读取目录并拼回与旧 ``sites.json`` **完全相同**的字典(``{"_global": {...}, "<域名>": {...}}``),
所以 ConfigEngine 以外的代码无需改动;
- 只重写内容有变化的站点文件(两阶段:先写全部临时文件,再逐个 ``os.replace``),保留信封元数据;
被删除的站点移入 ``.trash/``,不直接删除;
- 自动迁移旧的 ``config/sites.json``:目录为空时直接拆分;目录里已有发布版默认文件时(通过更新器升级的场景),
按更新器 ``merge_site_records`` 的语义合并(同一站点优先保留本地值、补入发布版新增字段);
迁移后旧文件改名为 ``sites.json.migrated-<时间>.bak``;
- 读不懂的文件(JSON 损坏或信封不完整)只告警跳过,并且**永远不会被保存逻辑覆盖或移入回收站**;
热重载时这类站点沿用上一次成功加载的配置。
"""
from __future__ import annotations
import json
import os
import time
from pathlib import Path
from typing import Any, Dict, List, Optional, Tuple
from app.core.config import get_logger
from app.services.config.site_schema import SITE_FILE_SCHEMA_VERSION, site_file_errors
logger = get_logger("SITE_STORE")
INDEX_FILE_NAME = "index.json"
TRASH_DIR_NAME = ".trash"
DEFAULT_MIN_APP_VERSION = "3.0.0"
_ENVELOPE_ORDER = ("schema_version", "site", "adapter_version", "min_app_version", "last_verified")
_WINDOWS_RESERVED = {"con", "prn", "aux", "nul", *(f"com{i}" for i in range(1, 10)), *(f"lpt{i}" for i in range(1, 10))}
def canonical_config_text(config: Any) -> str:
"""与格式、键顺序无关的规范化文本,用于变更检测与摘要(R1-3 的 config_sha256 也基于它)。"""
return json.dumps(config, ensure_ascii=False, sort_keys=True, separators=(",", ":"))
def site_filename(site: str) -> str:
"""站点键 -> 文件名。保留 ``[A-Za-z0-9._-]``,其余字节百分号编码;避开 Windows 保留名与 index.json。"""
encoded = "".join(
ch if (ch.isascii() and (ch.isalnum() or ch in "._-")) else "".join(f"%{b:02X}" for b in ch.encode("utf-8"))
for ch in str(site)
)
if not encoded or encoded.startswith(".") or encoded.split(".")[0].lower() in _WINDOWS_RESERVED | {"index"}:
encoded = "%5F" + encoded
return f"{encoded}.json"
def build_envelope(site: str, config: Dict[str, Any], meta: Optional[Dict[str, Any]] = None) -> Dict[str, Any]:
meta = dict(meta or {})
payload: Dict[str, Any] = {"schema_version": SITE_FILE_SCHEMA_VERSION, "site": site}
for key in _ENVELOPE_ORDER[2:]:
if meta.get(key) not in (None, ""):
payload[key] = meta[key]
for key, value in meta.items(): # 其余元数据(例如 R1-3 的 origin)原样保留
if key not in payload and key != "config":
payload[key] = value
payload["config"] = config
return payload
def _dump(payload: Dict[str, Any]) -> str:
return json.dumps(payload, indent=2, ensure_ascii=False) + "\n"
def _write_text_atomic(path: Path, text: str) -> None:
tmp = path.with_name(path.name + ".tmp")
with open(tmp, "w", encoding="utf-8", newline="\n") as handle:
handle.write(text)
handle.flush()
os.fsync(handle.fileno())
os.replace(tmp, path)
def _timestamp() -> str:
return time.strftime("%Y%m%d-%H%M%S")
class SiteStore:
"""``config/sites/`` 目录的读写与旧版迁移。线程安全由调用方(ConfigEngine 的 ``_io_lock``)保证。"""
def __init__(self, sites_dir: os.PathLike | str, legacy_file: os.PathLike | str):
self.sites_dir = Path(sites_dir)
self.legacy_file = Path(legacy_file)
self._files: Dict[str, Path] = {}
self._meta: Dict[str, Dict[str, Any]] = {}
self._on_disk: Dict[str, str] = {}
self._broken: Dict[Path, str] = {}
self._broken_sites: Dict[Path, str] = {} # 损坏文件 -> 上次成功加载时对应的站点
# ------------------------------------------------------------------ 查询
def site_files(self) -> List[Path]:
if not self.sites_dir.is_dir():
return []
return sorted(
path for path in self.sites_dir.glob("*.json")
if path.is_file() and path.name != INDEX_FILE_NAME and not path.name.startswith(".")
)
def exists(self) -> bool:
return bool(self.site_files()) or self.legacy_file.is_file()
def signature(self) -> Tuple:
"""目录内容签名(文件名、mtime、大小),用于热重载的变更检测。"""
entries = []
for path in self.site_files():
try:
stat = path.stat()
except OSError:
continue
entries.append((path.name, stat.st_mtime_ns, stat.st_size))
if self.legacy_file.is_file():
stat = self.legacy_file.stat()
entries.append(("<legacy>", stat.st_mtime_ns, stat.st_size))
return tuple(entries)
def metadata(self, site: str) -> Dict[str, Any]:
return dict(self._meta.get(site) or {})
def broken_files(self) -> Dict[str, str]:
return {str(path): reason for path, reason in self._broken.items()}
def file_for(self, site: str) -> Path:
return self._files.get(site) or (self.sites_dir / site_filename(site))
# ------------------------------------------------------------------ 读取
def read_envelope(self, path: Path) -> Dict[str, Any]:
payload = json.loads(path.read_text(encoding="utf-8-sig"))
if not isinstance(payload, dict) or not isinstance(payload.get("site"), str) or not isinstance(payload.get("config"), dict):
raise ValueError("不是有效的站点文件(缺少 site/config 字段)")
return payload
def load(self, previous: Optional[Dict[str, Any]] = None) -> Optional[Dict[str, Any]]:
"""读取全部站点,返回旧 sites.json 结构的字典;目录与旧文件都不存在时返回 None。
``previous`` 是上一次加载的结果:热重载时,暂时读不懂的文件沿用其中对应站点的配置。
"""
if self.legacy_file.is_file():
self.migrate_legacy()
files = self.site_files()
if not files:
return None
data: Dict[str, Any] = {}
files_map: Dict[str, Path] = {}
meta: Dict[str, Dict[str, Any]] = {}
on_disk: Dict[str, str] = {}
broken: Dict[Path, str] = {}
previous_sites_by_path = {path: site for site, path in self._files.items()}
previous_sites_by_path.update(self._broken_sites)
broken_sites: Dict[Path, str] = {}
for path in files:
try:
payload = self.read_envelope(path)
except (OSError, ValueError) as exc: # JSON 损坏、编码错误(均为 ValueError)、信封不完整、读文件失败
broken[path] = str(exc)
logger.error(f"站点配置文件无法读取,已跳过且不会被覆盖: {path.name}: {exc}")
old_site = previous_sites_by_path.get(path)
if old_site:
broken_sites[path] = old_site
if previous and old_site in previous:
data[old_site] = previous[old_site]
logger.warning(f"站点 {old_site} 暂时沿用上一次成功加载的配置")
continue
site = payload["site"]
if site in files_map:
broken[path] = f"与 {files_map[site].name} 重复声明站点 {site}"
logger.error(f"站点配置文件重复声明 {site},已忽略: {path.name}")
continue
for error in site_file_errors(payload)[:5]:
logger.warning(f"站点配置 Schema 告警({path.name}): {error}")
data[site] = payload["config"]
files_map[site] = path
meta[site] = {key: value for key, value in payload.items() if key != "config"}
on_disk[site] = canonical_config_text(payload["config"])
self._files, self._meta, self._on_disk = files_map, meta, on_disk
self._broken, self._broken_sites = broken, broken_sites
return data
# ------------------------------------------------------------------ 保存
def save(self, full_config: Dict[str, Any]) -> List[str]:
"""保存旧 sites.json 结构的字典,返回实际重写的站点列表。"""
self.sites_dir.mkdir(parents=True, exist_ok=True)
pending: List[Tuple[str, Path, Path, Dict[str, Any], str]] = []
taken = {path for path in self._files.values()} | set(self._broken)
try:
for site, config in full_config.items():
if not isinstance(config, dict):
continue
text = canonical_config_text(config)
path = self._files.get(site)
if path is not None and self._on_disk.get(site) == text and path.is_file():
continue
if path is None:
path = self._unique_path(site, taken)
taken.add(path)
if path in self._broken:
logger.error(f"站点 {site} 对应的文件 {path.name} 无法解析,为避免覆盖用户手改内容,本次跳过保存")
continue
payload = build_envelope(site, config, self._meta.get(site) or self._default_meta())
tmp = path.with_name(path.name + ".tmp")
with open(tmp, "w", encoding="utf-8", newline="\n") as handle:
handle.write(_dump(payload))
handle.flush()
os.fsync(handle.fileno())
pending.append((site, path, tmp, payload, text))
except Exception: # broad-except: 清理已写出的临时文件后原样重新抛出
for _, _, tmp, _, _ in pending:
try:
tmp.unlink()
except OSError:
pass
raise
written = []
for site, path, tmp, payload, text in pending:
os.replace(tmp, path)
self._files[site] = path
self._on_disk[site] = text
self._meta[site] = {key: value for key, value in payload.items() if key != "config"}
written.append(site)
for site in [s for s in self._files if s not in full_config]:
path = self._files.pop(site)
self._on_disk.pop(site, None)
self._meta.pop(site, None)
if path in self._broken or not path.is_file():
continue
trash = self.sites_dir / TRASH_DIR_NAME
trash.mkdir(exist_ok=True)
target = trash / f"{path.stem}.{_timestamp()}.json"
os.replace(path, target)
logger.info(f"站点 {site} 已删除,配置文件移入回收站: {target}")
return written
def _default_meta(self) -> Dict[str, Any]:
return {"min_app_version": DEFAULT_MIN_APP_VERSION}
def _unique_path(self, site: str, taken: set) -> Path:
base = self.sites_dir / site_filename(site)
path, counter = base, 2
while path in taken or (path.exists() and path not in self._files.values()):
path = base.with_name(f"{base.stem}~{counter}.json")
counter += 1
return path
# ------------------------------------------------------------------ 迁移
def migrate_legacy(self) -> Optional[Path]:
"""把旧的 config/sites.json 迁移进目录,返回备份文件路径;旧文件无法解析时保持原样并返回 None。"""
try:
raw = self.legacy_file.read_text(encoding="utf-8-sig").strip()
legacy = json.loads(raw) if raw else {}
if not isinstance(legacy, dict):
raise ValueError("顶层不是对象")
except (OSError, ValueError) as exc:
logger.error(f"旧版站点配置 {self.legacy_file} 无法解析,暂不迁移(原文件保持不变): {exc}")
return None
from updater import merge_site_records # 与更新器合并 sites.json 的语义保持一致
self.sites_dir.mkdir(parents=True, exist_ok=True)
existing: Dict[str, Tuple[Path, Dict[str, Any]]] = {}
for path in self.site_files():
try:
payload = self.read_envelope(path)
except (OSError, ValueError):
continue
existing[payload["site"]] = (path, payload)
taken = {path for path, _ in existing.values()}
merged_count = created_count = 0
for site, config in legacy.items():
if not isinstance(config, dict) or (site.startswith("_") and site != "_global"):
continue
if site in existing:
path, payload = existing[site]
payload = dict(payload)
merged_config = merge_site_records(config, payload["config"])
if (
site == "_global"
and config.get("selector_definitions") == []
and payload["config"].get("selector_definitions")
):
merged_config["selector_definitions"] = payload["config"]["selector_definitions"]
payload["config"] = merged_config
merged_count += 1
else:
path = self._unique_path(site, taken)
taken.add(path)
payload = build_envelope(site, config, self._default_meta())
created_count += 1
_write_text_atomic(path, _dump(payload))
backup = self.legacy_file.with_name(f"{self.legacy_file.name}.migrated-{_timestamp()}.bak")
os.replace(self.legacy_file, backup)
logger.warning(
f"已把旧版站点配置迁移到 {self.sites_dir}:新建 {created_count} 个、与发布版合并 {merged_count} 个;"
f"原文件已备份为 {backup.name}"
)
return backup
def load_sites_dict(sites_dir: os.PathLike | str) -> Dict[str, Any]:
"""只读地拼出目录里的全部站点(不迁移、不告警),供测试、脚本与导出使用。"""
store = SiteStore(sites_dir, Path(sites_dir) / "__no_legacy__.json")
return store.load() or {}
+17
View File
@@ -0,0 +1,17 @@
"""
app/services/config_engine.py - 兼容性代理文件
向后兼容:保持原有导入路径有效
实际实现已迁移到 app/services/config/ 模块
"""
# 从新模块导入所有内容
from app.services.config import (
config_engine,
ConfigEngine,
ConfigConstants,
DEFAULT_WORKFLOW
)
# 保持原有导出
__all__ = ['config_engine', 'ConfigEngine', 'ConfigConstants', 'DEFAULT_WORKFLOW']
+428
View File
@@ -0,0 +1,428 @@
"""
app/services/error_metadata.py - 标准异常元数据与响应构建器
职责:
- 解耦业务特定错误码(如 Arena / Gemini / Claude 拒绝与失败码)与主执行流
- 提供标准化的 ErrorMetadata 数据结构
- 提供基于元数据的 422 / 非可重试错误识别与统一 JSONResponse 构造
"""
from __future__ import annotations
import dataclasses
from typing import Any, Dict, Optional, Set
from fastapi.responses import JSONResponse
from app.services.arena_image_generation import (
ARENA_NON_RETRYABLE_CODES,
ARENA_PROMPT_REJECTED_CODE,
)
from app.core.workflow.arena_send_watchdog import (
ARENA_PAGE_ERROR,
ARENA_SEND_NO_TARGET_AFTER_RETRY,
)
@dataclasses.dataclass(frozen=True)
class ErrorMetadata:
code: str
message: str
status_code: int = 422
retryable: bool = False
error_type: str = "invalid_request_error"
param: Optional[str] = None
extra: Optional[Dict[str, Any]] = None
_MANUAL_TERMINATE_REASONS: Set[str] = {
"manual_terminate",
"manual_terminate_from_tab_pool",
}
_CANCEL_REASONS: Set[str] = {
"manual_terminate",
"manual_terminate_from_tab_pool",
"client_disconnected",
"coroutine_cancelled",
"cleanup",
"stream_done",
"stop_sequence",
"audio_media_fast_return",
"request_cancelled",
}
_PROMPT_REJECTION_CODES: Set[str] = {
ARENA_PROMPT_REJECTED_CODE,
"prompt_rejected",
}
_KNOWN_NON_RETRYABLE_CODES: Set[str] = {
*ARENA_NON_RETRYABLE_CODES,
ARENA_SEND_NO_TARGET_AFTER_RETRY,
ARENA_PAGE_ERROR,
*_PROMPT_REJECTION_CODES,
}
def is_manual_terminate(reason_or_ctx: Any) -> bool:
"""Check if the cancellation is due to manual termination."""
if hasattr(reason_or_ctx, "cancel_reason"):
reason = getattr(reason_or_ctx, "cancel_reason")
else:
reason = reason_or_ctx
return str(reason or "").strip() in _MANUAL_TERMINATE_REASONS
def is_prompt_rejection_code(code: Any) -> bool:
"""Check if code matches known prompt rejection codes."""
return str(code or "").strip() in _PROMPT_REJECTION_CODES
def _parse_bool(val: Any, default: bool = False) -> bool:
if val is None:
return default
if isinstance(val, bool):
return val
if isinstance(val, str):
s = val.strip().lower()
if s in {"1", "true", "yes"}:
return True
if s in {"0", "false", "no"}:
return False
return bool(val)
def _parse_int(val: Any, default: int = 422) -> int:
if val is None:
return default
try:
return int(val)
except (ValueError, TypeError):
return default
def _extract_json_dict_from_text(text: str) -> Optional[Dict[str, Any]]:
"""尝试从文本中提取最外层的 JSON 字典对象。"""
import json
if not text or "{" not in text or "}" not in text:
return None
start_idx = text.find("{")
end_idx = text.rfind("}")
if start_idx == -1 or end_idx == -1 or end_idx <= start_idx:
return None
candidate = text[start_idx : end_idx + 1].strip()
try:
data = json.loads(candidate)
if isinstance(data, dict):
return data
except Exception:
pass
return None
def _extract_status_code_from_text(text: str) -> Optional[int]:
"""从错误字符串中提取 HTTP 状态码。"""
import re
if not text:
return None
# 模式 1: 带有明确上下文前缀
match = re.search(
r"\b(?:http(?:\s+status)?\s+|status(?:\s*code)?\s*[:=]?\s*|failed\s+to\s+fetch(?:\s+image)?:\s*|code\s*[:=]\s*)([45]\d{2})\b",
text,
re.IGNORECASE,
)
if match:
try:
return int(match.group(1))
except Exception:
pass
# 模式 2: 常见 HTTP 状态码 + 描述词组合(如 "400 Bad Request", "429 Too Many Requests", "502 Bad Gateway")
match_status_desc = re.search(
r"\b([45]\d{2})\s+(?:Bad\s+Request|Unauthorized|Forbidden|Not\s+Found|Unprocessable\s+Entity|Too\s+Many\s+Requests|Internal\s+Server\s+Error|Bad\s+Gateway|Service\s+Unavailable|Gateway\s+Timeout)\b",
text,
re.IGNORECASE,
)
if match_status_desc:
try:
return int(match_status_desc.group(1))
except Exception:
pass
return None
def resolve_error_metadata(
reason_or_payload: Any,
default_message: Optional[str] = None,
) -> Optional[ErrorMetadata]:
"""Extract standard ErrorMetadata from a context, cancel reason, exception, or payload dict."""
if reason_or_payload is None:
return None
# Handle Exception objects
if isinstance(reason_or_payload, BaseException):
code = getattr(reason_or_payload, "code", None) or getattr(reason_or_payload, "error_code", None)
msg = (
getattr(reason_or_payload, "message", None)
or getattr(reason_or_payload, "error_msg", None)
or str(reason_or_payload)
)
if code:
return resolve_error_metadata(str(code), default_message=str(msg))
return resolve_error_metadata(str(reason_or_payload), default_message=default_message)
# Handle RequestContext-like objects
if hasattr(reason_or_payload, "cancel_reason"):
raw_reason = str(getattr(reason_or_payload, "cancel_reason") or "").strip()
if not raw_reason:
return None
if raw_reason in _CANCEL_REASONS:
return None
if raw_reason in _PROMPT_REJECTION_CODES:
return ErrorMetadata(
code=raw_reason,
message=default_message or "Arena 拒绝了该提示词:内容违反 Terms of Use",
status_code=422,
retryable=False,
error_type="invalid_request_error",
)
if raw_reason in _KNOWN_NON_RETRYABLE_CODES:
return ErrorMetadata(
code=raw_reason,
message=default_message or "Arena 图片生成失败,响应不可重试",
status_code=422,
retryable=False,
error_type="invalid_request_error",
)
return resolve_error_metadata(raw_reason, default_message=default_message)
# Handle payload dict like {"error": {...}} or SSE payload dict
if isinstance(reason_or_payload, dict):
error = reason_or_payload.get("error") if "error" in reason_or_payload else reason_or_payload
if isinstance(error, dict):
code = str(error.get("code") or "").strip()
msg = str(error.get("message") or "").strip()
raw_status_code = error.get("status_code")
raw_retryable = error.get("retryable")
param = error.get("param")
extra = {
k: v for k, v in error.items()
if k not in {"message", "type", "code", "status_code", "retryable", "param"}
}
status_code = _parse_int(raw_status_code, 422) if raw_status_code is not None else 422
retryable = _parse_bool(raw_retryable, False) if raw_retryable is not None else False
error_type = str(error.get("type") or "invalid_request_error").strip()
# If inner message itself contains a serialized JSON error or terminal error prefix, resolve it
if msg and ("{" in msg or "stream_terminal_error:" in msg):
nested_meta = resolve_error_metadata(msg, default_message=default_message)
if nested_meta:
merged_extra = dict(nested_meta.extra or {})
if extra:
merged_extra.update(extra)
return ErrorMetadata(
code=code if (code and code not in {"error", "workflow_failed", "arena_page_error"}) else nested_meta.code,
message=nested_meta.message,
status_code=status_code if raw_status_code is not None else nested_meta.status_code,
retryable=retryable if raw_retryable is not None else nested_meta.retryable,
error_type=error_type if error_type not in {"execution_error", "invalid_request_error"} else nested_meta.error_type,
param=param if param is not None else nested_meta.param,
extra=merged_extra or None,
)
if code in _PROMPT_REJECTION_CODES:
return ErrorMetadata(
code=code or ARENA_PROMPT_REJECTED_CODE,
message=msg or default_message or "Arena 拒绝了该提示词:内容违反 Terms of Use",
status_code=status_code,
retryable=retryable,
error_type=error_type,
param=param,
extra=extra or None,
)
if code in _KNOWN_NON_RETRYABLE_CODES:
return ErrorMetadata(
code=code,
message=msg or default_message or "Arena 图片生成失败,响应不可重试",
status_code=status_code,
retryable=retryable,
error_type=error_type,
param=param,
extra=extra or None,
)
if (raw_retryable is not None and not retryable) or (raw_status_code is not None and status_code in {400, 401, 403, 422}):
return ErrorMetadata(
code=code or "invalid_request_error",
message=msg or default_message or "请求不可重试",
status_code=status_code,
retryable=False,
error_type=error_type,
param=param,
extra=extra or None,
)
if msg or code:
return ErrorMetadata(
code=code or "error",
message=msg or default_message or "请求处理失败",
status_code=status_code if raw_status_code is not None else 500,
retryable=retryable,
error_type=error_type,
param=param,
extra=extra or None,
)
return None
# Handle string code / error message
if isinstance(reason_or_payload, str):
raw_str = reason_or_payload.strip()
if not raw_str:
return None
if raw_str in _CANCEL_REASONS:
return None
# 剥离内部标识前缀(支持多层嵌套剥离)
changed = True
while changed:
changed = False
for prefix in ("stream_terminal_error:", "execution_error:", "执行错误:", "[错误]", "[error]"):
if raw_str.lower().startswith(prefix.lower()):
raw_str = raw_str[len(prefix):].strip()
changed = True
break
# 1. 尝试从文本中解析嵌套的 JSON 对象
json_obj = _extract_json_dict_from_text(raw_str)
if json_obj:
nested_error = json_obj.get("error") if isinstance(json_obj.get("error"), dict) else json_obj
if isinstance(nested_error, dict):
extracted_msg = str(nested_error.get("message") or "").strip()
extracted_code = str(nested_error.get("code") or "").strip()
extracted_type = str(nested_error.get("type") or "").strip()
extracted_param = nested_error.get("param")
extra = {
k: v for k, v in nested_error.items()
if k not in {"message", "type", "code", "status_code", "retryable", "param"}
}
detected_status = _extract_status_code_from_text(raw_str) or _parse_int(nested_error.get("status_code"), 400)
is_retryable = detected_status >= 500 and extracted_code not in {"moderation_blocked", "content_policy_violation"}
return ErrorMetadata(
code=extracted_code or "upstream_error",
message=extracted_msg or raw_str or (default_message or "上游服务报错"),
status_code=detected_status or 400,
retryable=is_retryable,
error_type=extracted_type or "invalid_request_error",
param=str(extracted_param) if extracted_param is not None else None,
extra=extra or None,
)
# 2. 检查已知错误代码
if raw_str in _PROMPT_REJECTION_CODES:
return ErrorMetadata(
code=raw_str,
message=default_message or "Arena 拒绝了该提示词:内容违反 Terms of Use",
status_code=422,
retryable=False,
error_type="invalid_request_error",
)
if raw_str in _KNOWN_NON_RETRYABLE_CODES:
return ErrorMetadata(
code=raw_str,
message=default_message or "Arena 图片生成失败,响应不可重试",
status_code=422,
retryable=False,
error_type="invalid_request_error",
)
# 3. 检查是否含有 HTTP 状态码
status_code = _extract_status_code_from_text(raw_str)
if status_code is not None:
if status_code == 429:
return ErrorMetadata(
code="rate_limit_exceeded",
message=raw_str,
status_code=429,
retryable=False,
error_type="rate_limit_error",
)
if status_code in {400, 422}:
return ErrorMetadata(
code="bad_request" if status_code == 400 else "unprocessable_entity",
message=raw_str,
status_code=status_code,
retryable=False,
error_type="invalid_request_error",
)
if status_code in {401, 403}:
return ErrorMetadata(
code="permission_denied",
message=raw_str,
status_code=status_code,
retryable=False,
error_type="permission_error",
)
if status_code >= 500:
return ErrorMetadata(
code="upstream_service_error",
message=raw_str,
status_code=status_code,
retryable=True,
error_type="api_error",
)
# 4. 普通纯字符串
return ErrorMetadata(
code="workflow_failed",
message=raw_str or (default_message or "请求处理失败"),
status_code=500,
retryable=False,
error_type="execution_error",
)
return None
def is_non_retryable_error(reason_or_payload: Any) -> bool:
"""Check if a given reason or error payload represents a non-retryable 4xx error."""
meta = resolve_error_metadata(reason_or_payload)
return meta is not None and not meta.retryable and meta.status_code in {400, 401, 403, 422}
def build_error_response(
metadata: ErrorMetadata,
headers: Optional[Dict[str, str]] = None,
) -> JSONResponse:
"""Build a standard JSONResponse from ErrorMetadata."""
response_headers = dict(headers or {})
if not metadata.retryable:
response_headers["x-should-retry"] = "false"
error_body: Dict[str, Any] = {
"message": metadata.message,
"type": metadata.error_type,
"code": metadata.code,
"status_code": metadata.status_code,
"retryable": metadata.retryable,
}
if metadata.param is not None:
error_body["param"] = metadata.param
if metadata.extra:
for k, v in metadata.extra.items():
if k not in error_body:
error_body[k] = v
return JSONResponse(
content={
"error": error_body
},
status_code=metadata.status_code,
headers=response_headers,
)
class BrowserExecutionError(RuntimeError):
"""An upstream browser failure is not assistant text for tool parsing."""
def __init__(self, error):
error = error if isinstance(error, dict) else {"message": str(error)}
self.code = str(error.get("code") or "browser_execution_failed")
self.message = str(error.get("message") or self.code)
super().__init__(self.message)
+391
View File
@@ -0,0 +1,391 @@
"""
app/services/extractor_manager.py - 提取器配置管理器
职责:
- 加载 config/extractors.json
- 管理提取器配置
- 根据站点配置返回提取器实例
- 支持导入/导出提取器配置
"""
import os
import json
import copy
from app.core.config import atomic_write_json, get_logger
from typing import Dict, Optional, List, Any
from app.core.extractors import ExtractorRegistry, BaseExtractor
logger = get_logger("EXTRACT")
class ExtractorConfigManager:
"""
提取器配置管理器
负责:
1. 加载和保存 extractors.json
2. 根据配置动态加载提取器
3. 为站点提供提取器实例
"""
# 配置文件路径
_PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
CONFIG_FILE = os.path.join(_PROJECT_ROOT, "config", "extractors.json")
def __init__(self):
self._config: Dict[str, Any] = {}
self._extractors_config: Dict[str, Dict] = {}
self._default_id: str = "deep_mode_v1"
self._last_mtime: float = 0.0
# 加载配置
self._load_config()
def _load_config(self, *, preserve_on_error: bool = False):
"""加载 extractors.json 配置文件"""
if not os.path.exists(self.CONFIG_FILE):
logger.warning(f"提取器配置文件不存在: {self.CONFIG_FILE},使用默认配置")
if not preserve_on_error:
self._use_defaults()
return
try:
next_mtime = os.path.getmtime(self.CONFIG_FILE)
with open(self.CONFIG_FILE, "r", encoding="utf-8") as f:
loaded_config = json.load(f)
if not isinstance(loaded_config, dict):
raise ValueError("提取器配置必须是对象")
extractors_config = loaded_config.get("extractors", {})
if not isinstance(extractors_config, dict):
raise ValueError("提取器配置 extractors 必须是对象")
invalid_entries = [
extractor_id
for extractor_id, extractor_config in extractors_config.items()
if not isinstance(extractor_config, dict)
]
if invalid_entries:
raise ValueError(
"提取器配置项必须是对象: "
+ ", ".join(str(extractor_id) for extractor_id in invalid_entries)
)
self._config = loaded_config
self._extractors_config = extractors_config
self._default_id = str(loaded_config.get("default") or "deep_mode_v1").strip() or "deep_mode_v1"
self._config["extractors"] = self._extractors_config
self._config["default"] = self._default_id
self._config.setdefault("version", "1.0")
self._last_mtime = next_mtime
# 动态加载配置中定义的提取器
self._register_from_config()
logger.debug(
f"已加载提取器配置: {len(self._extractors_config)} 个提取器,"
f"默认: {self._default_id}"
)
except json.JSONDecodeError as e:
logger.error(f"提取器配置文件格式错误: {e}")
if not preserve_on_error:
self._use_defaults()
except Exception as e:
logger.error(f"加载提取器配置失败: {e}")
if not preserve_on_error:
self._use_defaults()
def _use_defaults(self):
"""使用默认配置"""
self._extractors_config = {
"deep_mode_v1": {
"id": "deep_mode_v1",
"name": "深度模式 (JS注入)",
"description": "通过 JavaScript 注入深度提取内容",
"class": "DeepBrowserExtractor",
"module": "app.core.extractors.deep_mode",
"enabled": True,
"config": {}
}
}
self._default_id = "deep_mode_v1"
self._config = {
"extractors": self._extractors_config,
"default": self._default_id,
"version": "1.0",
}
def _register_from_config(self):
"""根据配置文件动态注册提取器"""
for extractor_id, config in self._extractors_config.items():
if not isinstance(config, dict):
logger.warning(f"跳过无效提取器配置 [{extractor_id}]: 配置项必须是对象")
continue
if not config.get("enabled", True):
logger.debug(f"跳过禁用的提取器: {extractor_id}")
continue
# 检查是否已注册
if ExtractorRegistry.exists(extractor_id):
logger.debug(f"提取器已注册: {extractor_id}")
continue
# 尝试动态加载
module_path = config.get("module")
class_name = config.get("class")
if module_path and class_name:
try:
ExtractorRegistry.load_from_module(module_path, class_name, extractor_id)
except Exception as e:
logger.warning(f"动态加载提取器失败 [{extractor_id}]: {e}")
def refresh_if_changed(self):
"""检查配置文件是否变化,如果变化则重载"""
if not os.path.exists(self.CONFIG_FILE):
return
try:
current_mtime = os.path.getmtime(self.CONFIG_FILE)
if current_mtime != self._last_mtime:
logger.info("检测到提取器配置变化,重新加载")
self._load_config(preserve_on_error=True)
except Exception as e:
logger.error(f"检查配置变化失败: {e}")
def reload_config(self) -> None:
"""强制重新加载提取器配置,供配置备份恢复后热重载使用。"""
self._load_config(preserve_on_error=False)
# ================= 提取器获取 =================
def get_extractor(self, extractor_id: Optional[str] = None) -> BaseExtractor:
"""
获取提取器实例
Args:
extractor_id: 提取器 ID(None 则返回默认)
Returns:
提取器实例
"""
eid = extractor_id or self._default_id
try:
return ExtractorRegistry.get(eid)
except ValueError:
logger.warning(f"提取器不存在: {eid},使用默认: {self._default_id}")
return ExtractorRegistry.get(self._default_id)
def get_extractor_for_site(self, site_config: Dict) -> BaseExtractor:
"""
根据站点配置获取提取器
Args:
site_config: 站点配置字典(包含 extractor_id 字段)
Returns:
提取器实例
"""
extractor_id = site_config.get("extractor_id")
return self.get_extractor(extractor_id)
def get_default_id(self) -> str:
"""获取默认提取器 ID"""
return self._default_id
def set_default(self, extractor_id: str) -> bool:
"""
设置默认提取器
Args:
extractor_id: 提取器 ID
Returns:
是否成功
"""
if not ExtractorRegistry.exists(extractor_id):
logger.error(f"无法设置默认提取器:{extractor_id} 不存在")
return False
previous_default = self._default_id
previous_config = copy.deepcopy(self._config)
previous_extractors_config = copy.deepcopy(self._extractors_config)
try:
self._default_id = extractor_id
self._config["extractors"] = self._extractors_config
self._config["default"] = extractor_id
self._save_config()
except Exception as exc:
self._default_id = previous_default
self._config = previous_config
self._extractors_config = self._config.get("extractors", previous_extractors_config)
logger.error(f"设置默认提取器失败,已回滚: {exc}")
return False
return True
# ================= 配置查询 =================
def list_extractors(self) -> List[Dict[str, Any]]:
"""
列出所有可用的提取器
Returns:
提取器信息列表
"""
result = []
# 从注册表获取(确保包含运行时注册的)
for info in ExtractorRegistry.list_all():
extractor_id = info["id"]
# 合并配置文件中的信息
config = self._extractors_config.get(extractor_id, {})
if not isinstance(config, dict):
config = {}
result.append({
"id": extractor_id,
"name": config.get("name") or info.get("name", extractor_id),
"description": config.get("description") or info.get("description", ""),
"enabled": config.get("enabled", True),
"is_default": extractor_id == self._default_id
})
return result
def get_extractor_config(self, extractor_id: str) -> Optional[Dict]:
"""获取提取器配置"""
config = self._extractors_config.get(extractor_id)
return copy.deepcopy(config) if isinstance(config, dict) else None
def has_extractor(self, extractor_id: str) -> bool:
"""检查提取器是否已经注册。"""
return ExtractorRegistry.exists(extractor_id)
# ================= 导入/导出 =================
def export_config(self) -> Dict[str, Any]:
"""
导出提取器配置
Returns:
完整配置字典(可保存为 JSON)
"""
return {
"extractors": copy.deepcopy(self._extractors_config),
"default": self._default_id,
"version": self._config.get("version", "1.0")
}
def import_config(self, config: Dict[str, Any]) -> bool:
"""
导入提取器配置
Args:
config: 配置字典
Returns:
是否成功
"""
try:
# 验证格式
if not isinstance(config, dict):
logger.error("导入失败:配置必须是对象")
return False
extractors = config.get("extractors")
if "extractors" not in config:
logger.error("导入失败:缺少 extractors 字段")
return False
if not isinstance(extractors, dict):
logger.error("导入失败:extractors 必须是对象")
return False
invalid_entries = [
extractor_id
for extractor_id, extractor_config in extractors.items()
if not isinstance(extractor_config, dict)
]
if invalid_entries:
logger.error(
"导入失败:提取器配置必须是对象: "
+ ", ".join(str(extractor_id) for extractor_id in invalid_entries)
)
return False
default_id = str(config.get("default") or "deep_mode_v1").strip() or "deep_mode_v1"
default_entry = extractors.get(default_id)
if isinstance(default_entry, dict) and not default_entry.get("enabled", True):
logger.error(f"导入失败:默认提取器已禁用: {default_id}")
return False
if default_entry is None and not ExtractorRegistry.exists(default_id):
logger.error(f"导入失败:默认提取器不存在: {default_id}")
return False
previous_config = copy.deepcopy(self._config)
previous_extractors_config = copy.deepcopy(self._extractors_config)
previous_default_id = self._default_id
previous_registry = copy.copy(getattr(ExtractorRegistry, "_extractors", {}))
next_config = copy.deepcopy(config)
self._extractors_config = copy.deepcopy(next_config["extractors"])
self._default_id = default_id
next_config["extractors"] = self._extractors_config
next_config["default"] = self._default_id
self._config = next_config
# 保存到文件
saved_config = False
try:
self._save_config()
saved_config = True
self._register_from_config()
if not ExtractorRegistry.exists(self._default_id):
raise ValueError(f"默认提取器无法注册: {self._default_id}")
except Exception:
self._config = previous_config
self._extractors_config = self._config.get("extractors", previous_extractors_config)
self._default_id = previous_default_id
registry = getattr(ExtractorRegistry, "_extractors", None)
if isinstance(registry, dict):
registry.clear()
registry.update(previous_registry)
if saved_config:
try:
self._save_config()
except Exception as rollback_error:
logger.error(f"导入失败后恢复提取器配置文件失败: {rollback_error}")
raise
logger.info(f"成功导入提取器配置: {len(self._extractors_config)} 个")
return True
except Exception as e:
logger.error(f"导入配置失败: {e}")
return False
def _save_config(self):
"""保存配置到文件"""
try:
atomic_write_json(self.CONFIG_FILE, self._config)
try:
self._last_mtime = os.path.getmtime(self.CONFIG_FILE)
except Exception as mtime_error:
logger.warning(f"提取器配置已保存但更新时间戳失败: {mtime_error}")
logger.info("提取器配置已保存")
except Exception as e:
logger.error(f"保存配置失败: {e}")
raise
# ================= 单例 =================
extractor_manager = ExtractorConfigManager()
__all__ = ['ExtractorConfigManager', 'extractor_manager']
+214
View File
@@ -0,0 +1,214 @@
"""R2-7:可观测性——无第三方依赖的 Prometheus 指标与请求 ID。
- ``RequestMetricsMiddleware``(纯 ASGI 中间件,不缓冲流式响应):
- 为每个 HTTP 请求分配或透传 ``X-Request-ID``,写入响应头与 ``REQUEST_ID`` 上下文变量;
- 记录请求数、耗时直方图(含流式响应体发送完的总耗时),以及进行中的请求数。
- 标签用路由模板(如 ``/v1/chat/completions``),未匹配到路由的统一记为 ``<unmatched>``,避免标签基数爆炸。
- ``render_prometheus()``:Prometheus 文本格式(text/plain; version=0.0.4)。
抓取时顺带读取标签页池状态,但只在浏览器**已连接**时读取,抓取本身绝不触发连接。
"""
from __future__ import annotations
import contextvars
import re
import threading
import time
import uuid
from typing import Dict, Iterable, List, Optional, Tuple
REQUEST_ID: contextvars.ContextVar[str] = contextvars.ContextVar("uwapi_http_request_id", default="")
_REQUEST_ID_PATTERN = re.compile(r"^[A-Za-z0-9._:-]{1,64}$")
_STARTED_AT = time.time()
Labels = Tuple[Tuple[str, str], ...]
def _escape(value: str) -> str:
return str(value).replace("\\", "\\\\").replace("\n", "\\n").replace('"', '\\"')
def _format_labels(labels: Labels, extra: Iterable[Tuple[str, str]] = ()) -> str:
items = list(labels) + list(extra)
if not items:
return ""
return "{" + ",".join(f'{key}="{_escape(value)}"' for key, value in items) + "}"
class _Metric:
kind = ""
def __init__(self, name: str, help_text: str):
self.name = name
self.help = help_text
self._lock = threading.Lock()
def header(self) -> List[str]:
return [f"# HELP {self.name} {self.help}", f"# TYPE {self.name} {self.kind}"]
class Counter(_Metric):
kind = "counter"
def __init__(self, name: str, help_text: str):
super().__init__(name, help_text)
self._values: Dict[Labels, float] = {}
def inc(self, amount: float = 1.0, **labels: str) -> None:
key = tuple(sorted((k, str(v)) for k, v in labels.items()))
with self._lock:
self._values[key] = self._values.get(key, 0.0) + amount
def value(self, **labels: str) -> float:
return self._values.get(tuple(sorted((k, str(v)) for k, v in labels.items())), 0.0)
def render(self) -> List[str]:
with self._lock:
items = sorted(self._values.items())
return self.header() + [f"{self.name}{_format_labels(k)} {v:g}" for k, v in items]
class Gauge(Counter):
kind = "gauge"
def set(self, value: float, **labels: str) -> None:
key = tuple(sorted((k, str(v)) for k, v in labels.items()))
with self._lock:
self._values[key] = float(value)
def dec(self, amount: float = 1.0, **labels: str) -> None:
self.inc(-amount, **labels)
def clear(self) -> None:
with self._lock:
self._values.clear()
class Histogram(_Metric):
kind = "histogram"
def __init__(self, name: str, help_text: str, buckets: Iterable[float]):
super().__init__(name, help_text)
self.buckets = tuple(sorted(buckets))
self._data: Dict[Labels, List[float]] = {} # [bucket counts..., sum, count]
def observe(self, value: float, **labels: str) -> None:
key = tuple(sorted((k, str(v)) for k, v in labels.items()))
with self._lock:
data = self._data.setdefault(key, [0.0] * (len(self.buckets) + 2))
for index, bound in enumerate(self.buckets):
if value <= bound:
data[index] += 1
data[-2] += value
data[-1] += 1
def render(self) -> List[str]:
lines = self.header()
with self._lock:
items = sorted((k, list(v)) for k, v in self._data.items())
for labels, data in items:
for index, bound in enumerate(self.buckets):
lines.append(f"{self.name}_bucket{_format_labels(labels, [('le', f'{bound:g}')])} {data[index]:g}")
lines.append(f"{self.name}_bucket{_format_labels(labels, [('le', '+Inf')])} {data[-1]:g}")
lines.append(f"{self.name}_sum{_format_labels(labels)} {data[-2]:.6f}")
lines.append(f"{self.name}_count{_format_labels(labels)} {data[-1]:g}")
return lines
HTTP_REQUESTS = Counter("uwapi_http_requests_total", "HTTP 请求数(按方法、路由模板、状态码)")
HTTP_DURATION = Histogram(
"uwapi_http_request_duration_seconds",
"HTTP 请求耗时(秒,流式响应计到响应体发送完)",
buckets=(0.01, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60, 120, 300, 600),
)
HTTP_IN_FLIGHT = Gauge("uwapi_http_requests_in_flight", "进行中的 HTTP 请求数")
CHAT_REQUESTS = Counter("uwapi_chat_requests_total", "聊天请求数(按协议 openai.chat / anthropic.messages / openai.responses 与最终状态)")
CHAT_REQUEST_DURATION = Histogram(
"uwapi_chat_request_duration_seconds",
"聊天请求从创建到结束的耗时(秒,按协议)",
buckets=(0.5, 1, 2.5, 5, 10, 20, 30, 60, 120, 300, 600),
)
TAB_POOL_TABS = Gauge("uwapi_tab_pool_tabs", "标签页池中的标签页数(按状态;浏览器未连接时不输出)")
BUILD_INFO = Gauge("uwapi_build_info", "构建信息(值恒为 1)")
UPTIME = Gauge("uwapi_uptime_seconds", "进程已运行秒数")
REGISTRY: List[_Metric] = [
HTTP_REQUESTS, HTTP_DURATION, HTTP_IN_FLIGHT, CHAT_REQUESTS, CHAT_REQUEST_DURATION, TAB_POOL_TABS, BUILD_INFO, UPTIME,
]
def new_request_id() -> str:
return uuid.uuid4().hex[:16]
def _incoming_request_id(scope) -> Optional[str]:
for key, value in scope.get("headers") or []:
if key == b"x-request-id":
candidate = value.decode("latin-1").strip()
return candidate if _REQUEST_ID_PATTERN.fullmatch(candidate) else None
return None
class RequestMetricsMiddleware:
def __init__(self, app):
self.app = app
async def __call__(self, scope, receive, send):
if scope.get("type") != "http":
await self.app(scope, receive, send)
return
request_id = _incoming_request_id(scope) or new_request_id()
token = REQUEST_ID.set(request_id)
status = {"code": 500}
started = time.perf_counter()
async def send_with_request_id(message):
if message.get("type") == "http.response.start":
status["code"] = int(message.get("status") or 500)
headers = [(k, v) for k, v in (message.get("headers") or []) if k.lower() != b"x-request-id"]
headers.append((b"x-request-id", request_id.encode("latin-1")))
message = {**message, "headers": headers}
await send(message)
HTTP_IN_FLIGHT.inc()
try:
await self.app(scope, receive, send_with_request_id)
finally:
HTTP_IN_FLIGHT.dec()
route = getattr(scope.get("route"), "path", None) or "<unmatched>"
method = str(scope.get("method") or "GET")
HTTP_REQUESTS.inc(method=method, route=route, status=str(status["code"]))
HTTP_DURATION.observe(time.perf_counter() - started, method=method, route=route)
REQUEST_ID.reset(token)
def _collect_runtime_gauges() -> None:
from app import __version__
BUILD_INFO.set(1, version=__version__)
UPTIME.set(time.time() - _STARTED_AT)
TAB_POOL_TABS.clear()
try:
from app.core.browser import main as browser_main
browser = getattr(browser_main, "_browser_instance", None) # 不调用 get_browser:抓取不能创建或连接浏览器
if browser is None or not getattr(browser, "_connected", False):
return
pool = getattr(browser, "tab_pool", None)
if pool is None:
return
counts: Dict[str, int] = {}
for session in pool.get_sessions_snapshot():
state = str(getattr(getattr(session, "status", None), "value", "unknown") or "unknown")
counts[state] = counts.get(state, 0) + 1
for state, count in counts.items():
TAB_POOL_TABS.set(count, status=state)
except Exception: # broad-except: 指标抓取不能因为浏览器状态异常而失败
return
def render_prometheus() -> str:
_collect_runtime_gauges()
lines: List[str] = []
for metric in REGISTRY:
lines.extend(metric.render())
return "\n".join(lines) + "\n"
+522
View File
@@ -0,0 +1,522 @@
"""
app/services/parser_manager.py - Runtime parser installation manager.
Parser packages can be written into app/core/parsers and then
registered through ParserRegistry via config/parsers.json.
"""
from __future__ import annotations
import ast
import copy
import json
import os
import re
import tempfile
from datetime import datetime, timezone
from pathlib import Path
from typing import Any, Dict, List, Optional
from app.core.config import get_logger
from app.core.config import atomic_write_json
from app.core.parsers import ParserRegistry
logger = get_logger("PARSER_MGR")
# ---------------------------------------------------------------------------
# S3:运行时安装解析器 = 把一段 Python 写进 app/core/parsers 并立即 import。
# 1) 默认关闭,只有可信管理员显式设置 PARSER_INSTALL_ENABLED=true 才允许;
# 2) 导入期「无副作用」静态约束:模块顶层 / 类体只允许 import、函数/类定义、
# 字面量赋值,以及极少数纯函数调用(re.compile 等,且参数必须是字面量);
# 装饰器只允许 staticmethod/classmethod/property 等;默认参数必须是字面量。
# 这样 import 本身不会执行任意代码——解析逻辑只会在解析器被调用时运行;
# 3) 配置中的模块路径必须位于 app.core.parsers 包内。
# 这不是沙箱:解析器方法仍以服务进程权限运行,所以安装入口必须只对可信管理员开放。
# ---------------------------------------------------------------------------
PARSER_INSTALL_ENV = "PARSER_INSTALL_ENABLED"
PARSER_MODULE_PREFIX = "app.core.parsers."
_IMPORT_TIME_SAFE_CALLS = frozenset({
"re.compile", "set", "frozenset", "tuple", "list", "dict", "str.maketrans",
"get_logger", "logging.getLogger",
})
_SAFE_DECORATORS = frozenset({
"staticmethod", "classmethod", "property", "abstractmethod", "abc.abstractmethod",
"dataclass", "dataclasses.dataclass", "functools.cached_property", "cached_property",
})
def parser_install_enabled() -> bool:
value = str(os.getenv(PARSER_INSTALL_ENV, "") or "").strip().lower()
return value in ("true", "1", "yes", "on")
def _dotted_name(node: ast.AST) -> str:
if isinstance(node, ast.Name):
return node.id
if isinstance(node, ast.Attribute):
base = _dotted_name(node.value)
return f"{base}.{node.attr}" if base else ""
return ""
def _import_time_value_ok(node: Optional[ast.AST]) -> bool:
"""导入期会被求值的表达式:只允许字面量/名字引用/白名单纯函数调用。"""
if node is None:
return True
if isinstance(node, (ast.Constant, ast.Name)):
return True
if isinstance(node, ast.Attribute):
return not node.attr.startswith("__") and _import_time_value_ok(node.value)
if isinstance(node, (ast.Tuple, ast.List, ast.Set)):
return all(_import_time_value_ok(item) for item in node.elts)
if isinstance(node, ast.Dict):
return all(_import_time_value_ok(k) for k in node.keys if k is not None) and all(
_import_time_value_ok(v) for v in node.values
)
if isinstance(node, ast.UnaryOp):
return _import_time_value_ok(node.operand)
if isinstance(node, ast.BinOp):
return _import_time_value_ok(node.left) and _import_time_value_ok(node.right)
if isinstance(node, ast.JoinedStr):
return all(_import_time_value_ok(v) for v in node.values)
if isinstance(node, ast.FormattedValue):
return _import_time_value_ok(node.value)
if isinstance(node, ast.Call):
if _dotted_name(node.func) not in _IMPORT_TIME_SAFE_CALLS:
return False
return all(_import_time_value_ok(a) for a in node.args) and all(
_import_time_value_ok(k.value) for k in node.keywords
)
return False
def _decorator_ok(node: ast.AST) -> bool:
name = _dotted_name(node)
if name in _SAFE_DECORATORS:
return True
# @x.setter / @x.getter / @x.deleter(property 访问器)
return isinstance(node, ast.Attribute) and node.attr in {"setter", "getter", "deleter"} and bool(name)
def _function_signature_ok(node: ast.AST) -> bool:
args = node.args # type: ignore[attr-defined]
defaults = list(args.defaults) + [d for d in args.kw_defaults if d is not None]
annotations = [a.annotation for a in (*args.posonlyargs, *args.args, *args.kwonlyargs) if a.annotation]
if args.vararg and args.vararg.annotation:
annotations.append(args.vararg.annotation)
if args.kwarg and args.kwarg.annotation:
annotations.append(args.kwarg.annotation)
returns = getattr(node, "returns", None)
if returns is not None:
annotations.append(returns)
return all(_import_time_value_ok(d) for d in defaults) and all(
_annotation_ok(a) for a in annotations
)
def _annotation_ok(node: ast.AST) -> bool:
# 注解在没有 `from __future__ import annotations` 时也会在导入期求值;
# 允许 Name/Attribute/字符串/下标(Optional[str]、Dict[str, Any])/ | 联合。
if isinstance(node, (ast.Constant, ast.Name)):
return True
if isinstance(node, ast.Attribute):
return not node.attr.startswith("__") and _annotation_ok(node.value)
if isinstance(node, ast.Subscript):
return _annotation_ok(node.value) and _annotation_ok(node.slice)
if isinstance(node, (ast.Tuple, ast.List)):
return all(_annotation_ok(item) for item in node.elts)
if isinstance(node, ast.BinOp) and isinstance(node.op, ast.BitOr):
return _annotation_ok(node.left) and _annotation_ok(node.right)
return False
def check_import_time_side_effects(tree: ast.Module) -> List[str]:
"""返回导入期可能执行任意代码的位置描述;空列表表示通过。"""
problems: List[str] = []
def visit_body(body: List[ast.stmt], where: str) -> None:
for index, stmt in enumerate(body):
line = getattr(stmt, "lineno", "?")
if isinstance(stmt, (ast.Import, ast.ImportFrom, ast.Pass)):
continue
if isinstance(stmt, ast.Expr) and isinstance(stmt.value, ast.Constant) and isinstance(stmt.value.value, str):
continue # docstring
if isinstance(stmt, (ast.FunctionDef, ast.AsyncFunctionDef)):
if not all(_decorator_ok(d) for d in stmt.decorator_list):
problems.append(f"{where}:{line} 不允许的装饰器")
if not _function_signature_ok(stmt):
problems.append(f"{where}:{line} 默认参数/注解必须是字面量")
continue
if isinstance(stmt, ast.ClassDef):
if not all(_decorator_ok(d) for d in stmt.decorator_list):
problems.append(f"{where}:{line} 不允许的类装饰器")
if stmt.keywords or not all(isinstance(b, (ast.Name, ast.Attribute)) for b in stmt.bases):
problems.append(f"{where}:{line} 类继承列表只允许名字引用")
visit_body(stmt.body, f"class {stmt.name}")
continue
if isinstance(stmt, ast.Assign):
if all(isinstance(t, (ast.Name, ast.Tuple)) for t in stmt.targets) and _import_time_value_ok(stmt.value):
continue
if isinstance(stmt, ast.AnnAssign):
if isinstance(stmt.target, ast.Name) and _annotation_ok(stmt.annotation) and _import_time_value_ok(stmt.value):
continue
if isinstance(stmt, ast.If) and index == len(body) - 1 and where == "module":
test = stmt.test
# 允许末尾的 `if __name__ == "__main__":`(被 import 时不会执行)
if (
isinstance(test, ast.Compare)
and isinstance(test.left, ast.Name)
and test.left.id == "__name__"
and len(test.comparators) == 1
and isinstance(test.comparators[0], ast.Constant)
and test.comparators[0].value == "__main__"
):
continue
problems.append(f"{where}:{line} 导入期会执行的语句 ({type(stmt).__name__})")
visit_body(tree.body, "module")
return problems
class ParserConfigManager:
"""Manage runtime-installed response parsers."""
_PROJECT_ROOT = Path(__file__).resolve().parents[2]
CONFIG_FILE = Path(os.getenv("PARSERS_CONFIG_FILE", _PROJECT_ROOT / "config" / "parsers.json"))
PARSER_DIR = _PROJECT_ROOT / "app" / "core" / "parsers"
_ID_PATTERN = re.compile(r"^[A-Za-z0-9._-]+$")
_NAME_PATTERN = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$")
def __init__(self):
self._config: Dict[str, Any] = {}
self._parsers_config: Dict[str, Dict[str, Any]] = {}
self._last_mtime: float = 0.0
self._load_config()
def _load_config(self) -> None:
if not self.CONFIG_FILE.exists():
self._use_defaults()
return
try:
next_mtime = self.CONFIG_FILE.stat().st_mtime
with self.CONFIG_FILE.open("r", encoding="utf-8") as handle:
loaded_config = json.load(handle)
if not isinstance(loaded_config, dict):
logger.warning("Parsers config file must be an object; using defaults")
loaded_config = {"version": "1.0", "parsers": {}}
parsers_config = loaded_config.get("parsers", {})
if not isinstance(parsers_config, dict):
logger.warning("Parsers config field must be an object; ignoring invalid value")
parsers_config = {}
loaded_config["parsers"] = parsers_config
loaded_config.setdefault("version", "1.0")
self._config = loaded_config
self._parsers_config = parsers_config
self._last_mtime = next_mtime
self._register_from_config()
except json.JSONDecodeError as exc:
logger.error(f"Failed to decode parsers config: {exc}")
self._use_defaults()
except Exception as exc:
logger.error(f"Failed to load parsers config: {exc}")
self._use_defaults()
def _use_defaults(self) -> None:
self._config = {
"version": "1.0",
"parsers": {},
}
self._parsers_config = {}
def _register_from_config(self) -> None:
for parser_id, config in self._parsers_config.items():
if not isinstance(config, dict):
logger.warning(f"Skip invalid parser config [{parser_id}]: entry must be an object")
continue
if not config.get("enabled", True):
continue
try:
self._load_parser_entry(parser_id, config)
except Exception as exc:
logger.warning(f"Failed to register parser [{parser_id}]: {exc}")
def _save_config(self) -> None:
atomic_write_json(self.CONFIG_FILE, self._config)
try:
self._last_mtime = self.CONFIG_FILE.stat().st_mtime
except Exception as exc:
logger.warning(f"Parsers config saved but failed to update mtime: {exc}")
@staticmethod
def _atomic_write_text(path: Path, content: str) -> None:
target = Path(path)
target.parent.mkdir(parents=True, exist_ok=True)
fd = None
tmp_path: Optional[Path] = None
try:
fd, tmp_name = tempfile.mkstemp(
prefix=f".{target.name}.",
suffix=".tmp",
dir=str(target.parent),
)
tmp_path = Path(tmp_name)
with os.fdopen(fd, "w", encoding="utf-8", newline="\n") as handle:
fd = None
handle.write(content)
if not content.endswith("\n"):
handle.write("\n")
handle.flush()
os.fsync(handle.fileno())
os.replace(tmp_path, target)
except Exception:
if fd is not None:
try:
os.close(fd)
except Exception:
pass
if tmp_path is not None:
try:
tmp_path.unlink(missing_ok=True)
except Exception:
pass
raise
def _load_parser_entry(self, parser_id: str, config: Dict[str, Any]) -> None:
module_path = str(config.get("module") or "").strip()
class_name = str(config.get("class") or "").strip()
if not module_path or not class_name:
raise ValueError("Parser config is missing module/class")
# S3:配置里的模块只能来自解析器包本身,不能借 parsers.json 导入任意模块
suffix = module_path[len(PARSER_MODULE_PREFIX):] if module_path.startswith(PARSER_MODULE_PREFIX) else ""
if not suffix or not self._NAME_PATTERN.match(suffix):
raise ValueError(f"Parser module must be inside {PARSER_MODULE_PREFIX[:-1]}: {module_path}")
ParserRegistry.load_from_module(module_path, class_name, parser_id)
def list_parsers(self) -> List[Dict[str, Any]]:
result: List[Dict[str, Any]] = []
for info in ParserRegistry.list_all():
parser_id = str(info.get("id") or "").strip()
config = self._parsers_config.get(parser_id, {})
if not isinstance(config, dict):
config = {}
result.append({
**info,
"managed": bool(config),
"module": str(config.get("module") or ""),
"class": str(config.get("class") or ""),
"filename": str(config.get("filename") or ""),
"installed_at": str(config.get("installed_at") or ""),
})
result.sort(key=lambda item: str(item.get("id") or ""))
return result
def get_parser_config(self, parser_id: str) -> Optional[Dict[str, Any]]:
config = self._parsers_config.get(str(parser_id or "").strip())
return copy.deepcopy(config) if isinstance(config, dict) else None
def install_parser_package(self, payload: Dict[str, Any], overwrite: bool = False) -> Dict[str, Any]:
if not parser_install_enabled():
raise PermissionError(
f"运行时安装解析器默认关闭(会以服务进程权限执行源码);"
f"仅可信管理员可设置 {PARSER_INSTALL_ENV}=true 后使用"
)
normalized = self._normalize_parser_package(payload)
parser_id = normalized["parser_id"]
module_name = normalized["module_name"]
target_path = self.PARSER_DIR / normalized["filename"]
had_previous_entry = parser_id in self._parsers_config
previous_entry = copy.deepcopy(self._parsers_config.get(parser_id)) if had_previous_entry else None
existing_entry = previous_entry if isinstance(previous_entry, dict) else None
registry_snapshot = dict(getattr(ParserRegistry, "_parsers", {}))
if ParserRegistry.exists(parser_id) and existing_entry is None:
raise ValueError(f"解析器 ID 已被内置解析器占用: {parser_id}")
for other_id, other_entry in self._parsers_config.items():
if other_id == parser_id:
continue
if not isinstance(other_entry, dict):
logger.warning(f"Skip invalid parser config [{other_id}]: entry must be an object")
continue
if str(other_entry.get("filename") or "") == normalized["filename"]:
raise ValueError(f"解析器模块名已被 {other_id} 占用: {module_name}")
if target_path.exists():
owned_filename = str((existing_entry or {}).get("filename") or "")
if not owned_filename or owned_filename != normalized["filename"]:
raise ValueError(f"解析器文件已存在,且不受市场安装器管理: {target_path.name}")
if existing_entry and str(existing_entry.get("filename") or "") != normalized["filename"]:
raise ValueError("暂不支持修改已安装解析器的模块名")
if existing_entry and not overwrite:
raise FileExistsError(f"解析器已存在: {parser_id}")
previous_source: Optional[str] = None
if target_path.exists():
previous_source = target_path.read_text(encoding="utf-8")
self._validate_parser_source(
source_code=normalized["source_code"],
class_name=normalized["class_name"],
filename=target_path.name,
)
self.PARSER_DIR.mkdir(parents=True, exist_ok=True)
entry = {
"id": parser_id,
"name": normalized["name"],
"description": normalized["description"],
"module": normalized["module_path"],
"class": normalized["class_name"],
"module_name": module_name,
"filename": normalized["filename"],
"enabled": True,
"installed_at": datetime.now(timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z"),
"supported_patterns": normalized["supported_patterns"],
}
try:
self._atomic_write_text(target_path, normalized["source_code"])
self._parsers_config[parser_id] = entry
self._config["parsers"] = self._parsers_config
self._config.setdefault("version", "1.0")
self._save_config()
self._load_parser_entry(parser_id, entry)
except Exception:
self._rollback_install(
parser_id=parser_id,
target_path=target_path,
previous_entry=previous_entry,
previous_source=previous_source,
registry_snapshot=registry_snapshot,
had_previous_entry=had_previous_entry,
)
raise
return {
"id": parser_id,
"name": normalized["name"],
"description": normalized["description"],
"class_name": normalized["class_name"],
"module_name": module_name,
"module": normalized["module_path"],
"filename": normalized["filename"],
"supported_patterns": copy.deepcopy(normalized["supported_patterns"]),
}
def _rollback_install(
self,
parser_id: str,
target_path: Path,
previous_entry: Optional[Any],
previous_source: Optional[str],
registry_snapshot: Optional[Dict[str, Any]] = None,
had_previous_entry: bool = False,
) -> None:
try:
if registry_snapshot is not None:
ParserRegistry._parsers = dict(registry_snapshot)
if previous_source is None:
if target_path.exists():
target_path.unlink()
else:
self._atomic_write_text(target_path, previous_source)
if previous_entry is None and not had_previous_entry:
self._parsers_config.pop(parser_id, None)
else:
self._parsers_config[parser_id] = previous_entry
self._config["parsers"] = self._parsers_config
self._save_config()
if isinstance(previous_entry, dict) and previous_entry:
self._load_parser_entry(parser_id, previous_entry)
except Exception as rollback_exc:
logger.error(f"Failed to rollback parser install [{parser_id}]: {rollback_exc}")
def _normalize_parser_package(self, payload: Dict[str, Any]) -> Dict[str, Any]:
if not isinstance(payload, dict):
raise ValueError("解析器包必须是对象")
nested = payload.get("parser_package")
if isinstance(nested, dict):
return self._normalize_parser_package(nested)
parser_id = str(payload.get("parser_id") or payload.get("id") or "").strip()
class_name = str(payload.get("class_name") or payload.get("class") or "").strip()
module_name = str(payload.get("module_name") or payload.get("module") or payload.get("filename") or "").strip()
source_code = str(payload.get("source_code") or payload.get("content") or payload.get("code") or "")
name = str(payload.get("name") or parser_id or class_name or "").strip()
description = str(payload.get("description") or "").strip()
supported_patterns = payload.get("supported_patterns") or payload.get("patterns") or []
if module_name.endswith(".py"):
module_name = module_name[:-3]
module_name = module_name.replace("-", "_").strip()
if not parser_id:
raise ValueError("缺少 parser_id")
if not class_name:
raise ValueError("缺少 class_name")
if not module_name:
raise ValueError("缺少 module_name")
if not source_code.strip():
raise ValueError("缺少 source_code")
if not self._ID_PATTERN.match(parser_id):
raise ValueError(f"解析器 ID 不合法: {parser_id}")
if not self._NAME_PATTERN.match(class_name):
raise ValueError(f"解析器类名不合法: {class_name}")
if not self._NAME_PATTERN.match(module_name):
raise ValueError(f"解析器模块名不合法: {module_name}")
normalized_patterns = [
str(pattern).strip()
for pattern in (supported_patterns if isinstance(supported_patterns, list) else [])
if str(pattern).strip()
]
return {
"parser_id": parser_id,
"class_name": class_name,
"module_name": module_name,
"module_path": f"app.core.parsers.{module_name}",
"filename": f"{module_name}.py",
"name": name or parser_id,
"description": description,
"supported_patterns": normalized_patterns,
"source_code": source_code,
}
@staticmethod
def _validate_parser_source(source_code: str, class_name: str, filename: str) -> None:
try:
tree = ast.parse(source_code, filename=filename)
except SyntaxError as exc:
raise ValueError(f"解析器源码存在语法错误: {exc}") from exc
class_names = {node.name for node in tree.body if isinstance(node, ast.ClassDef)}
if class_name not in class_names:
raise ValueError(f"解析器源码里没有找到类: {class_name}")
problems = check_import_time_side_effects(tree)
if problems:
raise ValueError(
"解析器源码在导入期会执行代码,已拒绝安装: " + "; ".join(problems[:5])
)
try:
compile(source_code, filename, "exec")
except SyntaxError as exc:
raise ValueError(f"解析器源码编译失败: {exc}") from exc
parser_manager = ParserConfigManager()
__all__ = ["ParserConfigManager", "parser_manager"]
+559
View File
@@ -0,0 +1,559 @@
"""
Helpers for request execution lifetime, worker cleanup, and tab retirement.
"""
import asyncio
import anyio
from contextlib import contextmanager
import os
import queue
import threading
import time
from typing import Any, Callable, Dict, Optional
from app.core.config import BrowserConstants, _request_context, get_logger
from app.services.request_manager import RequestContext, RequestStatus, request_manager
logger = get_logger("REQUEST.LIFECYCLE")
DEFAULT_MAX_REQUEST_EXECUTE_TIME_SEC = 300.0
MIN_MAX_REQUEST_EXECUTE_TIME_SEC = 5.0
WORKER_QUEUE_FINAL_GRACE_SEC = 5.0
WORKER_QUEUE_MAX_PUT_BLOCK_SEC = 0.1
WORKER_QUEUE_CANCEL_PUT_BLOCK_SEC = 0.01
WORKER_QUEUE_WAIT_BACKOFF_FACTOR = 1.8
COMPLETED_WORKER_FAST_JOIN_SEC = 0.5
COMPLETED_WORKER_RETIRE_GRACE_SEC = 30.0
@contextmanager
def request_finalization(ctx: RequestContext, context_token=None):
"""Shield only bounded cleanup, never browser execution, from ASGI cancellation.
Synchronous finalization is guaranteed even if cleanup raises or a second raw
asyncio Task.cancel() interrupts it. A request is not retried here.
"""
try:
with anyio.CancelScope(shield=True):
yield
finally:
try:
with ctx._lock:
already_finalized = getattr(ctx, '_lifecycle_finalized', False)
ctx._lifecycle_finalized = True
if not already_finalized:
request_manager.finish_request(ctx, success=(ctx.status == RequestStatus.COMPLETED))
finally:
if context_token is not None:
try:
_request_context.reset(context_token)
except (ValueError, RuntimeError):
# Explicit close usually runs in the original response context;
# generator finalization during shutdown may use another one.
logger.debug("Request context already released or finalized in another context")
class TrackedWorkerExecutionCancelled(Exception):
"""Raised when a tracked blocking worker keeps running after cancellation."""
def _coerce_max_request_execute_time_sec(raw: Any, default: float, source: str) -> float:
try:
value = float(raw)
except (TypeError, ValueError, OverflowError):
logger.debug(f"Invalid {source} MAX_REQUEST_EXECUTE_TIME_SEC={raw!r}, using default {default}s")
return max(MIN_MAX_REQUEST_EXECUTE_TIME_SEC, float(default))
if value <= 0:
return 0.0
return max(MIN_MAX_REQUEST_EXECUTE_TIME_SEC, value)
def get_max_request_execute_time_sec(default: float = DEFAULT_MAX_REQUEST_EXECUTE_TIME_SEC) -> float:
"""Return the request hard timeout; set MAX_REQUEST_EXECUTE_TIME_SEC<=0 to disable."""
env_raw = os.getenv("MAX_REQUEST_EXECUTE_TIME_SEC")
if env_raw is not None and str(env_raw).strip() != "":
return _coerce_max_request_execute_time_sec(env_raw, default, "env")
try:
config_raw = BrowserConstants.get("MAX_REQUEST_EXECUTE_TIME_SEC")
except Exception as e:
logger.debug(f"Read browser config MAX_REQUEST_EXECUTE_TIME_SEC failed: {e}")
config_raw = default
if config_raw is None or str(config_raw).strip() == "":
config_raw = default
return _coerce_max_request_execute_time_sec(config_raw, default, "browser_config")
def mark_request_hard_timeout(
ctx: RequestContext,
started_at: float,
max_execute_time_sec: Optional[float],
*,
label: str = "",
) -> bool:
"""Cancel ctx when an application-level hard timeout is reached."""
try:
max_seconds = (
get_max_request_execute_time_sec()
if max_execute_time_sec is None
else float(max_execute_time_sec)
)
except Exception:
max_seconds = DEFAULT_MAX_REQUEST_EXECUTE_TIME_SEC
if max_seconds <= 0:
return False
ctx_started_at = getattr(ctx, "started_at_monotonic", None)
if ctx_started_at is None:
ctx_started_at = float(started_at or time.monotonic())
elapsed = time.monotonic() - ctx_started_at
if elapsed < max_seconds:
return False
if not ctx.should_stop():
logger.warning(
f"[{ctx.request_id}] request hard timeout reached "
f"(elapsed={elapsed:.1f}s, limit={max_seconds:.1f}s, {label or '-'})"
)
ctx.request_cancel("absolute_request_timeout")
return True
def put_worker_queue_item(
chunk_queue: queue.Queue,
ctx: RequestContext,
item: Any,
*,
final: bool = False,
poll_timeout: float = 0.5,
) -> bool:
"""Put worker output with bounded backpressure and cancellation awareness."""
deadline = time.monotonic() + WORKER_QUEUE_FINAL_GRACE_SEC if final else None
while final or not ctx.should_stop():
try:
chunk_queue.put(
item,
timeout=_coerce_worker_queue_put_timeout(
poll_timeout,
cancelled=ctx.should_stop(),
),
)
return True
except queue.Full:
if final:
if ctx.should_stop() or (deadline is not None and time.monotonic() >= deadline):
return False
continue
if ctx.should_stop():
return False
return False
def _coerce_worker_queue_put_timeout(raw_timeout: Any, *, cancelled: bool = False) -> float:
try:
value = float(raw_timeout)
except (TypeError, ValueError, OverflowError):
value = 0.5
if cancelled:
return WORKER_QUEUE_CANCEL_PUT_BLOCK_SEC
return max(
WORKER_QUEUE_CANCEL_PUT_BLOCK_SEC,
min(value if value > 0 else 0.5, WORKER_QUEUE_MAX_PUT_BLOCK_SEC),
)
async def wait_worker_queue_item(
chunk_queue: queue.Queue,
*,
timeout: float = 0.5,
poll_interval: float = 0.01,
max_poll_interval: float = 0.05,
) -> Any:
"""Wait for a worker queue item without occupying the default executor."""
try:
timeout_value = float(timeout)
except (TypeError, ValueError, OverflowError):
timeout_value = 0.5
timeout_value = max(0.0, timeout_value)
try:
interval = float(poll_interval)
except (TypeError, ValueError, OverflowError):
interval = 0.01
interval = max(0.001, min(interval, 0.05))
try:
max_interval = float(max_poll_interval)
except (TypeError, ValueError, OverflowError):
max_interval = 0.05
max_interval = max(interval, min(max_interval, 0.05))
deadline = time.monotonic() + timeout_value
current_interval = min(interval, timeout_value) if timeout_value > 0 else interval
while True:
try:
return chunk_queue.get_nowait()
except queue.Empty:
remaining = deadline - time.monotonic()
if remaining <= 0:
raise
sleep_for = min(current_interval, remaining)
await asyncio.sleep(sleep_for)
current_interval = _next_worker_queue_wait_interval(
current_interval,
max_interval,
remaining,
)
def _next_worker_queue_wait_interval(current: float, max_interval: float, remaining: float) -> float:
try:
current_value = float(current)
except (TypeError, ValueError, OverflowError):
current_value = 0.001
try:
max_value = float(max_interval)
except (TypeError, ValueError, OverflowError):
max_value = 0.05
try:
remaining_value = float(remaining)
except (TypeError, ValueError, OverflowError):
remaining_value = max_value
max_value = max(0.001, min(max_value, 0.05))
current_value = max(0.001, current_value)
remaining_value = max(0.0, remaining_value)
next_value = current_value * WORKER_QUEUE_WAIT_BACKOFF_FACTOR
return max(0.001, min(next_value, max_value, remaining_value or max_value))
def _set_worker_future_result(
future: "asyncio.Future[Any]",
*,
result: Any = None,
error: Optional[BaseException] = None,
) -> None:
if future.done():
return
if error is not None:
future.set_exception(error)
return
future.set_result(result)
def _consume_worker_future_exception(future: "asyncio.Future[Any]") -> None:
"""Retrieve late worker failures after the awaiting request has been cancelled."""
if future.cancelled():
return
try:
future.exception()
except (asyncio.CancelledError, Exception):
pass
def _get_cancel_reason(ctx: RequestContext, fallback: str = "worker_cancelled") -> str:
reason = str(getattr(ctx, "cancel_reason", "") or "").strip()
return reason or fallback
def run_in_request_context(ctx: RequestContext, worker_fn: Callable[[], Any]) -> Any:
"""Run a callable with the request id attached to all logs it emits."""
token = _request_context.set(str(getattr(ctx, "request_id", "") or "SYSTEM"))
try:
return worker_fn()
finally:
_request_context.reset(token)
async def run_tracked_blocking_call(
worker_fn: Callable[[], Any],
*,
ctx: RequestContext,
worker_state: Dict[str, Any],
label: str,
poll_timeout: float = 0.5,
max_execute_time_sec: Optional[float] = None,
) -> Any:
"""Run a blocking callable on a daemon thread and keep it visible for cleanup."""
loop = asyncio.get_running_loop()
result_future: "asyncio.Future[Any]" = loop.create_future()
started_at = time.monotonic()
def worker() -> None:
try:
result = run_in_request_context(ctx, worker_fn)
except Exception as exc:
try:
loop.call_soon_threadsafe(
lambda exc=exc: _set_worker_future_result(result_future, error=exc)
)
except RuntimeError:
pass
else:
try:
loop.call_soon_threadsafe(
lambda result=result: _set_worker_future_result(result_future, result=result)
)
except RuntimeError:
pass
worker_thread = threading.Thread(
target=worker,
daemon=True,
name=f"tracked-worker-{label}",
)
worker_state["thread"] = worker_thread
worker_state["label"] = label
ctx.worker_thread = worker_thread
def _clear_finished_worker_state() -> None:
if worker_state.get("thread") is worker_thread:
worker_state["thread"] = None
worker_state["label"] = None
def _handle_finished_worker(future: "asyncio.Future[Any]") -> None:
_clear_finished_worker_state()
_consume_worker_future_exception(future)
result_future.add_done_callback(_handle_finished_worker)
worker_thread.start()
try:
while True:
if mark_request_hard_timeout(
ctx,
started_at,
max_execute_time_sec,
label=f"worker={label}",
):
raise TrackedWorkerExecutionCancelled("absolute_request_timeout")
if ctx.should_stop():
raise TrackedWorkerExecutionCancelled(_get_cancel_reason(ctx))
try:
return await asyncio.wait_for(
asyncio.shield(result_future),
timeout=max(0.05, float(poll_timeout or 0.5)),
)
except asyncio.TimeoutError:
continue
finally:
if result_future.done():
_clear_finished_worker_state()
def _resolve_raw_tab_id(pool: Any, session: Any) -> str:
raw_tab_id = ""
if pool is not None:
try:
persistent_index = int(getattr(session, "persistent_index", 0) or 0)
for candidate_raw_id, candidate_index in getattr(pool, "_raw_id_to_persistent", {}).items():
if int(candidate_index or 0) == persistent_index:
raw_tab_id = str(candidate_raw_id or "").strip()
break
except Exception:
raw_tab_id = ""
if raw_tab_id:
return raw_tab_id
try:
return str(getattr(getattr(session, "tab", None), "tab_id", "") or "").strip()
except Exception:
return ""
def retire_bound_tab_after_worker_leak(ctx: RequestContext, reason: str) -> None:
"""Mark the tab bound to ctx as unhealthy without closing its browser target."""
tab_id = str(getattr(ctx, "tab_id", "") or "").strip()
if not tab_id:
return
try:
from app.core import get_browser
browser = get_browser(auto_connect=False)
pool = getattr(browser, "_tab_pool", None)
session = getattr(pool, "_tabs", {}).get(tab_id) if pool is not None else None
if session is None:
return
current_task = str(getattr(session, "current_task_id", "") or "").strip()
bound_request_id = str(getattr(session, "_bound_request_id", "") or "").strip()
if current_task and current_task != ctx.request_id:
logger.warning(
f"[{tab_id}] leaked worker found but tab has another task; skip retire "
f"(request={ctx.request_id}, current_task={current_task})"
)
return
if bound_request_id and bound_request_id != ctx.request_id:
logger.warning(
f"[{tab_id}] leaked worker found but bound request changed; skip retire "
f"(request={ctx.request_id}, bound={bound_request_id})"
)
return
if hasattr(session, "mark_error"):
session.mark_error(reason)
raw_tab_id = _resolve_raw_tab_id(pool, session)
if pool is not None:
try:
monitor = getattr(pool, "_global_network_monitor", None)
if monitor is not None and hasattr(monitor, "request_stop_for_session"):
monitor.request_stop_for_session(tab_id, reason=reason, detach=True)
except Exception as e:
logger.debug(f"[{tab_id}] stop leaked tab global monitor failed: {e}")
if pool is not None and hasattr(pool, "_condition"):
try:
with pool._condition:
pool._condition.notify_all()
except Exception:
pass
logger.warning(
f"[{tab_id}] worker did not exit in time; marked tab ERROR "
f"(request={ctx.request_id}, reason={reason}, raw={raw_tab_id or '-'}, "
"browser_tab_retained=True)"
)
except Exception as e:
logger.debug(f"retire leaked worker tab failed: {e}")
def cleanup_worker_thread(
worker_thread: Optional[threading.Thread],
ctx: RequestContext,
*,
cancel_reason: str = "cleanup",
join_timeout: float = 5.0,
retire_reason: str = "worker_cleanup_timeout",
) -> bool:
"""Cancel, join, and retire a worker thread that outlives its request."""
if not isinstance(worker_thread, threading.Thread) or not worker_thread.is_alive():
return False
ctx.request_cancel(cancel_reason)
worker_thread.join(timeout=max(0.0, float(join_timeout or 0.0)))
if worker_thread.is_alive():
retire_bound_tab_after_worker_leak(ctx, retire_reason)
return True
return False
def schedule_completed_worker_retire_check(
worker_thread: threading.Thread,
ctx: RequestContext,
reason: str,
*,
delay_sec: float = COMPLETED_WORKER_RETIRE_GRACE_SEC,
) -> None:
"""Watch a completed request's slow worker cleanup without blocking response finalization."""
try:
delay = max(0.1, float(delay_sec or COMPLETED_WORKER_RETIRE_GRACE_SEC))
except Exception:
delay = COMPLETED_WORKER_RETIRE_GRACE_SEC
def _wait_and_retire_if_still_alive() -> None:
started_at = time.monotonic()
worker_thread.join(timeout=delay)
elapsed = time.monotonic() - started_at
if worker_thread.is_alive():
logger.warning(
f"[{ctx.request_id}] completed request worker still alive after cleanup grace; "
f"retiring bound tab (tab={ctx.tab_id or '-'}, reason={reason}, "
f"grace={delay:.1f}s)"
)
retire_bound_tab_after_worker_leak(ctx, reason)
return
logger.debug(
f"[{ctx.request_id}] completed request worker cleanup finished in background "
f"(elapsed={elapsed:.2f}s, tab={ctx.tab_id or '-'})"
)
threading.Thread(
target=_wait_and_retire_if_still_alive,
daemon=True,
name=f"completed-worker-cleanup-{ctx.request_id[:8]}",
).start()
async def _wait_for_worker_exit(worker_thread: threading.Thread, timeout: float) -> None:
"""Bound cleanup wall time without queueing joins in the shared executor.
A saturated default executor must not postpone cleanup or starve the next
request's route discovery. is_alive() is nonblocking; cancellation is still
allowed here, and callers shield only this bounded cleanup window.
"""
deadline = time.monotonic() + max(0.0, timeout)
while worker_thread.is_alive():
remaining = deadline - time.monotonic()
if remaining <= 0:
break
await asyncio.sleep(min(.02, remaining))
async def cleanup_worker_thread_after_request(
worker_thread: Optional[threading.Thread],
ctx: RequestContext,
*,
completed: bool = False,
cancel_reason: str = "cleanup",
join_timeout: float = 5.0,
retire_reason: str = "worker_cleanup_timeout",
completed_join_timeout: float = COMPLETED_WORKER_FAST_JOIN_SEC,
completed_retire_delay: float = COMPLETED_WORKER_RETIRE_GRACE_SEC,
) -> bool:
"""Cleanup a request worker, deferring leak retirement for already-completed requests.
Returns True when the worker was still alive after the immediate cleanup window.
"""
if not isinstance(worker_thread, threading.Thread) or not worker_thread.is_alive():
return False
if completed:
try:
fast_join = max(0.0, float(completed_join_timeout or 0.0))
except Exception:
fast_join = COMPLETED_WORKER_FAST_JOIN_SEC
try:
await _wait_for_worker_exit(worker_thread, fast_join)
except asyncio.CancelledError:
if worker_thread.is_alive():
schedule_completed_worker_retire_check(worker_thread, ctx, retire_reason, delay_sec=completed_retire_delay)
raise
if worker_thread.is_alive():
logger.warning(
f"[{ctx.request_id}] completed request worker cleanup is still running; "
f"defer tab retirement check (tab={ctx.tab_id or '-'}, "
f"fast_join={fast_join:.1f}s, grace={completed_retire_delay:.1f}s)"
)
schedule_completed_worker_retire_check(
worker_thread,
ctx,
retire_reason,
delay_sec=completed_retire_delay,
)
return True
return False
ctx.request_cancel(cancel_reason)
try:
timeout = max(0.0, float(join_timeout or 0.0))
except Exception:
timeout = 5.0
try:
await _wait_for_worker_exit(worker_thread, timeout)
except asyncio.CancelledError:
if worker_thread.is_alive():
schedule_completed_worker_retire_check(worker_thread, ctx, retire_reason, delay_sec=timeout)
raise
if worker_thread.is_alive():
retire_bound_tab_after_worker_leak(ctx, retire_reason)
return True
return False
File diff suppressed because it is too large. Load diff
Loaded 100 of 7311 files, more files were not shown because too many files have changed in this diff. Show more