Files
universal-web-api/app/services/command_engine.py
T
2026-11-02 22:54:14 +08:00

650 lines
26 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""
app/services/command_engine.py - 命令引擎
职责:
- 命令的 CRUD 管理
- 触发条件检查(在标签页释放后调用)
- 动作执行调度
- 高级模式脚本执行(JavaScript / Python)
存储位置:config/commands.json
"""
import copy
import os
import threading
from concurrent.futures import ThreadPoolExecutor
from typing import Dict, List, Optional, Any, TYPE_CHECKING
from app.core.page_lifecycle import (
BACKGROUND_WAKE_JS_TIMEOUT,
)
from app.services.command_defs import ACTION_TYPES, TRIGGER_TYPES, get_default_command
from app.services.command_engine_actions import CommandEngineActionsMixin
from app.services.command_engine_results import CommandEngineResultsMixin
from app.services.command_engine_runtime import CommandEngineRuntimeMixin
from app.services.command_engine_pages import CommandEnginePagesMixin
from app.services.command_engine_scheduler import CommandEngineSchedulerMixin
from app.services.command_engine_store import CommandEngineStoreMixin
from app.services.command_engine_crud import CommandEngineCrudMixin
from app.services.command_engine_triggers import CommandEngineTriggersMixin
from app.services.command_engine_page_check import CommandEnginePageCheckMixin
if TYPE_CHECKING:
from app.core.tab_pool import TabSession
# R2-3:模块级常量、日志器与可选依赖迁到 command_engine_common,这里重新导出以保持兼容
from app.services.command_engine_common import ( # noqa: E402,F401
FOLLOW_DEFAULT_PRESET,
HAS_REQUESTS,
_NON_ASCII_RE,
_WHITESPACE_RE,
_WORD_LIKE_KEYWORD_RE,
logger,
)
# ================= 常量 =================
class CommandEngine(
CommandEnginePagesMixin,
CommandEngineSchedulerMixin,
CommandEngineStoreMixin,
CommandEngineCrudMixin,
CommandEngineTriggersMixin,
CommandEnginePageCheckMixin,
CommandEngineRuntimeMixin,
CommandEngineResultsMixin,
CommandEngineActionsMixin,
):
"""命令引擎"""
def __init__(self):
self._config_engine = None
self._browser = None
self._commands_file = None
self._commands_local_file = None
self._commands_mtime = 0.0
self._commands_local_mtime = 0.0
# B8:最近一次「读失败」的命令文件 mtime;同一 mtime 不重复读取/刷错误日志
self._commands_failed_mtime: Optional[float] = None
# (cache_key, compact_text):page_check 去空白页面文本的单槽缓存
self._compact_haystack_cache: Optional[tuple] = None
self._commands_loaded = False
self._commands_cache: List[Dict[str, Any]] = []
self._command_runtime_stats: Dict[str, Dict[str, Any]] = {}
self._lock = threading.RLock()
self._commands_lock = threading.RLock()
self._scheduler_lifecycle_lock = threading.RLock()
self._shutdown_requested = False
# 触发状态:{(command_id, tab_id): {"req": int, "err": int, ...}}
self._trigger_states: Dict[tuple, Dict[str, Any]] = {}
self._pending_async_trigger_meta: Dict[tuple, Dict[str, Any]] = {}
# 最近命令执行结果:{(source_command_id, tab_id): {...}}
self._command_results: Dict[tuple, Dict[str, Any]] = {}
# 命令结果事件:{tab_id: [event, ...]}
self._command_result_events: Dict[str, List[Dict[str, Any]]] = {}
# 最近网络事件:{tab_id: [event, ...]}
self._network_events: Dict[str, List[Dict[str, Any]]] = {}
# 正在执行的命令(防止重复触发)
self._executing: set = set()
self._periodic_next_run: Dict[tuple, float] = {}
self._periodic_stop_event = threading.Event()
self._periodic_thread: Optional[threading.Thread] = None
self._pending_high_by_session: Dict[str, int] = {}
self._running_high_by_session: Dict[str, int] = {}
self._pending_high_by_domain: Dict[str, int] = {}
self._running_high_by_domain: Dict[str, int] = {}
try:
_max_async_workers = int(os.getenv("CMD_ASYNC_MAX_WORKERS", "20"))
except Exception:
_max_async_workers = 20
self._command_executor = ThreadPoolExecutor(
max_workers=max(1, _max_async_workers),
thread_name_prefix="cmd-exec",
)
try:
_baseline = int(os.getenv("CMD_REQUEST_PRIORITY_BASELINE", "2"))
except Exception:
_baseline = 2
self._request_priority_baseline = _baseline
self._activate_tab_on_command = str(
os.getenv("CMD_ACTIVATE_TAB_ON_COMMAND", "false")
).strip().lower() in {"1", "true", "yes", "y", "on"}
self._use_focus_emulation_on_command = str(
os.getenv("CMD_USE_FOCUS_EMULATION_ON_COMMAND", "true")
).strip().lower() in {"1", "true", "yes", "y", "on"}
self._wake_tab_before_page_check = str(
os.getenv("CMD_WAKE_TAB_BEFORE_PAGE_CHECK", "true")
).strip().lower() in {"1", "true", "yes", "y", "on"}
# 唤醒标签页的 CDP 状态(focus / visibility / Page.setWebLifecycleState=active)
# 是粘性的,不需要每条 page_check 命令都重发一遍。
# 每次唤醒是 4~6 次 CDP 往返 + 一次整页可见性补丁 JS,
# 而一个会话上往往同时挂着多条 page_check 命令,实际调用频率是每秒数次。
# 默认 2s 限频:既能把唤醒风暴压下去,最坏也只比原来晚 2s 唤醒
# (page_check 自身的最小轮询间隔就是 1.5s,量级相当)。
# 设为 0 可恢复"每次都唤醒"的旧行为;若观察到后台页仍被冻结可调小。
try:
_wake_min_interval = float(os.getenv("CMD_WAKE_TAB_MIN_INTERVAL_SEC", "2"))
except Exception:
_wake_min_interval = 2.0
self._wake_tab_min_interval_sec = max(0.0, _wake_min_interval)
self._tab_pool_auto_refresh = str(
os.getenv("CMD_TAB_POOL_AUTO_REFRESH", "true")
).strip().lower() in {"1", "true", "yes", "y", "on"}
try:
_refresh_interval = float(os.getenv("CMD_TAB_POOL_REFRESH_INTERVAL_SEC", "5"))
except Exception:
_refresh_interval = 5.0
self._tab_pool_refresh_interval_sec = max(1.0, _refresh_interval)
self._last_tab_pool_refresh_at = 0.0
# In memory saver mode, do not wake every idle tab on a fixed cadence.
# A page_check or real workflow still wakes its target tab on demand.
memory_saver_enabled = str(
os.getenv("BROWSER_MEMORY_SAVER", "false")
).strip().lower() in {"1", "true", "yes", "y", "on"}
# P0-6:开启空闲冻结时,周期 keepalive 会不断唤醒标签页,默认关闭
# (显式设置 CMD_PERIODIC_KEEPALIVE_ENABLED 仍以用户为准)。
try:
from app.core.tab_pool_parts.idle_maintenance import is_idle_freeze_enabled
idle_freeze_enabled = is_idle_freeze_enabled()
except Exception:
idle_freeze_enabled = False
keepalive_default = "false" if (memory_saver_enabled or idle_freeze_enabled) else "true"
self._periodic_keepalive_enabled = str(
os.getenv("CMD_PERIODIC_KEEPALIVE_ENABLED", keepalive_default)
).strip().lower() in {"1", "true", "yes", "y", "on"}
try:
_keepalive_interval = float(os.getenv("CMD_PERIODIC_KEEPALIVE_INTERVAL_SEC", "20"))
except Exception:
_keepalive_interval = 20.0
self._periodic_keepalive_interval_sec = max(5.0, _keepalive_interval)
try:
_page_check_js_timeout = float(os.getenv("CMD_PAGE_CHECK_JS_TIMEOUT_SEC", str(BACKGROUND_WAKE_JS_TIMEOUT)))
except Exception:
_page_check_js_timeout = BACKGROUND_WAKE_JS_TIMEOUT
self._page_check_js_timeout_sec = max(0.1, min(2.0, _page_check_js_timeout))
try:
_page_check_failure_backoff = float(os.getenv("CMD_PAGE_CHECK_FAILURE_BACKOFF_SEC", "30"))
except Exception:
_page_check_failure_backoff = 30.0
self._page_check_failure_backoff_sec = max(1.0, _page_check_failure_backoff)
try:
_page_check_refresh_grace = float(
os.getenv("CMD_PAGE_CHECK_REFRESH_GRACE_SEC", "2")
)
except Exception:
_page_check_refresh_grace = 2.0
self._page_check_refresh_grace_sec = max(0.2, min(10.0, _page_check_refresh_grace))
self._last_keepalive_by_session: Dict[str, float] = {}
self._last_tab_pool_wait_log_at = 0.0
self._last_periodic_summary_log_at = 0.0
self._periodic_active_log_interval_sec = 8.0
self._periodic_idle_log_interval_sec = 30.0
self._periodic_summary_log_enabled = str(
os.getenv("CMD_PERIODIC_SUMMARY_LOG_ENABLED", "false")
).strip().lower() in {"1", "true", "yes", "y", "on"}
# page_check observer: {session_id: set_of_keywords_installed}
self._observer_keywords_by_session: Dict[str, set] = {}
logger.debug("命令引擎已初始化")
if self._should_auto_start_scheduler():
self._start_periodic_scheduler()
def evict_session(self, session_id: str):
"""Evict runtime caches and pending state associated with a removed tab session."""
session_key = str(session_id or "").strip()
if not session_key:
return
logger.info(f"逐出会话缓存: {session_key}")
with self._lock:
keys_to_remove = [
key
for key in self._trigger_states
if isinstance(key, tuple) and len(key) > 1 and key[1] == session_key
]
for key in keys_to_remove:
self._trigger_states.pop(key, None)
meta_keys_to_remove = [
key
for key in self._pending_async_trigger_meta
if isinstance(key, tuple) and len(key) > 1 and key[1] == session_key
]
for key in meta_keys_to_remove:
self._pending_async_trigger_meta.pop(key, None)
result_keys_to_remove = [
key
for key in self._command_results
if isinstance(key, tuple) and len(key) > 1 and key[1] == session_key
]
for key in result_keys_to_remove:
self._command_results.pop(key, None)
exec_keys_to_remove = [
key
for key in self._executing
if isinstance(key, tuple) and len(key) > 1 and key[1] == session_key
]
for key in exec_keys_to_remove:
self._executing.discard(key)
run_keys_to_remove = [
key
for key in self._periodic_next_run
if isinstance(key, tuple) and len(key) > 1 and key[1] == session_key
]
for key in run_keys_to_remove:
self._periodic_next_run.pop(key, None)
self._command_result_events.pop(session_key, None)
self._network_events.pop(session_key, None)
self._pending_high_by_session.pop(session_key, None)
self._running_high_by_session.pop(session_key, None)
self._last_keepalive_by_session.pop(session_key, None)
self._observer_keywords_by_session.pop(session_key, None)
def _get_config_engine(self):
if self._config_engine is None:
from app.services.config_engine import config_engine
self._config_engine = config_engine
return self._config_engine
def _get_browser(self):
if self._browser is None:
from app.core.browser import get_browser
self._browser = get_browser(auto_connect=False)
return self._browser
@staticmethod
def _format_scope_label(scope: str) -> str:
mapping = {
"all": "全部标签页",
"domain": "同域",
"tab": "当前标签页",
}
return mapping.get(str(scope or "").strip().lower(), str(scope or "未指定"))
def _set_pending_async_trigger_meta(
self,
command: Dict[str, Any],
session: 'TabSession',
meta: Optional[Dict[str, Any]],
):
key = (str(command.get("id", "")).strip(), str(getattr(session, "id", "") or ""))
if not all(key):
return
with self._lock:
if meta:
self._pending_async_trigger_meta[key] = copy.deepcopy(meta)
else:
self._pending_async_trigger_meta.pop(key, None)
def _take_pending_async_trigger_meta(
self,
command: Dict[str, Any],
session: 'TabSession',
) -> Optional[Dict[str, Any]]:
key = (str(command.get("id", "")).strip(), str(getattr(session, "id", "") or ""))
if not all(key):
return None
with self._lock:
meta = self._pending_async_trigger_meta.pop(key, None)
return copy.deepcopy(meta) if meta else None
_PAGE_CHECK_OBSERVER_JS = r"""
(function() {
var kws = %KEYWORDS%;
function newSeen() {
// Set 判重是 O(1);旧实现用数组 + indexOf 是 O(n),整页遍历退化成
// O(n^2),长对话页面每次快照要跑上千万次比较,是浏览器端 CPU 的主要来源。
try {
if (typeof Set === 'function') {
var s = new Set();
return {
has: function(n) { return s.has(n); },
add: function(n) { s.add(n); }
};
}
} catch (e) {}
var arr = [];
return {
has: function(n) { return arr.indexOf(n) !== -1; },
add: function(n) { arr.push(n); }
};
}
function appendText(parts, value) {
if (typeof value === 'string' && value) parts.push(value);
}
function collectText(node, parts, seen) {
if (!node || seen.has(node)) return;
seen.add(node);
if (node.nodeType === 1) {
var tag = String(node.tagName || '').toUpperCase();
if (
tag === 'SCRIPT' ||
tag === 'STYLE' ||
tag === 'NOSCRIPT' ||
tag === 'TEMPLATE' ||
tag === 'META' ||
tag === 'HEAD'
) {
return;
}
try {
if (node.shadowRoot) collectText(node.shadowRoot, parts, seen);
} catch (e) {}
if (tag === 'IFRAME') {
try {
var doc = node.contentDocument || (node.contentWindow && node.contentWindow.document);
if (doc) collectText(doc.documentElement || doc.body, parts, seen);
} catch (e) {}
}
}
if (node.nodeType === 3) {
appendText(parts, String(node.nodeValue || '').trim());
}
var child = null;
try {
child = node.firstChild;
} catch (e) {}
while (child) {
collectText(child, parts, seen);
try {
child = child.nextSibling;
} catch (e) {
child = null;
}
}
}
function collectRootText(root, parts, seen) {
collectText(root, parts, seen);
}
function isElementVisible(el) {
if (!el) return false;
try {
var style = window.getComputedStyle ? window.getComputedStyle(el) : null;
var rect = el.getBoundingClientRect ? el.getBoundingClientRect() : null;
if (style) {
if (style.display === 'none' || style.visibility === 'hidden') return false;
if (Number(style.opacity || 1) <= 0.02) return false;
}
return !!rect && rect.width > 5 && rect.height > 5;
} catch (e) {
return false;
}
}
function hasVisibleSelector(selector) {
try {
var els = document.querySelectorAll(selector);
for (var i = 0; i < els.length; i++) {
if (isElementVisible(els[i])) return true;
}
return false;
} catch (e) {
return false;
}
}
function buildSnapshot() {
var parts = [];
var seen = newSeen();
appendText(parts, document.title || '');
collectRootText(document.documentElement || document.body, parts, seen);
var text = parts.join('\n').toLowerCase();
var cfIndicators = [];
if (text.indexOf('security verification') !== -1) cfIndicators.push('security verification');
if (text.indexOf('protected by cloudflare') !== -1) cfIndicators.push('protected by cloudflare');
if (text.indexOf('verify you are human') !== -1) cfIndicators.push('verify you are human');
if (text.indexOf('checking your browser') !== -1) cfIndicators.push('checking your browser');
if (text.indexOf('确认您是真人') !== -1) cfIndicators.push('确认您是真人');
if (
hasVisibleSelector('iframe[src*="challenges.cloudflare.com"]') ||
hasVisibleSelector('iframe[src*="turnstile"]') ||
hasVisibleSelector('.cf-turnstile') ||
hasVisibleSelector('[name="cf-turnstile-response"]') ||
hasVisibleSelector('[data-testid*="cf" i]')
) {
cfIndicators.push('cloudflare');
} else if (text.indexOf('cloudflare') !== -1) {
cfIndicators.push('cloudflare');
}
if (cfIndicators.length) {
text += '\n' + cfIndicators.join('\n');
}
var recaptchaIndicators = [];
if (text.indexOf('protected by recaptcha') !== -1) recaptchaIndicators.push('protected by recaptcha');
if (
hasVisibleSelector('iframe[src*="google.com/recaptcha"]') ||
hasVisibleSelector('iframe[src*="recaptcha"]') ||
hasVisibleSelector('.g-recaptcha') ||
hasVisibleSelector('[name="g-recaptcha-response"]') ||
hasVisibleSelector('[title*="recaptcha" i]') ||
hasVisibleSelector('[aria-label*="recaptcha" i]')
) {
recaptchaIndicators.push('recaptcha');
recaptchaIndicators.push('\u4eba\u673a\u8eab\u4efd\u9a8c\u8bc1');
} else if (text.indexOf('recaptcha') !== -1) {
recaptchaIndicators.push('recaptcha');
}
if (recaptchaIndicators.length) {
text += '\n' + recaptchaIndicators.join('\n');
}
return text;
}
if (window.__pcObserver && window.__pcKeywords) {
var same = kws.length === window.__pcKeywords.length &&
kws.every(function(k){ return window.__pcKeywords.indexOf(k) !== -1; });
if (same) return 'already_installed';
window.__pcObserver.disconnect();
window.__pcObserver = null;
}
window.__pcKeywords = kws;
window.__pcHits = {};
var pending = null;
var MIN_DELAY_MS = 200;
var MAX_DELAY_MS = 800;
// 自适应节流:下一次扫描的间隔取"上次扫描耗时 × 4",钳在 200~800ms。
// 换成 Set 判重后,即使 6 万节点的页面单次扫描也只要 30ms 左右,
// 算出来仍是下限 200ms —— 正常页面行为完全不变。
// 只有扫描耗时超过 50ms 的病态页面才会退避,且最多退到 800ms,
// 不会拖慢 page_check 对 Cloudflare 盾 / 报错文案的识别。
var nextDelayMs = MIN_DELAY_MS;
function doCheck() {
pending = null;
var startedAt = 0;
try {
startedAt = (window.performance && performance.now) ? performance.now() : Date.now();
} catch (e) {}
try {
var text = buildSnapshot();
window.__pcSnapshot = text;
for (var i = 0; i < window.__pcKeywords.length; i++) {
var k = window.__pcKeywords[i];
window.__pcHits[k] = text.indexOf(k) !== -1;
}
} catch(e) {}
try {
var endedAt = (window.performance && performance.now) ? performance.now() : Date.now();
var cost = Math.max(0, endedAt - startedAt);
nextDelayMs = Math.min(MAX_DELAY_MS, Math.max(MIN_DELAY_MS, Math.round(cost * 4)));
} catch (e) {
nextDelayMs = MIN_DELAY_MS;
}
}
doCheck();
window.__pcObserver = new MutationObserver(function() {
if (!pending) pending = setTimeout(doCheck, nextDelayMs);
});
var target = document.body || document.documentElement;
if (target) {
window.__pcObserver.observe(target, {
childList: true, subtree: true, characterData: true
});
}
return 'installed';
})();
"""
_PAGE_CHECK_SNAPSHOT_JS = r"""
return (function() {
function newSeen() {
// 同 _PAGE_CHECK_OBSERVER_JS:数组 indexOf 判重会让整页遍历退化成 O(n^2)。
try {
if (typeof Set === 'function') {
var s = new Set();
return {
has: function(n) { return s.has(n); },
add: function(n) { s.add(n); }
};
}
} catch (e) {}
var arr = [];
return {
has: function(n) { return arr.indexOf(n) !== -1; },
add: function(n) { arr.push(n); }
};
}
function appendText(parts, value) {
if (typeof value === 'string' && value) parts.push(value);
}
function collectText(node, parts, seen) {
if (!node || seen.has(node)) return;
seen.add(node);
if (node.nodeType === 1) {
var tag = String(node.tagName || '').toUpperCase();
if (
tag === 'SCRIPT' ||
tag === 'STYLE' ||
tag === 'NOSCRIPT' ||
tag === 'TEMPLATE' ||
tag === 'META' ||
tag === 'HEAD'
) {
return;
}
try {
if (node.shadowRoot) collectText(node.shadowRoot, parts, seen);
} catch (e) {}
if (tag === 'IFRAME') {
try {
var doc = node.contentDocument || (node.contentWindow && node.contentWindow.document);
if (doc) collectText(doc.documentElement || doc.body, parts, seen);
} catch (e) {}
}
}
if (node.nodeType === 3) {
appendText(parts, String(node.nodeValue || '').trim());
}
var child = null;
try {
child = node.firstChild;
} catch (e) {}
while (child) {
collectText(child, parts, seen);
try {
child = child.nextSibling;
} catch (e) {
child = null;
}
}
}
function collectRootText(root, parts, seen) {
collectText(root, parts, seen);
}
function isElementVisible(el) {
if (!el) return false;
try {
var style = window.getComputedStyle ? window.getComputedStyle(el) : null;
var rect = el.getBoundingClientRect ? el.getBoundingClientRect() : null;
if (style) {
if (style.display === 'none' || style.visibility === 'hidden') return false;
if (Number(style.opacity || 1) <= 0.02) return false;
}
return !!rect && rect.width > 5 && rect.height > 5;
} catch (e) {
return false;
}
}
function hasVisibleSelector(selector) {
try {
var els = document.querySelectorAll(selector);
for (var i = 0; i < els.length; i++) {
if (isElementVisible(els[i])) return true;
}
return false;
} catch (e) {
return false;
}
}
var parts = [];
appendText(parts, document.title || '');
collectRootText(document.documentElement || document.body, parts, newSeen());
var text = parts.join('\n').toLowerCase();
var cfIndicators = [];
if (text.indexOf('security verification') !== -1) cfIndicators.push('security verification');
if (text.indexOf('protected by cloudflare') !== -1) cfIndicators.push('protected by cloudflare');
if (text.indexOf('verify you are human') !== -1) cfIndicators.push('verify you are human');
if (text.indexOf('checking your browser') !== -1) cfIndicators.push('checking your browser');
if (text.indexOf('确认您是真人') !== -1) cfIndicators.push('确认您是真人');
if (
hasVisibleSelector('iframe[src*="challenges.cloudflare.com"]') ||
hasVisibleSelector('iframe[src*="turnstile"]') ||
hasVisibleSelector('.cf-turnstile') ||
hasVisibleSelector('[name="cf-turnstile-response"]') ||
hasVisibleSelector('[data-testid*="cf" i]')
) {
cfIndicators.push('cloudflare');
} else if (text.indexOf('cloudflare') !== -1) {
cfIndicators.push('cloudflare');
}
if (cfIndicators.length) {
text += '\n' + cfIndicators.join('\n');
}
var recaptchaIndicators = [];
if (text.indexOf('protected by recaptcha') !== -1) recaptchaIndicators.push('protected by recaptcha');
if (
hasVisibleSelector('iframe[src*="google.com/recaptcha"]') ||
hasVisibleSelector('iframe[src*="recaptcha"]') ||
hasVisibleSelector('.g-recaptcha') ||
hasVisibleSelector('[name="g-recaptcha-response"]') ||
hasVisibleSelector('[title*="recaptcha" i]') ||
hasVisibleSelector('[aria-label*="recaptcha" i]')
) {
recaptchaIndicators.push('recaptcha');
recaptchaIndicators.push('\u4eba\u673a\u8eab\u4efd\u9a8c\u8bc1');
} else if (text.indexOf('recaptcha') !== -1) {
recaptchaIndicators.push('recaptcha');
}
if (recaptchaIndicators.length) {
text += '\n' + recaptchaIndicators.join('\n');
}
return text;
})();
"""
def shutdown(self):
with self._scheduler_lifecycle_lock:
self._shutdown_requested = True
self._periodic_stop_event.set()
thread = self._periodic_thread
if thread and thread.is_alive():
thread.join(timeout=1.0)
try:
self._command_executor.shutdown(wait=False, cancel_futures=True)
except TypeError:
self._command_executor.shutdown(wait=False)
except Exception as e:
logger.debug(f"[CMD] 命令线程池关闭失败(忽略): {e}")
# ================= 单例 =================
command_engine = CommandEngine()
__all__ = [
'CommandEngine',
'command_engine',
'TRIGGER_TYPES',
'ACTION_TYPES',
'get_default_command',
]