Files
universal-web-api/app/services/command_engine_scheduler.py
T
2026-11-02 22:54:14 +08:00

267 lines
11 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""CommandEngine 的一部分(R2-3 拆分):周期调度:保活、标签页池刷新、周期检查循环。
方法原样从 command_engine.py 迁出,行为不变;通过 mixin 组合回 CommandEngine,
类属性与实例状态都在 CommandEngine(command_engine.py)里定义。
"""
import os
import random
import threading
import time
from typing import Dict, List, Any, TYPE_CHECKING
from app.services.command_engine_common import logger
if TYPE_CHECKING:
from app.core.tab_pool import TabSession # noqa: F401
class CommandEngineSchedulerMixin:
"""周期调度:保活、标签页池刷新、周期检查循环"""
@staticmethod
def _should_auto_start_scheduler() -> bool:
# The application lifespan explicitly owns scheduler startup/shutdown.
# Starting here makes a plain module import (including pytest test
# collection) leak a background thread with no matching shutdown.
return str(os.getenv("CMD_ENGINE_AUTO_START", "false")).strip().lower() in {
"1",
"true",
"yes",
"y",
"on",
}
def _should_log_periodic_summary(self, now_ts: float, due_total: int) -> bool:
if not self._periodic_summary_log_enabled:
return False
interval = (
self._periodic_active_log_interval_sec
if due_total > 0
else self._periodic_idle_log_interval_sec
)
if (now_ts - self._last_periodic_summary_log_at) < interval:
return False
self._last_periodic_summary_log_at = now_ts
return True
def _refresh_tab_pool_if_due(self, pool: Any):
if not self._tab_pool_auto_refresh:
return
now = time.time()
if (now - self._last_tab_pool_refresh_at) < self._tab_pool_refresh_interval_sec:
return
self._last_tab_pool_refresh_at = now
try:
if hasattr(pool, "refresh_tabs"):
pool.refresh_tabs()
except Exception as e:
logger.debug(f"[CMD] 刷新标签页池失败(忽略): {e}")
def _maybe_periodic_keepalive(self, session: 'TabSession', now_ts: float):
if not self._periodic_keepalive_enabled:
return
sid = str(getattr(session, "id", "") or "")
if not sid:
return
last_at = float(self._last_keepalive_by_session.get(sid, 0.0) or 0.0)
if (now_ts - last_at) < self._periodic_keepalive_interval_sec:
return
self._last_keepalive_by_session[sid] = now_ts
self._try_wake_tab(session, reason="periodic_keepalive")
def _start_periodic_scheduler(self):
with self._scheduler_lifecycle_lock:
if self._shutdown_requested:
return False
if self._periodic_thread and self._periodic_thread.is_alive():
return True
self._periodic_stop_event.clear()
self._periodic_thread = threading.Thread(
target=self._periodic_loop,
daemon=True,
name="cmd-periodic-checker",
)
self._periodic_thread.start()
logger.debug("[CMD] 周期调度器已启动")
return True
def is_scheduler_running(self) -> bool:
with self._scheduler_lifecycle_lock:
thread = self._periodic_thread
return bool(
not self._shutdown_requested
and thread
and thread.is_alive()
)
def ensure_scheduler_running(self):
"""Best-effort watchdog: start periodic checker if it is not running."""
return self._start_periodic_scheduler()
def _periodic_loop(self):
while not self._periodic_stop_event.wait(1.0):
try:
self._run_periodic_checks()
except Exception as e:
logger.debug(f"[CMD] 周期调度循环异常(忽略): {e}")
def _get_periodic_trigger_timing(self, trigger: Dict, session_id: str) -> tuple[float, float]:
interval = max(1.0, self._coerce_float(trigger.get("periodic_interval_sec", 8), 8.0))
jitter = max(0.0, self._coerce_float(trigger.get("periodic_jitter_sec", 2), 2.0))
trigger_type = str(trigger.get("type", "")).strip().lower()
has_keyword_expression = bool(str(trigger.get("value", "") or "").strip())
if trigger_type == "page_check" and has_keyword_expression:
with self._lock:
has_observer = session_id in self._observer_keywords_by_session
if has_observer:
interval = min(interval, 1.5)
jitter = 0.0
return interval, jitter
def _run_periodic_checks(self):
try:
commands = self._load_commands_for_checks()
except Exception:
return
if not commands:
return
try:
browser = self._get_browser()
except Exception:
return
pool = getattr(browser, "_tab_pool", None)
if pool is None:
try:
pool = browser.tab_pool
logger.debug("[CMD] 周期调度器已初始化标签页池")
except Exception as e:
now = time.time()
if (now - self._last_tab_pool_wait_log_at) >= 10:
self._last_tab_pool_wait_log_at = now
logger.debug(f"[CMD] 周期调度器等待标签页池初始化: {e}")
return
if not hasattr(pool, "get_idle_sessions_snapshot"):
return
self._refresh_tab_pool_if_due(pool)
if hasattr(pool, "get_sessions_snapshot"):
sessions = pool.get_sessions_snapshot()
else:
sessions = pool.get_idle_sessions_snapshot()
if not sessions:
return
current_session_ids = {str(getattr(s, "id", "") or "") for s in sessions}
now = time.time()
active_keys = set()
due_total = 0
enabled_commands = [
(idx, cmd) for idx, cmd in enumerate(commands)
if cmd.get("enabled", True)
]
for session in sessions:
session_id = str(getattr(session, "id", "") or "")
if not session_id or session_id not in current_session_ids:
continue
session_status = self._session_status_value(session)
if session_status not in {"idle", "busy"}:
continue
is_busy_workflow = session_status == "busy" and self._has_active_workflow(session)
if session_status == "busy" and not is_busy_workflow:
continue
if session_status == "idle":
self._maybe_periodic_keepalive(session, now)
try:
self._sync_session_bootstrap_js_files(commands, session)
except Exception as error:
logger.debug(f"[CMD] 启动 JS 文件同步异常(忽略): {error}")
# Inject/update MutationObserver for page_check keywords on this session
try:
pc_keywords = self._collect_page_check_keywords(commands, session)
if pc_keywords:
self._ensure_page_check_observer(session, pc_keywords)
else:
self._clear_page_check_observer(session)
except Exception:
pass
due_commands: List[tuple[int, int, Dict[str, Any]]] = []
for idx, cmd in enabled_commands:
cmd_id = str(cmd.get("id", "")).strip()
if not cmd_id:
continue
trigger = cmd.get("trigger", {}) or {}
if not bool(trigger.get("periodic_enabled", True)):
continue
trigger_type = str(trigger.get("type", "")).strip().lower()
if (
is_busy_workflow
and trigger_type == "page_check"
and not self._should_evaluate_page_check_while_busy_workflow(cmd)
):
# While a workflow is actively running, low-priority page_check
# commands should not consume transient intermediate states.
continue
key = (cmd_id, session.id)
active_keys.add(key)
interval, jitter = self._get_periodic_trigger_timing(trigger, session_id)
with self._lock:
next_at = float(self._periodic_next_run.get(key, 0.0))
if now < next_at:
continue
delay = interval + (random.uniform(0.0, jitter) if jitter > 0 else 0.0)
with self._lock:
self._periodic_next_run[key] = now + delay
due_commands.append((self._get_command_priority(cmd), idx, cmd))
due_total += len(due_commands)
due_commands.sort(key=lambda item: (-item[0], item[1]))
for _, _, cmd in due_commands:
with self._command_logging_context(cmd):
current_status = str(getattr(getattr(session, "status", None), "value", "")).lower()
if current_status == "busy" and not self._has_active_workflow(session):
break
if current_status not in {"idle", "busy"}:
break
if self._should_trigger(cmd, session):
meta = self._take_pending_async_trigger_meta(cmd, session) or {}
if current_status == "busy":
scheduled = self._schedule_command_for_active_workflow(
cmd,
session,
interrupt_context=meta.get("interrupt_context"),
trigger_rollback=meta.get("rollback"),
)
if not scheduled:
if meta.get("rollback"):
self._rollback_trigger_consumption(cmd, session, meta.get("rollback"))
self._finalize_request_count_trigger_state(cmd, session, rollback=True)
self._reset_page_check_latch(cmd, session, reason="workflow_schedule_failed")
else:
self._execute_command_async(
cmd,
session,
interrupt_context=meta.get("interrupt_context"),
trigger_rollback=meta.get("rollback"),
)
if self._should_log_periodic_summary(now, due_total):
logger.debug(
f"[CMD] 周期检查: 会话数={len(sessions)}, "
f"已启用周期命令={len(active_keys)}, 本轮到点={due_total}"
)
with self._lock:
stale_keys = [k for k in self._periodic_next_run if k not in active_keys]
for key in stale_keys:
self._periodic_next_run.pop(key, None)
stale_keepalive_keys = [k for k in self._last_keepalive_by_session if k not in {s.id for s in sessions}]
for key in stale_keepalive_keys:
self._last_keepalive_by_session.pop(key, None)