Files
universal-web-api/app/api/system.py
T
2026-11-02 22:54:14 +08:00

2663 lines
101 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""
app/api/system.py - 系统功能 API
职责:
- 健康检查
- 日志管理
- 环境配置
- 浏览器常量
- 调试接口
"""
import json
import os
import re
import tempfile
import time
import asyncio
import threading as _threading
from pathlib import Path
from typing import Optional, Any, Dict, List
from app import __version__ as APP_VERSION
from fastapi import APIRouter, Request, HTTPException, Depends
from fastapi.responses import JSONResponse
from app.core.config import (
AppConfig,
_replace_file_with_retry,
atomic_write_json,
get_logger,
log_collector,
)
from app.core import get_browser, BrowserConnectionError
from app.api.deps import verify_dashboard_auth as verify_auth
from app.api.deps import verify_sensitive_admin_auth
from app.core.http_security import redact_env_for_backup, startup_security_errors
from app.services.config_engine import config_engine, ConfigConstants
from app.services.command_engine import command_engine
from app.services.extractor_manager import extractor_manager
from app.services.request_manager import request_manager
from app.utils.site_url import extract_remote_site_domain, route_domain_matches
from update_preserve import load_update_preserve_settings, save_update_preserve_settings
from app.core.driver import as_element, driver_for_tab
logger = get_logger("API.SYSTEM")
router = APIRouter()
async def _read_json_object_or_400(request: Request) -> Dict[str, Any]:
"""读取 JSON 请求体,并要求顶层必须是对象。"""
try:
data = await request.json()
except json.JSONDecodeError:
raise HTTPException(status_code=400, detail="请求体必须是有效 JSON")
except ValueError:
raise HTTPException(status_code=400, detail="请求体必须是有效 JSON")
if not isinstance(data, dict):
raise HTTPException(status_code=400, detail="请求体必须是 JSON 对象")
return data
# 修复:仅对数值语义的键做数字推断。此前对所有值 isdigit() 转 int,
# 会把 AUTH_TOKEN=00123456 这类字符串令牌静默改写成 123456;
# 同理 CURRENT_VERSION=1.10 会被转成 float 1.1 后回写成 1.1。
_NUMERIC_ENV_KEYS = {
"APP_PORT", "BROWSER_PORT", "MAX_HTML_CHARS", "CANVAS_IMAGE_MAX_SIZE",
"TOOL_CALLING_INTERNAL_RETRY_MAX", "TOOL_CALLING_MAX_TOOL_RESULT_CHARS",
"TAB_RECOVERY_MAX_ATTEMPTS", "TAB_RECOVERY_TIMEOUT_SEC",
"TAB_RECOVERY_WORKER_EXIT_WAIT_SEC",
"REQUEST_ZOMBIE_TTL", "LOG_MAX_BYTES", "LOG_BACKUP_COUNT",
"MAX_TABS", "MIN_TABS",
"CMD_ASYNC_MAX_WORKERS", "CMD_PAGE_CHECK_FAILURE_BACKOFF_SEC",
"CMD_PAGE_CHECK_JS_TIMEOUT_SEC", "CMD_PERIODIC_KEEPALIVE_INTERVAL_SEC",
"CMD_REQUEST_PRIORITY_BASELINE", "CMD_TAB_POOL_REFRESH_INTERVAL_SEC",
"TEMP_CLEANUP_MIN_AGE_SECONDS", "PASTE_TEMP_CLEANUP_MIN_AGE_SECONDS",
"ARENA_EVENT_BRIDGE_COMMAND_TEXT_LIMIT", "ARENA_EVENT_BRIDGE_DEDUPE_MAX_ENTRIES",
"SCHEDULED_RESTART_INTERVAL_SECONDS", "SCHEDULED_RESTART_DRAIN_TIMEOUT_SECONDS",
}
def _load_env_config_from_file() -> Dict[str, Any]:
"""读取 .env 文件配置。"""
env_path = Path(".env")
config: Dict[str, Any] = {}
if not env_path.exists():
return config
with open(env_path, "r", encoding="utf-8") as f:
for line in f:
line = line.strip()
if not line or line.startswith("#") or "=" not in line:
continue
key, value = line.split("=", 1)
key = key.strip()
value = value.strip()
if value.lower() == "true":
value = True
elif value.lower() == "false":
value = False
elif key in _NUMERIC_ENV_KEYS and value.isdigit():
value = int(value)
elif key in _NUMERIC_ENV_KEYS and re.match(r"^\d+\.\d+$", value):
value = float(value)
config[key] = value
# 修复:不再凭空合成 DASHBOARD_AUTH_* —— 前端会把合成值原样回写,
# 把「留空即沿用 AUTH_*」的动态回退语义固化成硬编码副本。
# 生效值改由 GET /api/settings/env 的只读 effective 字段返回。
return config
def _serialize_env_value(value: Any) -> str:
if isinstance(value, bool):
return "true" if value else "false"
if isinstance(value, (int, float)):
return str(value)
if value is None:
return ""
return str(value)
def _write_text_file_atomic(path: Path, lines: list[str]) -> None:
"""原子写入文本文件,避免写入中断时截断原文件。"""
path.parent.mkdir(parents=True, exist_ok=True)
fd = None
tmp_path: Optional[Path] = None
try:
fd, tmp_name = tempfile.mkstemp(
prefix=f".{path.name}.",
suffix=".tmp",
dir=str(path.parent),
)
tmp_path = Path(tmp_name)
with os.fdopen(fd, "w", encoding="utf-8", newline="") as f:
fd = None
f.writelines(lines)
f.flush()
os.fsync(f.fileno())
_replace_file_with_retry(tmp_path, path)
except Exception:
if fd is not None:
try:
os.close(fd)
except Exception:
pass
if tmp_path is not None:
try:
tmp_path.unlink(missing_ok=True)
except Exception:
pass
raise
def _write_env_config_file(new_config: Dict[str, Any]) -> None:
"""写入 .env 文件,尽量保留注释和现有顺序。"""
env_path = Path(".env")
lines = []
if env_path.exists():
with open(env_path, "r", encoding="utf-8") as f:
lines = f.readlines()
new_lines = []
existing_keys = set()
for line in lines:
stripped = line.strip()
if not stripped or stripped.startswith("#"):
new_lines.append(line)
continue
if "=" in stripped:
key = stripped.split("=", 1)[0].strip()
existing_keys.add(key)
if key in new_config:
value = _serialize_env_value(new_config[key])
new_lines.append(f"{key}={value}\n")
else:
new_lines.append(line)
else:
new_lines.append(line)
missing_items = []
for key, value in (new_config or {}).items():
if key in existing_keys:
continue
serialized = _serialize_env_value(value)
if serialized == "":
continue
missing_items.append((key, serialized))
if missing_items:
if new_lines and not new_lines[-1].endswith("\n"):
new_lines[-1] = new_lines[-1] + "\n"
if new_lines and new_lines[-1].strip():
new_lines.append("\n")
for key, value in missing_items:
new_lines.append(f"{key}={value}\n")
_write_text_file_atomic(env_path, new_lines)
def _read_json_file(path: Path, default: Any) -> Any:
if not path.exists():
return default
with open(path, "r", encoding="utf-8-sig") as f:
return json.load(f)
def _write_json_file(path: Path, payload: Any) -> None:
atomic_write_json(path, payload)
def _snapshot_file(path: Path) -> Optional[bytes]:
if not path.exists():
return None
return path.read_bytes()
def _restore_file_snapshot(path: Path, snapshot: Optional[bytes]) -> None:
if snapshot is None:
try:
path.unlink(missing_ok=True)
except FileNotFoundError:
pass
return
path.parent.mkdir(parents=True, exist_ok=True)
fd = None
tmp_path: Optional[Path] = None
try:
fd, tmp_name = tempfile.mkstemp(
prefix=f".{path.name}.rollback.",
suffix=".tmp",
dir=str(path.parent),
)
tmp_path = Path(tmp_name)
with os.fdopen(fd, "wb") as handle:
fd = None
handle.write(snapshot)
handle.flush()
os.fsync(handle.fileno())
_replace_file_with_retry(tmp_path, path)
finally:
if fd is not None:
try:
os.close(fd)
except Exception:
pass
if tmp_path is not None:
try:
tmp_path.unlink(missing_ok=True)
except Exception:
pass
def _restore_file_snapshots(snapshots: Dict[Path, Optional[bytes]]) -> None:
for path, snapshot in reversed(list(snapshots.items())):
try:
_restore_file_snapshot(path, snapshot)
except Exception as rollback_error:
logger.error(f"回滚配置文件失败: {path}: {rollback_error}")
def _schedule_service_restart(delay_seconds: float = 1.0) -> None:
async def trigger_restart():
await asyncio.sleep(max(0.1, float(delay_seconds or 1.0)))
logger.warning("=" * 60)
logger.warning("配置已更新,服务即将重启...")
logger.warning("=" * 60)
try:
from app.services.restart_guard import mark_service_restart
mark_service_restart()
except Exception:
pass
import os
os._exit(3)
asyncio.create_task(trigger_restart())
def _export_sites_config() -> Dict[str, Any]:
"""R1-2:从 config/sites/ 目录拼出与旧 sites.json 相同结构的字典(备份格式保持不变,旧备份仍可导入)。"""
from app.services.config.site_store import load_sites_dict
return load_sites_dict(config_engine.sites_dir)
def _build_settings_backup_bundle() -> Dict[str, Any]:
sites_local_file = Path(config_engine.local_sites_file)
commands_file = Path(ConfigConstants.COMMANDS_FILE)
commands_local_file = Path(ConfigConstants.COMMANDS_LOCAL_FILE)
browser_config_file = Path("config/browser_config.json")
extractors_file = Path(extractor_manager.CONFIG_FILE)
# S13:备份文件会被下载、转存、分享,默认绝不包含 .env 中的令牌/密钥/含凭据的代理地址。
# 被剔除的键只记录名字,导入时这些键保持目标机器现值不变。
safe_env, redacted_env_keys = redact_env_for_backup(_load_env_config_from_file())
return {
"bundle_version": 1,
"secrets_redacted": True,
"redacted_env_keys": redacted_env_keys,
"exported_at": int(time.time()),
"app_version": APP_VERSION,
"files": {
"sites": _export_sites_config(),
"sites_local": _read_json_file(sites_local_file, {"default_presets": {}}),
"commands": _read_json_file(commands_file, {"commands": []}),
"commands_local": _read_json_file(commands_local_file, {"commands": []}),
"browser_constants": _read_json_file(browser_config_file, {}),
"extractors": _read_json_file(extractors_file, extractor_manager.export_config()),
"update_preserve": load_update_preserve_settings(),
"env": safe_env,
},
}
_REDACTED_PLACEHOLDERS = {"***", "******", "<redacted>", "[redacted]", "__redacted__", "redacted"}
def _strip_redacted_env_import(env_payload: Dict[str, Any]) -> Dict[str, Any]:
"""导入备份时丢弃脱敏占位符,避免把 "***" 之类写回 .env 覆盖真实令牌。"""
cleaned: Dict[str, Any] = {}
for key, value in env_payload.items():
if isinstance(value, str) and value.strip().lower() in _REDACTED_PLACEHOLDERS:
continue
cleaned[key] = value
return cleaned
def _validate_settings_backup_files(files: Dict[str, Any]) -> Dict[str, Any]:
"""Validate all import sections before any target file is modified."""
validated: Dict[str, Any] = {}
if "sites" in files:
if not isinstance(files["sites"], dict):
raise HTTPException(status_code=400, detail="sites 配置格式无效")
validated["sites"] = files["sites"]
if "sites_local" in files:
if not isinstance(files["sites_local"], dict):
raise HTTPException(status_code=400, detail="sites_local 配置格式无效")
validated["sites_local"] = files["sites_local"]
if "commands" in files:
commands_payload = files["commands"]
if not isinstance(commands_payload, (dict, list)):
raise HTTPException(status_code=400, detail="commands 配置格式无效")
validated["commands"] = commands_payload
if "commands_local" in files:
commands_local_payload = files["commands_local"]
if not isinstance(commands_local_payload, dict):
raise HTTPException(status_code=400, detail="commands_local 配置格式无效")
validated["commands_local"] = commands_local_payload
if "browser_constants" in files:
if not isinstance(files["browser_constants"], dict):
raise HTTPException(status_code=400, detail="browser_constants 配置格式无效")
validated["browser_constants"] = files["browser_constants"]
if "extractors" in files:
extractors_payload = files["extractors"]
if not isinstance(extractors_payload, dict):
raise HTTPException(status_code=400, detail="extractors 配置格式无效")
if "extractors" not in extractors_payload:
raise HTTPException(status_code=400, detail="extractors 配置格式无效:缺少 extractors 字段")
extractors_config = extractors_payload.get("extractors")
if not isinstance(extractors_config, dict):
raise HTTPException(status_code=400, detail="extractors 配置格式无效:extractors 必须是对象")
invalid_entries = [
extractor_id
for extractor_id, extractor_config in extractors_config.items()
if not isinstance(extractor_config, dict)
]
if invalid_entries:
raise HTTPException(
status_code=400,
detail="extractors 配置格式无效:提取器配置项必须是对象",
)
validated["extractors"] = extractors_payload
if "update_preserve" in files:
preserve_payload = files["update_preserve"]
if isinstance(preserve_payload, dict):
selected_patterns = preserve_payload.get("selected_patterns", [])
else:
selected_patterns = preserve_payload
if selected_patterns is None:
selected_patterns = []
if not isinstance(selected_patterns, list):
raise HTTPException(status_code=400, detail="update_preserve 配置格式无效")
validated["update_preserve"] = selected_patterns
if "env" in files:
if not isinstance(files["env"], dict):
raise HTTPException(status_code=400, detail="env 配置格式无效")
env_payload = _strip_redacted_env_import(files["env"])
# 与 POST /api/settings/env 同一套校验:拦截换行注入、非法键名、认证自锁/公开无认证
_validate_env_config_payload(env_payload)
validated["env"] = env_payload
return validated
DEFAULT_BROWSER_CONSTANTS: Dict[str, Any] = {
"CONNECTION_TIMEOUT": 10,
"MAX_REQUEST_EXECUTE_TIME_SEC": 300.0,
"STEALTH_DELAY_MIN": 0.03,
"STEALTH_DELAY_MAX": 0.1,
"ACTION_DELAY_MIN": 0.06,
"ACTION_DELAY_MAX": 0.14,
"STEALTH_PAUSE_PROBABILITY": 0.0,
"STEALTH_PAUSE_EXTRA_MAX": 0.15,
"STEALTH_KEY_DOWN_UP_MIN": 0.015,
"STEALTH_KEY_DOWN_UP_MAX": 0.04,
"STEALTH_KEY_BETWEEN_MIN": 0.02,
"STEALTH_KEY_BETWEEN_MAX": 0.06,
"STEALTH_PASTE_SETTLE_MIN": 0.12,
"STEALTH_PASTE_SETTLE_MAX": 0.25,
"STEALTH_SKIP_PASTE_VERIFY": True,
"STEALTH_SEND_IMAGE_WAIT": 8.0,
"STEALTH_SEND_IMAGE_RETRY_INTERVAL": 1.2,
"STEALTH_MOUSE_WARMUP_ENABLED": False,
"STEALTH_CLICK_STRATEGY": "auto",
"STEALTH_DOM_CLICK_TARGETS": ["new_chat_btn", "input_box", "send_btn"],
"DEFAULT_ELEMENT_TIMEOUT": 3,
"FALLBACK_ELEMENT_TIMEOUT": 1,
"ELEMENT_CACHE_MAX_AGE": 5.0,
"LOG_INFO_CUTE_MODE": False,
"LOG_DEBUG_CUTE_MODE": False,
"LOG_CONSOLE_ENABLED": True,
"LOG_FILE_ENABLED": True,
"LOG_WEB_COLLECTOR_ENABLED": True,
"LOG_WEB_MAX_RECORDS": 500,
"STREAM_CHECK_INTERVAL_MIN": 0.1,
"STREAM_CHECK_INTERVAL_MAX": 1.0,
"STREAM_CHECK_INTERVAL_DEFAULT": 0.3,
"STREAM_SILENCE_THRESHOLD": 8.0,
"STREAM_MAX_TIMEOUT": 600,
"STREAM_INITIAL_WAIT": 180,
"STREAM_CONTENT_SHRINK_TOLERANCE": 3,
"STREAM_STABLE_COUNT_THRESHOLD": 8,
"STREAM_SILENCE_THRESHOLD_FALLBACK": 12,
"MAX_MESSAGE_LENGTH": 100000,
"MAX_MESSAGES_COUNT": 100,
"TEXT_INPUT_CHUNK_SIZE": 30000,
"GLOBAL_NETWORK_INTERCEPTION_ENABLED": False,
"GLOBAL_NETWORK_INTERCEPTION_LISTEN_PATTERN": "http",
"GLOBAL_NETWORK_INTERCEPTION_WAIT_TIMEOUT": 0.5,
"GLOBAL_NETWORK_INTERCEPTION_RETRY_DELAY": 1.0,
"GLOBAL_NETWORK_INTERCEPTION_RES_TYPES": "XHR,Fetch,EventSource,Document",
"NETWORK_DEBUG_CAPTURE_ENABLED": False,
"NETWORK_DEBUG_CAPTURE_MAX_BODY_CHARS": 50000,
"NETWORK_DEBUG_CAPTURE_MAX_FILES_PER_REQUEST": 3,
"NETWORK_DEBUG_CAPTURE_PARSER_FILTER": "",
"NETWORK_DEBUG_CAPTURE_MAX_TOTAL_MB": 50,
"NETWORK_DEBUG_CAPTURE_RETENTION_HOURS": 24,
"CONVERSATION_TIMEOUT_THRESHOLD": 0.0,
"FORCE_NEW_CONVERSATION": False,
"REQUEST_MONITOR_ENABLED": True,
"REQUEST_MONITOR_MAX_RECORDS": 200,
"REQUEST_MONITOR_DETAIL_ENABLED": True,
"REQUEST_MONITOR_SAVE_TO_FILE": True,
"REQUEST_MONITOR_MAX_CAPTURED_RESPONSE_CHARS": 30000,
# 修复:补齐缺失的入口,否则前端无法通过 /api/settings/browser-constants 配置它们
"REQUEST_ZOMBIE_TTL": 600,
"PAGE_INTERACTION_THROTTLE_ENABLED": True,
"PAGE_INTERACTION_MAX_CONCURRENT": 3,
"PAGE_INTERACTION_MAX_WAIT": 20.0,
"DASHBOARD_LOG_POLL_INTERVAL_MS": 1000,
"DASHBOARD_LOG_BACKGROUND_POLL_INTERVAL_MS": 5000,
"DASHBOARD_REQUEST_HISTORY_POLL_INTERVAL_MS": 3000,
"DASHBOARD_SYSTEM_STATS_ENABLED": True,
"DASHBOARD_SYSTEM_STATS_POLL_INTERVAL_MS": 3000,
"COMMAND_PERIODIC_CHECK_ENABLED": True,
"COMMAND_PERIODIC_CHECK_INTERVAL_SEC": 8.0,
"COMMAND_PERIODIC_CHECK_JITTER_SEC": 2.0,
"UPLOAD_HISTORY_IMAGES": False,
"tab_pool": {
"max_tabs": 5,
"min_tabs": 1,
"idle_timeout": 300,
"acquire_timeout": 60,
"stuck_timeout": 180,
"allocation_mode": "first_idle",
"excluded_urls": [],
"preserve_error_tabs": False,
"route_groups": [],
},
}
_DYNAMIC_SELECTOR_CLASS_RE = re.compile(r"^(?:_?[A-Fa-f0-9_-]{8,}|[A-Za-z0-9_-]{16,})$")
def _build_selector_test_diagnosis(
selector: str,
match_count: int,
top_candidates: Optional[list[dict[str, Any]]] = None,
) -> Dict[str, Any]:
normalized = str(selector or "").strip()
warnings: list[str] = []
tips: list[str] = []
if ":nth-child" in normalized or ":nth-of-type" in normalized:
warnings.append("当前写法用了 nth-child / nth-of-type,页面结构稍微变化就容易失效。")
if normalized.count(">") >= 3 or len(re.split(r"\s+", normalized)) >= 5:
warnings.append("当前选择器层级比较深,维护成本会偏高,建议优先收敛到稳定属性。")
if ":has(" in normalized:
warnings.append("当前选择器包含 :has(),兼容性和稳定性通常都不如稳定属性写法。")
if re.search(r"\.[A-Za-z0-9_-]{14,}", normalized):
warnings.append("当前写法里有很长的 class 名,看起来像动态类名,页面一改就可能失效。")
status = "missing"
if match_count <= 0:
summary = "当前没有命中任何元素。"
tips.extend([
"先检查是不是还停在正确页面,很多站点切到聊天页前后 DOM 差别很大。",
"优先试试 id、data-testid、aria-label、name 这类稳定属性,不要先从随机 class 开始。",
"如果页面刚切换完,适当把超时时间调高一点再测一次。",
])
elif match_count == 1:
status = "unique"
summary = "已经唯一命中 1 个元素,可以继续判断是不是命中了你真正想要的目标。"
tips.extend([
"下一步看元素摘要和候选选择器,确认它是不是你真正要找的输入框、发送按钮或回复容器。",
"如果当前写法能命中,但很依赖深层结构或长 class,优先换成下方更稳的候选。",
])
else:
status = "multiple"
summary = f"当前命中了 {match_count} 个元素,范围还不够收敛。"
tips.extend([
"先看下方每个命中元素的候选选择器,优先挑唯一命中的那种。",
"如果都是 button / div 这类大范围匹配,通常要补上 aria-label、data-testid、name 或更具体的父级信息。",
])
if top_candidates:
unique_count = sum(1 for item in top_candidates if item.get("unique"))
if unique_count > 0:
tips.insert(0, f"下面已经生成了 {unique_count} 个唯一候选,可以直接挑一个更稳的写法继续复测。")
return {
"status": status,
"summary": summary,
"warnings": warnings,
"tips": tips,
}
def _collect_selector_test_element_snapshot(ele: Any) -> Dict[str, Any]:
try:
snapshot = as_element(ele).run_js(
"""
return (function () {
const target = this;
const tag = String(target.tagName || '').toLowerCase();
const textValue = String(target.innerText || target.textContent || '')
.replace(/\\s+/g, ' ')
.trim();
const attrKeys = ['id', 'class', 'name', 'data-testid', 'aria-label', 'role', 'type', 'placeholder', 'href'];
const attributes = {};
for (const key of attrKeys) {
try {
const value = target.getAttribute ? target.getAttribute(key) : null;
if (value) {
attributes[key] = String(value).trim().slice(0, 160);
}
} catch (error) {}
}
function cssEscapeIdent(value) {
const raw = String(value || '');
if (!raw) return raw;
if (window.CSS && typeof window.CSS.escape === 'function') {
return window.CSS.escape(raw);
}
return raw.replace(/[^a-zA-Z0-9_-]/g, function (char) {
return '\\\\' + char;
});
}
function cssEscapeString(value) {
return String(value || '')
.replace(/\\\\/g, '\\\\\\\\')
.replace(/"/g, '\\\\\\"');
}
function looksStableClass(token) {
const cls = String(token || '').trim();
if (!cls || cls.length > 40) return false;
if (/^(css|jsx)-/i.test(cls)) return false;
if (/^_[A-Za-z0-9-]{8,}$/.test(cls)) return false;
if (/^[A-Fa-f0-9_-]{10,}$/.test(cls)) return false;
if (/[A-Z]/.test(cls) && /\\d/.test(cls) && cls.length >= 12) return false;
return /^[A-Za-z][A-Za-z0-9_-]*$/.test(cls);
}
function pushCandidate(bucket, selector, reason, score) {
if (!selector) return;
bucket.push({
selector,
reason,
score,
});
}
const candidates = [];
const rawClasses = String(attributes['class'] || '')
.split(/\\s+/)
.map(item => String(item || '').trim())
.filter(Boolean);
const stableClasses = rawClasses.filter(looksStableClass).slice(0, 3);
if (attributes.id) {
pushCandidate(candidates, `#${cssEscapeIdent(attributes.id)}`, 'ID 精确匹配', 120);
if (tag) {
pushCandidate(candidates, `${tag}#${cssEscapeIdent(attributes.id)}`, '标签 + ID', 112);
}
}
if (attributes['data-testid']) {
pushCandidate(candidates, `[data-testid="${cssEscapeString(attributes['data-testid'])}"]`, 'data-testid', 110);
if (tag) {
pushCandidate(candidates, `${tag}[data-testid="${cssEscapeString(attributes['data-testid'])}"]`, '标签 + data-testid', 104);
}
}
if (attributes['aria-label']) {
pushCandidate(candidates, `[aria-label="${cssEscapeString(attributes['aria-label'])}"]`, 'aria-label', 96);
if (tag) {
pushCandidate(candidates, `${tag}[aria-label="${cssEscapeString(attributes['aria-label'])}"]`, '标签 + aria-label', 92);
}
}
if (attributes.name) {
pushCandidate(candidates, `[name="${cssEscapeString(attributes.name)}"]`, 'name', 92);
if (tag) {
pushCandidate(candidates, `${tag}[name="${cssEscapeString(attributes.name)}"]`, '标签 + name', 88);
}
}
if (attributes.placeholder) {
pushCandidate(candidates, `[placeholder="${cssEscapeString(attributes.placeholder)}"]`, 'placeholder', 76);
if (tag) {
pushCandidate(candidates, `${tag}[placeholder="${cssEscapeString(attributes.placeholder)}"]`, '标签 + placeholder', 72);
}
}
if (attributes.type && ['input', 'button', 'textarea'].includes(tag)) {
pushCandidate(candidates, `${tag}[type="${cssEscapeString(attributes.type)}"]`, '标签 + type', 74);
}
if (attributes.role) {
pushCandidate(candidates, `[role="${cssEscapeString(attributes.role)}"]`, 'role', 62);
if (tag) {
pushCandidate(candidates, `${tag}[role="${cssEscapeString(attributes.role)}"]`, '标签 + role', 58);
}
}
if (attributes.href && tag === 'a') {
pushCandidate(candidates, `a[href="${cssEscapeString(attributes.href)}"]`, '精确 href', 82);
}
if (stableClasses.length > 0) {
const classSelector = stableClasses.map(item => `.${cssEscapeIdent(item)}`).join('');
pushCandidate(candidates, classSelector, '稳定 class 组合', 54);
if (tag) {
pushCandidate(candidates, `${tag}${classSelector}`, '标签 + 稳定 class 组合', 56);
}
}
if (tag) {
pushCandidate(candidates, tag, '仅标签名(通常过宽)', 8);
}
function evaluateSelector(selector) {
try {
const matches = Array.from(document.querySelectorAll(selector));
return {
count: matches.length,
exact: matches.includes(target),
};
} catch (error) {
return {
count: -1,
exact: false,
error: String(error && error.message ? error.message : error),
};
}
}
const seen = new Set();
const evaluated = [];
for (const candidate of candidates) {
if (!candidate.selector || seen.has(candidate.selector)) continue;
seen.add(candidate.selector);
const stats = evaluateSelector(candidate.selector);
if (stats.error || !stats.exact) continue;
evaluated.push({
...candidate,
count: stats.count,
unique: stats.count === 1,
});
}
evaluated.sort((left, right) => {
if (left.unique !== right.unique) {
return left.unique ? -1 : 1;
}
if (left.score !== right.score) {
return right.score - left.score;
}
if (left.count !== right.count) {
return left.count - right.count;
}
return String(left.selector).length - String(right.selector).length;
});
const rect = target.getBoundingClientRect ? target.getBoundingClientRect() : { x: 0, y: 0, width: 0, height: 0 };
const style = window.getComputedStyle ? window.getComputedStyle(target) : null;
const visible = !!(
rect.width > 0 &&
rect.height > 0 &&
(!style || (style.display !== 'none' && style.visibility !== 'hidden' && style.opacity !== '0'))
);
const warnings = [];
if (rawClasses.some(item => !looksStableClass(item))) {
warnings.push('这个元素带有疑似动态 class,优先考虑 id、data-testid、aria-label 这类属性。');
}
if (!evaluated.some(item => item.unique)) {
warnings.push('候选里还没有唯一命中的写法,通常需要加稳定属性或更具体的父级范围。');
}
return {
tag,
text: textValue.slice(0, 120),
visible,
rect: {
x: Math.round(rect.x || 0),
y: Math.round(rect.y || 0),
width: Math.round(rect.width || 0),
height: Math.round(rect.height || 0),
},
attributes,
html_preview: String(target.outerHTML || '').replace(/\\s+/g, ' ').trim().slice(0, 400),
candidate_selectors: evaluated.slice(0, 8),
warnings,
};
}).call(this);
"""
)
if isinstance(snapshot, dict):
return snapshot
except Exception as e:
logger.debug(f"收集元素快照失败: {e}")
return {}
def _build_selector_top_candidates(elements: list[Dict[str, Any]]) -> list[Dict[str, Any]]:
merged: Dict[str, Dict[str, Any]] = {}
for element_index, element in enumerate(elements):
for candidate in element.get("candidate_selectors") or []:
selector = str(candidate.get("selector") or "").strip()
if not selector:
continue
normalized = {
"selector": selector,
"reason": str(candidate.get("reason") or "").strip(),
"score": int(candidate.get("score") or 0),
"count": int(candidate.get("count") or 0),
"unique": bool(candidate.get("unique")),
"element_index": element_index,
}
existing = merged.get(selector)
if existing is None:
merged[selector] = normalized
continue
if normalized["unique"] and not existing["unique"]:
merged[selector] = normalized
continue
if normalized["score"] > existing["score"]:
merged[selector] = normalized
ranked = list(merged.values())
ranked.sort(
key=lambda item: (
0 if item.get("unique") else 1,
-int(item.get("score") or 0),
int(item.get("count") or 0),
int(item.get("element_index") or 0),
len(str(item.get("selector") or "")),
)
)
return ranked[:10]
# ================= 健康检查 =================
def _public_health_response() -> JSONResponse:
"""S7:匿名远程调用者的最小健康响应。
不调用 ``browser.health_check()``(它会主动连接/重连浏览器),只读当前连接标志;
保留控制面板登录前需要的 ``dashboard_auth_enabled``,以及原有 200/503 语义。
"""
try:
browser = get_browser(auto_connect=False)
connected = bool(getattr(browser, "_connected", False)) and bool(
getattr(browser, "browser_handle", None) is not None
)
except Exception:
connected = False
dashboard_auth = AppConfig.is_dashboard_auth_enabled()
response = {
"service": "healthy",
"browser": {"connected": connected},
"config": {
"auth_enabled": dashboard_auth,
"dashboard_auth_enabled": dashboard_auth,
},
"detail": "restricted",
"timestamp": int(time.time()),
}
return JSONResponse(content=response, status_code=200 if connected else 503)
@router.get("/health")
async def health_check(request: Request):
"""服务健康检查(本机直连或带有效令牌时返回详细诊断,否则最小响应)"""
from app.utils.diagnostics_access import request_is_privileged
if not request_is_privileged(request):
return _public_health_response()
try:
browser = get_browser(auto_connect=False)
browser_health = browser.health_check()
except Exception as e:
browser_health = {"connected": False, "error": str(e)}
rm_status = request_manager.get_status()
response = {
"service": "healthy",
"version": APP_VERSION,
"browser": browser_health,
"request_manager": rm_status,
"config": {
"sites_loaded": len(config_engine.sites),
"auth_enabled": AppConfig.is_dashboard_auth_enabled(),
"dashboard_auth_enabled": AppConfig.is_dashboard_auth_enabled(),
"service_auth_enabled": AppConfig.is_auth_enabled(),
},
"timestamp": int(time.time())
}
status_code = 200 if browser_health.get("connected") else 503
return JSONResponse(content=response, status_code=status_code)
# ================= 日志 API =================
@router.get("/api/logs")
async def get_logs(
since: float = 0,
after_seq: int = 0,
authenticated: bool = Depends(verify_auth),
):
"""获取日志"""
logs, next_seq, cleared = log_collector.get_recent(since=since, after_seq=after_seq)
return {"logs": logs, "timestamp": time.time(), "next_seq": next_seq, "cleared": cleared}
@router.delete("/api/logs")
async def clear_logs(authenticated: bool = Depends(verify_auth)):
"""清除日志"""
log_collector.clear()
return {"status": "success"}
# ================= 环境配置 API =================
@router.get("/api/settings/env")
async def get_env_config(authenticated: bool = Depends(verify_auth)):
"""读取 .env 文件配置"""
try:
# 修复:config 结构保持不变(前端只读 data.config);
# 另附只读 effective 字段暴露 DASHBOARD_AUTH_* 的实际生效值,前端不会回写它。
return {
"config": _load_env_config_from_file(),
"effective": {
"DASHBOARD_AUTH_ENABLED": AppConfig.is_dashboard_auth_enabled(),
"DASHBOARD_AUTH_TOKEN_SET": bool(AppConfig.get_dashboard_auth_token()),
},
}
except Exception as e:
logger.error(f"读取环境配置失败: {e}")
raise HTTPException(status_code=500, detail=f"读取失败: {str(e)}")
def _env_value_is_true(value: Any) -> bool:
"""与后端 AppConfig._env_bool 一致的真值白名单语义。"""
if isinstance(value, bool):
return value
if value is None:
return False
return str(value).strip().lower() in ("true", "1", "yes", "on")
def _validate_env_config_payload(new_config: Dict[str, Any]) -> None:
"""写入 .env 前校验:拦截非法键名、换行注入,以及会把控制面板锁死的认证组合。"""
# 修复:键名/值零校验时,值里带 \n 就能往 .env 注入额外行
# (例如注入 DASHBOARD_AUTH_ENABLED=false 关掉认证)。
#
# 注意:前端会把 GET 读到的整份 .env 原样回传,其中可能包含 http_proxy、no_proxy
# 这类小写的存量键。对这些"读进来又原样送回来、值也没变"的键做键名校验会让用户
# 完全无法保存任何设置,因此键名规则只对新增/变更的键生效。
try:
existing_env: Dict[str, Any] = _load_env_config_from_file()
except Exception:
existing_env = {}
changed_keys: list = []
for key, value in new_config.items():
serialized = _serialize_env_value(value)
if "\r" in serialized or "\n" in serialized:
raise HTTPException(status_code=400, detail=f"配置值不能包含换行: {key}")
unchanged_existing = (
key in existing_env
and _serialize_env_value(existing_env.get(key)) == serialized
)
if unchanged_existing:
continue
if not isinstance(key, str) or not re.match(r"^[A-Z][A-Z0-9_]*$", key):
raise HTTPException(status_code=400, detail=f"非法的配置键名: {key}")
changed_keys.append(key)
# R2-4:按环境变量登记表校验类型、范围与枚举取值(只校验新增/修改的键,存量值原样放行)
from app.core.settings_registry import validate_env_values
value_errors = validate_env_values(
{key: _serialize_env_value(new_config[key]) for key in changed_keys}
)
if value_errors:
raise HTTPException(status_code=400, detail="配置值不合法:" + ";".join(value_errors.values()))
# 修复:认证自锁保护。开启认证却留空令牌时,deps.py 会对每个 /api/* 抛 500,
# 控制面板此后打不开,只能手改 .env 恢复。按合并后的最终生效值判断。
merged: Dict[str, Any] = {**existing_env, **new_config}
auth_enabled = _env_value_is_true(merged.get("AUTH_ENABLED"))
auth_token = str(merged.get("AUTH_TOKEN") or "").strip()
if auth_enabled and not auth_token:
raise HTTPException(status_code=400, detail="启用 API 认证时必须设置 AUTH_TOKEN")
# DASHBOARD_AUTH_ENABLED 未设置/为空时沿用 AUTH_ENABLED
dashboard_raw = merged.get("DASHBOARD_AUTH_ENABLED")
if dashboard_raw is None or (not isinstance(dashboard_raw, bool) and str(dashboard_raw).strip() == ""):
dashboard_enabled = auth_enabled
else:
dashboard_enabled = _env_value_is_true(dashboard_raw)
dashboard_token = str(merged.get("DASHBOARD_AUTH_TOKEN") or "").strip()
if dashboard_enabled and not dashboard_token and not auth_token:
raise HTTPException(
status_code=400,
detail="启用控制面板认证时必须设置 DASHBOARD_AUTH_TOKEN 或 AUTH_TOKEN"
)
# H1 / S1:认证开关必须是合法布尔值;公开监听必须同时启用认证。
# 与启动期检查 startup_security_errors 同一套规则,避免保存后服务起不来。
merged_env = {str(k): _serialize_env_value(v) for k, v in merged.items()}
security_errors = startup_security_errors(env=merged_env)
if security_errors:
raise HTTPException(status_code=400, detail=";".join(security_errors))
@router.post("/api/settings/env")
async def save_env_config(
request: Request,
authenticated: bool = Depends(verify_auth)
):
"""保存 .env 配置"""
try:
data = await _read_json_object_or_400(request)
new_config = data.get("config", {})
if not isinstance(new_config, dict):
raise HTTPException(status_code=400, detail="环境配置必须是 JSON 对象")
_validate_env_config_payload(new_config)
_write_env_config_file(new_config)
logger.info(f"环境配置已保存: {len(new_config)} 项,准备触发重启...")
_schedule_service_restart(1.0)
return {
"status": "success",
"message": "环境配置已保存,服务将在 1 秒后重启...",
"updated_count": len(new_config),
"will_restart": True
}
except HTTPException:
raise
except Exception as e:
logger.error(f"保存环境配置失败: {e}")
raise HTTPException(status_code=500, detail=f"保存失败: {str(e)}")
@router.get("/api/settings/backup")
async def export_settings_backup(authenticated: bool = Depends(verify_sensitive_admin_auth)):
"""导出完整配置备份。"""
try:
return _build_settings_backup_bundle()
except Exception as e:
logger.error(f"导出配置备份失败: {e}")
raise HTTPException(status_code=500, detail=f"导出失败: {str(e)}")
@router.post("/api/settings/backup")
async def import_settings_backup(
request: Request,
authenticated: bool = Depends(verify_sensitive_admin_auth)
):
"""导入完整配置备份。"""
try:
data = await _read_json_object_or_400(request)
allowed_sections = {
"sites",
"sites_local",
"commands",
"commands_local",
"browser_constants",
"extractors",
"update_preserve",
"env",
}
raw_files = data.get("files") if isinstance(data, dict) else None
if isinstance(raw_files, dict):
files = {key: value for key, value in raw_files.items() if key in allowed_sections}
elif isinstance(data, dict):
files = {key: value for key, value in data.items() if key in allowed_sections}
else:
files = {}
if not files:
raise HTTPException(status_code=400, detail="备份文件格式无效")
files = _validate_settings_backup_files(files)
imported_sections = []
restart_required = False
file_snapshots: Dict[Path, Optional[bytes]] = {}
def remember_file(path: Path) -> Path:
target = Path(path)
if target not in file_snapshots:
file_snapshots[target] = _snapshot_file(target)
return target
def write_import_json(path: Path, payload: Any) -> None:
_write_json_file(remember_file(path), payload)
def write_import_env(payload: Dict[str, Any]) -> None:
remember_file(Path(".env"))
_write_env_config_file(payload)
def reload_imported_runtime() -> None:
if "sites" in files or "sites_local" in files:
config_engine.reload_config()
if "commands" in files or "commands_local" in files:
command_engine._refresh_commands_if_changed(force=True)
if "browser_constants" in files:
try:
from app.core.config import BrowserConstants
if hasattr(BrowserConstants, "reload"):
BrowserConstants.reload()
except Exception as reload_error:
logger.warning(f"导入后热重载浏览器常量失败: {reload_error}")
if "extractors" in files:
extractor_manager.reload_config()
try:
if "sites" in files:
# R1-2:备份里的 sites 是旧 sites.json 结构;按“整体替换”语义写回 config/sites/ 目录。
# 先记住目录里现有的和即将写入的每个文件,失败时由 _restore_file_snapshots 整体回滚。
store = config_engine.site_store
sites_to_save = dict(files["sites"])
if "_global" not in sites_to_save:
sites_to_save["_global"] = config_engine.global_config.to_dict()
for site_path in store.site_files():
remember_file(site_path)
for site_key in sites_to_save:
remember_file(store.file_for(str(site_key)))
with config_engine._io_lock:
store.save(sites_to_save)
imported_sections.append("sites")
if "sites_local" in files:
write_import_json(Path(config_engine.local_sites_file), files["sites_local"])
imported_sections.append("sites_local")
if "commands" in files:
write_import_json(Path(ConfigConstants.COMMANDS_FILE), files["commands"])
imported_sections.append("commands")
if "commands_local" in files:
write_import_json(Path(ConfigConstants.COMMANDS_LOCAL_FILE), files["commands_local"])
imported_sections.append("commands_local")
if "browser_constants" in files:
write_import_json(Path("config/browser_config.json"), files["browser_constants"])
imported_sections.append("browser_constants")
if "extractors" in files:
write_import_json(Path(extractor_manager.CONFIG_FILE), files["extractors"])
imported_sections.append("extractors")
if "update_preserve" in files:
remember_file(Path("config") / "update_settings.json")
save_update_preserve_settings(files["update_preserve"])
imported_sections.append("update_preserve")
if "env" in files:
write_import_env(files["env"])
imported_sections.append("env")
restart_required = True
except Exception:
_restore_file_snapshots(file_snapshots)
raise
try:
reload_imported_runtime()
except Exception:
_restore_file_snapshots(file_snapshots)
try:
reload_imported_runtime()
except Exception as rollback_reload_error:
logger.warning(f"导入失败回滚后热重载运行时状态失败: {rollback_reload_error}")
raise
logger.info(f"完整配置备份已导入: {', '.join(imported_sections)}")
if restart_required:
_schedule_service_restart(1.0)
return {
"success": True,
"message": "完整配置备份已导入",
"imported_sections": imported_sections,
"will_restart": restart_required,
}
except HTTPException:
raise
except Exception as e:
logger.error(f"导入配置备份失败: {e}")
raise HTTPException(status_code=500, detail=f"导入失败: {str(e)}")
@router.get("/api/settings/browser-constants")
async def get_browser_constants(authenticated: bool = Depends(verify_auth)):
"""读取浏览器常量配置"""
try:
config_path = Path("config/browser_config.json")
config = _read_json_file(config_path, DEFAULT_BROWSER_CONSTANTS)
return {"config": config}
except Exception as e:
logger.error(f"读取浏览器常量失败: {e}")
raise HTTPException(status_code=500, detail=f"读取失败: {str(e)}")
@router.post("/api/settings/browser-constants")
async def save_browser_constants(
request: Request,
authenticated: bool = Depends(verify_auth)
):
"""保存浏览器常量配置"""
try:
data = await _read_json_object_or_400(request)
config = data.get("config", {})
if not isinstance(config, dict):
raise HTTPException(status_code=400, detail="浏览器常量配置必须是 JSON 对象")
_write_json_file(Path("config/browser_config.json"), config)
try:
from app.core.config import BrowserConstants
if hasattr(BrowserConstants, 'reload'):
BrowserConstants.reload()
logger.info("浏览器常量已热重载")
else:
logger.warning("BrowserConstants 不支持热重载,需重启服务")
except Exception as reload_error:
logger.warning(f"热重载失败: {reload_error}")
tab_pool_synced = False
try:
tab_pool_config = config.get("tab_pool") or {}
if isinstance(tab_pool_config, dict):
import app.core.browser as browser_module
browser_instance = getattr(browser_module, "_browser_instance", None)
live_tab_pool = getattr(browser_instance, "_tab_pool", None) if browser_instance else None
if live_tab_pool is not None:
live_tab_pool.apply_runtime_config(
max_tabs=tab_pool_config.get("max_tabs"),
min_tabs=tab_pool_config.get("min_tabs"),
idle_timeout=tab_pool_config.get("idle_timeout"),
acquire_timeout=tab_pool_config.get("acquire_timeout"),
stuck_timeout=tab_pool_config.get("stuck_timeout"),
allocation_mode=tab_pool_config.get("allocation_mode"),
excluded_urls=tab_pool_config.get("excluded_urls"),
preserve_error_tabs=tab_pool_config.get("preserve_error_tabs"),
route_groups=tab_pool_config.get("route_groups"),
)
tab_pool_synced = True
logger.info("运行中的标签页池配置已同步")
except Exception as sync_error:
logger.warning(f"同步标签页池运行时配置失败: {sync_error}")
logger.info(f"浏览器常量已保存: {len(config)} 项")
return {
"status": "success",
"message": "浏览器常量已保存",
"updated_count": len(config),
"tab_pool_synced": tab_pool_synced,
}
except HTTPException:
raise
except Exception as e:
logger.error(f"保存浏览器常量失败: {e}")
raise HTTPException(status_code=500, detail=f"保存失败: {str(e)}")
@router.get("/api/settings/update-preserve")
async def get_update_preserve_settings(authenticated: bool = Depends(verify_auth)):
"""读取更新白名单配置。"""
try:
data = load_update_preserve_settings()
return {
"options": data.get("options", []),
"selected_patterns": data.get("selected_patterns", []),
}
except Exception as e:
logger.error(f"读取更新白名单失败: {e}")
raise HTTPException(status_code=500, detail=f"读取失败: {str(e)}")
@router.post("/api/settings/update-preserve")
async def save_update_preserve(
request: Request,
authenticated: bool = Depends(verify_auth)
):
"""保存更新白名单配置。"""
try:
data = await _read_json_object_or_400(request)
selected_patterns = data.get("selected_patterns", [])
if not isinstance(selected_patterns, list):
raise HTTPException(status_code=400, detail="更新白名单必须是数组")
result = save_update_preserve_settings(selected_patterns)
logger.info(f"更新白名单已保存: {len(result.get('selected_patterns', []))} 项")
return {
"status": "success",
"message": "更新白名单已保存,下次更新时生效",
"selected_patterns": result.get("selected_patterns", []),
}
except HTTPException:
raise
except Exception as e:
logger.error(f"保存更新白名单失败: {e}")
raise HTTPException(status_code=500, detail=f"保存失败: {str(e)}")
# ================= 调试 API =================
def _selector_test_tab_summary(session: Any, tab: Any) -> Dict[str, Any]:
url = ""
try:
url = str(getattr(tab, "url", "") or "")
except Exception:
pass
return {
"tab_id": str(getattr(tab, "tab_id", "") or ""),
"session_id": str(getattr(session, "id", "") or "") if session else "",
"tab_index": int(getattr(session, "persistent_index", 0) or 0) if session else 0,
"url": url,
}
def _selector_test_session_candidates(browser: Any) -> List[Any]:
"""Return a stable snapshot instead of letting the pool choose a random tab."""
pool = browser.tab_pool
pool.refresh_tabs()
sessions = pool.get_sessions_snapshot()
return sorted(
sessions,
key=lambda item: (
int(getattr(item, "persistent_index", 0) or 0),
str(getattr(item, "id", "") or ""),
),
)
def _run_selector_test(
selector: str,
timeout: Any = 2,
highlight: bool = False,
route_domain: str = "",
) -> Dict[str, Any]:
try:
browser = get_browser()
normalized_domain = str(route_domain or "").strip()
if selector.startswith(('tag:', '@', 'xpath:', 'css:')) or '@@' in selector:
query_selector = selector
else:
query_selector = f'css:{selector}'
detail_limit = 6
result = {
"success": False,
"selector": selector,
"locator_used": query_selector,
"route_domain": normalized_domain,
"count": 0,
"elements": [],
"tabs": [],
"tabs_tested": 0,
"matched_tabs": 0,
"skipped_busy_tabs": 0,
"inspected_count": 0,
"truncated": False,
}
def inspect_tab(tab: Any, session: Any = None) -> None:
tab_summary = _selector_test_tab_summary(session, tab)
actual_domain = extract_remote_site_domain(tab_summary.get("url", "")) or ""
if normalized_domain and not route_domain_matches(normalized_domain, actual_domain):
return
try:
elements = driver_for_tab(tab).find_all(query_selector, timeout=timeout)
except Exception as error:
tab_summary.update({"success": False, "count": 0, "message": str(error)})
result["tabs"].append(tab_summary)
return
if not isinstance(elements, list):
elements = [elements] if elements else []
valid_elements = []
for ele in elements:
try:
if ele and hasattr(ele, 'tag'):
valid_elements.append(ele)
except Exception:
pass
tab_summary.update({
"success": bool(valid_elements),
"count": len(valid_elements),
"message": "找到元素" if valid_elements else "元素未找到",
})
result["tabs"].append(tab_summary)
result["count"] += len(valid_elements)
if highlight:
for ele in valid_elements:
try:
as_element(ele).run_js("""
const token = `selector-test-${Date.now()}-${Math.random().toString(36).slice(2)}`;
const previous = {
outline: this.style.outline,
outlineOffset: this.style.outlineOffset,
boxShadow: this.style.boxShadow,
transition: this.style.transition,
};
this.dataset.selectorTestHighlightToken = token;
this.style.transition = 'outline .15s ease, box-shadow .15s ease';
this.style.outline = '3px solid rgba(239, 68, 68, 0.95)';
this.style.outlineOffset = '2px';
this.style.boxShadow = '0 0 0 6px rgba(251, 191, 36, 0.45)';
this.scrollIntoView({behavior: 'smooth', block: 'center', inline: 'center'});
setTimeout(() => {
if (this.dataset.selectorTestHighlightToken !== token) return;
this.style.outline = previous.outline;
this.style.outlineOffset = previous.outlineOffset;
this.style.boxShadow = previous.boxShadow;
this.style.transition = previous.transition;
delete this.dataset.selectorTestHighlightToken;
}, 5000);
""")
except Exception as error:
logger.debug(f"高亮失败: {error}")
remaining = max(0, detail_limit - len(result["elements"]))
start_index = len(result["elements"])
for offset, ele in enumerate(valid_elements[:remaining]):
try:
ele_info = {
"index": start_index + offset,
"tag": ele.tag if hasattr(ele, 'tag') else "unknown",
"text": "",
"candidate_selectors": [],
"warnings": [],
"tab_id": tab_summary.get("tab_id", ""),
"tab_index": tab_summary.get("tab_index", 0),
}
snapshot = _collect_selector_test_element_snapshot(ele)
if snapshot:
ele_info.update(snapshot)
try:
text = ele.text
if text:
ele_info["text"] = text[:100]
except:
pass
try:
attrs = {}
for attr in ['id', 'class', 'name', 'data-testid', 'aria-label']:
val = ele.attr(attr)
if val:
attrs[attr] = val[:50] if isinstance(val, str) else str(val)[:50]
if attrs:
ele_info["attributes"] = attrs
except:
pass
result["elements"].append(ele_info)
except Exception as e:
logger.debug(f"处理元素 {start_index + offset} 失败: {e}")
continue
if normalized_domain:
pool = browser.tab_pool
for candidate in _selector_test_session_candidates(browser):
status = str(getattr(getattr(candidate, "status", None), "value", ""))
cached_domain = str(getattr(candidate, "current_domain", "") or "")
if cached_domain and not route_domain_matches(normalized_domain, cached_domain):
continue
if status != "idle":
if cached_domain and route_domain_matches(normalized_domain, cached_domain):
result["skipped_busy_tabs"] += 1
continue
raw_tab_id = str(getattr(getattr(candidate, "tab", None), "tab_id", "") or "")
if not raw_tab_id:
continue
task_id = f"selector_test_{time.time_ns()}"
acquired = pool.acquire_by_raw_tab_id(
raw_tab_id,
task_id,
timeout=min(max(float(timeout or 2), 0.2), 2.0),
count_request=False,
activate=False,
)
if acquired is None:
continue
try:
inspect_tab(acquired.tab, acquired)
finally:
pool.release(
acquired.id,
check_triggers=False,
expected_task_id=task_id,
)
else:
with browser.get_temporary_tab() as tab:
inspect_tab(tab)
result["tabs_tested"] = len(result["tabs"])
result["matched_tabs"] = sum(1 for item in result["tabs"] if item.get("count", 0) > 0)
result["inspected_count"] = len(result["elements"])
result["truncated"] = result["count"] > len(result["elements"])
result["success"] = result["count"] > 0
if not result["tabs"]:
result["message"] = (
f"没有可测试的 {normalized_domain} 空闲页面"
if normalized_domain else "没有可测试的页面"
)
return result
if not result["success"]:
result["message"] = "已在所有匹配页面测试,元素均未找到"
return result
result["top_candidates"] = _build_selector_top_candidates(result["elements"])
per_tab_counts = [int(item.get("count", 0) or 0) for item in result["tabs"]]
result["diagnosis"] = _build_selector_test_diagnosis(
selector,
max(per_tab_counts) if per_tab_counts else result["count"],
result["top_candidates"],
)
if len(set(per_tab_counts)) > 1:
result["diagnosis"].setdefault("warnings", []).append(
"同一站点的不同页面命中数量不一致,请结合下方逐页结果确认页面状态。"
)
if result["elements"]:
first = result["elements"][0]
result["tag"] = first.get("tag", "unknown")
result["text"] = first.get("text", "")
result["attributes"] = first.get("attributes", {})
return result
except BrowserConnectionError as e:
return {"success": False, "count": 0, "message": f"浏览器未连接: {str(e)}"}
except Exception as e:
logger.error(f"测试选择器失败: {e}")
return {"success": False, "count": 0, "message": str(e)}
@router.post("/api/debug/test-selector")
async def test_selector(
request: Request,
authenticated: bool = Depends(verify_auth)
):
"""测试选择器是否有效"""
if not AppConfig.is_debug():
raise HTTPException(status_code=403, detail="调试功能未启用")
data = await _read_json_object_or_400(request)
selector = data.get("selector", "")
timeout = data.get("timeout", 2)
highlight = data.get("highlight", False)
route_domain = data.get("route_domain", "")
if not selector:
raise HTTPException(status_code=400, detail="缺少 selector")
return await asyncio.to_thread(
_run_selector_test,
selector,
timeout,
highlight,
route_domain,
)
@router.get("/api/debug/request-status")
async def request_status(authenticated: bool = Depends(verify_auth)):
"""查看请求管理器状态"""
if not AppConfig.DEBUG:
raise HTTPException(status_code=403, detail="调试功能未启用")
return request_manager.get_status()
@router.post("/api/debug/force-release")
async def force_release(authenticated: bool = Depends(verify_auth)):
"""强制释放锁"""
if not AppConfig.DEBUG:
raise HTTPException(status_code=403, detail="调试功能未启用")
was_locked = request_manager.is_locked()
cancelled_requests = request_manager.force_release()
released_tabs = 0
release_error = ""
try:
browser = get_browser(auto_connect=False)
pool = getattr(browser, "tab_pool", None)
if pool is not None and hasattr(pool, "force_release_all"):
released_tabs = int(pool.force_release_all() or 0)
except Exception as e:
release_error = str(e)
logger.warning(f"调试强制释放标签页失败: {e}")
released = bool(cancelled_requests or released_tabs)
is_now_locked = request_manager.is_locked()
logger.warning(
f"手动解锁: was={was_locked}, cancelled={cancelled_requests}, "
f"released_tabs={released_tabs}, released={released}, now={is_now_locked}"
)
result = {
"was_locked": was_locked,
"released": released,
"cancelled_requests": cancelled_requests,
"released_tabs": released_tabs,
"is_now_locked": is_now_locked
}
if release_error:
result["release_error"] = release_error
return result
@router.post("/api/debug/cancel-current")
async def cancel_current(
tab_id: Optional[str] = None,
authenticated: bool = Depends(verify_auth),
):
"""取消当前正在执行的请求"""
if not AppConfig.DEBUG:
raise HTTPException(status_code=403, detail="调试功能未启用")
running_requests = request_manager.get_running_requests(tab_id=tab_id)
if not running_requests:
return {"cancelled": False, "message": "没有正在执行的请求", "tab_id": tab_id}
if not tab_id and len(running_requests) > 1:
raise HTTPException(
status_code=400,
detail="存在多个运行中的请求,请指定 tab_id",
)
request_ids = [ctx.request_id for ctx in running_requests]
success = False
for request_id in request_ids:
success = request_manager.cancel_request(request_id, "manual_cancel") or success
return {
"cancelled": success,
"request_id": request_ids[0],
"request_ids": request_ids,
"tab_id": tab_id or running_requests[0].tab_id,
}
import shutil
import os as _os
import psutil as _psutil
def _panel_memory_metric() -> str:
"""PANEL_MEMORY_METRIC: fast(默认)| uss | rss。"""
value = str(_os.getenv("PANEL_MEMORY_METRIC", "fast") or "fast").strip().lower()
return value if value in {"fast", "uss", "rss"} else "fast"
def _get_process_private_memory_bytes(proc: _psutil.Process) -> int:
"""估算进程私有内存,避免多进程浏览器按 RSS 累加后虚高。
P0-7:默认不再调用 memory_full_info()(USS 需要读 smaps / 逐页扫描,
对几十个 Chrome 进程每次采样开销很大),改用 memory_info() 里已有的廉价字段:
- Windows:private(提交的私有内存)
- Linux:rss - shared(statm,近似私有常驻内存)
- 其他平台:rss
PANEL_MEMORY_METRIC=uss 恢复旧的 USS 统计;=rss 直接用 RSS。
"""
metric = _panel_memory_metric()
if metric == "uss":
try:
full_info = proc.memory_full_info()
for attr_name in ("uss", "private", "private_bytes"):
value = getattr(full_info, attr_name, None)
if isinstance(value, (int, float)) and value > 0:
return int(value)
except (_psutil.NoSuchProcess, _psutil.AccessDenied, AttributeError):
pass
try:
info = proc.memory_info()
except (_psutil.NoSuchProcess, _psutil.AccessDenied, AttributeError):
return 0
rss = int(getattr(info, "rss", 0) or 0)
if metric == "fast":
private = getattr(info, "private", None)
if isinstance(private, (int, float)) and private > 0:
return int(private)
shared = getattr(info, "shared", None)
if isinstance(shared, (int, float)) and 0 <= shared < rss:
return int(rss - shared)
return rss
def _collect_process_tree_memory_bytes(proc: _psutil.Process, seen_pids: set[int]) -> int:
"""统计进程树内存,并避免重复累计已经见过的 PID。"""
total_bytes = 0
try:
processes = [proc]
processes.extend(proc.children(recursive=True))
except (_psutil.NoSuchProcess, _psutil.AccessDenied):
processes = [proc]
for item in processes:
try:
pid = int(item.pid)
except (_psutil.NoSuchProcess, _psutil.AccessDenied, AttributeError, TypeError, ValueError):
continue
if pid in seen_pids:
continue
seen_pids.add(pid)
total_bytes += _get_process_private_memory_bytes(item)
return total_bytes
def _extract_process_flag_value(cmdline: list[str], flag_name: str) -> str:
"""从命令行参数中提取浏览器 flag 的值,兼容 --flag value / --flag=value。"""
normalized_flag = str(flag_name or "").strip()
if not normalized_flag:
return ""
for index, raw_part in enumerate(cmdline or []):
part = str(raw_part or "").strip().strip('"')
if not part:
continue
if part == normalized_flag:
if index + 1 < len(cmdline):
return str(cmdline[index + 1] or "").strip().strip('"')
return ""
prefix = f"{normalized_flag}="
if part.startswith(prefix):
return part[len(prefix):].strip().strip('"')
return ""
def _normalize_path_for_compare(path_value: Any) -> str:
"""路径比较前统一大小写和绝对路径,降低命令行写法差异的影响。"""
raw_text = str(path_value or "").strip().strip('"')
if not raw_text:
return ""
try:
return str(Path(raw_text).expanduser().resolve()).casefold()
except Exception:
try:
return str(Path(raw_text).expanduser().absolute()).casefold()
except Exception:
return raw_text.casefold()
def _is_chromium_family_process(process_name: str) -> bool:
lowered = str(process_name or "").strip().lower()
if not lowered:
return False
return any(
token in lowered
for token in ("chrome", "msedge", "brave", "vivaldi", "opera")
)
def _cmdline_matches_browser_profile(cmdline: list[str], profile_root_norm: str) -> bool:
if not profile_root_norm:
return False
user_data_dir = _extract_process_flag_value(cmdline, "--user-data-dir")
if not user_data_dir:
return False
return _normalize_path_for_compare(user_data_dir) == profile_root_norm
def _find_project_browser_root_processes(
main_proc: _psutil.Process,
*,
project_dir: Path,
python_descendant_pids: set[int],
) -> list[_psutil.Process]:
"""
只匹配当前项目启动的调试浏览器主进程。
必须同时命中项目 profile 目录和调试端口,避免把普通 Chrome 窗口算进来。
"""
profile_root = _resolve_browser_profile_root(project_dir)
profile_root_norm = _normalize_path_for_compare(profile_root)
debug_port = str(AppConfig.get_browser_port() or "").strip()
matched_processes: list[_psutil.Process] = []
for proc in _psutil.process_iter(['pid', 'name', 'cmdline']):
try:
pid = int(proc.pid)
if pid == main_proc.pid or pid in python_descendant_pids:
continue
proc_name = proc.info.get('name') or ""
if not _is_chromium_family_process(proc_name):
continue
cmdline = proc.info.get('cmdline') or []
if not cmdline:
continue
process_type = _extract_process_flag_value(cmdline, "--type")
if process_type:
continue
proc_debug_port = _extract_process_flag_value(cmdline, "--remote-debugging-port")
if not (
debug_port
and proc_debug_port == debug_port
):
continue
if not _cmdline_matches_browser_profile(cmdline, profile_root_norm):
continue
matched_processes.append(proc)
except (_psutil.NoSuchProcess, _psutil.AccessDenied, ValueError, TypeError):
continue
return matched_processes
def _collect_project_process_snapshot() -> Dict[str, Any]:
"""收集一次项目进程快照,供 CPU/内存统计共用。"""
try:
project_dir = Path(__file__).resolve().parents[2]
main_pid = _os.getpid()
main_proc = _psutil.Process(main_pid)
try:
python_descendants = main_proc.children(recursive=True)
except Exception:
python_descendants = []
python_descendant_pids = {
int(child.pid)
for child in python_descendants
if getattr(child, "pid", None) is not None
}
browser_roots = _find_project_browser_root_processes(
main_proc,
project_dir=project_dir,
python_descendant_pids=python_descendant_pids,
)
tracked_processes = [main_proc]
tracked_processes.extend(python_descendants)
tracked_pids = {main_pid}
tracked_pids.update(python_descendant_pids)
for browser_proc in browser_roots:
try:
browser_pid = int(browser_proc.pid)
except Exception:
browser_pid = None
if browser_pid is not None:
tracked_processes.append(browser_proc)
tracked_pids.add(browser_pid)
try:
browser_descendants = browser_proc.children(recursive=True)
except Exception:
browser_descendants = []
for child in browser_descendants:
try:
child_pid = int(child.pid)
except Exception:
continue
tracked_processes.append(child)
tracked_pids.add(child_pid)
return {
"project_dir": project_dir,
"main_pid": main_pid,
"main_proc": main_proc,
"python_descendants": python_descendants,
"python_descendant_pids": python_descendant_pids,
"browser_roots": browser_roots,
"tracked_processes": tracked_processes,
"tracked_pids": tracked_pids,
}
except Exception:
return {
"project_dir": Path(__file__).resolve().parents[2],
"main_pid": _os.getpid(),
"main_proc": None,
"python_descendants": [],
"python_descendant_pids": set(),
"browser_roots": [],
"tracked_processes": [],
"tracked_pids": set(),
}
def _get_project_memory_mb(process_snapshot: Optional[Dict[str, Any]] = None) -> float:
"""估算本项目相关进程的私有内存,尽量避免浏览器多进程重复计数。"""
try:
snapshot = process_snapshot or _get_project_process_snapshot_cached()
main_proc = snapshot.get("main_proc")
if main_proc is None:
return 0.0
seen_pids: set[int] = set()
total_bytes = 0
if "tracked_processes" in snapshot:
for proc in snapshot.get("tracked_processes") or []:
try:
pid = int(proc.pid)
except Exception:
continue
if pid in seen_pids:
continue
seen_pids.add(pid)
total_bytes += _get_process_private_memory_bytes(proc)
else:
total_bytes = _collect_process_tree_memory_bytes(main_proc, seen_pids)
for browser_proc in snapshot.get("browser_roots") or []:
total_bytes += _collect_process_tree_memory_bytes(browser_proc, seen_pids)
return round(total_bytes / (1024 * 1024), 1)
except Exception:
return 0.0
_PROJECT_PROCESS_CACHE: Dict[int, _psutil.Process] = {}
_PROJECT_PROCESS_CACHE_LOCK = _threading.Lock()
_PROJECT_PROCESS_SNAPSHOT_TTL_SECONDS = 5.0
_PROJECT_PROCESS_SNAPSHOT_CACHE = {
"expires_at": 0.0,
"snapshot": None,
}
_PROJECT_PROCESS_SNAPSHOT_CACHE_LOCK = _threading.Lock()
_PROJECT_MEMORY_CACHE_TTL_SECONDS = 10.0
_PROJECT_MEMORY_CACHE = {
"expires_at": 0.0,
"memory_mb": 0.0,
}
_PROJECT_MEMORY_CACHE_LOCK = _threading.Lock()
def _get_project_process_snapshot_cached(ttl_seconds: float = _PROJECT_PROCESS_SNAPSHOT_TTL_SECONDS) -> Dict[str, Any]:
"""复用项目进程树快照,避免频繁枚举系统进程。"""
now = time.monotonic()
cached_snapshot = _PROJECT_PROCESS_SNAPSHOT_CACHE.get("snapshot")
if cached_snapshot is not None and now < float(_PROJECT_PROCESS_SNAPSHOT_CACHE.get("expires_at", 0.0) or 0.0):
return cached_snapshot
with _PROJECT_PROCESS_SNAPSHOT_CACHE_LOCK:
now = time.monotonic()
cached_snapshot = _PROJECT_PROCESS_SNAPSHOT_CACHE.get("snapshot")
if cached_snapshot is not None and now < float(_PROJECT_PROCESS_SNAPSHOT_CACHE.get("expires_at", 0.0) or 0.0):
return cached_snapshot
snapshot = _collect_project_process_snapshot()
ttl = max(2.0, float(ttl_seconds or _PROJECT_PROCESS_SNAPSHOT_TTL_SECONDS))
_PROJECT_PROCESS_SNAPSHOT_CACHE["snapshot"] = snapshot
_PROJECT_PROCESS_SNAPSHOT_CACHE["expires_at"] = time.monotonic() + ttl
return snapshot
def _get_project_memory_mb_cached(
process_snapshot: Optional[Dict[str, Any]] = None,
ttl_seconds: float = _PROJECT_MEMORY_CACHE_TTL_SECONDS,
) -> float:
"""缓存项目内存采样,减少对多进程浏览器的频繁内存查询。"""
now = time.monotonic()
if now < float(_PROJECT_MEMORY_CACHE.get("expires_at", 0.0) or 0.0):
return float(_PROJECT_MEMORY_CACHE.get("memory_mb", 0.0) or 0.0)
with _PROJECT_MEMORY_CACHE_LOCK:
now = time.monotonic()
if now < float(_PROJECT_MEMORY_CACHE.get("expires_at", 0.0) or 0.0):
return float(_PROJECT_MEMORY_CACHE.get("memory_mb", 0.0) or 0.0)
snapshot = process_snapshot or _get_project_process_snapshot_cached()
memory_mb = float(_get_project_memory_mb(snapshot) or 0.0)
ttl = max(2.0, float(ttl_seconds or _PROJECT_MEMORY_CACHE_TTL_SECONDS))
_PROJECT_MEMORY_CACHE["memory_mb"] = memory_mb
_PROJECT_MEMORY_CACHE["expires_at"] = time.monotonic() + ttl
return memory_mb
def _get_project_cpu_percent(process_snapshot: Optional[Dict[str, Any]] = None) -> float:
"""估算本项目(包含主 Python 进程及其下所有 Chrome 调试浏览器进程)的总 CPU 占比,除以核心数折算为 0-100% 格式。"""
global _PROJECT_PROCESS_CACHE
try:
snapshot = process_snapshot or _collect_project_process_snapshot()
main_pid = int(snapshot.get("main_pid") or _os.getpid())
main_proc = snapshot.get("main_proc")
if main_proc is None:
return 0.0
with _PROJECT_PROCESS_CACHE_LOCK:
if main_pid not in _PROJECT_PROCESS_CACHE:
_PROJECT_PROCESS_CACHE[main_pid] = main_proc
tracked_by_pid = {}
def remember_snapshot_process(proc: Any) -> Optional[int]:
try:
pid = int(proc.pid)
except Exception:
return None
tracked_by_pid[pid] = proc
return pid
remember_snapshot_process(main_proc)
for proc in snapshot.get("tracked_processes") or []:
remember_snapshot_process(proc)
# 收集所有相关的当前活跃 PID。新快照阶段已遍历浏览器子进程,避免 CPU/内存路径重复 children()。
if "tracked_pids" in snapshot:
target_pids = set(snapshot.get("tracked_pids") or set())
target_pids.add(main_pid)
target_pids.update(tracked_by_pid.keys())
else:
target_pids = set()
remembered_pid = remember_snapshot_process(main_proc)
if remembered_pid is not None:
target_pids.add(remembered_pid)
for proc in snapshot.get("python_descendants") or []:
remembered_pid = remember_snapshot_process(proc)
if remembered_pid is not None:
target_pids.add(remembered_pid)
target_pids.update(snapshot.get("python_descendant_pids") or set())
for b_proc in snapshot.get("browser_roots") or []:
try:
remembered_pid = remember_snapshot_process(b_proc)
if remembered_pid is not None:
target_pids.add(remembered_pid)
for child in b_proc.children(recursive=True):
if getattr(child, "pid", None) is not None:
remembered_pid = remember_snapshot_process(child)
if remembered_pid is not None:
target_pids.add(remembered_pid)
except Exception:
pass
if not target_pids:
target_pids = {main_pid}
# 更新/清理 Process 缓存,只保留活跃的进程以防止内存泄露
new_cache = {}
for pid in target_pids:
if pid in tracked_by_pid:
new_cache[pid] = tracked_by_pid[pid]
elif pid in _PROJECT_PROCESS_CACHE:
new_cache[pid] = _PROJECT_PROCESS_CACHE[pid]
else:
try:
new_cache[pid] = _psutil.Process(pid)
except Exception:
pass
_PROJECT_PROCESS_CACHE = new_cache
# 复制一份用于在锁外进行计算,防阻塞
processes_to_measure = list(_PROJECT_PROCESS_CACHE.items())
# 在锁外调用 cpu_percent 以提升并发性能并减少锁定时间
seen_pids = set()
total_cpu = 0.0
for pid, proc in processes_to_measure:
try:
if pid in seen_pids:
continue
seen_pids.add(pid)
total_cpu += proc.cpu_percent(interval=None)
except Exception:
pass
cpu_count = _psutil.cpu_count() or 1
return round(total_cpu / cpu_count, 1)
except Exception as e:
logger.error(f"[CPU_DEBUG] 计算 CPU 失败: {e}")
return 0.0
def _resolve_browser_profile_root(project_dir: Path) -> Optional[Path]:
"""解析浏览器用户目录根路径。留空时默认使用项目内 chrome_profile。"""
try:
env_config = _load_env_config_from_file()
raw_value = str(
env_config.get("BROWSER_PROFILE_DIR")
or os.getenv("BROWSER_PROFILE_DIR", "")
or ""
).strip()
if not raw_value:
return project_dir / "chrome_profile"
profile_dir = Path(raw_value)
if not profile_dir.is_absolute():
profile_dir = project_dir / profile_dir
return profile_dir
except Exception:
return project_dir / "chrome_profile"
def _get_path_size_bytes(path: Path, *, skip_dir_names: Optional[set[str]] = None) -> int:
"""递归统计路径大小。"""
try:
resolved = path.resolve()
except Exception:
resolved = path
if not resolved.exists():
return 0
if resolved.is_file():
try:
return resolved.stat().st_size
except OSError:
return 0
total_size = 0
skip_names = set(skip_dir_names or set())
for dirpath, dirnames, filenames in os.walk(resolved):
if skip_names:
dirnames[:] = [d for d in dirnames if d not in skip_names]
for filename in filenames:
file_path = Path(dirpath) / filename
try:
total_size += file_path.stat().st_size
except OSError:
pass
return total_size
def _get_project_disk_usage_mb() -> float:
"""计算项目实际占用磁盘空间,包含 venv 和浏览器用户目录。"""
try:
project_dir = Path(__file__).resolve().parents[2]
total_size = _get_path_size_bytes(
project_dir,
skip_dir_names={".git", "__pycache__", "node_modules"},
)
profile_dir = _resolve_browser_profile_root(project_dir)
if profile_dir is not None:
try:
project_resolved = project_dir.resolve()
profile_resolved = profile_dir.resolve()
except Exception:
project_resolved = project_dir
profile_resolved = profile_dir
# 外部浏览器用户目录不在项目内时,额外计入一次。
if profile_resolved != project_resolved and project_resolved not in profile_resolved.parents:
total_size += _get_path_size_bytes(profile_resolved)
return round(total_size / (1024 * 1024), 1)
except Exception:
return 0.0
_SYSTEM_STATS_CACHE = {
"expires_at": 0.0,
"payload": {
"memory_mb": 0.0,
"disk_status": "0 MB",
"total_requests": 0,
"total_input_tokens": 0,
"total_output_tokens": 0,
"cpu_percent": 0.0,
"project_cpu": 0.0,
"memory_percent": 0.0,
"project_memory_percent": 0.0,
# 与实际构建的 payload 保持同一形状,避免消费方缺键
"running_count": 0,
"queued_count": 0,
},
}
_SYSTEM_STATS_CACHE_LOCK = _threading.Lock()
_DISK_USAGE_CACHE = {
"expires_at": 0.0,
"value": 0.0,
"last_success_value": 0.0,
}
_DISK_USAGE_REFRESH_LOCK = _threading.Lock()
_DISK_USAGE_REFRESH_WORKER: Optional[_threading.Thread] = None
def _reset_system_stats_cache_for_tests() -> None:
"""清理系统统计缓存,供回归测试隔离并避免后台刷新状态串扰。"""
global _PROJECT_PROCESS_CACHE, _DISK_USAGE_REFRESH_WORKER
with _SYSTEM_STATS_CACHE_LOCK:
_SYSTEM_STATS_CACHE["expires_at"] = 0.0
_SYSTEM_STATS_CACHE["payload"] = {
"memory_mb": 0.0,
"disk_status": "0 MB",
"total_requests": 0,
"total_input_tokens": 0,
"total_output_tokens": 0,
"cpu_percent": 0.0,
"project_cpu": 0.0,
"memory_percent": 0.0,
"project_memory_percent": 0.0,
# 与实际构建的 payload 保持同一形状,避免消费方缺键
"running_count": 0,
"queued_count": 0,
}
with _PROJECT_PROCESS_CACHE_LOCK:
_PROJECT_PROCESS_CACHE = {}
with _PROJECT_PROCESS_SNAPSHOT_CACHE_LOCK:
_PROJECT_PROCESS_SNAPSHOT_CACHE["expires_at"] = 0.0
_PROJECT_PROCESS_SNAPSHOT_CACHE["snapshot"] = None
with _PROJECT_MEMORY_CACHE_LOCK:
_PROJECT_MEMORY_CACHE["expires_at"] = 0.0
_PROJECT_MEMORY_CACHE["memory_mb"] = 0.0
_DISK_USAGE_CACHE["expires_at"] = 0.0
_DISK_USAGE_CACHE["value"] = 0.0
_DISK_USAGE_CACHE["last_success_value"] = 0.0
with _DISK_USAGE_REFRESH_LOCK:
_DISK_USAGE_REFRESH_WORKER = None
def _format_disk_usage(disk_mb: float) -> str:
if disk_mb >= 1024:
return f"{round(disk_mb / 1024, 2)} GB"
return f"{disk_mb} MB"
def _set_disk_usage_cache_result(disk_mb: float, ttl_seconds: float) -> float:
now = time.monotonic()
ttl = max(1.0, float(ttl_seconds or 60.0))
if disk_mb > 0:
_DISK_USAGE_CACHE["value"] = disk_mb
_DISK_USAGE_CACHE["last_success_value"] = disk_mb
_DISK_USAGE_CACHE["expires_at"] = now + ttl
return disk_mb
fallback_value = float(
_DISK_USAGE_CACHE.get("last_success_value")
or _DISK_USAGE_CACHE.get("value", 0.0)
or 0.0
)
if fallback_value > 0:
_DISK_USAGE_CACHE["value"] = fallback_value
_DISK_USAGE_CACHE["expires_at"] = now + min(ttl, 5.0)
return fallback_value
_DISK_USAGE_CACHE["value"] = 0.0
_DISK_USAGE_CACHE["expires_at"] = now + min(ttl, 5.0)
return 0.0
def _refresh_project_disk_usage_cache(ttl_seconds: float = 60.0) -> float:
try:
disk_mb = float(_get_project_disk_usage_mb() or 0.0)
except Exception:
disk_mb = 0.0
return _set_disk_usage_cache_result(disk_mb, ttl_seconds)
def _run_disk_usage_refresh_worker(ttl_seconds: float) -> None:
global _DISK_USAGE_REFRESH_WORKER
current_worker = _threading.current_thread()
try:
try:
_refresh_project_disk_usage_cache(ttl_seconds)
except Exception as exc:
try:
logger.warning(f"磁盘用量后台刷新失败: {exc}")
except Exception:
pass
finally:
with _DISK_USAGE_REFRESH_LOCK:
if _DISK_USAGE_REFRESH_WORKER is current_worker:
_DISK_USAGE_REFRESH_WORKER = None
def _schedule_disk_usage_refresh(ttl_seconds: float = 60.0) -> None:
global _DISK_USAGE_REFRESH_WORKER
with _DISK_USAGE_REFRESH_LOCK:
if _DISK_USAGE_REFRESH_WORKER and _DISK_USAGE_REFRESH_WORKER.is_alive():
return
worker = _threading.Thread(
target=_run_disk_usage_refresh_worker,
args=(ttl_seconds,),
daemon=True,
name="disk-usage-refresh",
)
_DISK_USAGE_REFRESH_WORKER = worker
worker.start()
def _get_project_disk_usage_mb_cached(ttl_seconds: float = 60.0) -> float:
now = time.monotonic()
cached_value = float(_DISK_USAGE_CACHE.get("value", 0.0) or 0.0)
if now < float(_DISK_USAGE_CACHE.get("expires_at", 0.0) or 0.0):
return cached_value
ttl = max(1.0, float(ttl_seconds or 60.0))
fallback_value = float(
_DISK_USAGE_CACHE.get("last_success_value")
or cached_value
or 0.0
)
if fallback_value > 0:
_DISK_USAGE_CACHE["value"] = fallback_value
_DISK_USAGE_CACHE["expires_at"] = now + min(ttl, 5.0)
_schedule_disk_usage_refresh(ttl)
return fallback_value
return _refresh_project_disk_usage_cache(ttl)
def _get_live_request_counts() -> Dict[str, int]:
"""实时请求计数(运行中 / 排队中)。stats 是高频轮询接口,取值失败一律回落 0,不能因此整体 500。"""
try:
status = request_manager.get_status() or {}
status_counts = status.get("status_counts") or {}
return {
"running_count": int(status.get("running_count", 0) or 0),
"queued_count": int(status_counts.get("queued", 0) or 0),
}
except Exception:
return {"running_count": 0, "queued_count": 0}
def _get_fresh_system_stats_payload() -> Optional[Dict[str, Any]]:
now = time.monotonic()
cached_payload = _SYSTEM_STATS_CACHE.get("payload") or {}
if now < float(_SYSTEM_STATS_CACHE.get("expires_at", 0.0) or 0.0):
payload = dict(cached_payload)
# 进程/内存采样缓存 5s;请求计数很便宜,保持实时
payload.update(_get_live_request_counts())
payload["total_requests"] = int(getattr(request_manager, "total_requests", 0) or 0)
payload["total_input_tokens"] = int(getattr(request_manager, "total_input_tokens", 0) or 0)
payload["total_output_tokens"] = int(getattr(request_manager, "total_output_tokens", 0) or 0)
return payload
return None
def _get_system_stats_payload_cached(ttl_seconds: float = 5.0) -> Dict[str, Any]:
fresh_payload = _get_fresh_system_stats_payload()
if fresh_payload is not None:
return fresh_payload
with _SYSTEM_STATS_CACHE_LOCK:
now = time.monotonic()
cached_payload = _SYSTEM_STATS_CACHE.get("payload") or {}
if now < float(_SYSTEM_STATS_CACHE.get("expires_at", 0.0) or 0.0):
return dict(cached_payload)
cpu_percent = 0.0
project_cpu = 0.0
memory_percent = 0.0
project_memory_percent = 0.0
process_snapshot = _get_project_process_snapshot_cached()
try:
cpu_percent = float(_psutil.cpu_percent(interval=None) or 0.0)
virtual_memory = _psutil.virtual_memory()
memory_percent = float(virtual_memory.percent or 0.0)
project_cpu = float(_get_project_cpu_percent(process_snapshot) or 0.0)
except Exception:
virtual_memory = None
project_memory_mb = _get_project_memory_mb_cached(process_snapshot)
try:
total_memory_bytes = int(getattr(virtual_memory, "total", 0) or 0)
if total_memory_bytes > 0:
project_memory_percent = round((project_memory_mb * 1024 * 1024 / total_memory_bytes) * 100, 1)
except Exception:
project_memory_percent = 0.0
# 追加实时请求计数:request-history 只落终态记录,无法反映“正在执行”
live_counts = _get_live_request_counts()
payload = {
"memory_mb": project_memory_mb,
"disk_status": _format_disk_usage(_get_project_disk_usage_mb_cached()),
"total_requests": int(getattr(request_manager, "total_requests", 0) or 0),
"total_input_tokens": int(getattr(request_manager, "total_input_tokens", 0) or 0),
"total_output_tokens": int(getattr(request_manager, "total_output_tokens", 0) or 0),
"cpu_percent": cpu_percent,
"project_cpu": project_cpu,
"memory_percent": memory_percent,
"project_memory_percent": project_memory_percent,
"running_count": live_counts["running_count"],
"queued_count": live_counts["queued_count"],
}
_SYSTEM_STATS_CACHE["payload"] = payload
_SYSTEM_STATS_CACHE["expires_at"] = now + max(0.8, float(ttl_seconds or 5.0))
return dict(payload)
@router.get("/api/system/stats")
async def get_system_stats(authenticated: bool = Depends(verify_auth)):
cached_payload = _get_fresh_system_stats_payload()
if cached_payload is not None:
return cached_payload
return await asyncio.to_thread(_get_system_stats_payload_cached, 1.0)
@router.get("/api/system/request-history")
async def get_request_history(
limit: int = 200,
detail: bool = False,
if_revision: Optional[str] = None,
authenticated: bool = Depends(verify_auth),
):
# RequestManager applies the configured REQUEST_MONITOR_MAX_RECORDS boundary.
# Do not impose a second 200-record ceiling at the API layer.
safe_limit = max(1, int(limit or 200))
return await asyncio.to_thread(
request_manager.get_request_history_payload,
safe_limit,
bool(detail),
if_revision,
)
@router.get("/api/system/request-history/{request_id}")
async def get_request_history_detail(
request_id: str,
authenticated: bool = Depends(verify_auth),
):
record = await asyncio.to_thread(request_manager.get_request_history_record, request_id)
if not record:
raise HTTPException(status_code=404, detail="请求历史不存在")
return record
# ================= 版本管理 API =================
_version_switch_lock = _threading.Lock()
_version_switch_state: dict = {
"running": False,
"tag": None,
"success": None,
"error": None,
}
_update_check_lock = _threading.Lock()
_update_check_state: dict = {
"checked": False,
"checking": False,
"available": False,
"current_version": APP_VERSION,
"latest_version": "",
"latest_tag": "",
"published_at": "",
"repo": "",
"checked_at": None,
"error": "",
}
_startup_update_check_scheduled = False
def _get_update_check_state() -> Dict[str, Any]:
with _update_check_lock:
return dict(_update_check_state)
def _set_update_check_state(**changes: Any) -> Dict[str, Any]:
with _update_check_lock:
_update_check_state.update(changes)
return dict(_update_check_state)
def _build_update_check_payload(release: Optional[dict], repo: str) -> Dict[str, Any]:
from updater import compare_versions, get_current_version, normalize_version
current_version = get_current_version()
latest_tag = ""
latest_version = ""
published_at = ""
if release:
latest_tag = str(release.get("tag_name") or "").strip()
published_at = str(release.get("published_at") or "").strip()
if latest_tag:
latest_version = normalize_version(latest_tag)
return {
"checked": True,
"checking": False,
"available": bool(latest_version and compare_versions(current_version, latest_version) < 0),
"current_version": current_version,
"latest_version": latest_version,
"latest_tag": latest_tag,
"published_at": published_at,
"repo": repo,
"checked_at": int(time.time()),
"error": "",
}
def _run_update_check(repo: Optional[str] = None) -> Dict[str, Any]:
from updater import DEFAULT_REPO, fetch_latest_release, get_current_version
target_repo = str(repo or os.getenv("GITHUB_REPO", DEFAULT_REPO) or DEFAULT_REPO).strip()
if not target_repo:
target_repo = DEFAULT_REPO
_set_update_check_state(checking=True, repo=target_repo, error="")
try:
release = fetch_latest_release(target_repo)
if not release:
raise RuntimeError("无法获取最新版本信息")
payload = _build_update_check_payload(release, target_repo)
logger.info(
f"[startup] 版本检查完成: 当前 v{payload['current_version']},"
f"最新 {payload['latest_tag'] or '未知'},"
f"{'有新版本' if payload['available'] else '已是最新'}"
)
return _set_update_check_state(**payload)
except Exception as exc:
logger.warning(f"版本检查失败: {exc}")
# 修复:_set_update_check_state 是 merge 语义,失败时若不显式清空这三项,
# 上一次成功检查的 latest_* 会残留下来,前端会同时显示"最新版本号"和"检查失败"。
return _set_update_check_state(
checked=True,
checking=False,
available=False,
current_version=get_current_version(),
latest_version="",
latest_tag="",
published_at="",
checked_at=int(time.time()),
error=str(exc),
)
def schedule_startup_update_check() -> None:
"""启动时只触发一次后台版本检查,供控制面板红点读取。"""
global _startup_update_check_scheduled
with _update_check_lock:
if _startup_update_check_scheduled or _update_check_state.get("checking"):
return
_startup_update_check_scheduled = True
_update_check_state["checking"] = True
_update_check_state["error"] = ""
t = _threading.Thread(
target=_run_update_check,
daemon=True,
name="startup-update-check",
)
t.start()
def _apply_release_list_update_check(releases_raw: list, repo: str) -> Dict[str, Any]:
release = releases_raw[0] if releases_raw else None
try:
payload = _build_update_check_payload(release, repo)
except Exception as exc:
logger.warning(f"更新版本提示状态失败: {exc}")
return _get_update_check_state()
return _set_update_check_state(**payload)
@router.get("/api/update/check")
async def get_update_check(authenticated: bool = Depends(verify_auth)):
"""读取启动时缓存的版本检查状态。不会访问 GitHub。"""
return _get_update_check_state()
@router.post("/api/update/check")
async def refresh_update_check(
repo: Optional[str] = None,
authenticated: bool = Depends(verify_auth),
):
"""手动触发一次版本检查。"""
return await asyncio.to_thread(_run_update_check, repo)
@router.get("/api/update/releases")
async def get_releases(
repo: Optional[str] = None,
authenticated: bool = Depends(verify_auth),
):
"""获取 GitHub Release 列表(供控制面板手动选择版本)"""
try:
from updater import fetch_all_releases, get_current_version, normalize_version, DEFAULT_REPO
target_repo = repo or os.getenv("GITHUB_REPO", DEFAULT_REPO)
releases_raw = await asyncio.to_thread(fetch_all_releases, target_repo)
current_version = get_current_version()
update_check = _apply_release_list_update_check(releases_raw, target_repo)
result = []
for r in releases_raw:
tag = str(r.get("tag_name") or "").strip()
if not tag:
continue
result.append({
"tag": tag,
"published_at": r.get("published_at") or "",
"body": r.get("body") or "",
"zipball_url": r.get("zipball_url") or "",
"is_current": normalize_version(tag) == normalize_version(current_version),
})
return {"releases": result, "current_version": current_version, "update_check": update_check}
except Exception as e:
logger.error(f"获取 Release 列表失败: {e}")
raise HTTPException(status_code=500, detail=f"获取失败: {str(e)}")
@router.post("/api/update/switch")
async def switch_version(
request: Request,
authenticated: bool = Depends(verify_auth),
):
"""切换到指定版本(在后台线程中执行,完成后触发服务重启)"""
global _version_switch_state
with _version_switch_lock:
if _version_switch_state.get("running"):
raise HTTPException(status_code=409, detail="已有版本切换任务正在运行,请稍候")
data = await _read_json_object_or_400(request)
tag = str(data.get("tag") or "").strip()
if not tag:
raise HTTPException(status_code=400, detail="缺少 tag 参数")
def _run_switch():
global _version_switch_state
try:
from updater import update_to_version
success = update_to_version(tag)
with _version_switch_lock:
_version_switch_state = {"running": False, "tag": tag, "success": success, "error": None}
if success:
import time as _time
logger.warning(f"版本切换成功: {tag},2 秒后重启服务")
_time.sleep(2.0)
try:
from app.services.restart_guard import mark_service_restart
mark_service_restart()
except Exception:
pass
import os as _os
_os._exit(3)
except Exception as exc:
logger.error(f"版本切换失败: {exc}")
with _version_switch_lock:
_version_switch_state = {"running": False, "tag": tag, "success": False, "error": str(exc)}
with _version_switch_lock:
_version_switch_state = {"running": True, "tag": tag, "success": None, "error": None}
t = _threading.Thread(target=_run_switch, daemon=True, name=f"version-switch-{tag}")
t.start()
logger.info(f"版本切换任务已启动: {tag}")
return {"status": "started", "tag": tag}
@router.get("/api/update/status")
async def get_version_switch_status(authenticated: bool = Depends(verify_auth)):
"""查询当前版本切换任务状态"""
with _version_switch_lock:
return dict(_version_switch_state)