Initial commit
This commit is contained in:
commit
57fc3525eb
7311 files changed
+1291398
No files matched your search
@@ -0,0 +1,13 @@
|
||||
"""
|
||||
app/services/config - 配置管理模块
|
||||
|
||||
职责:
|
||||
- 导出配置引擎单例(保持向后兼容)
|
||||
"""
|
||||
|
||||
from .engine import ConfigEngine, ConfigConstants, DEFAULT_WORKFLOW
|
||||
|
||||
# 创建单例
|
||||
config_engine = ConfigEngine()
|
||||
|
||||
__all__ = ['config_engine', 'ConfigEngine', 'ConfigConstants', 'DEFAULT_WORKFLOW']
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -0,0 +1,48 @@
|
||||
"""
|
||||
app/services/config/cache.py - lightweight in-memory cache for config reads.
|
||||
"""
|
||||
|
||||
import threading
|
||||
import time
|
||||
from typing import Any, Dict, Tuple
|
||||
|
||||
|
||||
_CACHE_MISS = object()
|
||||
|
||||
|
||||
class ConfigCache:
|
||||
"""Small TTL cache used only for derived config read results."""
|
||||
|
||||
def __init__(self, ttl: float = 5.0):
|
||||
try:
|
||||
ttl_value = float(ttl)
|
||||
except (TypeError, ValueError):
|
||||
ttl_value = 5.0
|
||||
self._ttl = max(0.0, ttl_value)
|
||||
self._cache: Dict[str, Tuple[Any, float]] = {}
|
||||
self._lock = threading.RLock()
|
||||
|
||||
def get(self, key: str, default: Any = _CACHE_MISS) -> Any:
|
||||
now = time.monotonic()
|
||||
with self._lock:
|
||||
entry = self._cache.get(key)
|
||||
if entry is None:
|
||||
return default
|
||||
|
||||
value, created_at = entry
|
||||
if self._ttl and now - created_at > self._ttl:
|
||||
self._cache.pop(key, None)
|
||||
return default
|
||||
|
||||
return value
|
||||
|
||||
def set(self, key: str, value: Any) -> None:
|
||||
with self._lock:
|
||||
self._cache[key] = (value, time.monotonic())
|
||||
|
||||
def invalidate(self, key: str = None) -> None:
|
||||
with self._lock:
|
||||
if key is None:
|
||||
self._cache.clear()
|
||||
else:
|
||||
self._cache.pop(key, None)
|
||||
@@ -0,0 +1,298 @@
|
||||
"""
|
||||
app/services/config/engine.py - 配置引擎主类
|
||||
|
||||
职责:
|
||||
- 配置文件读写
|
||||
- 站点配置管理
|
||||
- 配置缓存与热重载
|
||||
- 图片配置、提取器管理
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import copy
|
||||
import re
|
||||
import threading
|
||||
import time
|
||||
from app.core.config import get_logger
|
||||
from typing import Dict, Optional, List, Any, Set, Union, Callable
|
||||
from urllib.parse import urljoin
|
||||
from app.core.parsers import ParserRegistry
|
||||
from app.models.schemas import (
|
||||
SiteConfig,
|
||||
WorkflowStep,
|
||||
SelectorDefinition,
|
||||
ADVANCED_FIELDS,
|
||||
PRESET_ADVANCED_FIELDS,
|
||||
SITE_ADVANCED_FIELDS,
|
||||
get_default_image_extraction_config,
|
||||
get_default_file_paste_config,
|
||||
get_default_prompt_padding_config,
|
||||
get_default_site_advanced_config,
|
||||
get_default_attachment_monitor_config,
|
||||
get_default_send_confirmation_config,
|
||||
get_modality_policy,
|
||||
get_enabled_modalities,
|
||||
is_modality_enabled,
|
||||
normalize_modalities_config,
|
||||
normalize_modality_policy,
|
||||
)
|
||||
from app.services.extractor_manager import extractor_manager
|
||||
from app.services.parser_manager import parser_manager
|
||||
from app.utils.site_rules import derive_site_card_id, get_site_rule
|
||||
from app.utils.site_discovery import automatic_discovery_allowed, specific_chat_selectors, selectors_match_chat_html
|
||||
from app.core.request_transport import (
|
||||
get_default_request_transport_config,
|
||||
normalize_request_transport_config,
|
||||
)
|
||||
from .cache import ConfigCache
|
||||
from .site_store import SiteStore
|
||||
from .managers import GlobalConfigManager, ImagePresetsManager
|
||||
from .processors import HTMLCleaner, SelectorValidator, AIAnalyzer
|
||||
|
||||
|
||||
# R2-3:模块级定义迁到 engine_common,这里重新导出以保持兼容(外部与测试都从 engine 导入这些名字)
|
||||
from .engine_common import ( # noqa: E402,F401
|
||||
ConfigConstants,
|
||||
DEFAULT_PRESET_NAME,
|
||||
DEFAULT_WORKFLOW,
|
||||
PRESET_FIELDS,
|
||||
_MISSING,
|
||||
_SITE_STARTUP_JS_PATTERNS,
|
||||
_coerce_bool,
|
||||
_merge_config_patch,
|
||||
get_default_network_config,
|
||||
get_default_stream_config,
|
||||
logger,
|
||||
)
|
||||
from .engine_local_overrides import ConfigLocalOverridesMixin # noqa: E402
|
||||
from .engine_migrations import ConfigMigrationsMixin # noqa: E402
|
||||
from .engine_presets import ConfigPresetsMixin # noqa: E402
|
||||
from .engine_sites import ConfigSitesMixin # noqa: E402
|
||||
from .engine_media import ConfigMediaMixin # noqa: E402
|
||||
from .engine_stream import ConfigStreamMixin # noqa: E402
|
||||
|
||||
# ================= 配置引擎主类 =================
|
||||
|
||||
class ConfigEngine(
|
||||
ConfigLocalOverridesMixin,
|
||||
ConfigMigrationsMixin,
|
||||
ConfigPresetsMixin,
|
||||
ConfigSitesMixin,
|
||||
ConfigMediaMixin,
|
||||
ConfigStreamMixin,
|
||||
):
|
||||
"""配置引擎主类"""
|
||||
def __init__(self):
|
||||
self.config_file = ConfigConstants.CONFIG_FILE # 旧版单文件,仅用于自动迁移
|
||||
self.sites_dir = ConfigConstants.SITES_DIR
|
||||
self.site_store = SiteStore(self.sites_dir, self.config_file)
|
||||
self.local_sites_file = ConfigConstants.SITES_LOCAL_FILE
|
||||
self._io_lock = threading.RLock()
|
||||
self.last_mtime = 0.0
|
||||
self.last_signature: tuple = ()
|
||||
self.last_local_mtime = 0.0
|
||||
self.sites: Dict[str, SiteConfig] = {}
|
||||
self._global_default_presets: Dict[str, str] = {}
|
||||
self._local_default_presets: Dict[str, str] = {}
|
||||
self._local_sites_payload: Dict[str, Any] = {}
|
||||
self._cache = ConfigCache(ttl=5.0)
|
||||
|
||||
# 子管理器
|
||||
self.global_config = GlobalConfigManager()
|
||||
self.image_presets = ImagePresetsManager(ConfigConstants.IMAGE_PRESETS_FILE)
|
||||
|
||||
# 加载配置
|
||||
self._load_config()
|
||||
self._migrate_global_commands()
|
||||
|
||||
# 处理器
|
||||
self.html_cleaner = HTMLCleaner()
|
||||
self.validator = SelectorValidator(self.global_config.get_fallback_selectors())
|
||||
self.ai_analyzer = AIAnalyzer(self.global_config)
|
||||
|
||||
# 迁移旧配置(顺序重要:先转预设格式,再补缺失字段,最后清理残留)
|
||||
self._migrate_loaded_config()
|
||||
self._apply_local_site_overrides()
|
||||
|
||||
logger.debug(f"配置引擎已初始化,已加载 {len(self.sites)} 个站点配置")
|
||||
def _load_config(self):
|
||||
with self._io_lock:
|
||||
return self._load_config_locked()
|
||||
def _load_config_locked(self):
|
||||
"""初始化加载站点配置(config/sites/ 目录;发现旧版 sites.json 时自动迁移)"""
|
||||
try:
|
||||
data = self.site_store.load()
|
||||
self.last_signature = self.site_store.signature()
|
||||
if data is None:
|
||||
logger.info(f"站点配置目录 {self.sites_dir} 为空,将在首次保存时创建")
|
||||
self._apply_local_site_overrides()
|
||||
return
|
||||
self.last_mtime = time.time()
|
||||
|
||||
# 提取并加载 _global;缺失时重置为默认全局配置。
|
||||
global_section = data.pop("_global", {})
|
||||
self.global_config.load(global_section if isinstance(global_section, dict) else {})
|
||||
|
||||
# 过滤内部键
|
||||
self.sites = {
|
||||
k: v for k, v in data.items()
|
||||
if not k.startswith('_')
|
||||
}
|
||||
self._refresh_global_default_presets_from_sites()
|
||||
self._apply_local_site_overrides()
|
||||
logger.debug(f"已加载站点配置: {self.sites_dir}({len(self.sites)} 个站点)")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"加载配置失败: {e}")
|
||||
def refresh_if_changed(self):
|
||||
"""检查文件是否变化,如果变化则重载"""
|
||||
if not self.site_store.exists() and not os.path.exists(self.local_sites_file):
|
||||
return
|
||||
|
||||
try:
|
||||
current_signature = self.site_store.signature()
|
||||
current_local_mtime = os.path.getmtime(self.local_sites_file) if os.path.exists(self.local_sites_file) else 0.0
|
||||
if current_signature != self.last_signature or current_local_mtime != self.last_local_mtime:
|
||||
logger.debug("⚡ 检测到站点配置变化,热重载")
|
||||
self.reload_config()
|
||||
except Exception as e:
|
||||
logger.error(f"检查文件变化失败: {e}")
|
||||
def reload_config(self):
|
||||
with self._io_lock:
|
||||
return self._reload_config_locked()
|
||||
def _reload_config_locked(self):
|
||||
"""重新加载配置(Hot Reload)"""
|
||||
self._cache.invalidate()
|
||||
|
||||
if not self.site_store.exists():
|
||||
logger.warning("重载失败:站点配置目录为空")
|
||||
return
|
||||
|
||||
try:
|
||||
previous = {"_global": self.global_config.to_dict(), **self.sites}
|
||||
data = self.site_store.load(previous=previous)
|
||||
if data is None:
|
||||
logger.warning("重载失败:站点配置目录为空")
|
||||
return
|
||||
signature = self.site_store.signature()
|
||||
|
||||
# 提取并加载 _global;缺失时重置为默认全局配置。
|
||||
global_section = data.pop("_global", {})
|
||||
self.global_config.load(global_section if isinstance(global_section, dict) else {})
|
||||
self.validator.fallback_selectors = self.global_config.get_fallback_selectors()
|
||||
|
||||
# 过滤内部键
|
||||
self.sites = {
|
||||
k: v for k, v in data.items()
|
||||
if not k.startswith('_')
|
||||
}
|
||||
self.last_signature = signature
|
||||
self.last_mtime = time.time()
|
||||
self._refresh_global_default_presets_from_sites()
|
||||
self._migrate_loaded_config()
|
||||
self._apply_local_site_overrides()
|
||||
logger.debug(f"✅ 配置已热重载 (Sites: {len(self.sites)})")
|
||||
|
||||
except json.JSONDecodeError as e:
|
||||
logger.error(f"❌ 重载配置失败(JSON格式错误),保留旧配置: {e}")
|
||||
except Exception as e:
|
||||
logger.error(f"❌ 重载配置失败: {e}")
|
||||
def save_config(self):
|
||||
"""公开的保存方法(供 API 调用)"""
|
||||
return self._save_config()
|
||||
def _save_config(self) -> bool:
|
||||
with self._io_lock:
|
||||
return self._save_config_locked()
|
||||
def _save_config_locked(self) -> bool:
|
||||
"""保存站点配置(只重写有变化的站点文件,每个文件原子替换)"""
|
||||
local_snapshot: Optional[tuple[bool, bytes, Dict[str, Any], Dict[str, str]]] = None
|
||||
local_overrides_written = False
|
||||
default_maps_snapshot = (
|
||||
copy.deepcopy(self._global_default_presets),
|
||||
copy.deepcopy(self._local_default_presets),
|
||||
)
|
||||
|
||||
try:
|
||||
self._prune_default_preset_maps()
|
||||
persisted_sites = {}
|
||||
for domain, site in self.sites.items():
|
||||
if domain.startswith('_') or not isinstance(site, dict):
|
||||
continue
|
||||
|
||||
site_copy = copy.deepcopy(site)
|
||||
from app.utils.site_url import route_domain_matches
|
||||
if route_domain_matches("arena.ai", domain):
|
||||
try:
|
||||
from app.services.arena_model_catalog import load_arena_model_catalog_data
|
||||
cat_data = load_arena_model_catalog_data()
|
||||
d_key = str(domain).strip().lower().strip(".")
|
||||
domain_cat = cat_data.get(d_key, {})
|
||||
except Exception:
|
||||
domain_cat = None
|
||||
|
||||
presets_dict = site_copy.get("presets")
|
||||
if isinstance(presets_dict, dict):
|
||||
for p_key, p_val in presets_dict.items():
|
||||
if isinstance(p_val, dict) and "model_catalog" in p_val:
|
||||
if domain_cat is not None and str(p_key).strip() in domain_cat:
|
||||
p_val.pop("model_catalog", None)
|
||||
# 同步清理内存中的残留
|
||||
mem_presets = site.get("presets")
|
||||
if isinstance(mem_presets, dict):
|
||||
for p_key, p_val in mem_presets.items():
|
||||
if isinstance(p_val, dict) and "model_catalog" in p_val:
|
||||
if domain_cat is not None and str(p_key).strip() in domain_cat:
|
||||
p_val.pop("model_catalog", None)
|
||||
|
||||
persisted_default = self._get_persisted_default_preset(domain, site_copy)
|
||||
if persisted_default:
|
||||
site_copy["default_preset"] = persisted_default
|
||||
else:
|
||||
site_copy.pop("default_preset", None)
|
||||
persisted_sites[domain] = site_copy
|
||||
|
||||
# 构建完整配置(包含 _global)
|
||||
full_config = {
|
||||
"_global": self.global_config.to_dict(),
|
||||
**persisted_sites
|
||||
}
|
||||
|
||||
local_snapshot = self._snapshot_local_site_overrides_locked()
|
||||
if local_snapshot is None:
|
||||
self._global_default_presets, self._local_default_presets = default_maps_snapshot
|
||||
return False
|
||||
if not self._save_local_site_overrides():
|
||||
self._restore_local_site_overrides_locked(local_snapshot)
|
||||
self._global_default_presets, self._local_default_presets = default_maps_snapshot
|
||||
return False
|
||||
local_overrides_written = True
|
||||
|
||||
written = self.site_store.save(full_config)
|
||||
self.last_signature = self.site_store.signature()
|
||||
self.last_mtime = time.time()
|
||||
|
||||
self._cache.invalidate()
|
||||
logger.info(f"配置已保存: {self.sites_dir}(重写 {len(written)} 个站点文件)")
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"保存配置失败: {e}")
|
||||
if local_overrides_written:
|
||||
self._restore_local_site_overrides_locked(local_snapshot)
|
||||
self._global_default_presets, self._local_default_presets = default_maps_snapshot
|
||||
return False
|
||||
def _migrate_loaded_config(self):
|
||||
"""Run migrations that apply to freshly loaded site config data."""
|
||||
self._migrate_to_presets()
|
||||
self.migrate_site_configs()
|
||||
self._migrate_site_advanced_to_presets()
|
||||
self._cleanup_preset_residuals()
|
||||
try:
|
||||
from app.services.arena_model_catalog import migrate_and_cleanup_sites_model_catalog
|
||||
migrate_and_cleanup_sites_model_catalog(self)
|
||||
except Exception as exc:
|
||||
logger.warning(f"Arena 模型目录自动迁移失败: {exc}")
|
||||
|
||||
|
||||
__all__ = ['ConfigEngine', 'ConfigConstants', 'DEFAULT_WORKFLOW', 'DEFAULT_PRESET_NAME']
|
||||
@@ -0,0 +1,128 @@
|
||||
"""ConfigEngine 及其 mixin 共用的模块级定义(R2-3 拆分时从 engine.py 原样迁出,engine.py 仍重新导出)。"""
|
||||
|
||||
import os
|
||||
import copy
|
||||
import re
|
||||
from app.core.config import get_logger
|
||||
from typing import Dict, List, Any
|
||||
from app.models.schemas import (
|
||||
WorkflowStep,
|
||||
get_default_attachment_monitor_config,
|
||||
get_default_send_confirmation_config,
|
||||
)
|
||||
from app.core.request_transport import get_default_request_transport_config
|
||||
|
||||
|
||||
logger = get_logger("CFG_ENG")
|
||||
|
||||
|
||||
# ================= 常量配置 =================
|
||||
|
||||
class ConfigConstants:
|
||||
"""配置引擎常量"""
|
||||
_PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
|
||||
# R1-2:站点配置的事实来源是 config/sites/ 目录(每站点一个文件);CONFIG_FILE 只作为旧版单文件的迁移来源
|
||||
CONFIG_FILE = os.getenv("SITES_CONFIG_FILE", os.path.join(_PROJECT_ROOT, "config", "sites.json"))
|
||||
SITES_DIR = os.getenv("SITES_CONFIG_DIR", os.path.join(_PROJECT_ROOT, "config", "sites"))
|
||||
SITES_LOCAL_FILE = os.getenv("SITES_LOCAL_FILE", os.path.join(_PROJECT_ROOT, "config", "sites.local.json"))
|
||||
COMMANDS_FILE = os.getenv("COMMANDS_CONFIG_FILE", os.path.join(_PROJECT_ROOT, "config", "commands.json"))
|
||||
COMMANDS_LOCAL_FILE = os.getenv("COMMANDS_LOCAL_FILE", os.path.join(_PROJECT_ROOT, "config", "commands.local.json"))
|
||||
IMAGE_PRESETS_FILE = os.path.join(_PROJECT_ROOT, "config", "image_presets.json")
|
||||
|
||||
MAX_HTML_CHARS = int(os.getenv("MAX_HTML_CHARS", "120000"))
|
||||
TEXT_TRUNCATE_LENGTH = 80
|
||||
|
||||
AI_MAX_RETRIES = 3
|
||||
AI_RETRY_BASE_DELAY = 1.0
|
||||
AI_RETRY_MAX_DELAY = 10.0
|
||||
AI_REQUEST_TIMEOUT = 120
|
||||
|
||||
|
||||
_MISSING = object()
|
||||
|
||||
|
||||
def _coerce_bool(value: Any, default: bool = False) -> bool:
|
||||
"""Parse bool-like config values without treating every non-empty string as true."""
|
||||
if isinstance(value, bool):
|
||||
return value
|
||||
if value is None:
|
||||
return default
|
||||
if isinstance(value, str):
|
||||
lowered = value.strip().lower()
|
||||
if lowered in {"1", "true", "yes", "y", "on"}:
|
||||
return True
|
||||
if lowered in {"0", "false", "no", "n", "off"}:
|
||||
return False
|
||||
return default
|
||||
if isinstance(value, (int, float)):
|
||||
return value != 0
|
||||
return bool(value)
|
||||
|
||||
|
||||
# ================= 预设常量 =================
|
||||
|
||||
DEFAULT_PRESET_NAME = "主预设"
|
||||
|
||||
# 预设内包含的配置字段(用于迁移和校验)
|
||||
PRESET_FIELDS = [
|
||||
"selectors", "workflow", "stream_config",
|
||||
"image_extraction", "file_paste", "prompt_padding", "stealth",
|
||||
"extractor_id", "extractor_verified"
|
||||
]
|
||||
|
||||
# 默认工作流
|
||||
DEFAULT_WORKFLOW: List[WorkflowStep] = [
|
||||
{"action": "CLICK", "target": "new_chat_btn", "optional": True, "value": None},
|
||||
{"action": "WAIT", "target": "", "optional": False, "value": "0.5"},
|
||||
{"action": "FILL_INPUT", "target": "input_box", "optional": False, "value": None},
|
||||
{"action": "CLICK", "target": "send_btn", "optional": True, "value": None},
|
||||
{"action": "KEY_PRESS", "target": "Enter", "optional": True, "value": None},
|
||||
{"action": "STREAM_WAIT", "target": "result_container", "optional": False, "value": None}
|
||||
]
|
||||
|
||||
def get_default_stream_config() -> Dict[str, Any]:
|
||||
"""获取默认流式配置"""
|
||||
return {
|
||||
"mode": "dom", # dom / network
|
||||
"request_transport": get_default_request_transport_config(),
|
||||
"hard_timeout": 300, # 全局硬超时(秒)
|
||||
"send_confirmation": get_default_send_confirmation_config(),
|
||||
"attachment_monitor": get_default_attachment_monitor_config(),
|
||||
|
||||
# 网络监听配置(可选)
|
||||
"network": None
|
||||
}
|
||||
|
||||
|
||||
def get_default_network_config() -> Dict[str, Any]:
|
||||
"""获取默认网络监听配置"""
|
||||
return {
|
||||
"listen_pattern": "", # URL 匹配模式(必填)
|
||||
"stream_match_mode": "keyword", # 流目标匹配模式(keyword / regex)
|
||||
"stream_match_pattern": "", # 流目标匹配表达式(为空时回退到 listen_pattern)
|
||||
"parser": "", # 解析器 ID(必填)
|
||||
"first_response_timeout": 300.0, # 等待首个匹配目标流量的最大时间
|
||||
"silence_threshold": 3.0, # 静默超时(秒)
|
||||
"response_interval": 0.5 # 轮询间隔(秒)
|
||||
}
|
||||
|
||||
|
||||
def _merge_config_patch(base: Any, patch: Any) -> Any:
|
||||
"""Deep-merge object patches while letting scalars/lists replace existing values."""
|
||||
if not isinstance(base, dict) or not isinstance(patch, dict):
|
||||
return copy.deepcopy(patch)
|
||||
|
||||
merged = dict(base)
|
||||
for key, value in patch.items():
|
||||
existing = base.get(key)
|
||||
if isinstance(value, dict) and isinstance(existing, dict):
|
||||
merged[key] = _merge_config_patch(existing, value)
|
||||
else:
|
||||
merged[key] = copy.deepcopy(value)
|
||||
return merged
|
||||
|
||||
|
||||
_SITE_STARTUP_JS_PATTERNS = (
|
||||
re.compile(r"""location\.(?:assign|replace)\(\s*['"]([^'"]+)['"]\s*\)""", re.IGNORECASE),
|
||||
re.compile(r"""location\.href\s*=\s*['"]([^'"]+)['"]""", re.IGNORECASE),
|
||||
)
|
||||
@@ -0,0 +1,226 @@
|
||||
"""ConfigEngine 的一部分(R2-3 拆分):本地站点覆盖(sites.local.json)与默认预设状态。
|
||||
|
||||
方法原样从 engine.py 迁出,行为不变;通过 mixin 组合回 ConfigEngine,
|
||||
实例状态都在 ConfigEngine.__init__(engine.py)里初始化。
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import copy
|
||||
from typing import Dict, Optional, Any
|
||||
from .engine_common import logger
|
||||
|
||||
|
||||
class ConfigLocalOverridesMixin:
|
||||
"""本地站点覆盖(sites.local.json)与默认预设状态"""
|
||||
|
||||
def _load_local_site_overrides(self) -> Dict[str, str]:
|
||||
with self._io_lock:
|
||||
return self._load_local_site_overrides_locked()
|
||||
def _load_local_site_overrides_locked(self) -> Dict[str, str]:
|
||||
"""加载本地站点覆盖配置,并保留未识别字段。"""
|
||||
if not os.path.exists(self.local_sites_file):
|
||||
self.last_local_mtime = 0.0
|
||||
self._local_sites_payload = {}
|
||||
return {}
|
||||
|
||||
try:
|
||||
with open(self.local_sites_file, "r", encoding="utf-8-sig") as f:
|
||||
data = json.load(f)
|
||||
|
||||
self.last_local_mtime = os.path.getmtime(self.local_sites_file)
|
||||
self._local_sites_payload = data if isinstance(data, dict) else {}
|
||||
defaults = self._local_sites_payload.get("default_presets", {})
|
||||
if not isinstance(defaults, dict):
|
||||
return {}
|
||||
|
||||
return {
|
||||
str(domain).strip(): str(preset).strip()
|
||||
for domain, preset in defaults.items()
|
||||
if str(domain).strip() and str(preset).strip()
|
||||
}
|
||||
except json.JSONDecodeError as e:
|
||||
logger.error(f"本地站点覆盖配置格式错误: {e}")
|
||||
return copy.deepcopy(self._local_default_presets or {})
|
||||
except Exception as e:
|
||||
logger.error(f"加载本地站点覆盖配置失败: {e}")
|
||||
return copy.deepcopy(self._local_default_presets or {})
|
||||
def _prune_default_preset_maps(self) -> None:
|
||||
active_domains = {
|
||||
domain for domain, site in self.sites.items()
|
||||
if not domain.startswith('_') and isinstance(site, dict)
|
||||
}
|
||||
self._global_default_presets = {
|
||||
domain: preset
|
||||
for domain, preset in self._global_default_presets.items()
|
||||
if domain in active_domains and str(preset or "").strip()
|
||||
}
|
||||
self._local_default_presets = {
|
||||
domain: preset
|
||||
for domain, preset in self._local_default_presets.items()
|
||||
if domain in active_domains and str(preset or "").strip()
|
||||
}
|
||||
def _refresh_global_default_presets_from_sites(self) -> None:
|
||||
next_defaults: Dict[str, str] = {}
|
||||
for domain, site in self.sites.items():
|
||||
if domain.startswith('_') or not isinstance(site, dict):
|
||||
continue
|
||||
resolved = self._resolve_default_preset_name(site)
|
||||
if resolved:
|
||||
next_defaults[domain] = resolved
|
||||
self._global_default_presets = next_defaults
|
||||
def _get_persisted_default_preset(self, domain: str, site: Dict[str, Any]) -> Optional[str]:
|
||||
presets = site.get("presets", {})
|
||||
if not isinstance(presets, dict) or not presets:
|
||||
self._global_default_presets.pop(domain, None)
|
||||
return None
|
||||
|
||||
candidate = str(self._global_default_presets.get(domain, "") or "").strip()
|
||||
if candidate and candidate in presets:
|
||||
return candidate
|
||||
|
||||
resolved = self._resolve_default_preset_name(site)
|
||||
if resolved:
|
||||
self._global_default_presets[domain] = resolved
|
||||
return resolved
|
||||
|
||||
self._global_default_presets.pop(domain, None)
|
||||
return None
|
||||
def _sync_site_default_preset_state(self, domain: str, site: Dict[str, Any]) -> bool:
|
||||
presets = site.get("presets", {})
|
||||
if not isinstance(presets, dict) or not presets:
|
||||
self._global_default_presets.pop(domain, None)
|
||||
self._local_default_presets.pop(domain, None)
|
||||
if "default_preset" in site:
|
||||
del site["default_preset"]
|
||||
return True
|
||||
return False
|
||||
|
||||
persisted_default = self._get_persisted_default_preset(domain, site)
|
||||
|
||||
local_default = str(self._local_default_presets.get(domain, "") or "").strip()
|
||||
if local_default and local_default not in presets:
|
||||
self._local_default_presets.pop(domain, None)
|
||||
local_default = ""
|
||||
|
||||
if local_default and persisted_default and local_default == persisted_default:
|
||||
self._local_default_presets.pop(domain, None)
|
||||
local_default = ""
|
||||
|
||||
effective_default = local_default or persisted_default
|
||||
if not effective_default:
|
||||
effective_default = self._resolve_default_preset_name(site)
|
||||
|
||||
if effective_default and site.get("default_preset") != effective_default:
|
||||
site["default_preset"] = effective_default
|
||||
return True
|
||||
|
||||
if not effective_default and "default_preset" in site:
|
||||
del site["default_preset"]
|
||||
return True
|
||||
|
||||
return False
|
||||
def _apply_local_site_overrides(self):
|
||||
"""将本地默认预设选择覆盖到当前站点配置。"""
|
||||
self._local_default_presets = self._load_local_site_overrides()
|
||||
applied = 0
|
||||
for domain, site in self.sites.items():
|
||||
if not isinstance(site, dict):
|
||||
continue
|
||||
self._sync_site_default_preset_state(domain, site)
|
||||
local_default = self._local_default_presets.get(domain)
|
||||
if local_default and site.get("default_preset") == local_default:
|
||||
applied += 1
|
||||
|
||||
if applied > 0:
|
||||
logger.debug(f"已应用 {applied} 个本地默认预设覆盖")
|
||||
def _save_local_site_overrides(self) -> bool:
|
||||
with self._io_lock:
|
||||
return self._save_local_site_overrides_locked()
|
||||
def _save_local_site_overrides_locked(self) -> bool:
|
||||
"""保存本地站点覆盖配置。"""
|
||||
tmp_file = self.local_sites_file + ".tmp"
|
||||
self._prune_default_preset_maps()
|
||||
defaults = {}
|
||||
for domain, preset_name in self._local_default_presets.items():
|
||||
site = self.sites.get(domain)
|
||||
if not isinstance(site, dict):
|
||||
continue
|
||||
presets = site.get("presets", {})
|
||||
preset_value = str(preset_name or "").strip()
|
||||
if isinstance(presets, dict) and preset_value and preset_value in presets:
|
||||
defaults[domain] = preset_value
|
||||
|
||||
payload = dict(self._local_sites_payload or {})
|
||||
if os.path.exists(self.local_sites_file):
|
||||
try:
|
||||
with open(self.local_sites_file, "r", encoding="utf-8-sig") as f:
|
||||
latest_payload = json.load(f)
|
||||
if isinstance(latest_payload, dict):
|
||||
payload.update(latest_payload)
|
||||
except Exception:
|
||||
pass
|
||||
payload["default_presets"] = defaults
|
||||
|
||||
try:
|
||||
os.makedirs(os.path.dirname(self.local_sites_file), exist_ok=True)
|
||||
with open(tmp_file, "w", encoding="utf-8", newline="\n") as f:
|
||||
json.dump(payload, f, indent=2, ensure_ascii=False)
|
||||
f.flush()
|
||||
os.fsync(f.fileno())
|
||||
|
||||
os.replace(tmp_file, self.local_sites_file)
|
||||
try:
|
||||
self.last_local_mtime = os.path.getmtime(self.local_sites_file)
|
||||
except Exception as e:
|
||||
logger.warning(f"本地站点覆盖已保存但更新时间戳失败: {e}")
|
||||
self._local_sites_payload = payload
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.error(f"保存本地站点覆盖配置失败: {e}")
|
||||
try:
|
||||
if os.path.exists(tmp_file):
|
||||
os.remove(tmp_file)
|
||||
except Exception:
|
||||
pass
|
||||
return False
|
||||
def _snapshot_local_site_overrides_locked(self) -> Optional[tuple[bool, bytes, Dict[str, Any], Dict[str, str]]]:
|
||||
try:
|
||||
payload_snapshot = copy.deepcopy(self._local_sites_payload or {})
|
||||
defaults_snapshot = copy.deepcopy(self._local_default_presets or {})
|
||||
if not os.path.exists(self.local_sites_file):
|
||||
return (False, b"", payload_snapshot, defaults_snapshot)
|
||||
with open(self.local_sites_file, "rb") as f:
|
||||
return (True, f.read(), payload_snapshot, defaults_snapshot)
|
||||
except Exception as e:
|
||||
logger.error(f"读取本地站点覆盖快照失败: {e}")
|
||||
return None
|
||||
def _restore_local_site_overrides_locked(self, snapshot: Optional[tuple[bool, bytes, Dict[str, Any], Dict[str, str]]]) -> None:
|
||||
if snapshot is None:
|
||||
return
|
||||
|
||||
tmp_file = self.local_sites_file + ".restore.tmp"
|
||||
existed, payload, payload_snapshot, defaults_snapshot = snapshot
|
||||
try:
|
||||
if existed:
|
||||
os.makedirs(os.path.dirname(self.local_sites_file), exist_ok=True)
|
||||
with open(tmp_file, "wb") as f:
|
||||
f.write(payload)
|
||||
f.flush()
|
||||
os.fsync(f.fileno())
|
||||
os.replace(tmp_file, self.local_sites_file)
|
||||
elif os.path.exists(self.local_sites_file):
|
||||
os.remove(self.local_sites_file)
|
||||
try:
|
||||
self.last_local_mtime = os.path.getmtime(self.local_sites_file) if os.path.exists(self.local_sites_file) else 0.0
|
||||
except Exception as e:
|
||||
logger.warning(f"本地站点覆盖已恢复但更新时间戳失败: {e}")
|
||||
self._local_sites_payload = copy.deepcopy(payload_snapshot)
|
||||
self._local_default_presets = copy.deepcopy(defaults_snapshot)
|
||||
except Exception as e:
|
||||
logger.error(f"恢复本地站点覆盖失败: {e}")
|
||||
try:
|
||||
if os.path.exists(tmp_file):
|
||||
os.remove(tmp_file)
|
||||
except Exception:
|
||||
pass
|
||||
@@ -0,0 +1,685 @@
|
||||
"""ConfigEngine 的一部分(R2-3 拆分):图片提取、文件粘贴、提示词填充配置与图片预设。
|
||||
|
||||
方法原样从 engine.py 迁出,行为不变;通过 mixin 组合回 ConfigEngine,
|
||||
实例状态都在 ConfigEngine.__init__(engine.py)里初始化。
|
||||
"""
|
||||
|
||||
import copy
|
||||
from typing import Dict, Optional, Any
|
||||
from app.models.schemas import (
|
||||
get_default_image_extraction_config,
|
||||
get_default_file_paste_config,
|
||||
get_default_prompt_padding_config,
|
||||
get_default_attachment_monitor_config,
|
||||
get_default_send_confirmation_config,
|
||||
get_enabled_modalities,
|
||||
is_modality_enabled,
|
||||
normalize_modalities_config,
|
||||
normalize_modality_policy,
|
||||
)
|
||||
from .engine_common import DEFAULT_PRESET_NAME, _coerce_bool, _merge_config_patch, logger
|
||||
|
||||
|
||||
class ConfigMediaMixin:
|
||||
"""图片提取、文件粘贴、提示词填充配置与图片预设"""
|
||||
|
||||
def get_site_image_config(self, domain: str, preset_name: str = None) -> Dict:
|
||||
"""获取站点的图片提取配置"""
|
||||
self.refresh_if_changed()
|
||||
|
||||
default_config = get_default_image_extraction_config()
|
||||
|
||||
data = self._get_site_data(domain, preset_name)
|
||||
if data is None:
|
||||
return copy.deepcopy(default_config)
|
||||
|
||||
image_config = data.get("image_extraction", {})
|
||||
return self._validate_image_config(image_config)
|
||||
def set_site_image_config(self, domain: str, config: Dict,
|
||||
preset_name: str = None) -> bool:
|
||||
"""设置站点的图片提取配置"""
|
||||
self.refresh_if_changed()
|
||||
|
||||
data = self._get_site_data(domain, preset_name)
|
||||
if data is None:
|
||||
logger.warning(f"站点或预设不存在: {domain}/{preset_name}")
|
||||
return False
|
||||
|
||||
current_config = self._validate_image_config(data.get("image_extraction", {}))
|
||||
merged_config = _merge_config_patch(current_config, config if isinstance(config, dict) else {})
|
||||
validated = self._validate_image_config(merged_config)
|
||||
|
||||
if not self._assign_preset_field_and_save(data, "image_extraction", validated):
|
||||
return False
|
||||
|
||||
logger.info(f"站点 {domain} [{preset_name or DEFAULT_PRESET_NAME}] 多模态提取配置已更新")
|
||||
return True
|
||||
def _validate_image_config(self, config: Dict) -> Dict:
|
||||
"""验证并规范化多模态提取配置"""
|
||||
default = get_default_image_extraction_config()
|
||||
result = copy.deepcopy(default)
|
||||
|
||||
if not config:
|
||||
return result
|
||||
|
||||
raw_modalities = config.get("modalities")
|
||||
if isinstance(raw_modalities, dict):
|
||||
result["modalities"] = normalize_modalities_config(raw_modalities)
|
||||
|
||||
legacy_enabled = None
|
||||
if "enabled" in config:
|
||||
legacy_enabled = _coerce_bool(config.get("enabled"), False)
|
||||
result["enabled"] = legacy_enabled
|
||||
|
||||
if (
|
||||
legacy_enabled is not None
|
||||
and not isinstance(raw_modalities, dict)
|
||||
):
|
||||
result["modalities"]["image"] = normalize_modality_policy("image", legacy_enabled)
|
||||
elif legacy_enabled is not None and isinstance(raw_modalities, dict):
|
||||
if not legacy_enabled:
|
||||
for key in ("image", "audio", "video"):
|
||||
result["modalities"][key] = normalize_modality_policy(key, False)
|
||||
elif not get_enabled_modalities(result.get("modalities")):
|
||||
result["modalities"]["image"] = normalize_modality_policy("image", True)
|
||||
|
||||
if "selector" in config and config["selector"]:
|
||||
result["selector"] = str(config["selector"]).strip()
|
||||
if not result["selector"]:
|
||||
result["selector"] = "img"
|
||||
|
||||
if "audio_selector" in config and config["audio_selector"]:
|
||||
result["audio_selector"] = str(config["audio_selector"]).strip()
|
||||
if not result["audio_selector"]:
|
||||
result["audio_selector"] = default["audio_selector"]
|
||||
|
||||
if "video_selector" in config and config["video_selector"]:
|
||||
result["video_selector"] = str(config["video_selector"]).strip()
|
||||
if not result["video_selector"]:
|
||||
result["video_selector"] = default["video_selector"]
|
||||
|
||||
if "container_selector" in config:
|
||||
val = config["container_selector"]
|
||||
result["container_selector"] = str(val).strip() if val else None
|
||||
|
||||
if "final_target_strategy" in config:
|
||||
val = str(config["final_target_strategy"] or "").strip().lower()
|
||||
if val in ("container", "latest_reply", "latest_visual_reply"):
|
||||
result["final_target_strategy"] = val
|
||||
|
||||
if "latest_visual_column" in config:
|
||||
val = str(config["latest_visual_column"] or "").strip().lower()
|
||||
if val in ("left", "right"):
|
||||
result["latest_visual_column"] = val
|
||||
|
||||
if "allow_container_fallback" in config:
|
||||
result["allow_container_fallback"] = _coerce_bool(config.get("allow_container_fallback"), False)
|
||||
|
||||
if "force_postprocess" in config:
|
||||
result["force_postprocess"] = _coerce_bool(config.get("force_postprocess"), False)
|
||||
|
||||
if "arena_image_generation" in config:
|
||||
result["arena_image_generation"] = _coerce_bool(
|
||||
config.get("arena_image_generation"), False
|
||||
)
|
||||
|
||||
if "direct_postprocess_modalities" in config:
|
||||
raw_direct_modalities = config.get("direct_postprocess_modalities")
|
||||
if isinstance(raw_direct_modalities, (list, tuple, set)):
|
||||
allowed_direct_modalities = []
|
||||
for item in raw_direct_modalities:
|
||||
media_type = str(item or "").strip().lower()
|
||||
if (
|
||||
media_type in {"image", "audio", "video"}
|
||||
and is_modality_enabled(result.get("modalities"), media_type)
|
||||
and media_type not in allowed_direct_modalities
|
||||
):
|
||||
allowed_direct_modalities.append(media_type)
|
||||
if allowed_direct_modalities:
|
||||
result["direct_postprocess_modalities"] = allowed_direct_modalities
|
||||
|
||||
if "debounce_seconds" in config:
|
||||
try:
|
||||
val = float(config["debounce_seconds"])
|
||||
result["debounce_seconds"] = max(0, min(val, 30))
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
|
||||
if "wait_for_load" in config:
|
||||
result["wait_for_load"] = _coerce_bool(config.get("wait_for_load"), True)
|
||||
|
||||
if "load_timeout_seconds" in config:
|
||||
try:
|
||||
val = float(config["load_timeout_seconds"])
|
||||
result["load_timeout_seconds"] = max(1, min(val, 60))
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
|
||||
if "download_blobs" in config:
|
||||
result["download_blobs"] = _coerce_bool(config.get("download_blobs"), False)
|
||||
|
||||
if "max_size_mb" in config:
|
||||
try:
|
||||
val = int(config["max_size_mb"])
|
||||
result["max_size_mb"] = max(1, min(val, 100))
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
|
||||
if "canvas_export_mime" in config:
|
||||
val = str(config["canvas_export_mime"] or "").strip().lower()
|
||||
if val in {"image/jpeg", "image/webp", "image/png"}:
|
||||
result["canvas_export_mime"] = val
|
||||
|
||||
if "canvas_export_quality" in config:
|
||||
try:
|
||||
val = float(config["canvas_export_quality"])
|
||||
result["canvas_export_quality"] = max(0.1, min(val, 1.0))
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
|
||||
if "src_allow_patterns" in config:
|
||||
raw_patterns = config.get("src_allow_patterns")
|
||||
if isinstance(raw_patterns, str):
|
||||
raw_patterns = raw_patterns.replace("\r\n", "\n").replace(";", "\n").split("\n")
|
||||
if isinstance(raw_patterns, (list, tuple, set)):
|
||||
patterns = []
|
||||
seen = set()
|
||||
for item in raw_patterns:
|
||||
text = str(item or "").strip()
|
||||
if not text or text in seen:
|
||||
continue
|
||||
seen.add(text)
|
||||
patterns.append(text)
|
||||
result["src_allow_patterns"] = patterns
|
||||
|
||||
if "mode" in config:
|
||||
val = str(config["mode"]).lower()
|
||||
if val in ("all", "first", "last"):
|
||||
result["mode"] = val
|
||||
|
||||
if "audio_capture_enabled" in config:
|
||||
result["audio_capture_enabled"] = _coerce_bool(config.get("audio_capture_enabled"), False)
|
||||
|
||||
if "audio_capture_mute_playback" in config:
|
||||
result["audio_capture_mute_playback"] = _coerce_bool(config.get("audio_capture_mute_playback"), False)
|
||||
|
||||
if "audio_capture_preload_enabled" in config:
|
||||
result["audio_capture_preload_enabled"] = _coerce_bool(config.get("audio_capture_preload_enabled"), False)
|
||||
|
||||
if "audio_capture_reload_before_workflow" in config:
|
||||
result["audio_capture_reload_before_workflow"] = _coerce_bool(config.get("audio_capture_reload_before_workflow"), False)
|
||||
|
||||
if "audio_capture_preserve_graph" in config:
|
||||
result["audio_capture_preserve_graph"] = _coerce_bool(config.get("audio_capture_preserve_graph"), False)
|
||||
|
||||
if "audio_capture_terminal_settle_seconds" in config:
|
||||
try:
|
||||
val = float(config["audio_capture_terminal_settle_seconds"])
|
||||
result["audio_capture_terminal_settle_seconds"] = max(0.0, min(val, 5.0))
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
|
||||
if "audio_trigger_selector" in config:
|
||||
result["audio_trigger_selector"] = str(config["audio_trigger_selector"] or "").strip()
|
||||
|
||||
if "audio_trigger_labels" in config:
|
||||
raw_labels = config.get("audio_trigger_labels")
|
||||
if isinstance(raw_labels, (list, tuple)):
|
||||
labels = [
|
||||
str(item).strip()
|
||||
for item in raw_labels
|
||||
if str(item).strip()
|
||||
]
|
||||
if labels:
|
||||
result["audio_trigger_labels"] = labels
|
||||
|
||||
if "audio_capture_max_wait_seconds" in config:
|
||||
try:
|
||||
val = float(config["audio_capture_max_wait_seconds"])
|
||||
result["audio_capture_max_wait_seconds"] = max(1.0, min(val, 120.0))
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
|
||||
if "audio_capture_min_wait_seconds" in config:
|
||||
try:
|
||||
val = float(config["audio_capture_min_wait_seconds"])
|
||||
result["audio_capture_min_wait_seconds"] = max(0.2, min(val, 30.0))
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
|
||||
if "audio_capture_hard_max_wait_seconds" in config:
|
||||
try:
|
||||
val = float(config["audio_capture_hard_max_wait_seconds"])
|
||||
result["audio_capture_hard_max_wait_seconds"] = max(1.0, min(val, 180.0))
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
|
||||
if "audio_capture_estimated_chars_per_second" in config:
|
||||
try:
|
||||
val = float(config["audio_capture_estimated_chars_per_second"])
|
||||
result["audio_capture_estimated_chars_per_second"] = max(1.0, min(val, 20.0))
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
|
||||
if "audio_capture_wait_padding_seconds" in config:
|
||||
try:
|
||||
val = float(config["audio_capture_wait_padding_seconds"])
|
||||
result["audio_capture_wait_padding_seconds"] = max(0.0, min(val, 10.0))
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
|
||||
network_capture = dict(result.get("audio_network_capture") or {})
|
||||
raw_network_capture = config.get("audio_network_capture")
|
||||
if isinstance(raw_network_capture, dict):
|
||||
if "enabled" in raw_network_capture:
|
||||
network_capture["enabled"] = _coerce_bool(raw_network_capture.get("enabled"), False)
|
||||
if "timeout_seconds" in raw_network_capture:
|
||||
try:
|
||||
val = float(raw_network_capture["timeout_seconds"])
|
||||
network_capture["timeout_seconds"] = max(0.1, min(val, 15.0))
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
if "transport" in raw_network_capture:
|
||||
val = str(raw_network_capture["transport"] or "").strip()
|
||||
if val in {"page_websocket_probe"}:
|
||||
network_capture["transport"] = val
|
||||
if "extractor" in raw_network_capture:
|
||||
val = str(raw_network_capture["extractor"] or "").strip()
|
||||
if val in {"voicegenie_ogg_pages", "voicegenie_binary_stream"}:
|
||||
network_capture["extractor"] = val
|
||||
if "settle_seconds" in raw_network_capture:
|
||||
try:
|
||||
val = float(raw_network_capture["settle_seconds"])
|
||||
network_capture["settle_seconds"] = max(0.05, min(val, 5.0))
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
if "url_patterns" in raw_network_capture:
|
||||
raw_patterns = raw_network_capture.get("url_patterns")
|
||||
if isinstance(raw_patterns, (list, tuple)):
|
||||
patterns = [
|
||||
str(item).strip()
|
||||
for item in raw_patterns
|
||||
if str(item).strip()
|
||||
]
|
||||
if patterns:
|
||||
network_capture["url_patterns"] = patterns
|
||||
if "max_payload_bytes" in raw_network_capture:
|
||||
# 修复:此前漏处理该键,导致前端设置的单条载荷上限被静默重置为默认 10MB
|
||||
try:
|
||||
val = int(raw_network_capture["max_payload_bytes"])
|
||||
network_capture["max_payload_bytes"] = max(65536, min(val, 104857600))
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
|
||||
# 兼容旧平铺字段,最终统一收口到新对象
|
||||
if "audio_network_capture_enabled" in config:
|
||||
network_capture["enabled"] = _coerce_bool(config.get("audio_network_capture_enabled"), False)
|
||||
|
||||
if "audio_network_capture_timeout_seconds" in config:
|
||||
try:
|
||||
val = float(config["audio_network_capture_timeout_seconds"])
|
||||
network_capture["timeout_seconds"] = max(0.1, min(val, 15.0))
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
|
||||
if "audio_network_url_patterns" in config:
|
||||
raw_patterns = config.get("audio_network_url_patterns")
|
||||
if isinstance(raw_patterns, (list, tuple)):
|
||||
patterns = [
|
||||
str(item).strip()
|
||||
for item in raw_patterns
|
||||
if str(item).strip()
|
||||
]
|
||||
if patterns:
|
||||
network_capture["url_patterns"] = patterns
|
||||
|
||||
result["audio_network_capture"] = network_capture
|
||||
|
||||
browser_tts_fallback = dict(result.get("audio_browser_tts_fallback") or {})
|
||||
raw_browser_tts_fallback = config.get("audio_browser_tts_fallback")
|
||||
if isinstance(raw_browser_tts_fallback, dict):
|
||||
if "enabled" in raw_browser_tts_fallback:
|
||||
browser_tts_fallback["enabled"] = _coerce_bool(raw_browser_tts_fallback.get("enabled"), False)
|
||||
if "provider" in raw_browser_tts_fallback:
|
||||
val = str(raw_browser_tts_fallback["provider"] or "").strip()
|
||||
if val in {"doubao_samantha"}:
|
||||
browser_tts_fallback["provider"] = val
|
||||
if "speaker" in raw_browser_tts_fallback:
|
||||
val = str(raw_browser_tts_fallback["speaker"] or "").strip()
|
||||
if val:
|
||||
browser_tts_fallback["speaker"] = val
|
||||
if "speech_rate" in raw_browser_tts_fallback:
|
||||
try:
|
||||
val = int(raw_browser_tts_fallback["speech_rate"])
|
||||
browser_tts_fallback["speech_rate"] = max(-100, min(val, 100))
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
if "pitch" in raw_browser_tts_fallback:
|
||||
try:
|
||||
val = int(raw_browser_tts_fallback["pitch"])
|
||||
browser_tts_fallback["pitch"] = max(-100, min(val, 100))
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
if "format" in raw_browser_tts_fallback:
|
||||
val = str(raw_browser_tts_fallback["format"] or "").strip().lower()
|
||||
if val in {"aac"}:
|
||||
browser_tts_fallback["format"] = val
|
||||
if "timeout_seconds" in raw_browser_tts_fallback:
|
||||
try:
|
||||
val = float(raw_browser_tts_fallback["timeout_seconds"])
|
||||
browser_tts_fallback["timeout_seconds"] = max(3.0, min(val, 120.0))
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
for key in (
|
||||
"pc_version",
|
||||
"aid",
|
||||
"real_aid",
|
||||
"language",
|
||||
"device_platform",
|
||||
"pkg_type",
|
||||
"region",
|
||||
"sys_region",
|
||||
"use_olympus_account",
|
||||
"samantha_web",
|
||||
):
|
||||
if key in raw_browser_tts_fallback:
|
||||
val = str(raw_browser_tts_fallback[key] or "").strip()
|
||||
if val:
|
||||
browser_tts_fallback[key] = val
|
||||
|
||||
result["audio_browser_tts_fallback"] = browser_tts_fallback
|
||||
|
||||
if "audio_capture_poll_seconds" in config:
|
||||
try:
|
||||
val = float(config["audio_capture_poll_seconds"])
|
||||
result["audio_capture_poll_seconds"] = max(0.05, min(val, 5.0))
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
|
||||
if "audio_capture_silence_seconds" in config:
|
||||
try:
|
||||
val = float(config["audio_capture_silence_seconds"])
|
||||
result["audio_capture_silence_seconds"] = max(0.2, min(val, 30.0))
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
|
||||
if "audio_capture_activity_threshold" in config:
|
||||
try:
|
||||
val = float(config["audio_capture_activity_threshold"])
|
||||
result["audio_capture_activity_threshold"] = max(0.0001, min(val, 0.2))
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
|
||||
if "audio_capture_activity_silence_seconds" in config:
|
||||
try:
|
||||
val = float(config["audio_capture_activity_silence_seconds"])
|
||||
result["audio_capture_activity_silence_seconds"] = max(0.2, min(val, 10.0))
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
|
||||
result["enabled"] = bool(get_enabled_modalities(result.get("modalities")))
|
||||
|
||||
return result
|
||||
def get_site_file_paste_config(self, domain: str, preset_name: str = None) -> dict:
|
||||
"""获取站点的文件粘贴配置"""
|
||||
self.refresh_if_changed()
|
||||
|
||||
default_config = get_default_file_paste_config()
|
||||
|
||||
data = self._get_site_data(domain, preset_name)
|
||||
if data is None:
|
||||
result = copy.deepcopy(default_config)
|
||||
result["send_confirmation"] = get_default_send_confirmation_config()
|
||||
result["attachment_monitor"] = get_default_attachment_monitor_config()
|
||||
return result
|
||||
|
||||
return self._validate_file_paste_config(
|
||||
data.get("file_paste", {}),
|
||||
legacy_stream_config=data.get("stream_config"),
|
||||
include_attachment_defaults=True,
|
||||
)
|
||||
def set_site_file_paste_config(self, domain: str, config: dict,
|
||||
preset_name: str = None) -> bool:
|
||||
"""设置站点的文件粘贴配置"""
|
||||
self.refresh_if_changed()
|
||||
|
||||
data = self._get_site_data(domain, preset_name)
|
||||
if data is None:
|
||||
logger.warning(f"站点或预设不存在: {domain}/{preset_name}")
|
||||
return False
|
||||
|
||||
current_config = self._validate_file_paste_config(
|
||||
data.get("file_paste", {}),
|
||||
legacy_stream_config=data.get("stream_config"),
|
||||
)
|
||||
merged_config = _merge_config_patch(current_config, config if isinstance(config, dict) else {})
|
||||
validated = self._validate_file_paste_config(
|
||||
merged_config,
|
||||
legacy_stream_config=data.get("stream_config"),
|
||||
)
|
||||
|
||||
if not self._assign_preset_field_and_save(data, "file_paste", validated):
|
||||
return False
|
||||
|
||||
logger.info(f"站点 {domain} [{preset_name or DEFAULT_PRESET_NAME}] 文件粘贴配置已更新")
|
||||
return True
|
||||
def get_all_file_paste_configs(self) -> dict:
|
||||
"""获取所有站点的文件粘贴配置(使用各站点当前默认预设)"""
|
||||
self.refresh_if_changed()
|
||||
result = {}
|
||||
|
||||
for domain in self.sites:
|
||||
if domain.startswith('_'):
|
||||
continue
|
||||
|
||||
data = self._get_site_data(domain)
|
||||
if data is None:
|
||||
continue
|
||||
|
||||
result[domain] = self._validate_file_paste_config(
|
||||
data.get("file_paste", {}),
|
||||
legacy_stream_config=data.get("stream_config"),
|
||||
include_attachment_defaults=True,
|
||||
)
|
||||
|
||||
return result
|
||||
def get_site_prompt_padding_config(self, domain: str, preset_name: str = None) -> Dict[str, Any]:
|
||||
"""获取站点的提示词开头注入配置"""
|
||||
self.refresh_if_changed()
|
||||
|
||||
data = self._get_site_data(domain, preset_name)
|
||||
if data is None:
|
||||
return copy.deepcopy(get_default_prompt_padding_config())
|
||||
|
||||
return self._validate_prompt_padding_config(data.get("prompt_padding", {}))
|
||||
def set_site_prompt_padding_config(
|
||||
self,
|
||||
domain: str,
|
||||
config: Dict[str, Any],
|
||||
preset_name: str = None,
|
||||
) -> bool:
|
||||
"""设置站点的提示词开头注入配置"""
|
||||
self.refresh_if_changed()
|
||||
|
||||
data = self._get_site_data(domain, preset_name)
|
||||
if data is None:
|
||||
logger.warning(f"站点或预设不存在: {domain}/{preset_name}")
|
||||
return False
|
||||
|
||||
current_config = self._validate_prompt_padding_config(data.get("prompt_padding", {}))
|
||||
merged_config = _merge_config_patch(current_config, config if isinstance(config, dict) else {})
|
||||
validated = self._validate_prompt_padding_config(merged_config)
|
||||
if not self._assign_preset_field_and_save(data, "prompt_padding", validated):
|
||||
return False
|
||||
|
||||
logger.info(f"站点 {domain} [{preset_name or DEFAULT_PRESET_NAME}] 提示词开头注入配置已更新")
|
||||
return True
|
||||
def _validate_file_paste_config(
|
||||
self,
|
||||
config: dict,
|
||||
*,
|
||||
legacy_stream_config: Optional[Dict[str, Any]] = None,
|
||||
include_attachment_defaults: bool = False,
|
||||
) -> dict:
|
||||
"""验证并规范化文件粘贴配置"""
|
||||
default = get_default_file_paste_config()
|
||||
result = copy.deepcopy(default)
|
||||
|
||||
if not isinstance(config, dict):
|
||||
config = {}
|
||||
|
||||
if "enabled" in config:
|
||||
result["enabled"] = _coerce_bool(config.get("enabled"), False)
|
||||
|
||||
if "threshold" in config:
|
||||
try:
|
||||
val = int(config["threshold"])
|
||||
result["threshold"] = max(1000, min(val, 10000000))
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
|
||||
if "temp_file_type" in config:
|
||||
val = str(config.get("temp_file_type") or "").strip().lower().lstrip(".")
|
||||
if val in {"txt", "pdf", "chunk", "error"}:
|
||||
result["temp_file_type"] = val
|
||||
|
||||
if "hint_text" in config:
|
||||
val = str(config["hint_text"]).strip()
|
||||
# 限制长度,避免过长的引导文本
|
||||
hint_val = val[:500] if val else ""
|
||||
result["hint_text"] = hint_val
|
||||
|
||||
# 智能向后兼容:结合老配置的策略类型,防止新字段被无关的旧数据污染
|
||||
old_temp_type = config.get("temp_file_type", "txt")
|
||||
|
||||
if "txt_hint_text" not in config:
|
||||
result["txt_hint_text"] = hint_val if old_temp_type == "txt" else "完全专注于文件内容"
|
||||
if "pdf_hint_text" not in config:
|
||||
result["pdf_hint_text"] = hint_val if old_temp_type == "pdf" else "完全专注于文件内容"
|
||||
if "error_hint_text" not in config:
|
||||
result["error_hint_text"] = hint_val if old_temp_type == "error" else "输入文本长度超过限制,已中止发送"
|
||||
|
||||
if "txt_hint_text" in config:
|
||||
val = str(config["txt_hint_text"]).strip()
|
||||
result["txt_hint_text"] = val[:500] if val else ""
|
||||
|
||||
if "pdf_hint_text" in config:
|
||||
val = str(config["pdf_hint_text"]).strip()
|
||||
result["pdf_hint_text"] = val[:500] if val else ""
|
||||
|
||||
if "error_hint_text" in config:
|
||||
val = str(config["error_hint_text"]).strip()
|
||||
result["error_hint_text"] = val[:500] if val else ""
|
||||
|
||||
if "reacquire_input_after_upload" in config:
|
||||
result["reacquire_input_after_upload"] = _coerce_bool(config.get("reacquire_input_after_upload"), False)
|
||||
|
||||
if "post_upload_input_selector" in config:
|
||||
val = str(config["post_upload_input_selector"] or "").strip()
|
||||
result["post_upload_input_selector"] = val[:500] if val else ""
|
||||
|
||||
if "post_upload_settle" in config:
|
||||
try:
|
||||
val = float(config["post_upload_settle"])
|
||||
result["post_upload_settle"] = max(0.0, min(val, 30.0))
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
|
||||
if "upload_signal_timeout" in config:
|
||||
try:
|
||||
val = float(config["upload_signal_timeout"])
|
||||
result["upload_signal_timeout"] = max(0.5, min(val, 120.0))
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
|
||||
if "upload_signal_grace" in config:
|
||||
try:
|
||||
val = float(config["upload_signal_grace"])
|
||||
result["upload_signal_grace"] = max(0.0, min(val, 120.0))
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
|
||||
default_state_probe = copy.deepcopy(default.get("state_probe") or {})
|
||||
raw_state_probe = config.get("state_probe")
|
||||
state_probe = copy.deepcopy(default_state_probe)
|
||||
if isinstance(raw_state_probe, dict):
|
||||
if "enabled" in raw_state_probe:
|
||||
state_probe["enabled"] = _coerce_bool(raw_state_probe.get("enabled"), False)
|
||||
if "code" in raw_state_probe:
|
||||
code = str(raw_state_probe["code"] or "").strip()
|
||||
state_probe["code"] = code[:20000] if code else ""
|
||||
if state_probe:
|
||||
result["state_probe"] = state_probe
|
||||
|
||||
legacy_send_confirmation = {}
|
||||
if isinstance(legacy_stream_config, dict) and isinstance(legacy_stream_config.get("send_confirmation"), dict):
|
||||
legacy_send_confirmation.update(legacy_stream_config.get("send_confirmation") or {})
|
||||
raw_send_confirmation = config.get("send_confirmation")
|
||||
if isinstance(raw_send_confirmation, dict):
|
||||
legacy_send_confirmation.update(raw_send_confirmation)
|
||||
if legacy_send_confirmation or include_attachment_defaults:
|
||||
result["send_confirmation"] = self._validate_send_confirmation_config(legacy_send_confirmation)
|
||||
|
||||
legacy_attachment_monitor = {}
|
||||
if isinstance(legacy_stream_config, dict) and isinstance(legacy_stream_config.get("attachment_monitor"), dict):
|
||||
legacy_attachment_monitor.update(legacy_stream_config.get("attachment_monitor") or {})
|
||||
raw_attachment_monitor = config.get("attachment_monitor")
|
||||
if isinstance(raw_attachment_monitor, dict):
|
||||
legacy_attachment_monitor.update(raw_attachment_monitor)
|
||||
if legacy_attachment_monitor or include_attachment_defaults:
|
||||
result["attachment_monitor"] = self._validate_attachment_monitor_config(legacy_attachment_monitor)
|
||||
|
||||
from app.utils.attachments import attachment_config
|
||||
if "attachments" in config or include_attachment_defaults:
|
||||
result["attachments"] = attachment_config(config.get("attachments"))
|
||||
|
||||
return result
|
||||
def _validate_prompt_padding_config(self, config: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"""验证并规范化提示词开头注入配置。"""
|
||||
result = copy.deepcopy(get_default_prompt_padding_config())
|
||||
|
||||
if not isinstance(config, dict):
|
||||
return result
|
||||
|
||||
if "enabled" in config:
|
||||
result["enabled"] = _coerce_bool(config.get("enabled"), False)
|
||||
|
||||
if "marker_text" in config:
|
||||
marker_text = str(config.get("marker_text") or "").strip()
|
||||
result["marker_text"] = marker_text
|
||||
|
||||
if "segments_per_side" in config:
|
||||
try:
|
||||
segments_per_side = int(config.get("segments_per_side"))
|
||||
except (TypeError, ValueError):
|
||||
segments_per_side = int(result["segments_per_side"])
|
||||
result["segments_per_side"] = max(0, min(segments_per_side, 64))
|
||||
|
||||
if "random_insert_enabled" in config:
|
||||
result["random_insert_enabled"] = _coerce_bool(
|
||||
config.get("random_insert_enabled"), False
|
||||
)
|
||||
|
||||
if "random_insert_chars" in config:
|
||||
result["random_insert_chars"] = str(
|
||||
config.get("random_insert_chars") or ""
|
||||
)
|
||||
|
||||
return result
|
||||
def list_image_presets(self):
|
||||
"""列出所有可用的图片配置预设"""
|
||||
return self.image_presets.list_presets()
|
||||
def get_image_preset(self, domain: str):
|
||||
"""获取指定站点的预设信息"""
|
||||
return self.image_presets.get_preset_for_display(domain)
|
||||
def apply_image_preset(self, domain: str, preset_domain: str):
|
||||
"""将预设配置应用到站点"""
|
||||
preset_config = self.image_presets.get_preset(preset_domain)
|
||||
|
||||
if not preset_config:
|
||||
raise ValueError(f"找不到预设: {preset_domain}")
|
||||
|
||||
return self.set_site_image_config(domain, preset_config)
|
||||
def reload_presets(self):
|
||||
"""重新加载图片预设"""
|
||||
self.image_presets.reload()
|
||||
@@ -0,0 +1,374 @@
|
||||
"""ConfigEngine 的一部分(R2-3 拆分):历史配置迁移与命令文件读写。
|
||||
|
||||
方法原样从 engine.py 迁出,行为不变;通过 mixin 组合回 ConfigEngine,
|
||||
实例状态都在 ConfigEngine.__init__(engine.py)里初始化。
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import copy
|
||||
from typing import Dict, List, Any
|
||||
from app.models.schemas import (
|
||||
PRESET_ADVANCED_FIELDS,
|
||||
SITE_ADVANCED_FIELDS,
|
||||
get_default_image_extraction_config,
|
||||
get_default_file_paste_config,
|
||||
get_default_prompt_padding_config,
|
||||
)
|
||||
from app.utils.site_rules import get_site_rule
|
||||
from .engine_common import ConfigConstants, DEFAULT_PRESET_NAME, PRESET_FIELDS, logger
|
||||
|
||||
|
||||
class ConfigMigrationsMixin:
|
||||
"""历史配置迁移与命令文件读写"""
|
||||
|
||||
def _load_commands_file(self) -> List[Dict[str, Any]]:
|
||||
"""加载独立命令配置文件"""
|
||||
commands_file = ConfigConstants.COMMANDS_FILE
|
||||
if not os.path.exists(commands_file):
|
||||
return []
|
||||
|
||||
try:
|
||||
with open(commands_file, "r", encoding="utf-8") as f:
|
||||
data = json.load(f)
|
||||
|
||||
if isinstance(data, dict):
|
||||
data = data.get("commands", [])
|
||||
|
||||
if isinstance(data, list):
|
||||
return data
|
||||
|
||||
logger.warning(f"命令配置文件格式无效: {commands_file}")
|
||||
return []
|
||||
except json.JSONDecodeError as e:
|
||||
logger.error(f"命令配置文件格式错误: {e}")
|
||||
return []
|
||||
except Exception as e:
|
||||
logger.error(f"加载命令配置失败: {e}")
|
||||
return []
|
||||
def _save_commands_file(self, commands: List[Dict[str, Any]]) -> bool:
|
||||
"""保存独立命令配置文件"""
|
||||
commands_file = ConfigConstants.COMMANDS_FILE
|
||||
tmp_file = commands_file + ".tmp"
|
||||
|
||||
try:
|
||||
os.makedirs(os.path.dirname(commands_file), exist_ok=True)
|
||||
payload = {"commands": commands}
|
||||
|
||||
with open(tmp_file, "w", encoding="utf-8", newline="\n") as f:
|
||||
json.dump(payload, f, indent=2, ensure_ascii=False)
|
||||
f.flush()
|
||||
os.fsync(f.fileno())
|
||||
|
||||
os.replace(tmp_file, commands_file)
|
||||
logger.info(f"命令配置已保存: {commands_file}")
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.error(f"保存命令配置失败: {e}")
|
||||
try:
|
||||
if os.path.exists(tmp_file):
|
||||
os.remove(tmp_file)
|
||||
except Exception:
|
||||
pass
|
||||
return False
|
||||
@staticmethod
|
||||
def _merge_commands(existing: List[Dict[str, Any]], incoming: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
|
||||
"""保留已有命令,仅追加不存在的命令"""
|
||||
merged = list(existing or [])
|
||||
seen_ids = {
|
||||
str(cmd.get("id", "")).strip()
|
||||
for cmd in merged
|
||||
if isinstance(cmd, dict) and str(cmd.get("id", "")).strip()
|
||||
}
|
||||
seen_names = {
|
||||
str(cmd.get("name", "")).strip()
|
||||
for cmd in merged
|
||||
if isinstance(cmd, dict) and str(cmd.get("name", "")).strip()
|
||||
}
|
||||
|
||||
for cmd in incoming or []:
|
||||
if not isinstance(cmd, dict):
|
||||
continue
|
||||
|
||||
command_id = str(cmd.get("id", "")).strip()
|
||||
command_name = str(cmd.get("name", "")).strip()
|
||||
|
||||
if command_id and command_id in seen_ids:
|
||||
continue
|
||||
if command_name and command_name in seen_names:
|
||||
continue
|
||||
|
||||
merged.append(cmd)
|
||||
if command_id:
|
||||
seen_ids.add(command_id)
|
||||
if command_name:
|
||||
seen_names.add(command_name)
|
||||
|
||||
return merged
|
||||
def _migrate_global_commands(self):
|
||||
"""将旧版 _global.commands 迁移到独立文件"""
|
||||
legacy_commands = self.global_config.get("commands", [])
|
||||
existing_commands = self._load_commands_file()
|
||||
|
||||
if not isinstance(legacy_commands, list):
|
||||
legacy_commands = []
|
||||
|
||||
merged_commands = self._merge_commands(existing_commands, legacy_commands)
|
||||
|
||||
if legacy_commands or not os.path.exists(ConfigConstants.COMMANDS_FILE):
|
||||
self._save_commands_file(merged_commands)
|
||||
|
||||
if self.global_config.remove("commands"):
|
||||
self._save_config()
|
||||
def _migrate_to_presets(self):
|
||||
"""
|
||||
将旧格式(扁平)站点配置迁移为预设格式
|
||||
|
||||
旧格式: { "selectors": {...}, "workflow": [...], ... }
|
||||
新格式: { "presets": { "主预设": { "selectors": {...}, ... } } }
|
||||
"""
|
||||
migrated_count = 0
|
||||
|
||||
for domain in list(self.sites.keys()):
|
||||
if domain.startswith('_'):
|
||||
continue
|
||||
|
||||
site_config = self.sites[domain]
|
||||
|
||||
# 已经是预设格式,跳过
|
||||
if "presets" in site_config:
|
||||
continue
|
||||
|
||||
# 将所有已知配置字段提取到主预设中
|
||||
preset_data = {}
|
||||
remaining = {}
|
||||
|
||||
for key, value in site_config.items():
|
||||
if key in PRESET_FIELDS:
|
||||
preset_data[key] = value
|
||||
elif key == "advanced":
|
||||
remaining[key] = value
|
||||
else:
|
||||
# 未知字段也放入预设(保留用户自定义数据)
|
||||
preset_data[key] = value
|
||||
|
||||
# 构建新格式
|
||||
self.sites[domain] = {
|
||||
"default_preset": DEFAULT_PRESET_NAME,
|
||||
"presets": {
|
||||
DEFAULT_PRESET_NAME: preset_data
|
||||
},
|
||||
**remaining,
|
||||
}
|
||||
|
||||
migrated_count += 1
|
||||
logger.debug(f"迁移站点配置: {domain} → 预设格式")
|
||||
|
||||
if migrated_count > 0:
|
||||
self._save_config()
|
||||
logger.info(f"✅ 已迁移 {migrated_count} 个站点配置为预设格式")
|
||||
def _cleanup_preset_residuals(self):
|
||||
"""
|
||||
清理站点配置中预设外的残留字段
|
||||
|
||||
当站点已有 presets 结构时,顶层不应再有 selectors/workflow/file_paste 等字段。
|
||||
这些残留通常由旧版 bug 或手动编辑产生。
|
||||
"""
|
||||
cleaned_count = 0
|
||||
default_fixed_count = 0
|
||||
|
||||
for domain in list(self.sites.keys()):
|
||||
if domain.startswith('_'):
|
||||
continue
|
||||
|
||||
site_config = self.sites[domain]
|
||||
|
||||
# 只处理已有 presets 结构的站点
|
||||
if "presets" not in site_config:
|
||||
continue
|
||||
|
||||
# 找出预设外的残留字段
|
||||
residual_keys = []
|
||||
for key in list(site_config.keys()):
|
||||
if key == "presets":
|
||||
continue
|
||||
if key in PRESET_FIELDS:
|
||||
residual_keys.append(key)
|
||||
|
||||
# 删除残留
|
||||
for key in residual_keys:
|
||||
del site_config[key]
|
||||
cleaned_count += 1
|
||||
logger.debug(f"清理残留: {domain}.{key}")
|
||||
|
||||
if self._normalize_site_default_preset(domain, site_config):
|
||||
default_fixed_count += 1
|
||||
logger.debug(f"修正默认预设: {domain} -> {site_config.get('default_preset')}")
|
||||
|
||||
if cleaned_count > 0 or default_fixed_count > 0:
|
||||
self._save_config()
|
||||
logger.info(
|
||||
f"✅ 已清理 {cleaned_count} 个预设外残留字段,"
|
||||
f"修正 {default_fixed_count} 个站点默认预设"
|
||||
)
|
||||
def _migrate_site_advanced_to_presets(self):
|
||||
"""
|
||||
规范化已有预设级 advanced,避免启动时改写站点级 advanced 语义。
|
||||
|
||||
站点根级 advanced 仍作为所有预设的共享基线;预设级 advanced 只保存
|
||||
覆盖项。因此这里不再把站点级时序字段复制到所有预设,也不从站点级
|
||||
删除这些字段,只清理明显放错位置的 Cookie 字段并规范化已有覆盖项。
|
||||
"""
|
||||
cleaned_count = 0
|
||||
changed = False
|
||||
|
||||
for domain, site_config in self.sites.items():
|
||||
if domain.startswith("_") or not isinstance(site_config, dict):
|
||||
continue
|
||||
|
||||
presets = site_config.get("presets", {})
|
||||
if not isinstance(presets, dict) or not presets:
|
||||
continue
|
||||
|
||||
for preset_name, preset_data in presets.items():
|
||||
if not isinstance(preset_data, dict):
|
||||
continue
|
||||
|
||||
preset_advanced = preset_data.get("advanced")
|
||||
if not isinstance(preset_advanced, dict):
|
||||
continue
|
||||
|
||||
for key in list(SITE_ADVANCED_FIELDS):
|
||||
if key in preset_advanced:
|
||||
del preset_advanced[key]
|
||||
cleaned_count += 1
|
||||
changed = True
|
||||
|
||||
normalized_preset_advanced = self._normalize_site_advanced_config(
|
||||
preset_advanced
|
||||
)
|
||||
for key in list(preset_advanced.keys()):
|
||||
if key in PRESET_ADVANCED_FIELDS:
|
||||
value = normalized_preset_advanced[key]
|
||||
if preset_advanced.get(key) != value:
|
||||
preset_advanced[key] = value
|
||||
cleaned_count += 1
|
||||
changed = True
|
||||
|
||||
if preset_advanced != preset_data.get("advanced"):
|
||||
logger.debug(f"规范化预设高级配置: {domain}/{preset_name}")
|
||||
|
||||
if not preset_advanced:
|
||||
del preset_data["advanced"]
|
||||
cleaned_count += 1
|
||||
changed = True
|
||||
|
||||
if changed:
|
||||
self._save_config()
|
||||
logger.info(
|
||||
f"✅ 已规范化预设高级配置,清理 {cleaned_count} 个残留项"
|
||||
)
|
||||
def _guess_stealth(self, domain: str) -> bool:
|
||||
"""Guess whether stealth mode should default to enabled."""
|
||||
rule = get_site_rule(domain)
|
||||
if "stealth_default" in rule:
|
||||
enabled = bool(rule.get("stealth_default"))
|
||||
if enabled:
|
||||
logger.info(f"检测到默认启用低熵模式的域名: {domain}")
|
||||
return enabled
|
||||
return False
|
||||
def migrate_site_configs(self):
|
||||
"""迁移旧版站点配置,补充各预设中缺失的字段"""
|
||||
migrated_count = 0
|
||||
default_image_config = get_default_image_extraction_config()
|
||||
default_file_paste = get_default_file_paste_config()
|
||||
default_prompt_padding = get_default_prompt_padding_config()
|
||||
obsolete_stream_keys = {
|
||||
"silence_threshold",
|
||||
"initial_wait",
|
||||
"enable_wrapper_search",
|
||||
"rerender_wait",
|
||||
"content_shrink_tolerance",
|
||||
}
|
||||
obsolete_network_keys = set()
|
||||
|
||||
for domain, site_config in self.sites.items():
|
||||
if domain.startswith("_"):
|
||||
continue
|
||||
|
||||
presets = site_config.get("presets", {})
|
||||
|
||||
for preset_name, preset_data in presets.items():
|
||||
if "image_extraction" not in preset_data:
|
||||
preset_data["image_extraction"] = copy.deepcopy(default_image_config)
|
||||
migrated_count += 1
|
||||
logger.debug(f"迁移: {domain}/{preset_name} (添加 image_extraction)")
|
||||
else:
|
||||
normalized_image_config = self._validate_image_config(preset_data.get("image_extraction", {}))
|
||||
if normalized_image_config != preset_data.get("image_extraction"):
|
||||
preset_data["image_extraction"] = normalized_image_config
|
||||
migrated_count += 1
|
||||
logger.debug(f"迁移: {domain}/{preset_name} (规范化 image_extraction)")
|
||||
|
||||
if "file_paste" not in preset_data:
|
||||
preset_data["file_paste"] = copy.deepcopy(default_file_paste)
|
||||
migrated_count += 1
|
||||
logger.debug(f"迁移: {domain}/{preset_name} (添加 file_paste)")
|
||||
|
||||
if "prompt_padding" not in preset_data:
|
||||
preset_data["prompt_padding"] = copy.deepcopy(default_prompt_padding)
|
||||
migrated_count += 1
|
||||
logger.debug(f"迁移: {domain}/{preset_name} (添加 prompt_padding)")
|
||||
else:
|
||||
normalized_prompt_padding = self._validate_prompt_padding_config(
|
||||
preset_data.get("prompt_padding", {})
|
||||
)
|
||||
if normalized_prompt_padding != preset_data.get("prompt_padding"):
|
||||
preset_data["prompt_padding"] = normalized_prompt_padding
|
||||
migrated_count += 1
|
||||
logger.debug(f"迁移: {domain}/{preset_name} (规范化 prompt_padding)")
|
||||
|
||||
stream_config = preset_data.get("stream_config")
|
||||
moved_attachment_rules = False
|
||||
if not isinstance(stream_config, dict):
|
||||
stream_config = {}
|
||||
preset_data["stream_config"] = stream_config
|
||||
|
||||
normalized_file_paste = self._validate_file_paste_config(
|
||||
preset_data.get("file_paste", {}),
|
||||
legacy_stream_config=stream_config,
|
||||
)
|
||||
if normalized_file_paste != preset_data.get("file_paste"):
|
||||
preset_data["file_paste"] = normalized_file_paste
|
||||
migrated_count += 1
|
||||
logger.debug(f"迁移: {domain}/{preset_name} (规范化 file_paste)")
|
||||
|
||||
for legacy_key in ("send_confirmation", "attachment_monitor"):
|
||||
if legacy_key in stream_config:
|
||||
del stream_config[legacy_key]
|
||||
moved_attachment_rules = True
|
||||
|
||||
if isinstance(stream_config, dict):
|
||||
removed_stream = False
|
||||
for key in list(obsolete_stream_keys):
|
||||
if key in stream_config:
|
||||
del stream_config[key]
|
||||
removed_stream = True
|
||||
|
||||
network_config = stream_config.get("network")
|
||||
removed_network = False
|
||||
if isinstance(network_config, dict):
|
||||
for key in list(obsolete_network_keys):
|
||||
if key in network_config:
|
||||
del network_config[key]
|
||||
removed_network = True
|
||||
|
||||
if removed_stream or removed_network or moved_attachment_rules:
|
||||
migrated_count += 1
|
||||
logger.debug(f"迁移: {domain}/{preset_name} (清理废弃的流式配置字段)")
|
||||
|
||||
if migrated_count > 0:
|
||||
self._save_config()
|
||||
logger.info(f"已迁移 {migrated_count} 个预设配置")
|
||||
|
||||
return migrated_count
|
||||
@@ -0,0 +1,667 @@
|
||||
"""ConfigEngine 的一部分(R2-3 拆分):预设的解析、创建、改名、删除及引用同步。
|
||||
|
||||
方法原样从 engine.py 迁出,行为不变;通过 mixin 组合回 ConfigEngine,
|
||||
实例状态都在 ConfigEngine.__init__(engine.py)里初始化。
|
||||
"""
|
||||
|
||||
import json
|
||||
import copy
|
||||
from typing import Dict, Optional, List, Any
|
||||
from .engine_common import DEFAULT_PRESET_NAME, DEFAULT_WORKFLOW, _MISSING, logger
|
||||
|
||||
|
||||
class ConfigPresetsMixin:
|
||||
"""预设的解析、创建、改名、删除及引用同步"""
|
||||
|
||||
def _resolve_default_preset_name(self, site: Dict[str, Any]) -> Optional[str]:
|
||||
"""解析站点有效默认预设名(不修改原对象)"""
|
||||
presets = site.get("presets", {})
|
||||
if not presets:
|
||||
return None
|
||||
|
||||
configured_default = site.get("default_preset")
|
||||
if isinstance(configured_default, str):
|
||||
configured_default = configured_default.strip()
|
||||
else:
|
||||
configured_default = None
|
||||
|
||||
if configured_default and configured_default in presets:
|
||||
return configured_default
|
||||
|
||||
if DEFAULT_PRESET_NAME in presets:
|
||||
return DEFAULT_PRESET_NAME
|
||||
|
||||
return next(iter(presets))
|
||||
def _normalize_site_default_preset(self, domain: str, site: Dict[str, Any]) -> bool:
|
||||
"""
|
||||
规范化站点 default_preset 字段
|
||||
|
||||
Returns:
|
||||
是否发生修改
|
||||
"""
|
||||
return self._sync_site_default_preset_state(domain, site)
|
||||
def _get_site_data(self, domain: str, preset_name: str = None) -> Optional[Dict]:
|
||||
"""
|
||||
获取指定站点的预设配置数据(可变引用)
|
||||
|
||||
查找顺序:
|
||||
- preset_name 显式提供时:只匹配该预设(含别名)
|
||||
- preset_name 为空时:按站点默认预设 / 主预设 / 第一个可用预设回退
|
||||
|
||||
Args:
|
||||
domain: 站点域名
|
||||
preset_name: 预设名称,None 则使用默认
|
||||
|
||||
Returns:
|
||||
预设配置字典的引用(可直接修改),或 None
|
||||
"""
|
||||
if domain not in self.sites:
|
||||
return None
|
||||
|
||||
site = self.sites[domain]
|
||||
presets = site.get("presets", {})
|
||||
|
||||
if not presets:
|
||||
return None
|
||||
|
||||
requested_preset = str(preset_name or "").strip()
|
||||
if requested_preset:
|
||||
resolved_target = self._resolve_preset_alias_key(requested_preset, presets)
|
||||
if resolved_target != requested_preset:
|
||||
logger.debug(f"预设别名命中: '{requested_preset}' -> '{resolved_target}'")
|
||||
if resolved_target in presets:
|
||||
return presets[resolved_target]
|
||||
return None
|
||||
|
||||
resolved_default = self._resolve_default_preset_name(site)
|
||||
if resolved_default and resolved_default in presets:
|
||||
return presets[resolved_default]
|
||||
|
||||
if DEFAULT_PRESET_NAME in presets:
|
||||
return presets[DEFAULT_PRESET_NAME]
|
||||
|
||||
first_key = next(iter(presets))
|
||||
logger.warning(f"默认预设不存在,使用第一个预设: '{first_key}'")
|
||||
return presets[first_key]
|
||||
@staticmethod
|
||||
def _resolve_preset_alias_key(target: Any, presets: Dict[str, Any]) -> str:
|
||||
"""兼容历史命名:允许不带“预设_”前缀的名字命中真实预设键。"""
|
||||
normalized = str(target or "").strip()
|
||||
if not normalized or not isinstance(presets, dict):
|
||||
return normalized
|
||||
|
||||
if normalized in presets:
|
||||
return normalized
|
||||
|
||||
candidates = []
|
||||
if normalized.startswith("预设_"):
|
||||
stripped = normalized[len("预设_"):].strip()
|
||||
if stripped:
|
||||
candidates.append(stripped)
|
||||
else:
|
||||
candidates.append(f"预设_{normalized}")
|
||||
|
||||
for candidate in candidates:
|
||||
if candidate in presets:
|
||||
return candidate
|
||||
|
||||
return normalized
|
||||
def _get_site_data_readonly(self, domain: str, preset_name: str = None) -> Optional[Dict]:
|
||||
"""获取预设配置的深拷贝(只读用途)"""
|
||||
with self._io_lock:
|
||||
data = self._get_site_data(domain, preset_name)
|
||||
if data is None:
|
||||
return None
|
||||
return copy.deepcopy(data)
|
||||
def _get_preset_data_exact(self, domain: str, preset_name: str = None) -> Optional[Dict]:
|
||||
"""获取预设配置引用;显式预设不存在时不回退到默认预设。"""
|
||||
site = self.sites.get(domain)
|
||||
if not isinstance(site, dict):
|
||||
return None
|
||||
|
||||
presets = site.get("presets", {})
|
||||
if not isinstance(presets, dict) or not presets:
|
||||
return None
|
||||
|
||||
if preset_name is None or not str(preset_name).strip():
|
||||
target = self._resolve_default_preset_name(site)
|
||||
else:
|
||||
target = str(preset_name).strip()
|
||||
|
||||
resolved_target = self._resolve_preset_alias_key(target, presets)
|
||||
if resolved_target and resolved_target in presets:
|
||||
return presets[resolved_target]
|
||||
|
||||
return None
|
||||
def list_presets(self, domain: str) -> List[str]:
|
||||
"""获取指定站点的所有预设名称"""
|
||||
self.refresh_if_changed()
|
||||
|
||||
if domain not in self.sites:
|
||||
return []
|
||||
|
||||
site = self.sites[domain]
|
||||
presets = site.get("presets", {})
|
||||
return list(presets.keys())
|
||||
def get_default_preset(self, domain: str) -> Optional[str]:
|
||||
"""获取指定站点的默认预设名称(已解析回退)"""
|
||||
self.refresh_if_changed()
|
||||
|
||||
cache_key = f"default_preset:{domain}"
|
||||
cached = self._cache.get(cache_key, _MISSING)
|
||||
if cached is not _MISSING:
|
||||
return cached
|
||||
|
||||
site = self.sites.get(domain)
|
||||
if not site:
|
||||
self._cache.set(cache_key, None)
|
||||
return None
|
||||
|
||||
result = self._resolve_default_preset_name(site)
|
||||
self._cache.set(cache_key, result)
|
||||
return result
|
||||
def _assign_preset_field_and_save(
|
||||
self,
|
||||
preset_data: Dict[str, Any],
|
||||
field_name: str,
|
||||
value: Any,
|
||||
) -> bool:
|
||||
previous_value = preset_data.get(field_name, _MISSING)
|
||||
if previous_value is not _MISSING:
|
||||
previous_value = copy.deepcopy(previous_value)
|
||||
preset_data[field_name] = value
|
||||
if self._save_config():
|
||||
return True
|
||||
if previous_value is _MISSING:
|
||||
preset_data.pop(field_name, None)
|
||||
else:
|
||||
preset_data[field_name] = previous_value
|
||||
return False
|
||||
def set_default_preset(self, domain: str, preset_name: str) -> bool:
|
||||
"""设置指定站点的默认预设"""
|
||||
self.refresh_if_changed()
|
||||
|
||||
site = self.sites.get(domain)
|
||||
if not site:
|
||||
return False
|
||||
|
||||
presets = site.get("presets", {})
|
||||
if preset_name not in presets:
|
||||
logger.warning(f"默认预设设置失败,预设不存在: {domain}/{preset_name}")
|
||||
return False
|
||||
|
||||
previous_default = site.get("default_preset", _MISSING)
|
||||
if previous_default is not _MISSING:
|
||||
previous_default = copy.deepcopy(previous_default)
|
||||
previous_default_maps = (
|
||||
copy.deepcopy(getattr(self, "_global_default_presets", {})),
|
||||
copy.deepcopy(getattr(self, "_local_default_presets", {})),
|
||||
)
|
||||
persisted_default = self._get_persisted_default_preset(domain, site)
|
||||
if persisted_default == preset_name:
|
||||
self._local_default_presets.pop(domain, None)
|
||||
else:
|
||||
self._local_default_presets[domain] = preset_name
|
||||
site["default_preset"] = preset_name
|
||||
if not self._save_local_site_overrides():
|
||||
if previous_default is _MISSING:
|
||||
site.pop("default_preset", None)
|
||||
else:
|
||||
site["default_preset"] = previous_default
|
||||
self._global_default_presets, self._local_default_presets = previous_default_maps
|
||||
return False
|
||||
logger.info(f"✅ 站点 {domain} 默认预设已设置为: '{preset_name}'(仅本地覆盖)")
|
||||
return True
|
||||
def create_preset(self, domain: str, new_name: str,
|
||||
source_name: str = None) -> bool:
|
||||
"""
|
||||
创建新预设(克隆自现有预设)
|
||||
|
||||
Args:
|
||||
domain: 站点域名
|
||||
new_name: 新预设名称
|
||||
source_name: 要克隆的源预设名称,None 则克隆主预设
|
||||
|
||||
Returns:
|
||||
是否成功
|
||||
"""
|
||||
self.refresh_if_changed()
|
||||
|
||||
if domain not in self.sites:
|
||||
logger.warning(f"站点不存在: {domain}")
|
||||
return False
|
||||
|
||||
site = self.sites[domain]
|
||||
presets = site.get("presets", {})
|
||||
|
||||
if new_name in presets:
|
||||
logger.warning(f"预设已存在: {new_name}")
|
||||
return False
|
||||
|
||||
# 获取源预设
|
||||
if source_name is not None and str(source_name or "").strip():
|
||||
requested_source = str(source_name or "").strip()
|
||||
source = self._resolve_preset_alias_key(requested_source, presets)
|
||||
source_data = presets.get(source)
|
||||
if not source_data:
|
||||
logger.warning(f"源预设不存在: {requested_source}")
|
||||
return False
|
||||
else:
|
||||
source = DEFAULT_PRESET_NAME
|
||||
source_data = presets.get(source)
|
||||
if not source_data:
|
||||
# 未显式指定源时才尝试第一个可用预设
|
||||
if presets:
|
||||
source = next(iter(presets))
|
||||
source_data = presets[source]
|
||||
else:
|
||||
logger.warning(f"没有可克隆的源预设")
|
||||
return False
|
||||
|
||||
previous_default = site.get("default_preset", _MISSING)
|
||||
if previous_default is not _MISSING:
|
||||
previous_default = copy.deepcopy(previous_default)
|
||||
previous_presets = copy.deepcopy(presets)
|
||||
previous_global_default = copy.deepcopy(self._global_default_presets)
|
||||
previous_local_default = copy.deepcopy(self._local_default_presets)
|
||||
|
||||
# 深拷贝创建新预设
|
||||
presets[new_name] = copy.deepcopy(source_data)
|
||||
self._normalize_site_default_preset(domain, site)
|
||||
if not self._save_config():
|
||||
site["presets"] = previous_presets
|
||||
if previous_default is _MISSING:
|
||||
site.pop("default_preset", None)
|
||||
else:
|
||||
site["default_preset"] = previous_default
|
||||
self._global_default_presets = previous_global_default
|
||||
self._local_default_presets = previous_local_default
|
||||
return False
|
||||
|
||||
logger.info(f"✅ 站点 {domain} 创建预设: '{new_name}' (克隆自 '{source}')")
|
||||
return True
|
||||
@staticmethod
|
||||
def _build_preset_rename_map(old_name: Any, new_name: Any) -> Dict[str, str]:
|
||||
old_raw = str(old_name or "").strip()
|
||||
new_raw = str(new_name or "").strip()
|
||||
if not old_raw or not new_raw:
|
||||
return {}
|
||||
|
||||
def _strip_prefix(value: str) -> str:
|
||||
if value.startswith("预设_"):
|
||||
return value[len("预设_"):].strip()
|
||||
return value
|
||||
|
||||
def _add_prefix(value: str) -> str:
|
||||
return value if value.startswith("预设_") else f"预设_{value}"
|
||||
|
||||
mapping: Dict[str, str] = {}
|
||||
old_plain = _strip_prefix(old_raw)
|
||||
new_plain = _strip_prefix(new_raw)
|
||||
old_prefixed = _add_prefix(old_raw)
|
||||
new_prefixed = _add_prefix(new_raw)
|
||||
|
||||
for src, dst in (
|
||||
(old_raw, new_raw),
|
||||
(old_plain, new_plain),
|
||||
(old_prefixed, new_prefixed),
|
||||
):
|
||||
src_text = str(src or "").strip()
|
||||
dst_text = str(dst or "").strip()
|
||||
if src_text and dst_text:
|
||||
mapping[src_text] = dst_text
|
||||
|
||||
return mapping
|
||||
@staticmethod
|
||||
def _command_targets_domain(command: Dict[str, Any], domain: str) -> bool:
|
||||
normalized_domain = str(domain or "").strip().lower()
|
||||
if not normalized_domain or not isinstance(command, dict):
|
||||
return False
|
||||
|
||||
trigger = command.get("trigger", {}) or {}
|
||||
command_domain = str(trigger.get("domain", "") or "").strip().lower()
|
||||
if not command_domain:
|
||||
return False
|
||||
|
||||
return (
|
||||
command_domain == normalized_domain
|
||||
or command_domain.endswith(f".{normalized_domain}")
|
||||
or normalized_domain.endswith(f".{command_domain}")
|
||||
)
|
||||
def _rename_preset_references_in_commands(
|
||||
self,
|
||||
domain: str,
|
||||
rename_map: Dict[str, str],
|
||||
) -> int:
|
||||
if not rename_map:
|
||||
return 0
|
||||
|
||||
commands = self._load_commands_file()
|
||||
if not commands:
|
||||
return 0
|
||||
|
||||
updated = 0
|
||||
for command in commands:
|
||||
if not self._command_targets_domain(command, domain):
|
||||
continue
|
||||
|
||||
actions = command.get("actions", [])
|
||||
if not isinstance(actions, list):
|
||||
continue
|
||||
|
||||
for action in actions:
|
||||
if not isinstance(action, dict):
|
||||
continue
|
||||
current_preset = str(action.get("preset_name", "") or "").strip()
|
||||
replacement = rename_map.get(current_preset)
|
||||
if not replacement or replacement == current_preset:
|
||||
continue
|
||||
action["preset_name"] = replacement
|
||||
updated += 1
|
||||
|
||||
if updated > 0:
|
||||
self._save_commands_file(commands)
|
||||
|
||||
return updated
|
||||
def _rename_preset_references_in_active_tabs(
|
||||
self,
|
||||
domain: str,
|
||||
rename_map: Dict[str, str],
|
||||
) -> int:
|
||||
if not rename_map:
|
||||
return 0
|
||||
|
||||
try:
|
||||
from app.core import browser as browser_module
|
||||
|
||||
instance = getattr(browser_module, "_browser_instance", None)
|
||||
if instance is None:
|
||||
return 0
|
||||
|
||||
pool = getattr(instance, "_tab_pool", None)
|
||||
if pool is None or not hasattr(pool, "get_sessions_snapshot"):
|
||||
return 0
|
||||
|
||||
updated = 0
|
||||
for session in pool.get_sessions_snapshot():
|
||||
session_domain = str(getattr(session, "current_domain", "") or "").strip().lower()
|
||||
if session_domain != str(domain or "").strip().lower():
|
||||
continue
|
||||
|
||||
current_preset = str(getattr(session, "preset_name", "") or "").strip()
|
||||
replacement = rename_map.get(current_preset)
|
||||
if not replacement or replacement == current_preset:
|
||||
continue
|
||||
|
||||
if hasattr(session, "set_preset"):
|
||||
session.set_preset(replacement, source=getattr(session, "preset_override_source", None))
|
||||
else:
|
||||
session.preset_name = replacement
|
||||
updated += 1
|
||||
|
||||
return updated
|
||||
except Exception as e:
|
||||
logger.debug(f"同步活动标签页预设引用失败(忽略): {e}")
|
||||
return 0
|
||||
def _clear_preset_references_in_active_tabs(self, domain: str, preset_name: str) -> int:
|
||||
"""预设被删除后,把在用标签页里指向它的 preset_name 置为 None(回落站点默认预设)。"""
|
||||
target = str(preset_name or "").strip()
|
||||
if not target:
|
||||
return 0
|
||||
|
||||
# 兼容历史命名:带/不带“预设_”前缀都视为命中同一个预设
|
||||
targets = {target}
|
||||
if target.startswith("预设_"):
|
||||
stripped = target[len("预设_"):].strip()
|
||||
if stripped:
|
||||
targets.add(stripped)
|
||||
else:
|
||||
targets.add(f"预设_{target}")
|
||||
|
||||
try:
|
||||
from app.core import browser as browser_module
|
||||
|
||||
instance = getattr(browser_module, "_browser_instance", None)
|
||||
if instance is None:
|
||||
return 0
|
||||
|
||||
pool = getattr(instance, "_tab_pool", None)
|
||||
if pool is None or not hasattr(pool, "get_sessions_snapshot"):
|
||||
return 0
|
||||
|
||||
cleared = 0
|
||||
for session in pool.get_sessions_snapshot():
|
||||
session_domain = str(getattr(session, "current_domain", "") or "").strip().lower()
|
||||
if session_domain != str(domain or "").strip().lower():
|
||||
continue
|
||||
|
||||
current_preset = str(getattr(session, "preset_name", "") or "").strip()
|
||||
if current_preset not in targets:
|
||||
continue
|
||||
|
||||
if hasattr(session, "set_preset"):
|
||||
session.set_preset(None, source=None)
|
||||
else:
|
||||
session.preset_name = None
|
||||
cleared += 1
|
||||
|
||||
return cleared
|
||||
except Exception as e:
|
||||
logger.debug(f"清理活动标签页预设引用失败(忽略): {e}")
|
||||
return 0
|
||||
def _sync_preset_overrides_file(self, domain: str, old_preset: str, new_preset: Optional[str] = None) -> int:
|
||||
"""当预设重命名或删除时,同步清理/更新 preset_overrides.local.json"""
|
||||
target = str(old_preset or "").strip()
|
||||
if not target:
|
||||
return 0
|
||||
from pathlib import Path
|
||||
config_path = Path("config/preset_overrides.local.json")
|
||||
if not config_path.exists():
|
||||
return 0
|
||||
from app.utils.site_url import extract_remote_site_domain
|
||||
import json
|
||||
try:
|
||||
with open(config_path, "r", encoding="utf-8-sig") as f:
|
||||
payload = json.load(f)
|
||||
if not isinstance(payload, dict):
|
||||
return 0
|
||||
urls = payload.get("urls", {})
|
||||
if not isinstance(urls, dict):
|
||||
return 0
|
||||
|
||||
domain_norm = str(domain or "").strip().lower()
|
||||
changed = 0
|
||||
keys_to_delete = []
|
||||
for url_key, p_name in list(urls.items()):
|
||||
url_domain = str(extract_remote_site_domain(url_key) or "").strip().lower()
|
||||
if domain_norm and url_domain != domain_norm:
|
||||
continue
|
||||
if p_name == target or p_name == f"预设_{target}" or (target.startswith("预设_") and p_name == target[3:]):
|
||||
if new_preset:
|
||||
urls[url_key] = new_preset
|
||||
changed += 1
|
||||
else:
|
||||
keys_to_delete.append(url_key)
|
||||
changed += 1
|
||||
for k in keys_to_delete:
|
||||
urls.pop(k, None)
|
||||
|
||||
if changed > 0:
|
||||
from app.core.config import atomic_write_json
|
||||
atomic_write_json(config_path, payload)
|
||||
try:
|
||||
from app.core import browser as browser_module
|
||||
instance = getattr(browser_module, "_browser_instance", None)
|
||||
pool = getattr(instance, "_tab_pool", None)
|
||||
if pool is not None and hasattr(pool, "apply_runtime_config"):
|
||||
pool.apply_runtime_config(preset_overrides=payload)
|
||||
except Exception:
|
||||
pass
|
||||
return changed
|
||||
except Exception as e:
|
||||
logger.debug(f"同步预设本地配置失败: {e}")
|
||||
return 0
|
||||
def delete_preset(self, domain: str, preset_name: str) -> bool:
|
||||
"""
|
||||
删除预设(不允许删除最后一个预设)
|
||||
|
||||
Args:
|
||||
domain: 站点域名
|
||||
preset_name: 要删除的预设名称
|
||||
|
||||
Returns:
|
||||
是否成功
|
||||
"""
|
||||
self.refresh_if_changed()
|
||||
|
||||
if domain not in self.sites:
|
||||
return False
|
||||
|
||||
site = self.sites[domain]
|
||||
presets = site.get("presets", {})
|
||||
|
||||
resolved_preset_name = self._resolve_preset_alias_key(preset_name, presets)
|
||||
if resolved_preset_name not in presets:
|
||||
logger.warning(f"预设不存在: {preset_name}")
|
||||
return False
|
||||
|
||||
if len(presets) <= 1:
|
||||
logger.warning(f"不能删除最后一个预设")
|
||||
return False
|
||||
|
||||
previous_default = site.get("default_preset", _MISSING)
|
||||
if previous_default is not _MISSING:
|
||||
previous_default = copy.deepcopy(previous_default)
|
||||
removed_preset = copy.deepcopy(presets[resolved_preset_name])
|
||||
previous_global_default = copy.deepcopy(self._global_default_presets)
|
||||
previous_local_default = copy.deepcopy(self._local_default_presets)
|
||||
|
||||
del presets[resolved_preset_name]
|
||||
self._normalize_site_default_preset(domain, site)
|
||||
if not self._save_config():
|
||||
presets[resolved_preset_name] = removed_preset
|
||||
if previous_default is _MISSING:
|
||||
site.pop("default_preset", None)
|
||||
else:
|
||||
site["default_preset"] = previous_default
|
||||
self._global_default_presets = previous_global_default
|
||||
self._local_default_presets = previous_local_default
|
||||
return False
|
||||
|
||||
# 预设已删除并落盘:清理在用标签页对它的残留引用,否则 session.preset_name 会指向不存在的预设
|
||||
cleared_tab_refs = 0
|
||||
try:
|
||||
cleared_tab_refs = self._clear_preset_references_in_active_tabs(domain, resolved_preset_name)
|
||||
except Exception as e:
|
||||
logger.warning(f"清理活动标签页预设引用失败(忽略): {e}")
|
||||
|
||||
# 同步清理持久化 preset_overrides.local.json
|
||||
try:
|
||||
self._sync_preset_overrides_file(domain, resolved_preset_name, new_preset=None)
|
||||
except Exception as e:
|
||||
logger.warning(f"清理本地预设记忆文件失败(忽略): {e}")
|
||||
|
||||
logger.info(
|
||||
f"✅ 站点 {domain} 删除预设: '{resolved_preset_name}' "
|
||||
f"(活动标签页同步 {cleared_tab_refs} 处)"
|
||||
)
|
||||
return True
|
||||
def rename_preset(self, domain: str, old_name: str, new_name: str) -> bool:
|
||||
"""重命名预设"""
|
||||
self.refresh_if_changed()
|
||||
|
||||
if domain not in self.sites:
|
||||
return False
|
||||
|
||||
site = self.sites[domain]
|
||||
presets = site.get("presets", {})
|
||||
|
||||
resolved_old_name = self._resolve_preset_alias_key(old_name, presets)
|
||||
if resolved_old_name not in presets:
|
||||
return False
|
||||
|
||||
if new_name in presets:
|
||||
logger.warning(f"预设名已存在: {new_name}")
|
||||
return False
|
||||
|
||||
rename_map = self._build_preset_rename_map(resolved_old_name, new_name)
|
||||
default_preset = site.get("default_preset")
|
||||
previous_default = site.get("default_preset", _MISSING)
|
||||
if previous_default is not _MISSING:
|
||||
previous_default = copy.deepcopy(previous_default)
|
||||
previous_presets = copy.deepcopy(presets)
|
||||
previous_global_default = copy.deepcopy(self._global_default_presets)
|
||||
previous_local_default = copy.deepcopy(self._local_default_presets)
|
||||
local_default = str(self._local_default_presets.get(domain, "") or "").strip()
|
||||
|
||||
# 保持顺序:创建有序副本
|
||||
new_presets = {}
|
||||
for key, value in presets.items():
|
||||
if key == resolved_old_name:
|
||||
new_presets[new_name] = value
|
||||
else:
|
||||
new_presets[key] = value
|
||||
|
||||
site["presets"] = new_presets
|
||||
if default_preset == resolved_old_name:
|
||||
site["default_preset"] = new_name
|
||||
if local_default:
|
||||
local_replacement = rename_map.get(local_default)
|
||||
if local_replacement:
|
||||
self._local_default_presets[domain] = local_replacement
|
||||
self._normalize_site_default_preset(domain, site)
|
||||
if not self._save_config():
|
||||
site["presets"] = previous_presets
|
||||
if previous_default is _MISSING:
|
||||
site.pop("default_preset", None)
|
||||
else:
|
||||
site["default_preset"] = previous_default
|
||||
self._global_default_presets = previous_global_default
|
||||
self._local_default_presets = previous_local_default
|
||||
return False
|
||||
|
||||
updated_command_refs = self._rename_preset_references_in_commands(domain, rename_map)
|
||||
updated_tab_refs = self._rename_preset_references_in_active_tabs(domain, rename_map)
|
||||
try:
|
||||
self._sync_preset_overrides_file(domain, resolved_old_name, new_preset=new_name)
|
||||
except Exception as e:
|
||||
logger.warning(f"同步本地预设记忆文件失败(忽略): {e}")
|
||||
|
||||
logger.info(
|
||||
f"站点 {domain} 重命名预设: '{resolved_old_name}' → '{new_name}' "
|
||||
f"(命令引用同步 {updated_command_refs} 处, 活动标签页同步 {updated_tab_refs} 处)"
|
||||
)
|
||||
return True
|
||||
def get_preset_selectors(self, domain: str, preset_name: str = None) -> Dict:
|
||||
"""获取指定预设的选择器配置"""
|
||||
data = self._get_site_data_readonly(domain, preset_name)
|
||||
return data.get("selectors", {}) if data else {}
|
||||
def set_preset_selectors(self, domain: str, selectors: Dict,
|
||||
preset_name: str = None) -> bool:
|
||||
"""设置指定预设的选择器配置"""
|
||||
self.refresh_if_changed()
|
||||
data = self._get_site_data(domain, preset_name)
|
||||
if data is None:
|
||||
return False
|
||||
if not self._assign_preset_field_and_save(data, "selectors", selectors):
|
||||
return False
|
||||
logger.info(f"站点 {domain} [{preset_name or DEFAULT_PRESET_NAME}] 选择器已更新")
|
||||
return True
|
||||
def get_preset_workflow(self, domain: str, preset_name: str = None) -> List:
|
||||
"""获取指定预设的工作流配置"""
|
||||
data = self._get_site_data_readonly(domain, preset_name)
|
||||
return data.get("workflow", DEFAULT_WORKFLOW) if data else DEFAULT_WORKFLOW
|
||||
def set_preset_workflow(self, domain: str, workflow: List,
|
||||
preset_name: str = None) -> bool:
|
||||
"""设置指定预设的工作流配置"""
|
||||
from app.core.workflow.flow_runtime import has_control_flow, validate_workflow
|
||||
if has_control_flow(workflow):
|
||||
validate_workflow(workflow)
|
||||
self.refresh_if_changed()
|
||||
data = self._get_site_data(domain, preset_name)
|
||||
if data is None:
|
||||
return False
|
||||
if not self._assign_preset_field_and_save(data, "workflow", workflow):
|
||||
return False
|
||||
logger.info(f"站点 {domain} [{preset_name or DEFAULT_PRESET_NAME}] 工作流已更新")
|
||||
return True
|
||||
@@ -0,0 +1,535 @@
|
||||
"""ConfigEngine 的一部分(R2-3 拆分):站点目录、起始页推断、站点高级配置与选择器定义。
|
||||
|
||||
方法原样从 engine.py 迁出,行为不变;通过 mixin 组合回 ConfigEngine,
|
||||
实例状态都在 ConfigEngine.__init__(engine.py)里初始化。
|
||||
"""
|
||||
|
||||
import copy
|
||||
from typing import Dict, Optional, List, Any, Set, Union, Callable
|
||||
from urllib.parse import urljoin
|
||||
from app.models.schemas import (
|
||||
SiteConfig,
|
||||
SelectorDefinition,
|
||||
ADVANCED_FIELDS,
|
||||
PRESET_ADVANCED_FIELDS,
|
||||
SITE_ADVANCED_FIELDS,
|
||||
get_default_image_extraction_config,
|
||||
get_default_file_paste_config,
|
||||
get_default_prompt_padding_config,
|
||||
get_default_site_advanced_config,
|
||||
)
|
||||
from app.utils.site_rules import derive_site_card_id, get_site_rule
|
||||
from app.utils.site_discovery import (
|
||||
automatic_discovery_allowed,
|
||||
specific_chat_selectors,
|
||||
selectors_match_chat_html,
|
||||
)
|
||||
from .engine_common import (
|
||||
DEFAULT_PRESET_NAME,
|
||||
DEFAULT_WORKFLOW,
|
||||
_MISSING,
|
||||
_SITE_STARTUP_JS_PATTERNS,
|
||||
_coerce_bool,
|
||||
get_default_stream_config,
|
||||
logger,
|
||||
)
|
||||
|
||||
|
||||
class ConfigSitesMixin:
|
||||
"""站点目录、起始页推断、站点高级配置与选择器定义"""
|
||||
|
||||
def _extract_startup_url_from_script(self, script: str) -> str:
|
||||
text = str(script or "").strip()
|
||||
if not text:
|
||||
return ""
|
||||
|
||||
for pattern in _SITE_STARTUP_JS_PATTERNS:
|
||||
match = pattern.search(text)
|
||||
if match:
|
||||
return str(match.group(1) or "").strip()
|
||||
return ""
|
||||
def _normalize_startup_url(self, domain: str, raw_url: str) -> str:
|
||||
normalized_domain = str(domain or "").strip().lower().strip(".")
|
||||
text = str(raw_url or "").strip()
|
||||
if not normalized_domain:
|
||||
return text
|
||||
if not text:
|
||||
return f"https://{normalized_domain}"
|
||||
if text.startswith(("http://", "https://")):
|
||||
return text
|
||||
return urljoin(f"https://{normalized_domain}", text)
|
||||
def _infer_site_startup_url(self, domain: str, site: Dict[str, Any]) -> str:
|
||||
preset_name = self._resolve_default_preset_name(site)
|
||||
preset_data = self._get_site_data_readonly(domain, preset_name)
|
||||
workflow = preset_data.get("workflow", []) if isinstance(preset_data, dict) else []
|
||||
|
||||
if isinstance(workflow, list):
|
||||
for step in workflow:
|
||||
if str((step or {}).get("action") or "").strip().upper() != "JS_EXEC":
|
||||
continue
|
||||
startup_url = self._extract_startup_url_from_script((step or {}).get("value"))
|
||||
if startup_url:
|
||||
return self._normalize_startup_url(domain, startup_url)
|
||||
|
||||
return self._normalize_startup_url(domain, "")
|
||||
def get_site_catalog_entry(self, domain: str, fallback_order: int = 0) -> Optional[Dict[str, Any]]:
|
||||
self.refresh_if_changed()
|
||||
|
||||
site = self.sites.get(domain)
|
||||
if not isinstance(site, dict) or str(domain or "").startswith("_"):
|
||||
return None
|
||||
|
||||
rule = get_site_rule(domain)
|
||||
startup_url = rule.get("startup_url") or self._infer_site_startup_url(domain, site)
|
||||
display_name = str(rule.get("display_name") or domain).strip() or str(domain)
|
||||
card_id = str(rule.get("card_id") or derive_site_card_id(domain)).strip() or derive_site_card_id(domain)
|
||||
guide_priority = rule.get("guide_priority")
|
||||
if not isinstance(guide_priority, int):
|
||||
guide_priority = int(fallback_order)
|
||||
|
||||
entry = {
|
||||
"domain": str(domain),
|
||||
"display_name": display_name,
|
||||
"url": self._normalize_startup_url(domain, startup_url),
|
||||
"card_id": card_id,
|
||||
"guide_priority": guide_priority,
|
||||
}
|
||||
|
||||
if isinstance(rule.get("route_aliases"), list):
|
||||
entry["route_aliases"] = [str(item) for item in rule.get("route_aliases", [])]
|
||||
if "stealth_default" in rule:
|
||||
entry["stealth_default"] = bool(rule.get("stealth_default"))
|
||||
|
||||
return entry
|
||||
def list_site_catalog(self) -> List[Dict[str, Any]]:
|
||||
self.refresh_if_changed()
|
||||
|
||||
ordered_domains = [
|
||||
domain for domain, site in self.sites.items()
|
||||
if not domain.startswith("_") and isinstance(site, dict)
|
||||
]
|
||||
entries: List[Dict[str, Any]] = []
|
||||
|
||||
for index, domain in enumerate(ordered_domains):
|
||||
entry = self.get_site_catalog_entry(domain, fallback_order=index)
|
||||
if entry:
|
||||
entries.append(entry)
|
||||
|
||||
entries.sort(key=lambda item: (int(item.get("guide_priority", 0)), str(item.get("domain") or "")))
|
||||
return entries
|
||||
def _normalize_site_advanced_config(
|
||||
self,
|
||||
raw_config: Optional[Dict[str, Any]] = None,
|
||||
*,
|
||||
base_config: Optional[Dict[str, Any]] = None,
|
||||
) -> Dict[str, Any]:
|
||||
"""合并并规范化高级配置。"""
|
||||
normalized = {
|
||||
**get_default_site_advanced_config(),
|
||||
}
|
||||
if isinstance(base_config, dict):
|
||||
normalized.update(copy.deepcopy(base_config))
|
||||
if isinstance(raw_config, dict):
|
||||
normalized.update(copy.deepcopy(raw_config))
|
||||
|
||||
normalized["independent_cookies"] = _coerce_bool(normalized.get("independent_cookies"), False)
|
||||
normalized["independent_cookies_auto_takeover"] = _coerce_bool(
|
||||
normalized.get("independent_cookies_auto_takeover"), False
|
||||
)
|
||||
normalized["input_box_stability_wait_enabled"] = _coerce_bool(
|
||||
normalized.get("input_box_stability_wait_enabled"), False
|
||||
)
|
||||
normalized["input_box_stability_wait_after_new_chat_only"] = _coerce_bool(
|
||||
normalized.get("input_box_stability_wait_after_new_chat_only"), True
|
||||
)
|
||||
try:
|
||||
timeout_value = float(normalized.get("input_box_stability_wait_timeout", 1.5))
|
||||
except Exception:
|
||||
timeout_value = 1.5
|
||||
normalized["input_box_stability_wait_timeout"] = max(0.1, min(timeout_value, 10.0))
|
||||
normalized["url_transition_wait_on_new_chat"] = _coerce_bool(
|
||||
normalized.get("url_transition_wait_on_new_chat"), False
|
||||
)
|
||||
raw_patterns = normalized.get("url_transition_wait_patterns") or []
|
||||
if isinstance(raw_patterns, str):
|
||||
raw_patterns = raw_patterns.replace("\n", ",").replace(";", ",").split(",")
|
||||
if not isinstance(raw_patterns, (list, tuple, set)):
|
||||
raw_patterns = []
|
||||
normalized["url_transition_wait_patterns"] = [
|
||||
str(pattern or "").strip()
|
||||
for pattern in raw_patterns
|
||||
if str(pattern or "").strip()
|
||||
]
|
||||
normalized["send_confirmation_check_enabled"] = _coerce_bool(
|
||||
normalized.get("send_confirmation_check_enabled"), False
|
||||
)
|
||||
try:
|
||||
send_timeout_value = float(normalized.get("send_confirmation_check_timeout", 1.5))
|
||||
except Exception:
|
||||
send_timeout_value = 1.5
|
||||
normalized["send_confirmation_check_timeout"] = max(0.1, min(send_timeout_value, 10.0))
|
||||
normalized["skip_new_chat_on_retry"] = _coerce_bool(
|
||||
normalized.get("skip_new_chat_on_retry"), False
|
||||
)
|
||||
return normalized
|
||||
def get_site_advanced_config(self, domain: str, preset_name: str = None) -> Dict[str, Any]:
|
||||
"""获取站点高级配置;传入 preset_name 时叠加对应预设的覆盖项。"""
|
||||
self.refresh_if_changed()
|
||||
|
||||
site = self.sites.get(domain)
|
||||
if not site:
|
||||
return self._normalize_site_advanced_config()
|
||||
|
||||
raw_config = site.get("advanced") if isinstance(site, dict) else None
|
||||
normalized = self._normalize_site_advanced_config(
|
||||
raw_config if isinstance(raw_config, dict) else {}
|
||||
)
|
||||
|
||||
requested_preset = str(preset_name or "").strip()
|
||||
if requested_preset:
|
||||
preset_data = self._get_preset_data_exact(domain, requested_preset)
|
||||
if preset_data is None:
|
||||
logger.warning(f"站点高级配置预设不存在: {domain}/{requested_preset}")
|
||||
return normalized
|
||||
preset_advanced = (
|
||||
preset_data.get("advanced")
|
||||
if isinstance(preset_data, dict)
|
||||
else None
|
||||
)
|
||||
if isinstance(preset_advanced, dict):
|
||||
preset_advanced = {
|
||||
key: value
|
||||
for key, value in preset_advanced.items()
|
||||
if key in PRESET_ADVANCED_FIELDS
|
||||
}
|
||||
normalized = self._normalize_site_advanced_config(
|
||||
preset_advanced,
|
||||
base_config=normalized,
|
||||
)
|
||||
|
||||
return normalized
|
||||
def set_site_advanced_config(self, domain: str, config: Dict[str, Any]) -> bool:
|
||||
"""设置站点级高级配置。"""
|
||||
with self._io_lock:
|
||||
self.refresh_if_changed()
|
||||
|
||||
site = self.sites.get(domain)
|
||||
if not site:
|
||||
logger.warning(f"站点不存在: {domain}")
|
||||
return False
|
||||
|
||||
previous_advanced = site.get("advanced", _MISSING)
|
||||
if previous_advanced is not _MISSING:
|
||||
previous_advanced = copy.deepcopy(previous_advanced)
|
||||
|
||||
existing = site.get("advanced") if isinstance(site.get("advanced"), dict) else {}
|
||||
stored = copy.deepcopy(existing)
|
||||
normalized = self._normalize_site_advanced_config(config or {}, base_config=existing)
|
||||
|
||||
provided_keys = {
|
||||
key
|
||||
for key in ((config or {}).keys() if isinstance(config, dict) else set())
|
||||
if key in ADVANCED_FIELDS
|
||||
}
|
||||
if not provided_keys and previous_advanced is _MISSING:
|
||||
return True
|
||||
for key in ADVANCED_FIELDS:
|
||||
if key in provided_keys:
|
||||
stored[key] = normalized[key]
|
||||
|
||||
site["advanced"] = stored
|
||||
if self._save_config_locked():
|
||||
return True
|
||||
if previous_advanced is _MISSING:
|
||||
site.pop("advanced", None)
|
||||
else:
|
||||
site["advanced"] = previous_advanced
|
||||
return False
|
||||
def set_preset_advanced_config(
|
||||
self,
|
||||
domain: str,
|
||||
config: Dict[str, Any],
|
||||
preset_name: str = None,
|
||||
*,
|
||||
prune_inherited_fields: Optional[Set[str]] = None,
|
||||
) -> bool:
|
||||
"""设置当前预设的高级配置;拒绝混入站点级字段。"""
|
||||
with self._io_lock:
|
||||
self.refresh_if_changed()
|
||||
|
||||
site = self.sites.get(domain)
|
||||
if not isinstance(site, dict):
|
||||
logger.warning(f"站点不存在: {domain}")
|
||||
return False
|
||||
|
||||
requested_preset = str(preset_name or "").strip()
|
||||
if not requested_preset:
|
||||
logger.warning(f"预设级高级配置缺少 preset_name: {domain}")
|
||||
return False
|
||||
|
||||
data = self._get_preset_data_exact(domain, requested_preset)
|
||||
if data is None:
|
||||
logger.warning(f"站点或预设不存在: {domain}/{requested_preset}")
|
||||
return False
|
||||
|
||||
raw_config = config if isinstance(config, dict) else {}
|
||||
|
||||
invalid_site_keys = {
|
||||
key
|
||||
for key in raw_config.keys()
|
||||
if key in SITE_ADVANCED_FIELDS
|
||||
}
|
||||
if invalid_site_keys:
|
||||
joined = ", ".join(sorted(invalid_site_keys))
|
||||
logger.warning(f"预设级高级配置不能包含站点级字段: {domain}/{requested_preset} ({joined})")
|
||||
return False
|
||||
|
||||
previous_advanced = data.get("advanced", _MISSING)
|
||||
if previous_advanced is not _MISSING:
|
||||
previous_advanced = copy.deepcopy(previous_advanced)
|
||||
|
||||
stored = copy.deepcopy(data.get("advanced") or {})
|
||||
if not isinstance(stored, dict):
|
||||
stored = {}
|
||||
|
||||
for key in list(SITE_ADVANCED_FIELDS):
|
||||
stored.pop(key, None)
|
||||
|
||||
site_advanced = site.get("advanced") if isinstance(site.get("advanced"), dict) else {}
|
||||
inherited = self._normalize_site_advanced_config(site_advanced)
|
||||
|
||||
normalized = self._normalize_site_advanced_config(
|
||||
raw_config,
|
||||
base_config=stored,
|
||||
)
|
||||
|
||||
provided_keys = {
|
||||
key
|
||||
for key in raw_config.keys()
|
||||
if key in PRESET_ADVANCED_FIELDS
|
||||
}
|
||||
if not provided_keys and previous_advanced is _MISSING:
|
||||
return True
|
||||
prune_inherited_keys = set(prune_inherited_fields or set())
|
||||
for key in PRESET_ADVANCED_FIELDS:
|
||||
if key in provided_keys:
|
||||
if (
|
||||
key in prune_inherited_keys
|
||||
and key not in stored
|
||||
and normalized[key] == inherited.get(key)
|
||||
):
|
||||
stored.pop(key, None)
|
||||
else:
|
||||
stored[key] = normalized[key]
|
||||
|
||||
if not stored and previous_advanced is _MISSING:
|
||||
return True
|
||||
|
||||
data["advanced"] = stored
|
||||
if self._save_config_locked():
|
||||
return True
|
||||
if previous_advanced is _MISSING:
|
||||
data.pop("advanced", None)
|
||||
else:
|
||||
data["advanced"] = previous_advanced
|
||||
return False
|
||||
def list_sites(self) -> Dict[str, Any]:
|
||||
"""获取所有站点配置(过滤内部键)"""
|
||||
self.refresh_if_changed()
|
||||
|
||||
with self._io_lock:
|
||||
return copy.deepcopy({
|
||||
domain: config
|
||||
for domain, config in self.sites.items()
|
||||
if not domain.startswith('_')
|
||||
})
|
||||
def get_site_config(self, domain: str, html_content: Optional[Union[str, Callable[[], str]]] = None,
|
||||
preset_name: str = None) -> Optional[SiteConfig]:
|
||||
"""
|
||||
获取站点配置(缓存 + AI 分析)
|
||||
|
||||
Args:
|
||||
domain: 站点域名
|
||||
html_content: 页面 HTML 或用于延迟获取 HTML 的可调用对象(用于 AI 分析未知站点)
|
||||
preset_name: 预设名称,None 则使用默认预设
|
||||
"""
|
||||
self.refresh_if_changed()
|
||||
|
||||
if domain in self.sites:
|
||||
cache_key = f"site_config:{domain}:{preset_name or ''}"
|
||||
cached = self._cache.get(cache_key, _MISSING)
|
||||
if cached is not _MISSING:
|
||||
return copy.deepcopy(cached)
|
||||
|
||||
site = self.sites.get(domain, {})
|
||||
config = self._get_site_data(domain, preset_name)
|
||||
|
||||
if config is None:
|
||||
logger.warning(f"站点 {domain} 无可用预设")
|
||||
return None
|
||||
|
||||
used_preset = preset_name or self._resolve_default_preset_name(site) or DEFAULT_PRESET_NAME
|
||||
previous_config = None
|
||||
|
||||
# 补充缺失字段
|
||||
changed = False
|
||||
if "workflow" not in config:
|
||||
if previous_config is None:
|
||||
previous_config = copy.deepcopy(config)
|
||||
config["workflow"] = DEFAULT_WORKFLOW
|
||||
changed = True
|
||||
|
||||
if "image_extraction" not in config:
|
||||
if previous_config is None:
|
||||
previous_config = copy.deepcopy(config)
|
||||
config["image_extraction"] = get_default_image_extraction_config()
|
||||
changed = True
|
||||
|
||||
if "file_paste" not in config:
|
||||
if previous_config is None:
|
||||
previous_config = copy.deepcopy(config)
|
||||
config["file_paste"] = get_default_file_paste_config()
|
||||
changed = True
|
||||
else:
|
||||
normalized_file_paste = self._validate_file_paste_config(
|
||||
config.get("file_paste", {}),
|
||||
legacy_stream_config=config.get("stream_config"),
|
||||
)
|
||||
if normalized_file_paste != config.get("file_paste"):
|
||||
if previous_config is None:
|
||||
previous_config = copy.deepcopy(config)
|
||||
config["file_paste"] = normalized_file_paste
|
||||
changed = True
|
||||
|
||||
if "prompt_padding" not in config:
|
||||
if previous_config is None:
|
||||
previous_config = copy.deepcopy(config)
|
||||
config["prompt_padding"] = get_default_prompt_padding_config()
|
||||
changed = True
|
||||
else:
|
||||
normalized_prompt_padding = self._validate_prompt_padding_config(
|
||||
config.get("prompt_padding", {})
|
||||
)
|
||||
if normalized_prompt_padding != config.get("prompt_padding"):
|
||||
if previous_config is None:
|
||||
previous_config = copy.deepcopy(config)
|
||||
config["prompt_padding"] = normalized_prompt_padding
|
||||
changed = True
|
||||
|
||||
if changed:
|
||||
completed_config = copy.deepcopy(config)
|
||||
if not self._save_config():
|
||||
config.clear()
|
||||
config.update(previous_config)
|
||||
logger.warning(
|
||||
f"站点 {domain} [{used_preset}] 配置自动补全保存失败,"
|
||||
"已仅对当前请求返回补全配置"
|
||||
)
|
||||
return completed_config
|
||||
|
||||
logger.debug(f"使用缓存配置: {domain} [预设: {used_preset}]")
|
||||
result = copy.deepcopy(config)
|
||||
self._cache.set(cache_key, result)
|
||||
return copy.deepcopy(result)
|
||||
|
||||
if not automatic_discovery_allowed(domain):
|
||||
logger.info(f"跳过非聊天站点的自动收录: {domain}(已有手动配置不受影响)")
|
||||
return None
|
||||
|
||||
resolved_html = ""
|
||||
if callable(html_content):
|
||||
try:
|
||||
resolved_html = str(html_content() or "")
|
||||
except Exception as e:
|
||||
logger.warning(f"获取页面 HTML 失败: {e}")
|
||||
resolved_html = ""
|
||||
elif html_content is not None:
|
||||
resolved_html = str(html_content or "")
|
||||
|
||||
if resolved_html.strip():
|
||||
logger.info(f"🔍 未知域名 {domain},启动 AI 识别...")
|
||||
clean_html = self.html_cleaner.clean(resolved_html)
|
||||
selectors = self.ai_analyzer.analyze(clean_html)
|
||||
|
||||
if specific_chat_selectors(selectors):
|
||||
selectors = self.validator.validate(selectors)
|
||||
if selectors_match_chat_html(selectors, clean_html):
|
||||
new_preset: SiteConfig = {
|
||||
"selectors": selectors,
|
||||
"workflow": DEFAULT_WORKFLOW,
|
||||
"stealth": self._guess_stealth(domain),
|
||||
"stream_config": copy.deepcopy(get_default_stream_config()),
|
||||
"image_extraction": get_default_image_extraction_config(),
|
||||
"file_paste": get_default_file_paste_config(),
|
||||
"prompt_padding": get_default_prompt_padding_config(),
|
||||
}
|
||||
|
||||
self.sites[domain] = {
|
||||
"default_preset": DEFAULT_PRESET_NAME,
|
||||
"presets": {
|
||||
DEFAULT_PRESET_NAME: new_preset
|
||||
}
|
||||
}
|
||||
if self._save_config():
|
||||
logger.info(f"✅ 配置已生成并保存: {domain}")
|
||||
else:
|
||||
logger.warning(f"⚠️ 配置已生成但保存失败,仅保留在当前运行内存: {domain}")
|
||||
return copy.deepcopy(new_preset)
|
||||
|
||||
logger.warning(f"⚠️ AI 分析失败,使用通用回退配置: {domain}")
|
||||
else:
|
||||
logger.warning(f"⚠️ 未知域名 {domain} 且未提供网页 HTML,使用通用回退配置")
|
||||
|
||||
fallback_selectors = self.global_config.get_fallback_selectors()
|
||||
|
||||
fallback_preset: SiteConfig = {
|
||||
"selectors": fallback_selectors,
|
||||
"workflow": DEFAULT_WORKFLOW,
|
||||
"stealth": False,
|
||||
"stream_config": copy.deepcopy(get_default_stream_config()),
|
||||
"image_extraction": get_default_image_extraction_config(),
|
||||
"file_paste": get_default_file_paste_config(),
|
||||
"prompt_padding": get_default_prompt_padding_config(),
|
||||
}
|
||||
|
||||
# A generic fallback is only for this caller. It is not proof that this
|
||||
# domain hosts an AI chat product, and must never enter the site catalog.
|
||||
logger.info(f"通用回退仅用于当前请求,未自动添加站点: {domain}")
|
||||
return copy.deepcopy(fallback_preset)
|
||||
def delete_site_config(self, domain: str) -> bool:
|
||||
"""删除指定站点配置"""
|
||||
self.refresh_if_changed()
|
||||
|
||||
if domain in self.sites:
|
||||
removed_site = copy.deepcopy(self.sites[domain])
|
||||
previous_global_default = copy.deepcopy(self._global_default_presets)
|
||||
previous_local_default = copy.deepcopy(self._local_default_presets)
|
||||
del self.sites[domain]
|
||||
if not self._save_config():
|
||||
self.sites[domain] = removed_site
|
||||
self._global_default_presets = previous_global_default
|
||||
self._local_default_presets = previous_local_default
|
||||
return False
|
||||
logger.info(f"已删除配置: {domain}")
|
||||
return True
|
||||
return False
|
||||
def get_selector_definitions(self) -> List[SelectorDefinition]:
|
||||
"""获取元素定义列表"""
|
||||
return self.global_config.get_selector_definitions()
|
||||
def set_selector_definitions(self, definitions: List[SelectorDefinition]) -> bool:
|
||||
"""设置元素定义列表并保存"""
|
||||
previous_definitions = self.global_config.get_selector_definitions()
|
||||
previous_fallback_selectors = copy.deepcopy(self.validator.fallback_selectors)
|
||||
self.global_config.set_selector_definitions(definitions)
|
||||
|
||||
# 更新验证器的回退选择器
|
||||
self.validator.fallback_selectors = self.global_config.get_fallback_selectors()
|
||||
|
||||
# 保存配置
|
||||
if not self._save_config():
|
||||
self.global_config.set_selector_definitions(previous_definitions)
|
||||
self.validator.fallback_selectors = previous_fallback_selectors
|
||||
return False
|
||||
|
||||
logger.info(f"元素定义已更新: {len(definitions)} 个")
|
||||
return True
|
||||
@@ -0,0 +1,485 @@
|
||||
"""ConfigEngine 的一部分(R2-3 拆分):提取器、流式/网络监听配置及其校验、解析器列表。
|
||||
|
||||
方法原样从 engine.py 迁出,行为不变;通过 mixin 组合回 ConfigEngine,
|
||||
实例状态都在 ConfigEngine.__init__(engine.py)里初始化。
|
||||
"""
|
||||
|
||||
import copy
|
||||
from typing import Dict, Optional, List, Any
|
||||
from app.core.parsers import ParserRegistry
|
||||
from app.models.schemas import get_default_attachment_monitor_config, get_default_send_confirmation_config
|
||||
from app.services.extractor_manager import extractor_manager
|
||||
from app.services.parser_manager import parser_manager
|
||||
from app.core.request_transport import normalize_request_transport_config
|
||||
from .engine_common import (
|
||||
DEFAULT_PRESET_NAME,
|
||||
_MISSING,
|
||||
_coerce_bool,
|
||||
_merge_config_patch,
|
||||
get_default_network_config,
|
||||
get_default_stream_config,
|
||||
logger,
|
||||
)
|
||||
|
||||
|
||||
class ConfigStreamMixin:
|
||||
"""提取器、流式/网络监听配置及其校验、解析器列表"""
|
||||
|
||||
def get_site_extractor(self, domain: str, preset_name: str = None):
|
||||
"""获取站点的提取器实例"""
|
||||
self.refresh_if_changed()
|
||||
|
||||
data = self._get_site_data(domain, preset_name)
|
||||
if data is not None:
|
||||
return extractor_manager.get_extractor_for_site(data)
|
||||
|
||||
return extractor_manager.get_extractor()
|
||||
def set_site_extractor(self, domain: str, extractor_id: str,
|
||||
preset_name: str = None) -> bool:
|
||||
"""为站点设置提取器"""
|
||||
self.refresh_if_changed()
|
||||
|
||||
data = self._get_site_data(domain, preset_name)
|
||||
if data is None:
|
||||
logger.warning(f"站点或预设不存在: {domain}/{preset_name}")
|
||||
return False
|
||||
|
||||
from app.core.extractors import ExtractorRegistry
|
||||
if not ExtractorRegistry.exists(extractor_id):
|
||||
logger.error(f"提取器不存在: {extractor_id}")
|
||||
return False
|
||||
|
||||
previous_extractor_id = data.get("extractor_id", _MISSING)
|
||||
if previous_extractor_id is not _MISSING:
|
||||
previous_extractor_id = copy.deepcopy(previous_extractor_id)
|
||||
previous_verified = data.get("extractor_verified", _MISSING)
|
||||
if previous_verified is not _MISSING:
|
||||
previous_verified = copy.deepcopy(previous_verified)
|
||||
data["extractor_id"] = extractor_id
|
||||
data["extractor_verified"] = False
|
||||
if not self._save_config():
|
||||
if previous_extractor_id is _MISSING:
|
||||
data.pop("extractor_id", None)
|
||||
else:
|
||||
data["extractor_id"] = previous_extractor_id
|
||||
if previous_verified is _MISSING:
|
||||
data.pop("extractor_verified", None)
|
||||
else:
|
||||
data["extractor_verified"] = previous_verified
|
||||
return False
|
||||
|
||||
logger.info(f"站点 {domain} [{preset_name or DEFAULT_PRESET_NAME}] 已绑定提取器: {extractor_id}")
|
||||
return True
|
||||
def set_site_extractor_verified(self, domain: str, verified: bool = True,
|
||||
preset_name: str = None) -> bool:
|
||||
"""设置站点提取器验证状态"""
|
||||
self.refresh_if_changed()
|
||||
|
||||
data = self._get_site_data(domain, preset_name)
|
||||
if data is None:
|
||||
return False
|
||||
|
||||
if not self._assign_preset_field_and_save(data, "extractor_verified", verified):
|
||||
return False
|
||||
|
||||
return True
|
||||
def get_site_stream_config(self, domain: str, preset_name: str = None) -> Dict[str, Any]:
|
||||
"""
|
||||
获取站点的流式配置
|
||||
|
||||
Args:
|
||||
domain: 站点域名
|
||||
preset_name: 预设名称
|
||||
|
||||
Returns:
|
||||
完整的流式配置(包含默认值)
|
||||
"""
|
||||
self.refresh_if_changed()
|
||||
|
||||
cache_key = f"stream_config:{domain}:{preset_name or ''}"
|
||||
cached = self._cache.get(cache_key, _MISSING)
|
||||
if cached is not _MISSING:
|
||||
return copy.deepcopy(cached)
|
||||
|
||||
default_config = get_default_stream_config()
|
||||
|
||||
data = self._get_site_data(domain, preset_name)
|
||||
if data is None:
|
||||
result = copy.deepcopy(default_config)
|
||||
self._cache.set(cache_key, result)
|
||||
return copy.deepcopy(result)
|
||||
|
||||
stream_config = data.get("stream_config", {})
|
||||
|
||||
# 合并默认值
|
||||
result = copy.deepcopy(default_config)
|
||||
|
||||
# 更新顶层字段
|
||||
for key in ["mode", "hard_timeout"]:
|
||||
if key in stream_config:
|
||||
result[key] = stream_config[key]
|
||||
|
||||
if isinstance(stream_config.get("request_transport"), dict):
|
||||
result["request_transport"] = normalize_request_transport_config(
|
||||
stream_config.get("request_transport")
|
||||
)
|
||||
|
||||
# 处理 send_confirmation 配置
|
||||
if isinstance(stream_config.get("send_confirmation"), dict):
|
||||
result["send_confirmation"].update(stream_config["send_confirmation"])
|
||||
|
||||
# 处理 attachment_monitor 配置
|
||||
if isinstance(stream_config.get("attachment_monitor"), dict):
|
||||
result["attachment_monitor"].update(stream_config["attachment_monitor"])
|
||||
|
||||
file_paste_config = data.get("file_paste", {})
|
||||
if isinstance(file_paste_config, dict):
|
||||
if isinstance(file_paste_config.get("send_confirmation"), dict):
|
||||
result["send_confirmation"].update(file_paste_config["send_confirmation"])
|
||||
if isinstance(file_paste_config.get("attachment_monitor"), dict):
|
||||
result["attachment_monitor"].update(file_paste_config["attachment_monitor"])
|
||||
|
||||
# 处理 network 配置
|
||||
if stream_config.get("network"):
|
||||
network_default = get_default_network_config()
|
||||
network_config = stream_config["network"]
|
||||
|
||||
result["network"] = network_default.copy()
|
||||
for key in [
|
||||
"listen_pattern",
|
||||
"stream_match_mode",
|
||||
"stream_match_pattern",
|
||||
"parser",
|
||||
"hard_timeout",
|
||||
"first_response_timeout",
|
||||
"first_content_timeout",
|
||||
"initial_target_body_wait",
|
||||
"silence_threshold",
|
||||
"response_interval",
|
||||
]:
|
||||
if key in network_config:
|
||||
result["network"][key] = network_config[key]
|
||||
|
||||
self._cache.set(cache_key, copy.deepcopy(result))
|
||||
return result
|
||||
def set_site_stream_config(self, domain: str, config: Dict[str, Any],
|
||||
preset_name: str = None) -> bool:
|
||||
"""
|
||||
设置站点的流式配置
|
||||
|
||||
Args:
|
||||
domain: 站点域名
|
||||
config: 流式配置(部分或完整)
|
||||
preset_name: 预设名称
|
||||
|
||||
Returns:
|
||||
是否成功
|
||||
"""
|
||||
self.refresh_if_changed()
|
||||
|
||||
data = self._get_site_data(domain, preset_name)
|
||||
if data is None:
|
||||
logger.warning(f"站点或预设不存在: {domain}/{preset_name}")
|
||||
return False
|
||||
|
||||
previous_file_paste = data.get("file_paste", _MISSING)
|
||||
if previous_file_paste is not _MISSING:
|
||||
previous_file_paste = copy.deepcopy(previous_file_paste)
|
||||
previous_stream = data.get("stream_config", _MISSING)
|
||||
if previous_stream is not _MISSING:
|
||||
previous_stream = copy.deepcopy(previous_stream)
|
||||
|
||||
legacy_file_paste_updates: Dict[str, Any] = {}
|
||||
if isinstance(config.get("send_confirmation"), dict):
|
||||
legacy_file_paste_updates["send_confirmation"] = config.get("send_confirmation") or {}
|
||||
if isinstance(config.get("attachment_monitor"), dict):
|
||||
legacy_file_paste_updates["attachment_monitor"] = config.get("attachment_monitor") or {}
|
||||
|
||||
# 验证并规范化配置
|
||||
current_config = self.get_site_stream_config(domain, preset_name)
|
||||
merged_config = _merge_config_patch(current_config, config if isinstance(config, dict) else {})
|
||||
validated = self._validate_stream_config(merged_config)
|
||||
validated.pop("send_confirmation", None)
|
||||
validated.pop("attachment_monitor", None)
|
||||
|
||||
if legacy_file_paste_updates:
|
||||
existing_file_paste = self._validate_file_paste_config(
|
||||
data.get("file_paste", {}),
|
||||
legacy_stream_config=data.get("stream_config"),
|
||||
)
|
||||
merged_file_paste = _merge_config_patch(existing_file_paste, legacy_file_paste_updates)
|
||||
data["file_paste"] = self._validate_file_paste_config(
|
||||
merged_file_paste,
|
||||
legacy_stream_config=data.get("stream_config"),
|
||||
)
|
||||
|
||||
data["stream_config"] = validated
|
||||
if not self._save_config():
|
||||
if previous_stream is _MISSING:
|
||||
data.pop("stream_config", None)
|
||||
else:
|
||||
data["stream_config"] = previous_stream
|
||||
if previous_file_paste is _MISSING:
|
||||
data.pop("file_paste", None)
|
||||
else:
|
||||
data["file_paste"] = previous_file_paste
|
||||
return False
|
||||
|
||||
logger.info(f"站点 {domain} [{preset_name or DEFAULT_PRESET_NAME}] 流式配置已更新 (mode={validated.get('mode')})")
|
||||
return True
|
||||
def _validate_stream_config(self, config: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"""
|
||||
验证并规范化流式配置
|
||||
|
||||
Args:
|
||||
config: 原始配置
|
||||
|
||||
Returns:
|
||||
规范化后的配置
|
||||
"""
|
||||
result = get_default_stream_config()
|
||||
|
||||
if not config:
|
||||
return result
|
||||
|
||||
mode_explicitly_set = False
|
||||
|
||||
# 验证 mode
|
||||
if "mode" in config:
|
||||
mode = str(config["mode"]).lower()
|
||||
if mode in ("dom", "network"):
|
||||
result["mode"] = mode
|
||||
mode_explicitly_set = True
|
||||
|
||||
# 验证数值字段
|
||||
for key in ["hard_timeout"]:
|
||||
if key in config:
|
||||
try:
|
||||
val = float(config[key])
|
||||
if key == "hard_timeout":
|
||||
result[key] = max(10, min(val, 600))
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
|
||||
# 验证 send_confirmation 配置
|
||||
if isinstance(config.get("send_confirmation"), dict):
|
||||
result["send_confirmation"] = self._validate_send_confirmation_config(
|
||||
config["send_confirmation"]
|
||||
)
|
||||
|
||||
if isinstance(config.get("request_transport"), dict):
|
||||
result["request_transport"] = normalize_request_transport_config(
|
||||
config["request_transport"]
|
||||
)
|
||||
|
||||
# 验证 attachment_monitor 配置
|
||||
if isinstance(config.get("attachment_monitor"), dict):
|
||||
result["attachment_monitor"] = self._validate_attachment_monitor_config(
|
||||
config["attachment_monitor"]
|
||||
)
|
||||
|
||||
# 验证 network 配置
|
||||
if config.get("network"):
|
||||
network_config = self._validate_network_config(config["network"])
|
||||
if network_config:
|
||||
result["network"] = network_config
|
||||
# 仅在调用方没有显式指定 mode 时,才根据 network 配置自动切到 network。
|
||||
# 这样切回 DOM 时可以保留 parser/listen_pattern,而不会被后端强制改回 network。
|
||||
if (
|
||||
not mode_explicitly_set
|
||||
and network_config.get("parser")
|
||||
and network_config.get("listen_pattern")
|
||||
):
|
||||
result["mode"] = "network"
|
||||
|
||||
return result
|
||||
def _validate_send_confirmation_config(self, config: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"""验证发送成功判定配置。"""
|
||||
result = get_default_send_confirmation_config()
|
||||
|
||||
if not isinstance(config, dict):
|
||||
return result
|
||||
|
||||
numeric_ranges = {
|
||||
"post_click_observe_window": (0.0, 15.0),
|
||||
"pre_retry_probe_window": (0.0, 5.0),
|
||||
"retry_observe_window": (0.0, 15.0),
|
||||
"attachment_observe_window": (0.0, 30.0),
|
||||
"retry_interval": (0.0, 30.0),
|
||||
"retry_cooldown_window": (0.0, 30.0),
|
||||
}
|
||||
for key, (minimum, maximum) in numeric_ranges.items():
|
||||
if key not in config:
|
||||
continue
|
||||
try:
|
||||
value = float(config[key])
|
||||
except (TypeError, ValueError):
|
||||
continue
|
||||
result[key] = max(minimum, min(value, maximum))
|
||||
|
||||
if "max_retry_count" in config:
|
||||
try:
|
||||
value = int(config["max_retry_count"])
|
||||
except (TypeError, ValueError):
|
||||
value = None
|
||||
if value is not None:
|
||||
result["max_retry_count"] = max(0, min(value, 10))
|
||||
|
||||
retry_action = str(config.get("retry_action") or "").strip().lower()
|
||||
if retry_action in {"click_send_btn", "key_press"}:
|
||||
result["retry_action"] = retry_action
|
||||
|
||||
if "retry_key_combo" in config:
|
||||
retry_key_combo = str(config.get("retry_key_combo") or "").strip()
|
||||
if retry_key_combo:
|
||||
result["retry_key_combo"] = retry_key_combo[:64]
|
||||
|
||||
bool_fields = [
|
||||
"retry_on_unconfirmed_send",
|
||||
"accept_attachment_change",
|
||||
"accept_attachment_disappear",
|
||||
"accept_probe_confirmation",
|
||||
"retry_block_on_stop_button",
|
||||
"retry_block_if_generating",
|
||||
"trust_network_activity",
|
||||
"trust_generating_indicator",
|
||||
"trust_send_disabled_with_input_shrink",
|
||||
]
|
||||
for key in bool_fields:
|
||||
if key not in config:
|
||||
continue
|
||||
result[key] = _coerce_bool(config.get(key), bool(result.get(key, False)))
|
||||
|
||||
sensitivity = str(config.get("attachment_sensitivity") or "").strip().lower()
|
||||
if sensitivity in {"low", "medium", "high"}:
|
||||
result["attachment_sensitivity"] = sensitivity
|
||||
|
||||
return result
|
||||
def _validate_attachment_monitor_config(self, config: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"""验证附件上传/发送前判定规则。"""
|
||||
result = get_default_attachment_monitor_config()
|
||||
|
||||
if not isinstance(config, dict):
|
||||
return result
|
||||
|
||||
list_fields = [
|
||||
"root_selectors",
|
||||
"attachment_selectors",
|
||||
"pending_selectors",
|
||||
"status_selectors",
|
||||
"content_exclusion_selectors",
|
||||
"busy_text_markers",
|
||||
"ignored_busy_text_markers",
|
||||
"send_button_disabled_markers",
|
||||
]
|
||||
for key in list_fields:
|
||||
raw_value = config.get(key)
|
||||
if raw_value is None:
|
||||
continue
|
||||
if not isinstance(raw_value, list):
|
||||
continue
|
||||
cleaned = []
|
||||
for item in raw_value:
|
||||
value = str(item or "").strip()
|
||||
if value and value not in cleaned:
|
||||
cleaned.append(value)
|
||||
result[key] = cleaned
|
||||
|
||||
numeric_ranges = {
|
||||
"idle_timeout": (0.5, 60.0),
|
||||
"hard_max_wait": (1.0, 300.0),
|
||||
}
|
||||
for key, (minimum, maximum) in numeric_ranges.items():
|
||||
if key not in config:
|
||||
continue
|
||||
try:
|
||||
value = float(config[key])
|
||||
except (TypeError, ValueError):
|
||||
continue
|
||||
result[key] = max(minimum, min(value, maximum))
|
||||
|
||||
bool_fields = [
|
||||
"require_attachment_present",
|
||||
"require_upload_signal_before_ready",
|
||||
"continue_once_on_unconfirmed_send",
|
||||
]
|
||||
for key in bool_fields:
|
||||
if key not in config:
|
||||
continue
|
||||
result[key] = _coerce_bool(config.get(key), bool(result.get(key, False)))
|
||||
|
||||
return result
|
||||
def _validate_network_config(self, config: Dict[str, Any]) -> Optional[Dict[str, Any]]:
|
||||
"""
|
||||
验证网络监听配置
|
||||
|
||||
Args:
|
||||
config: 原始网络配置
|
||||
|
||||
Returns:
|
||||
规范化后的配置,无效则返回 None
|
||||
"""
|
||||
if not config:
|
||||
return None
|
||||
|
||||
result = get_default_network_config()
|
||||
|
||||
# listen_pattern(必填)
|
||||
if "listen_pattern" in config:
|
||||
pattern = str(config["listen_pattern"]).strip()
|
||||
if pattern:
|
||||
result["listen_pattern"] = pattern
|
||||
|
||||
if "stream_match_mode" in config:
|
||||
match_mode = str(config["stream_match_mode"]).strip().lower()
|
||||
if match_mode in {"keyword", "regex"}:
|
||||
result["stream_match_mode"] = match_mode
|
||||
|
||||
if "stream_match_pattern" in config:
|
||||
result["stream_match_pattern"] = str(config["stream_match_pattern"]).strip()
|
||||
|
||||
# parser(必填,需验证存在性)
|
||||
if "parser" in config:
|
||||
parser_id = str(config["parser"]).strip()
|
||||
if parser_id:
|
||||
# 验证解析器是否存在
|
||||
if ParserRegistry.exists(parser_id):
|
||||
result["parser"] = parser_id
|
||||
else:
|
||||
logger.warning(f"解析器不存在: {parser_id}")
|
||||
# 仍然保存,允许后续添加解析器
|
||||
result["parser"] = parser_id
|
||||
|
||||
# 验证数值字段
|
||||
for key in ["first_response_timeout", "silence_threshold", "response_interval"]:
|
||||
if key in config:
|
||||
try:
|
||||
val = float(config[key])
|
||||
if key == "first_response_timeout":
|
||||
result[key] = max(0.5, min(val, 300))
|
||||
elif key == "silence_threshold":
|
||||
result[key] = max(0.5, min(val, 30))
|
||||
elif key == "response_interval":
|
||||
result[key] = max(0.1, min(val, 5))
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
|
||||
# 检查是否有有效配置
|
||||
if not result["listen_pattern"] or not result["parser"]:
|
||||
return None
|
||||
|
||||
return result
|
||||
def list_available_parsers(self) -> List[Dict[str, str]]:
|
||||
"""
|
||||
列出所有可用的响应解析器
|
||||
|
||||
Returns:
|
||||
解析器信息列表
|
||||
"""
|
||||
return parser_manager.list_parsers()
|
||||
def get_extractor_manager(self):
|
||||
"""获取提取器管理器实例"""
|
||||
return extractor_manager
|
||||
def get_parser_manager(self):
|
||||
"""获取解析器管理器实例"""
|
||||
return parser_manager
|
||||
@@ -0,0 +1,440 @@
|
||||
"""
|
||||
app/services/config/managers.py - 配置子管理器
|
||||
|
||||
职责:
|
||||
- 全局配置管理(元素定义)
|
||||
- 图片预设管理
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import copy
|
||||
from app.core.config import get_logger
|
||||
from typing import Dict, List, Any, Optional
|
||||
|
||||
from app.models.schemas import (
|
||||
SelectorDefinition,
|
||||
get_default_selector_definitions,
|
||||
get_default_image_extraction_config
|
||||
)
|
||||
|
||||
|
||||
logger = get_logger("CFG_MGR")
|
||||
|
||||
|
||||
# ================= 全局配置管理器 =================
|
||||
|
||||
class GlobalConfigManager:
|
||||
"""
|
||||
全局配置管理器
|
||||
|
||||
管理 _global 节点中的配置,包括:
|
||||
- selector_definitions: 元素定义列表
|
||||
- 以及任意扩展字段(通过 get/set 访问)
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
self._selector_definitions: List[SelectorDefinition] = get_default_selector_definitions()
|
||||
self._extra: Dict[str, Any] = {} # 通用扩展存储
|
||||
|
||||
def load(self, global_section: Dict[str, Any]):
|
||||
"""从 _global 节点加载配置"""
|
||||
self._extra = {}
|
||||
self._selector_definitions = get_default_selector_definitions()
|
||||
if not global_section:
|
||||
return
|
||||
|
||||
if "selector_definitions" in global_section:
|
||||
defs = global_section["selector_definitions"]
|
||||
if isinstance(defs, list):
|
||||
self._selector_definitions = self._merge_selector_definitions(defs)
|
||||
logger.debug(f"已加载 {len(defs)} 个元素定义")
|
||||
|
||||
# 加载所有非 selector_definitions 的字段到通用存储
|
||||
for key, value in global_section.items():
|
||||
if key != "selector_definitions":
|
||||
self._extra[key] = copy.deepcopy(value)
|
||||
|
||||
if "commands" in self._extra:
|
||||
cmd_count = len(self._extra["commands"]) if isinstance(self._extra["commands"], list) else 0
|
||||
logger.debug(f"已加载 {cmd_count} 个自动化命令")
|
||||
|
||||
def get_selector_definitions(self) -> List[SelectorDefinition]:
|
||||
"""获取元素定义列表"""
|
||||
return copy.deepcopy(self._selector_definitions)
|
||||
|
||||
def set_selector_definitions(self, definitions: List[SelectorDefinition]):
|
||||
"""设置元素定义列表"""
|
||||
self._selector_definitions = self._merge_selector_definitions(definitions)
|
||||
|
||||
@staticmethod
|
||||
def _normalize_selector_definition(raw_item: Any) -> Optional[SelectorDefinition]:
|
||||
if not isinstance(raw_item, dict):
|
||||
return None
|
||||
key = raw_item.get("key")
|
||||
if not isinstance(key, str):
|
||||
return None
|
||||
key = key.strip()
|
||||
if not key:
|
||||
return None
|
||||
|
||||
description = raw_item.get("description", "")
|
||||
if not isinstance(description, str):
|
||||
description = str(description) if description is not None else ""
|
||||
|
||||
return {
|
||||
"key": key,
|
||||
"description": description,
|
||||
"enabled": bool(raw_item.get("enabled", True)),
|
||||
"required": bool(raw_item.get("required", False)),
|
||||
}
|
||||
|
||||
def _merge_selector_definitions(self, definitions: List[SelectorDefinition]) -> List[SelectorDefinition]:
|
||||
"""合并内置默认字段,确保升级后新增的选择器能自动出现。"""
|
||||
merged = []
|
||||
existing_keys = set()
|
||||
|
||||
for item in definitions or []:
|
||||
normalized = self._normalize_selector_definition(item)
|
||||
if not normalized:
|
||||
continue
|
||||
key = normalized["key"]
|
||||
if key in existing_keys:
|
||||
continue
|
||||
merged.append(normalized)
|
||||
existing_keys.add(key)
|
||||
|
||||
for default_def in get_default_selector_definitions():
|
||||
normalized_default = self._normalize_selector_definition(default_def)
|
||||
if not normalized_default:
|
||||
continue
|
||||
key = normalized_default.get("key")
|
||||
if key not in existing_keys:
|
||||
merged.append(normalized_default)
|
||||
existing_keys.add(key)
|
||||
|
||||
return merged
|
||||
|
||||
def get_enabled_definitions(self) -> List[SelectorDefinition]:
|
||||
"""获取启用的元素定义"""
|
||||
return [
|
||||
copy.deepcopy(d)
|
||||
for d in self._selector_definitions
|
||||
if isinstance(d, dict) and d.get("enabled", True)
|
||||
]
|
||||
|
||||
def get_fallback_selectors(self) -> Dict[str, Optional[str]]:
|
||||
"""
|
||||
生成回退选择器字典
|
||||
|
||||
基于元素定义生成,用于 SelectorValidator
|
||||
"""
|
||||
fallback_map = {
|
||||
"input_box": "textarea",
|
||||
"send_btn": 'button[type="submit"]',
|
||||
"result_container": "div",
|
||||
"new_chat_btn": None,
|
||||
"retry_send_btn": None,
|
||||
"message_wrapper": None,
|
||||
"generating_indicator": None,
|
||||
"upload_btn": None,
|
||||
"file_input": 'input[type="file"]',
|
||||
"drop_zone": None,
|
||||
}
|
||||
|
||||
result = {}
|
||||
for d in self._selector_definitions:
|
||||
if not isinstance(d, dict):
|
||||
continue
|
||||
key = d.get("key")
|
||||
if not isinstance(key, str) or not key:
|
||||
continue
|
||||
result[key] = fallback_map.get(key, None)
|
||||
|
||||
return result
|
||||
|
||||
def build_prompt_selector_list(self) -> str:
|
||||
"""
|
||||
生成 AI 提示词中的元素查找列表
|
||||
|
||||
只包含 enabled=True 的元素
|
||||
"""
|
||||
lines = []
|
||||
for d in self._selector_definitions:
|
||||
if not isinstance(d, dict):
|
||||
continue
|
||||
if not d.get("enabled", True):
|
||||
continue
|
||||
|
||||
key = str(d.get("key") or "").strip()
|
||||
if not key:
|
||||
continue
|
||||
desc = str(d.get("description") or "")
|
||||
required = d.get("required", False)
|
||||
|
||||
if required:
|
||||
tag = "[REQUIRED]"
|
||||
else:
|
||||
tag = "[OPTIONAL, return null if not found]"
|
||||
|
||||
lines.append(f"- `{key}`: {desc} {tag}")
|
||||
|
||||
return "\n".join(lines)
|
||||
|
||||
def build_prompt_json_keys(self) -> str:
|
||||
"""
|
||||
生成 AI 提示词中的 JSON 输出格式说明
|
||||
"""
|
||||
lines = []
|
||||
for d in self._selector_definitions:
|
||||
if not isinstance(d, dict):
|
||||
continue
|
||||
if not d.get("enabled", True):
|
||||
continue
|
||||
|
||||
key = str(d.get("key") or "").strip()
|
||||
if not key:
|
||||
continue
|
||||
desc = str(d.get("description") or "")
|
||||
lines.append(f'- `{key}`: {desc}')
|
||||
|
||||
return "\n".join(lines)
|
||||
|
||||
def get(self, key: str, default: Any = None) -> Any:
|
||||
"""获取通用配置值"""
|
||||
return copy.deepcopy(self._extra.get(key, default))
|
||||
|
||||
def set(self, key: str, value: Any):
|
||||
"""设置通用配置值"""
|
||||
self._extra[key] = copy.deepcopy(value)
|
||||
|
||||
def remove(self, key: str) -> bool:
|
||||
"""删除通用配置值"""
|
||||
if key not in self._extra:
|
||||
return False
|
||||
del self._extra[key]
|
||||
return True
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
"""导出为字典(用于保存)"""
|
||||
result = {
|
||||
"selector_definitions": self._selector_definitions
|
||||
}
|
||||
# 合并通用存储中的所有字段
|
||||
result.update(self._extra)
|
||||
return copy.deepcopy(result)
|
||||
|
||||
|
||||
# ================= 图片预设管理器 =================
|
||||
|
||||
class ImagePresetsManager:
|
||||
"""
|
||||
图片提取预设管理器
|
||||
|
||||
职责:
|
||||
- 加载预设配置文件
|
||||
- 提供站点预设查询
|
||||
- 应用预设到站点配置
|
||||
"""
|
||||
|
||||
def __init__(self, presets_file: str):
|
||||
self.presets_file = presets_file
|
||||
self.presets: Dict[str, Any] = {}
|
||||
self._load_presets()
|
||||
|
||||
@staticmethod
|
||||
def _default_presets_payload() -> Dict[str, Any]:
|
||||
return {
|
||||
"_default": {
|
||||
"name": "Default",
|
||||
"description": "Built-in fallback image extraction preset",
|
||||
"image_extraction": get_default_image_extraction_config(),
|
||||
}
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _get_preset_image_config(preset_data: Any) -> Optional[Dict[str, Any]]:
|
||||
if not isinstance(preset_data, dict):
|
||||
return None
|
||||
config = preset_data.get("image_extraction")
|
||||
return copy.deepcopy(config) if isinstance(config, dict) else None
|
||||
|
||||
@staticmethod
|
||||
def _get_preset_text(preset_data: Dict[str, Any], key: str, default: str = "") -> str:
|
||||
value = preset_data.get(key, default)
|
||||
if value is None:
|
||||
return default
|
||||
if isinstance(value, (str, int, float, bool)):
|
||||
return str(value)
|
||||
return default
|
||||
|
||||
def _find_matching_preset(self, domain: str) -> Optional[tuple[str, Dict[str, Any]]]:
|
||||
domain = str(domain or "").strip().lower().rstrip(".")
|
||||
if not domain:
|
||||
return None
|
||||
|
||||
normalized_presets = []
|
||||
for preset_domain, preset_data in self.presets.items():
|
||||
if preset_domain.startswith("_") or not isinstance(preset_data, dict):
|
||||
continue
|
||||
normalized_preset_domain = str(preset_domain).strip().lower().rstrip(".")
|
||||
if not normalized_preset_domain:
|
||||
continue
|
||||
normalized_presets.append((preset_domain, preset_data, normalized_preset_domain))
|
||||
|
||||
for preset_domain, preset_data, normalized_preset_domain in normalized_presets:
|
||||
if domain == normalized_preset_domain:
|
||||
return preset_domain, preset_data
|
||||
|
||||
suffix_matches = [
|
||||
item
|
||||
for item in normalized_presets
|
||||
if domain.endswith(f".{item[2]}")
|
||||
]
|
||||
if suffix_matches:
|
||||
preset_domain, preset_data, _ = max(
|
||||
suffix_matches,
|
||||
key=lambda item: len(item[2]),
|
||||
)
|
||||
logger.debug(f"使用模糊匹配预设: {domain} -> {preset_domain}")
|
||||
return preset_domain, preset_data
|
||||
|
||||
return None
|
||||
|
||||
def _use_default_presets_or_keep_existing(self, keep_existing: bool) -> None:
|
||||
if keep_existing and self.presets:
|
||||
logger.warning("图片预设加载失败,保留当前内存中的预设")
|
||||
return
|
||||
self.presets = self._default_presets_payload()
|
||||
|
||||
def _load_presets(self, keep_existing_on_error: bool = False):
|
||||
"""加载预设配置文件"""
|
||||
if not os.path.exists(self.presets_file):
|
||||
logger.warning(f"图片预设文件不存在: {self.presets_file}")
|
||||
self._use_default_presets_or_keep_existing(keep_existing_on_error)
|
||||
return
|
||||
|
||||
try:
|
||||
with open(self.presets_file, 'r', encoding='utf-8') as f:
|
||||
data = json.load(f)
|
||||
|
||||
if not isinstance(data, dict):
|
||||
logger.error("预设文件格式错误: 顶层必须是对象")
|
||||
self._use_default_presets_or_keep_existing(keep_existing_on_error)
|
||||
return
|
||||
|
||||
# 移除元数据
|
||||
loaded_presets = {
|
||||
str(k): v
|
||||
for k, v in data.items()
|
||||
if k != "_meta" and isinstance(v, dict)
|
||||
}
|
||||
skipped_count = len([k for k, v in data.items() if k != "_meta" and not isinstance(v, dict)])
|
||||
if skipped_count:
|
||||
logger.warning(f"已跳过 {skipped_count} 个格式无效的图片预设")
|
||||
if not loaded_presets:
|
||||
self._use_default_presets_or_keep_existing(keep_existing_on_error)
|
||||
logger.debug(f"已加载 {len(self.presets)} 个图片预设")
|
||||
return
|
||||
|
||||
self.presets = loaded_presets
|
||||
|
||||
logger.debug(f"已加载 {len(self.presets)} 个图片预设")
|
||||
|
||||
except json.JSONDecodeError as e:
|
||||
logger.error(f"预设文件格式错误: {e}")
|
||||
self._use_default_presets_or_keep_existing(keep_existing_on_error)
|
||||
except Exception as e:
|
||||
logger.error(f"加载预设失败: {e}")
|
||||
self._use_default_presets_or_keep_existing(keep_existing_on_error)
|
||||
|
||||
def get_preset(self, domain: str) -> Optional[Dict]:
|
||||
"""
|
||||
获取站点的预设配置
|
||||
|
||||
Args:
|
||||
domain: 站点域名
|
||||
|
||||
Returns:
|
||||
预设配置字典,不存在返回 None
|
||||
"""
|
||||
match = self._find_matching_preset(domain)
|
||||
if match is None:
|
||||
return None
|
||||
return self._get_preset_image_config(match[1])
|
||||
|
||||
def list_presets(self) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
列出所有可用预设
|
||||
|
||||
Returns:
|
||||
预设列表,每项包含 domain、name、description、enabled
|
||||
"""
|
||||
result = []
|
||||
|
||||
for domain, data in self.presets.items():
|
||||
if domain == "_meta":
|
||||
continue
|
||||
if not isinstance(data, dict):
|
||||
continue
|
||||
|
||||
config = self._get_preset_image_config(data) or {}
|
||||
|
||||
item = {
|
||||
"domain": domain,
|
||||
"name": self._get_preset_text(data, "name", domain),
|
||||
"description": self._get_preset_text(data, "description", ""),
|
||||
"enabled": config.get("enabled", False),
|
||||
"notes": self._get_preset_text(data, "notes", ""),
|
||||
"config": config
|
||||
}
|
||||
|
||||
result.append(item)
|
||||
|
||||
result.sort(key=lambda x: x["domain"])
|
||||
|
||||
return result
|
||||
|
||||
def get_preset_for_display(self, domain: str) -> Dict[str, Any]:
|
||||
"""
|
||||
获取用于显示的预设信息
|
||||
|
||||
Args:
|
||||
domain: 站点域名
|
||||
|
||||
Returns:
|
||||
包含 available、preset_domain、config 的字典
|
||||
"""
|
||||
match = self._find_matching_preset(domain)
|
||||
if match is None:
|
||||
preset_config = None
|
||||
else:
|
||||
preset_config = self._get_preset_image_config(match[1])
|
||||
|
||||
if preset_config:
|
||||
matched_domain, matched_data = match
|
||||
|
||||
return {
|
||||
"available": True,
|
||||
"preset_domain": matched_domain,
|
||||
"name": self._get_preset_text(
|
||||
matched_data,
|
||||
"name",
|
||||
"",
|
||||
),
|
||||
"config": preset_config
|
||||
}
|
||||
|
||||
return {
|
||||
"available": False,
|
||||
"preset_domain": None,
|
||||
"name": None,
|
||||
"config": None
|
||||
}
|
||||
|
||||
def reload(self):
|
||||
"""重新加载预设文件"""
|
||||
self._load_presets(keep_existing_on_error=True)
|
||||
|
||||
|
||||
__all__ = ['GlobalConfigManager', 'ImagePresetsManager']
|
||||
@@ -0,0 +1,661 @@
|
||||
"""
|
||||
app/services/config/processors.py - 配置处理器
|
||||
|
||||
职责:
|
||||
- HTML 清理
|
||||
- 选择器验证
|
||||
- AI 页面分析
|
||||
"""
|
||||
|
||||
import json
|
||||
import importlib.util
|
||||
import os
|
||||
import re
|
||||
import time
|
||||
from app.core.config import get_logger
|
||||
from app.utils.site_discovery import admitted_selectors
|
||||
from typing import Dict, Optional
|
||||
from urllib import request, error
|
||||
|
||||
import bs4
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
from app.core.config import AppConfig
|
||||
|
||||
from .managers import GlobalConfigManager
|
||||
|
||||
|
||||
logger = get_logger("CFG_PRC")
|
||||
|
||||
|
||||
_BS4_PARSER = "lxml" if importlib.util.find_spec("lxml") is not None else "html.parser"
|
||||
|
||||
|
||||
# ================= 常量 =================
|
||||
|
||||
# HTML 清理配置
|
||||
MAX_HTML_CHARS = int(os.getenv("MAX_HTML_CHARS", "120000"))
|
||||
TEXT_TRUNCATE_LENGTH = 80
|
||||
|
||||
# AI 请求配置
|
||||
AI_MAX_RETRIES = 3
|
||||
AI_RETRY_BASE_DELAY = 1.0
|
||||
AI_RETRY_MAX_DELAY = 10.0
|
||||
AI_REQUEST_TIMEOUT = 120
|
||||
|
||||
# 无效选择器语法模式
|
||||
INVALID_SYNTAX_PATTERNS = [
|
||||
(r'~\s*\.\.', '~ .. 无效语法'),
|
||||
(r'\.\.\s*$', '结尾 .. 无效'),
|
||||
(r'>>\s', '>> 无效语法'),
|
||||
(r':has\(', ':has() 兼容性差'),
|
||||
(r'\s~\s*$', '结尾 ~ 无效'),
|
||||
]
|
||||
|
||||
|
||||
# ================= HTML 清理器 =================
|
||||
|
||||
class HTMLCleaner:
|
||||
"""HTML 清理器"""
|
||||
|
||||
TAGS_TO_REMOVE = [
|
||||
'script', 'style', 'meta', 'link', 'noscript',
|
||||
'img', 'video', 'audio', 'iframe', 'canvas',
|
||||
'path', 'rect', 'circle', 'polygon', 'defs', 'clipPath',
|
||||
'header', 'footer', 'nav', 'aside',
|
||||
]
|
||||
|
||||
ALLOWED_ATTRS = [
|
||||
'id', 'class', 'name', 'placeholder', 'aria-label', 'role',
|
||||
'data-testid', 'type', 'disabled', 'value', 'title', 'tabindex',
|
||||
'contenteditable', 'href'
|
||||
]
|
||||
|
||||
INTERACTIVE_TAGS = ['input', 'textarea', 'button', 'form', 'a']
|
||||
|
||||
CORE_AREA_SELECTORS = [
|
||||
'[role="main"]',
|
||||
'main',
|
||||
'#app',
|
||||
'#root',
|
||||
'.chat',
|
||||
'.conversation',
|
||||
'.message',
|
||||
]
|
||||
|
||||
def __init__(self, max_chars: int = None, text_truncate: int = None):
|
||||
self.max_chars = max_chars or MAX_HTML_CHARS
|
||||
self.text_truncate = text_truncate or TEXT_TRUNCATE_LENGTH
|
||||
|
||||
def clean(self, html: str) -> str:
|
||||
"""深度清理 HTML"""
|
||||
logger.debug("开始 HTML 清理...")
|
||||
original_length = len(html)
|
||||
|
||||
soup = BeautifulSoup(html, _BS4_PARSER)
|
||||
try:
|
||||
interactive_elements = self._extract_interactive_elements(soup)
|
||||
|
||||
for tag in soup(self.TAGS_TO_REMOVE):
|
||||
tag.decompose()
|
||||
|
||||
for element in soup(text=lambda t: isinstance(t, bs4.element.Comment)):
|
||||
element.extract()
|
||||
|
||||
for tag in soup.find_all(True):
|
||||
if tag.string and len(tag.string) > self.text_truncate:
|
||||
tag.string = tag.string[:self.text_truncate] + "..."
|
||||
|
||||
attrs = dict(tag.attrs)
|
||||
for attr in attrs:
|
||||
if attr not in self.ALLOWED_ATTRS:
|
||||
del tag.attrs[attr]
|
||||
|
||||
if 'class' in tag.attrs and isinstance(tag.attrs['class'], list):
|
||||
tag.attrs['class'] = " ".join(tag.attrs['class'])
|
||||
|
||||
clean_html = str(soup.body) if soup.body else str(soup)
|
||||
clean_html = re.sub(r'\s+', ' ', clean_html).strip()
|
||||
|
||||
if len(clean_html) > self.max_chars:
|
||||
logger.warning(f"HTML 过长 ({len(clean_html)}),执行智能截断...")
|
||||
clean_html = self._smart_truncate(clean_html, interactive_elements)
|
||||
|
||||
final_length = len(clean_html)
|
||||
reduction = 100 - (final_length / original_length * 100) if original_length > 0 else 0
|
||||
logger.info(f"HTML 清理完成: {original_length} → {final_length} 字符 (减少 {reduction:.1f}%)")
|
||||
|
||||
return clean_html
|
||||
finally:
|
||||
soup.decompose()
|
||||
|
||||
def _extract_interactive_elements(self, soup: BeautifulSoup) -> str:
|
||||
elements = []
|
||||
|
||||
for tag_name in self.INTERACTIVE_TAGS:
|
||||
for element in soup.find_all(tag_name):
|
||||
context = self._get_element_with_context(element, levels=2)
|
||||
if context:
|
||||
elements.append(context)
|
||||
|
||||
unique_elements = list(dict.fromkeys(elements))
|
||||
return "\n".join(unique_elements)
|
||||
|
||||
def _get_element_with_context(self, element, levels: int = 2) -> str:
|
||||
try:
|
||||
current = element
|
||||
for _ in range(levels):
|
||||
if current.parent and current.parent.name not in ['body', 'html', '[document]']:
|
||||
current = current.parent
|
||||
else:
|
||||
break
|
||||
|
||||
html_str = str(current)
|
||||
if len(html_str) > 2000:
|
||||
html_str = html_str[:2000] + "..."
|
||||
return html_str
|
||||
except Exception:
|
||||
return str(element)
|
||||
|
||||
def _smart_truncate(self, html: str, interactive_html: str) -> str:
|
||||
interactive_budget = int(self.max_chars * 0.3)
|
||||
if len(interactive_html) > interactive_budget:
|
||||
interactive_html = interactive_html[:interactive_budget]
|
||||
|
||||
remaining_budget = self.max_chars - len(interactive_html) - 100
|
||||
|
||||
if remaining_budget <= 0:
|
||||
logger.warning("交互元素已占满预算")
|
||||
return interactive_html
|
||||
|
||||
core_html = self._extract_core_area(html)
|
||||
if core_html and len(core_html) <= remaining_budget:
|
||||
result = core_html + "\n<!-- INTERACTIVE ELEMENTS -->\n" + interactive_html
|
||||
return result[:self.max_chars]
|
||||
|
||||
head_budget = remaining_budget // 3
|
||||
tail_budget = remaining_budget // 3
|
||||
|
||||
head_part = self._truncate_at_tag_boundary(html[:head_budget * 2], head_budget, from_end=False)
|
||||
tail_part = self._truncate_at_tag_boundary(html[-tail_budget * 2:], tail_budget, from_end=True)
|
||||
|
||||
result = (
|
||||
head_part +
|
||||
"\n<!-- TRUNCATED: MIDDLE SECTION -->\n" +
|
||||
"<!-- INTERACTIVE ELEMENTS START -->\n" +
|
||||
interactive_html +
|
||||
"\n<!-- INTERACTIVE ELEMENTS END -->\n" +
|
||||
tail_part
|
||||
)
|
||||
|
||||
if len(result) > self.max_chars:
|
||||
result = result[:self.max_chars]
|
||||
|
||||
return result
|
||||
|
||||
def _extract_core_area(self, html: str) -> Optional[str]:
|
||||
soup = None
|
||||
try:
|
||||
soup = BeautifulSoup(html, _BS4_PARSER)
|
||||
|
||||
for selector in self.CORE_AREA_SELECTORS:
|
||||
try:
|
||||
element = soup.select_one(selector)
|
||||
if element:
|
||||
core_html = str(element)
|
||||
logger.debug(f"找到核心区域: {selector} ({len(core_html)} chars)")
|
||||
return core_html
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
return None
|
||||
except Exception as e:
|
||||
logger.debug(f"提取核心区域失败: {e}")
|
||||
return None
|
||||
finally:
|
||||
if soup is not None:
|
||||
soup.decompose()
|
||||
|
||||
def _truncate_at_tag_boundary(self, html: str, max_len: int, from_end: bool = False) -> str:
|
||||
if len(html) <= max_len:
|
||||
return html
|
||||
|
||||
if from_end:
|
||||
start_pos = len(html) - max_len
|
||||
tag_start = html.find('<', start_pos)
|
||||
if tag_start != -1 and tag_start < len(html) - 100:
|
||||
return html[tag_start:]
|
||||
return html[-max_len:]
|
||||
else:
|
||||
tag_end = html.rfind('>', 0, max_len)
|
||||
if tag_end != -1 and tag_end > 100:
|
||||
return html[:tag_end + 1]
|
||||
return html[:max_len]
|
||||
|
||||
|
||||
# ================= 选择器验证器 =================
|
||||
|
||||
class SelectorValidator:
|
||||
"""选择器验证器"""
|
||||
|
||||
def __init__(self, fallback_selectors: Dict[str, Optional[str]] = None):
|
||||
self.fallback_selectors = fallback_selectors or {}
|
||||
|
||||
def validate(self, selectors: Dict[str, Optional[str]]) -> Dict[str, Optional[str]]:
|
||||
"""验证并修复选择器"""
|
||||
fixed = {}
|
||||
|
||||
for key, selector in selectors.items():
|
||||
if selector is None:
|
||||
fixed[key] = self.fallback_selectors.get(key)
|
||||
continue
|
||||
|
||||
is_invalid = False
|
||||
invalid_reason = ""
|
||||
|
||||
for pattern, reason in INVALID_SYNTAX_PATTERNS:
|
||||
if re.search(pattern, selector):
|
||||
is_invalid = True
|
||||
invalid_reason = reason
|
||||
break
|
||||
|
||||
if is_invalid:
|
||||
logger.warning(f"❌ 无效选择器 [{key}]: {selector}")
|
||||
logger.warning(f" 原因: {invalid_reason}")
|
||||
|
||||
repaired = self._try_repair(selector)
|
||||
if repaired:
|
||||
logger.info(f" ✅ 修复为: {repaired}")
|
||||
fixed[key] = repaired
|
||||
else:
|
||||
fallback = self.fallback_selectors.get(key)
|
||||
logger.info(f" 🔄 回退为: {fallback}")
|
||||
fixed[key] = fallback
|
||||
else:
|
||||
if re.search(r'\._[a-f0-9]{5,}|^\.[a-f0-9]{6,}', selector):
|
||||
logger.info(f"ℹ️ 哈希类名 [{key}]: {selector} (可能不稳定,但保留)")
|
||||
|
||||
fixed[key] = selector
|
||||
|
||||
return fixed
|
||||
|
||||
def _try_repair(self, selector: str) -> Optional[str]:
|
||||
tag_match = re.match(r'^(\w+)', selector)
|
||||
if not tag_match:
|
||||
return None
|
||||
|
||||
tag = tag_match.group(1)
|
||||
|
||||
attr_patterns = [
|
||||
r'(\[name=["\']?\w+["\']?\])',
|
||||
r'(\[type=["\']?\w+["\']?\])',
|
||||
r'(\[role=["\']?\w+["\']?\])',
|
||||
r'(#[\w-]+)',
|
||||
]
|
||||
|
||||
for pattern in attr_patterns:
|
||||
match = re.search(pattern, selector)
|
||||
if match:
|
||||
return tag + match.group(1)
|
||||
|
||||
return tag
|
||||
|
||||
|
||||
# ================= AI 分析器 =================
|
||||
|
||||
class AIAnalyzer:
|
||||
"""AI 页面分析器(支持 OpenAI 兼容 / Gemini 官方 / Claude 官方)"""
|
||||
|
||||
# ===== Provider 注册表 =====
|
||||
# 每个 provider 定义如何构建请求和解析响应
|
||||
# 新增 provider 只需在此处注册,无需修改其他逻辑
|
||||
PROVIDERS = {
|
||||
"openai": {
|
||||
"build_url": "_build_url_openai",
|
||||
"build_headers": "_build_headers_openai",
|
||||
"build_body": "_build_body_openai",
|
||||
"parse_response": "_parse_response_openai",
|
||||
},
|
||||
"gemini": {
|
||||
"build_url": "_build_url_gemini",
|
||||
"build_headers": "_build_headers_gemini",
|
||||
"build_body": "_build_body_gemini",
|
||||
"parse_response": "_parse_response_gemini",
|
||||
},
|
||||
"claude": {
|
||||
"build_url": "_build_url_claude",
|
||||
"build_headers": "_build_headers_claude",
|
||||
"build_body": "_build_body_claude",
|
||||
"parse_response": "_parse_response_claude",
|
||||
},
|
||||
}
|
||||
|
||||
# URL 关键词 → provider 类型的映射(用于自动检测)
|
||||
URL_PROVIDER_HINTS = [
|
||||
("generativelanguage.googleapis.com", "gemini"),
|
||||
("api.anthropic.com", "claude"),
|
||||
("openai.azure.com", "azure"), # 未来可扩展
|
||||
]
|
||||
|
||||
def __init__(self, global_config: GlobalConfigManager = None):
|
||||
self.api_key = AppConfig.get_helper_api_key()
|
||||
|
||||
base_url = AppConfig.get_helper_base_url()
|
||||
self.base_url = base_url.rstrip('/') if base_url else "http://127.0.0.1:5104/v1"
|
||||
|
||||
self.model = AppConfig.get_helper_model()
|
||||
self.global_config = global_config
|
||||
|
||||
# 确定 provider 类型
|
||||
configured_provider = AppConfig.get_helper_api_provider()
|
||||
if configured_provider == "auto" or configured_provider not in self.PROVIDERS:
|
||||
self.provider = self._detect_provider(self.base_url)
|
||||
else:
|
||||
self.provider = configured_provider
|
||||
|
||||
if not self.api_key:
|
||||
logger.warning("⚠️ 未配置 HELPER_API_KEY,AI 分析功能将不可用")
|
||||
else:
|
||||
logger.debug(f"AI 分析器已配置: Provider={self.provider}, URL={self.base_url}, Model={self.model}")
|
||||
|
||||
# ================= 公共接口 =================
|
||||
|
||||
def analyze(self, html: str) -> Optional[Dict[str, str]]:
|
||||
"""分析 HTML 并返回选择器"""
|
||||
if not self.api_key:
|
||||
logger.error("API Key 未配置")
|
||||
return None
|
||||
|
||||
prompt = self._build_prompt(html)
|
||||
|
||||
for attempt in range(AI_MAX_RETRIES):
|
||||
try:
|
||||
logger.info(f"正在请求 AI 分析(尝试 {attempt + 1}/{AI_MAX_RETRIES},Provider: {self.provider})...")
|
||||
|
||||
response = self._request_ai(prompt)
|
||||
if response:
|
||||
analysis = self._extract_json(response)
|
||||
if analysis is not None:
|
||||
selectors = admitted_selectors(analysis)
|
||||
if selectors:
|
||||
logger.info("✅ 已确认 AI 聊天页并识别到专用定位器")
|
||||
return selectors
|
||||
logger.info("未确认 AI 聊天界面或缺少专用回复容器,不自动收录")
|
||||
return None
|
||||
|
||||
logger.warning(f"第 {attempt + 1} 次分析失败")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"AI 请求异常: {e}")
|
||||
|
||||
if attempt < AI_MAX_RETRIES - 1:
|
||||
delay = min(
|
||||
AI_RETRY_BASE_DELAY * (2 ** attempt),
|
||||
AI_RETRY_MAX_DELAY
|
||||
)
|
||||
jitter = delay * 0.1 * (0.5 - os.urandom(1)[0] / 255)
|
||||
sleep_time = delay + jitter
|
||||
|
||||
logger.info(f"等待 {sleep_time:.2f}s 后重试...")
|
||||
time.sleep(sleep_time)
|
||||
|
||||
logger.error("❌ AI 分析失败(已达最大重试次数)")
|
||||
return None
|
||||
|
||||
# ================= Provider 自动检测 =================
|
||||
|
||||
def _detect_provider(self, base_url: str) -> str:
|
||||
"""根据 URL 自动推断 API 类型"""
|
||||
if not base_url:
|
||||
return "openai"
|
||||
|
||||
url_lower = base_url.lower()
|
||||
for keyword, provider in self.URL_PROVIDER_HINTS:
|
||||
if keyword in url_lower:
|
||||
logger.info(f"自动检测到 API 类型: {provider}(基于 URL 包含 '{keyword}')")
|
||||
return provider
|
||||
|
||||
# 默认回退到 OpenAI 兼容格式
|
||||
return "openai"
|
||||
|
||||
# ================= 统一请求调度 =================
|
||||
|
||||
def _request_ai(self, prompt: str) -> Optional[str]:
|
||||
"""向 AI API 发送请求(根据 provider 自动适配格式)"""
|
||||
provider_config = self.PROVIDERS.get(self.provider)
|
||||
if not provider_config:
|
||||
logger.error(f"未知的 Provider: {self.provider},回退到 openai")
|
||||
provider_config = self.PROVIDERS["openai"]
|
||||
|
||||
# 通过注册表动态调用对应的构建/解析方法
|
||||
url = getattr(self, provider_config["build_url"])(prompt)
|
||||
headers = getattr(self, provider_config["build_headers"])()
|
||||
body = getattr(self, provider_config["build_body"])(prompt)
|
||||
parse_fn = getattr(self, provider_config["parse_response"])
|
||||
|
||||
try:
|
||||
req = request.Request(
|
||||
url,
|
||||
data=json.dumps(body).encode('utf-8'),
|
||||
headers=headers
|
||||
)
|
||||
|
||||
with request.urlopen(req, timeout=AI_REQUEST_TIMEOUT) as response:
|
||||
response_text = response.read().decode('utf-8')
|
||||
|
||||
try:
|
||||
json_resp = json.loads(response_text)
|
||||
return parse_fn(json_resp)
|
||||
except json.JSONDecodeError:
|
||||
logger.error(f"AI 响应 JSON 解析失败,原始响应前 500 字符: {response_text[:500]}")
|
||||
return None
|
||||
|
||||
except error.HTTPError as e:
|
||||
error_body = ""
|
||||
try:
|
||||
error_body = e.read().decode('utf-8')[:500]
|
||||
except Exception:
|
||||
pass
|
||||
logger.error(f"HTTP 错误 {e.code}: {e.reason}" + (f",响应: {error_body}" if error_body else ""))
|
||||
return None
|
||||
except error.URLError as e:
|
||||
logger.error(f"网络错误: {e.reason}")
|
||||
return None
|
||||
except Exception as e:
|
||||
# 兜底:捕获 socket.timeout 等未被上层覆盖的异常
|
||||
logger.error(f"请求异常: {type(e).__name__}: {e}")
|
||||
return None
|
||||
|
||||
# ================= OpenAI 兼容格式 =================
|
||||
|
||||
def _build_url_openai(self, prompt: str) -> str:
|
||||
return f"{self.base_url}/chat/completions"
|
||||
|
||||
def _build_headers_openai(self) -> Dict[str, str]:
|
||||
return {
|
||||
"Content-Type": "application/json",
|
||||
"Authorization": f"Bearer {self.api_key}",
|
||||
}
|
||||
|
||||
def _build_body_openai(self, prompt: str) -> Dict:
|
||||
return {
|
||||
"model": self.model,
|
||||
"messages": [{"role": "user", "content": prompt}],
|
||||
"temperature": 0.1,
|
||||
}
|
||||
|
||||
def _parse_response_openai(self, json_resp: Dict) -> Optional[str]:
|
||||
try:
|
||||
if "choices" in json_resp and len(json_resp["choices"]) > 0:
|
||||
return json_resp["choices"][0]["message"]["content"]
|
||||
except (KeyError, IndexError, TypeError) as e:
|
||||
logger.error(f"OpenAI 响应解析失败: {e}")
|
||||
return None
|
||||
|
||||
# ================= Gemini 官方格式 =================
|
||||
|
||||
def _build_url_gemini(self, prompt: str) -> str:
|
||||
# Gemini 官方 API: base_url 应为 https://generativelanguage.googleapis.com
|
||||
# 完整端点: /v1beta/models/{model}:generateContent?key={api_key}
|
||||
base = self.base_url.rstrip('/')
|
||||
# 如果用户配置的 base_url 已经包含 /v1beta 或 /v1,不再追加
|
||||
if "/v1beta" not in base and "/v1/" not in base:
|
||||
base = f"{base}/v1beta"
|
||||
return f"{base}/models/{self.model}:generateContent?key={self.api_key}"
|
||||
|
||||
def _build_headers_gemini(self) -> Dict[str, str]:
|
||||
# Gemini 官方通过 URL 参数传递 key,不需要 Authorization 头
|
||||
return {
|
||||
"Content-Type": "application/json",
|
||||
}
|
||||
|
||||
def _build_body_gemini(self, prompt: str) -> Dict:
|
||||
return {
|
||||
"contents": [
|
||||
{
|
||||
"parts": [
|
||||
{"text": prompt}
|
||||
]
|
||||
}
|
||||
],
|
||||
"generationConfig": {
|
||||
"temperature": 0.1,
|
||||
},
|
||||
}
|
||||
|
||||
def _parse_response_gemini(self, json_resp: Dict) -> Optional[str]:
|
||||
try:
|
||||
candidates = json_resp.get("candidates", [])
|
||||
if candidates:
|
||||
parts = candidates[0].get("content", {}).get("parts", [])
|
||||
if parts:
|
||||
return parts[0].get("text")
|
||||
except (KeyError, IndexError, TypeError) as e:
|
||||
logger.error(f"Gemini 响应解析失败: {e}")
|
||||
|
||||
# 检查是否有错误信息
|
||||
if "error" in json_resp:
|
||||
err = json_resp["error"]
|
||||
logger.error(f"Gemini API 错误: [{err.get('code')}] {err.get('message', '')}")
|
||||
|
||||
return None
|
||||
|
||||
# ================= Claude 官方格式 =================
|
||||
|
||||
def _build_url_claude(self, prompt: str) -> str:
|
||||
base = self.base_url.rstrip('/')
|
||||
# Claude 官方端点: /v1/messages
|
||||
if not base.endswith("/v1"):
|
||||
base = f"{base}/v1"
|
||||
return f"{base}/messages"
|
||||
|
||||
def _build_headers_claude(self) -> Dict[str, str]:
|
||||
return {
|
||||
"Content-Type": "application/json",
|
||||
"x-api-key": self.api_key,
|
||||
"anthropic-version": "2023-06-01",
|
||||
}
|
||||
|
||||
def _build_body_claude(self, prompt: str) -> Dict:
|
||||
return {
|
||||
"model": self.model,
|
||||
"max_tokens": 4096,
|
||||
"messages": [{"role": "user", "content": prompt}],
|
||||
"temperature": 0.1,
|
||||
}
|
||||
|
||||
def _parse_response_claude(self, json_resp: Dict) -> Optional[str]:
|
||||
try:
|
||||
content = json_resp.get("content", [])
|
||||
if content:
|
||||
# Claude 响应的 content 是列表,每项有 type 和 text
|
||||
text_parts = [block["text"] for block in content if block.get("type") == "text"]
|
||||
if text_parts:
|
||||
return "\n".join(text_parts)
|
||||
except (KeyError, IndexError, TypeError) as e:
|
||||
logger.error(f"Claude 响应解析失败: {e}")
|
||||
|
||||
# 检查是否有错误信息
|
||||
if "error" in json_resp:
|
||||
err = json_resp["error"]
|
||||
logger.error(f"Claude API 错误: [{err.get('type')}] {err.get('message', '')}")
|
||||
|
||||
return None
|
||||
|
||||
# ================= Prompt 构建(不变)=================
|
||||
|
||||
def _build_prompt(self, clean_html: str) -> str:
|
||||
"""构建 AI 提示词(动态版)"""
|
||||
if self.global_config:
|
||||
selector_list = self.global_config.build_prompt_selector_list()
|
||||
json_keys = self.global_config.build_prompt_json_keys()
|
||||
else:
|
||||
selector_list = self._build_default_selector_list()
|
||||
json_keys = self._build_default_json_keys()
|
||||
|
||||
lines = [
|
||||
"You are a web interface classifier. First determine whether this is an AI CONVERSATION interface; do not assume every page is a chat app.",
|
||||
"Return {\"is_chat_site\": false} for search engines, search results, login/consent pages, articles, ordinary forms, and pages with insufficient evidence.",
|
||||
"A textarea, submit button, generic div, or the word AI alone is NOT evidence of an AI conversation interface.",
|
||||
"Only for a genuine AI chat page with a composer and a specific assistant-reply container, return is_chat_site: true and the selector keys below. Do not invent selectors for missing elements.",
|
||||
"",
|
||||
"## CRITICAL RULES:",
|
||||
"1. **Uniqueness is Key**: Ensure selectors matches ONLY the intended element.",
|
||||
"2. **Distinguish AI vs User**: For `result_container`, specify the selector to target the **AI's response text** only. It MUST exclude user prompts, sidebars, or chat history.",
|
||||
"3. **Use Hierarchy**: If a class like `.prose` or `.markdown` is used for both User and AI, you MUST find a unique parent class to differentiate (e.g., `.bot-msg .prose`).",
|
||||
"4. **Syntax**: Use standard CSS selectors. Spaces for descendants (e.g., `div.bot p`) are encouraged for precision.",
|
||||
"5. **No Invalid Syntax**: Do NOT use `xpath`, `~`, `:has()`, or `text()`.",
|
||||
"",
|
||||
"## PREFERENCE ORDER:",
|
||||
"1. `id`, `name`, `data-testid` (Most preferred)",
|
||||
"2. `button[type=\"submit\"]`",
|
||||
"3. Unique parent class + target class (e.g., `.response-area .content`)",
|
||||
"4. Hashed classes (only if no other option exists)",
|
||||
"",
|
||||
"## ELEMENTS TO FIND:",
|
||||
selector_list,
|
||||
"",
|
||||
"## REQUIRED OUTPUT (JSON ONLY):",
|
||||
"Return a JSON object with boolean is_chat_site plus these keys ONLY when is_chat_site is true:",
|
||||
json_keys,
|
||||
"",
|
||||
"## HTML:",
|
||||
clean_html
|
||||
]
|
||||
return "\n".join(lines)
|
||||
|
||||
def _build_default_selector_list(self) -> str:
|
||||
"""默认的元素列表(回退用)"""
|
||||
return """- `input_box`: The text input area (textarea/input). [REQUIRED]
|
||||
- `send_btn`: The button that sends the message. [REQUIRED]
|
||||
- `result_container`: The container for the AI's generated text response. [REQUIRED]
|
||||
- `new_chat_btn`: Button to start a fresh conversation. [OPTIONAL, return null if not found]"""
|
||||
|
||||
def _build_default_json_keys(self) -> str:
|
||||
"""默认的 JSON 键说明(回退用)"""
|
||||
return """- `input_box`: The text input area
|
||||
- `send_btn`: The send button
|
||||
- `result_container`: AI response container
|
||||
- `new_chat_btn`: New chat button (or null)"""
|
||||
|
||||
# ================= JSON 提取(不变)=================
|
||||
|
||||
def _extract_json(self, text: str) -> Optional[Dict]:
|
||||
"""从 AI 响应中提取 JSON"""
|
||||
try:
|
||||
match = re.search(r'```(?:json)?\s*(\{[\s\S]*?\})\s*```', text)
|
||||
if match:
|
||||
return json.loads(match.group(1))
|
||||
|
||||
match = re.search(r'(\{[\s\S]*\})', text)
|
||||
if match:
|
||||
return json.loads(match.group(1))
|
||||
|
||||
return json.loads(text)
|
||||
|
||||
except json.JSONDecodeError as e:
|
||||
logger.error(f"JSON 解析失败: {e}")
|
||||
return None
|
||||
|
||||
|
||||
__all__ = ['HTMLCleaner', 'SelectorValidator', 'AIAnalyzer']
|
||||
@@ -0,0 +1,154 @@
|
||||
"""R1-1:站点适配器 Schema v1。
|
||||
|
||||
只校验“形状”:类型、必填项、嵌套关系,以及 default_preset 必须指向已有预设。
|
||||
语义校验(工作流 DSL、选择器能否命中等)仍由 ``flow_runtime.validate_workflow`` 和
|
||||
ConfigEngine 的各个 ``_validate_*`` 负责。为了不误伤现有配置和用户自建预设,未知字段一律放行,
|
||||
预设内部也不设必填项。
|
||||
|
||||
单站点文件(R1-2,``config/sites/<站点>.json``)的外层信封::
|
||||
|
||||
{"schema_version": 1, "site": "chat.deepseek.com", "adapter_version": "2026.09.26",
|
||||
"min_app_version": "3.0.0", "last_verified": "2026-09-26", "config": {...}}
|
||||
|
||||
``config`` 与旧版 ``config/sites.json`` 中对应节点的内容完全相同。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Dict, Iterable, List
|
||||
|
||||
from jsonschema import Draft202012Validator
|
||||
|
||||
SITE_FILE_SCHEMA_VERSION = 1
|
||||
GLOBAL_KEY = "_global"
|
||||
|
||||
_OBJECT = {"type": "object"}
|
||||
|
||||
WORKFLOW_STEP_SCHEMA: Dict[str, Any] = {
|
||||
"type": "object",
|
||||
"required": ["action"],
|
||||
"properties": {
|
||||
"action": {"type": "string", "minLength": 1},
|
||||
"target": {"type": ["string", "null"]},
|
||||
"optional": {"type": "boolean"},
|
||||
"label": {"type": "string"},
|
||||
"selector": {"type": "string"},
|
||||
"execution": _OBJECT,
|
||||
"flow_version": {"type": "integer"},
|
||||
"retry_safe": {"type": "boolean"},
|
||||
},
|
||||
}
|
||||
|
||||
PRESET_SCHEMA: Dict[str, Any] = {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"selectors": {"type": "object", "additionalProperties": {"type": ["string", "null"]}},
|
||||
"workflow": {"type": "array", "items": WORKFLOW_STEP_SCHEMA},
|
||||
"stealth": {"type": "boolean"},
|
||||
"stream_config": _OBJECT,
|
||||
"image_extraction": _OBJECT,
|
||||
"file_paste": _OBJECT,
|
||||
"prompt_padding": _OBJECT,
|
||||
"advanced": _OBJECT,
|
||||
},
|
||||
}
|
||||
|
||||
SITE_CONFIG_SCHEMA: Dict[str, Any] = {
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"type": "object",
|
||||
"required": ["presets"],
|
||||
"properties": {
|
||||
"presets": {"type": "object", "minProperties": 1, "additionalProperties": PRESET_SCHEMA},
|
||||
"default_preset": {"type": "string", "minLength": 1},
|
||||
"advanced": _OBJECT,
|
||||
},
|
||||
}
|
||||
|
||||
GLOBAL_CONFIG_SCHEMA: Dict[str, Any] = {
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"selector_definitions": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"required": ["key"],
|
||||
"properties": {
|
||||
"key": {"type": "string", "minLength": 1},
|
||||
"description": {"type": "string"},
|
||||
"enabled": {"type": "boolean"},
|
||||
"required": {"type": "boolean"},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
SITE_FILE_SCHEMA: Dict[str, Any] = {
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"type": "object",
|
||||
"required": ["schema_version", "site", "config"],
|
||||
"properties": {
|
||||
"schema_version": {"const": SITE_FILE_SCHEMA_VERSION},
|
||||
"site": {"type": "string", "minLength": 1},
|
||||
"adapter_version": {"type": "string"},
|
||||
"min_app_version": {"type": "string", "pattern": r"^\d+(\.\d+){0,3}$"},
|
||||
"last_verified": {"type": "string", "pattern": r"^\d{4}-\d{2}-\d{2}$"},
|
||||
"config": _OBJECT,
|
||||
},
|
||||
}
|
||||
|
||||
_VALIDATORS = {
|
||||
name: Draft202012Validator(schema)
|
||||
for name, schema in (
|
||||
("site", SITE_CONFIG_SCHEMA),
|
||||
("global", GLOBAL_CONFIG_SCHEMA),
|
||||
("file", SITE_FILE_SCHEMA),
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
def _format_path(prefix: str, parts: Iterable[Any]) -> str:
|
||||
path = prefix
|
||||
for part in parts:
|
||||
path += f"[{part}]" if isinstance(part, int) else f".{part}"
|
||||
return path
|
||||
|
||||
|
||||
def _schema_errors(kind: str, payload: Any, prefix: str) -> List[str]:
|
||||
errors = sorted(_VALIDATORS[kind].iter_errors(payload), key=lambda e: list(map(str, e.absolute_path)))
|
||||
return [f"{_format_path(prefix, error.absolute_path)}: {error.message}" for error in errors]
|
||||
|
||||
|
||||
def site_config_errors(site: str, config: Any) -> List[str]:
|
||||
"""校验单个站点(或 ``_global``)的配置,返回可读的路径级错误列表;空列表表示通过。"""
|
||||
prefix = f"sites[{site}]"
|
||||
if site == GLOBAL_KEY:
|
||||
return _schema_errors("global", config, prefix)
|
||||
errors = _schema_errors("site", config, prefix)
|
||||
if not errors and isinstance(config, dict):
|
||||
default = config.get("default_preset")
|
||||
presets = config.get("presets") or {}
|
||||
if default and default not in presets:
|
||||
errors.append(f"{prefix}.default_preset: '{default}' 不在 presets 中(可选:{', '.join(presets)})")
|
||||
return errors
|
||||
|
||||
|
||||
def site_file_errors(payload: Any) -> List[str]:
|
||||
"""校验单站点文件(信封 + config)。"""
|
||||
errors = _schema_errors("file", payload, "file")
|
||||
if errors:
|
||||
return errors
|
||||
return site_config_errors(payload["site"], payload["config"])
|
||||
|
||||
|
||||
def sites_errors(sites: Any) -> Dict[str, List[str]]:
|
||||
"""校验整份站点配置字典(旧 sites.json 结构),返回 {站点: [错误...]},只包含有错误的站点。"""
|
||||
if not isinstance(sites, dict):
|
||||
return {"*": [f"sites: 应为对象,实际为 {type(sites).__name__}"]}
|
||||
result: Dict[str, List[str]] = {}
|
||||
for site, config in sites.items():
|
||||
errors = site_config_errors(str(site), config)
|
||||
if errors:
|
||||
result[str(site)] = errors
|
||||
return result
|
||||
@@ -0,0 +1,312 @@
|
||||
"""R1-2:站点配置目录存储(``config/sites/<站点>.json``)。
|
||||
|
||||
旧版把全部站点放在一个 271KB 的 ``config/sites.json`` 里;现在每个站点(以及 ``_global``)一个文件,
|
||||
外层是 R1-1 定义的信封(见 ``site_schema.py``)。本模块负责:
|
||||
|
||||
- 读取目录并拼回与旧 ``sites.json`` **完全相同**的字典(``{"_global": {...}, "<域名>": {...}}``),
|
||||
所以 ConfigEngine 以外的代码无需改动;
|
||||
- 只重写内容有变化的站点文件(两阶段:先写全部临时文件,再逐个 ``os.replace``),保留信封元数据;
|
||||
被删除的站点移入 ``.trash/``,不直接删除;
|
||||
- 自动迁移旧的 ``config/sites.json``:目录为空时直接拆分;目录里已有发布版默认文件时(通过更新器升级的场景),
|
||||
按更新器 ``merge_site_records`` 的语义合并(同一站点优先保留本地值、补入发布版新增字段);
|
||||
迁移后旧文件改名为 ``sites.json.migrated-<时间>.bak``;
|
||||
- 读不懂的文件(JSON 损坏或信封不完整)只告警跳过,并且**永远不会被保存逻辑覆盖或移入回收站**;
|
||||
热重载时这类站点沿用上一次成功加载的配置。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
|
||||
from app.core.config import get_logger
|
||||
from app.services.config.site_schema import SITE_FILE_SCHEMA_VERSION, site_file_errors
|
||||
|
||||
logger = get_logger("SITE_STORE")
|
||||
|
||||
INDEX_FILE_NAME = "index.json"
|
||||
TRASH_DIR_NAME = ".trash"
|
||||
DEFAULT_MIN_APP_VERSION = "3.0.0"
|
||||
_ENVELOPE_ORDER = ("schema_version", "site", "adapter_version", "min_app_version", "last_verified")
|
||||
_WINDOWS_RESERVED = {"con", "prn", "aux", "nul", *(f"com{i}" for i in range(1, 10)), *(f"lpt{i}" for i in range(1, 10))}
|
||||
|
||||
|
||||
def canonical_config_text(config: Any) -> str:
|
||||
"""与格式、键顺序无关的规范化文本,用于变更检测与摘要(R1-3 的 config_sha256 也基于它)。"""
|
||||
return json.dumps(config, ensure_ascii=False, sort_keys=True, separators=(",", ":"))
|
||||
|
||||
|
||||
def site_filename(site: str) -> str:
|
||||
"""站点键 -> 文件名。保留 ``[A-Za-z0-9._-]``,其余字节百分号编码;避开 Windows 保留名与 index.json。"""
|
||||
encoded = "".join(
|
||||
ch if (ch.isascii() and (ch.isalnum() or ch in "._-")) else "".join(f"%{b:02X}" for b in ch.encode("utf-8"))
|
||||
for ch in str(site)
|
||||
)
|
||||
if not encoded or encoded.startswith(".") or encoded.split(".")[0].lower() in _WINDOWS_RESERVED | {"index"}:
|
||||
encoded = "%5F" + encoded
|
||||
return f"{encoded}.json"
|
||||
|
||||
|
||||
def build_envelope(site: str, config: Dict[str, Any], meta: Optional[Dict[str, Any]] = None) -> Dict[str, Any]:
|
||||
meta = dict(meta or {})
|
||||
payload: Dict[str, Any] = {"schema_version": SITE_FILE_SCHEMA_VERSION, "site": site}
|
||||
for key in _ENVELOPE_ORDER[2:]:
|
||||
if meta.get(key) not in (None, ""):
|
||||
payload[key] = meta[key]
|
||||
for key, value in meta.items(): # 其余元数据(例如 R1-3 的 origin)原样保留
|
||||
if key not in payload and key != "config":
|
||||
payload[key] = value
|
||||
payload["config"] = config
|
||||
return payload
|
||||
|
||||
|
||||
def _dump(payload: Dict[str, Any]) -> str:
|
||||
return json.dumps(payload, indent=2, ensure_ascii=False) + "\n"
|
||||
|
||||
|
||||
def _write_text_atomic(path: Path, text: str) -> None:
|
||||
tmp = path.with_name(path.name + ".tmp")
|
||||
with open(tmp, "w", encoding="utf-8", newline="\n") as handle:
|
||||
handle.write(text)
|
||||
handle.flush()
|
||||
os.fsync(handle.fileno())
|
||||
os.replace(tmp, path)
|
||||
|
||||
|
||||
def _timestamp() -> str:
|
||||
return time.strftime("%Y%m%d-%H%M%S")
|
||||
|
||||
|
||||
class SiteStore:
|
||||
"""``config/sites/`` 目录的读写与旧版迁移。线程安全由调用方(ConfigEngine 的 ``_io_lock``)保证。"""
|
||||
|
||||
def __init__(self, sites_dir: os.PathLike | str, legacy_file: os.PathLike | str):
|
||||
self.sites_dir = Path(sites_dir)
|
||||
self.legacy_file = Path(legacy_file)
|
||||
self._files: Dict[str, Path] = {}
|
||||
self._meta: Dict[str, Dict[str, Any]] = {}
|
||||
self._on_disk: Dict[str, str] = {}
|
||||
self._broken: Dict[Path, str] = {}
|
||||
self._broken_sites: Dict[Path, str] = {} # 损坏文件 -> 上次成功加载时对应的站点
|
||||
|
||||
# ------------------------------------------------------------------ 查询
|
||||
def site_files(self) -> List[Path]:
|
||||
if not self.sites_dir.is_dir():
|
||||
return []
|
||||
return sorted(
|
||||
path for path in self.sites_dir.glob("*.json")
|
||||
if path.is_file() and path.name != INDEX_FILE_NAME and not path.name.startswith(".")
|
||||
)
|
||||
|
||||
def exists(self) -> bool:
|
||||
return bool(self.site_files()) or self.legacy_file.is_file()
|
||||
|
||||
def signature(self) -> Tuple:
|
||||
"""目录内容签名(文件名、mtime、大小),用于热重载的变更检测。"""
|
||||
entries = []
|
||||
for path in self.site_files():
|
||||
try:
|
||||
stat = path.stat()
|
||||
except OSError:
|
||||
continue
|
||||
entries.append((path.name, stat.st_mtime_ns, stat.st_size))
|
||||
if self.legacy_file.is_file():
|
||||
stat = self.legacy_file.stat()
|
||||
entries.append(("<legacy>", stat.st_mtime_ns, stat.st_size))
|
||||
return tuple(entries)
|
||||
|
||||
def metadata(self, site: str) -> Dict[str, Any]:
|
||||
return dict(self._meta.get(site) or {})
|
||||
|
||||
def broken_files(self) -> Dict[str, str]:
|
||||
return {str(path): reason for path, reason in self._broken.items()}
|
||||
|
||||
def file_for(self, site: str) -> Path:
|
||||
return self._files.get(site) or (self.sites_dir / site_filename(site))
|
||||
|
||||
# ------------------------------------------------------------------ 读取
|
||||
def read_envelope(self, path: Path) -> Dict[str, Any]:
|
||||
payload = json.loads(path.read_text(encoding="utf-8-sig"))
|
||||
if not isinstance(payload, dict) or not isinstance(payload.get("site"), str) or not isinstance(payload.get("config"), dict):
|
||||
raise ValueError("不是有效的站点文件(缺少 site/config 字段)")
|
||||
return payload
|
||||
|
||||
def load(self, previous: Optional[Dict[str, Any]] = None) -> Optional[Dict[str, Any]]:
|
||||
"""读取全部站点,返回旧 sites.json 结构的字典;目录与旧文件都不存在时返回 None。
|
||||
|
||||
``previous`` 是上一次加载的结果:热重载时,暂时读不懂的文件沿用其中对应站点的配置。
|
||||
"""
|
||||
if self.legacy_file.is_file():
|
||||
self.migrate_legacy()
|
||||
files = self.site_files()
|
||||
if not files:
|
||||
return None
|
||||
|
||||
data: Dict[str, Any] = {}
|
||||
files_map: Dict[str, Path] = {}
|
||||
meta: Dict[str, Dict[str, Any]] = {}
|
||||
on_disk: Dict[str, str] = {}
|
||||
broken: Dict[Path, str] = {}
|
||||
previous_sites_by_path = {path: site for site, path in self._files.items()}
|
||||
previous_sites_by_path.update(self._broken_sites)
|
||||
broken_sites: Dict[Path, str] = {}
|
||||
|
||||
for path in files:
|
||||
try:
|
||||
payload = self.read_envelope(path)
|
||||
except (OSError, ValueError) as exc: # JSON 损坏、编码错误(均为 ValueError)、信封不完整、读文件失败
|
||||
broken[path] = str(exc)
|
||||
logger.error(f"站点配置文件无法读取,已跳过且不会被覆盖: {path.name}: {exc}")
|
||||
old_site = previous_sites_by_path.get(path)
|
||||
if old_site:
|
||||
broken_sites[path] = old_site
|
||||
if previous and old_site in previous:
|
||||
data[old_site] = previous[old_site]
|
||||
logger.warning(f"站点 {old_site} 暂时沿用上一次成功加载的配置")
|
||||
continue
|
||||
site = payload["site"]
|
||||
if site in files_map:
|
||||
broken[path] = f"与 {files_map[site].name} 重复声明站点 {site}"
|
||||
logger.error(f"站点配置文件重复声明 {site},已忽略: {path.name}")
|
||||
continue
|
||||
for error in site_file_errors(payload)[:5]:
|
||||
logger.warning(f"站点配置 Schema 告警({path.name}): {error}")
|
||||
data[site] = payload["config"]
|
||||
files_map[site] = path
|
||||
meta[site] = {key: value for key, value in payload.items() if key != "config"}
|
||||
on_disk[site] = canonical_config_text(payload["config"])
|
||||
|
||||
self._files, self._meta, self._on_disk = files_map, meta, on_disk
|
||||
self._broken, self._broken_sites = broken, broken_sites
|
||||
return data
|
||||
|
||||
# ------------------------------------------------------------------ 保存
|
||||
def save(self, full_config: Dict[str, Any]) -> List[str]:
|
||||
"""保存旧 sites.json 结构的字典,返回实际重写的站点列表。"""
|
||||
self.sites_dir.mkdir(parents=True, exist_ok=True)
|
||||
pending: List[Tuple[str, Path, Path, Dict[str, Any], str]] = []
|
||||
taken = {path for path in self._files.values()} | set(self._broken)
|
||||
try:
|
||||
for site, config in full_config.items():
|
||||
if not isinstance(config, dict):
|
||||
continue
|
||||
text = canonical_config_text(config)
|
||||
path = self._files.get(site)
|
||||
if path is not None and self._on_disk.get(site) == text and path.is_file():
|
||||
continue
|
||||
if path is None:
|
||||
path = self._unique_path(site, taken)
|
||||
taken.add(path)
|
||||
if path in self._broken:
|
||||
logger.error(f"站点 {site} 对应的文件 {path.name} 无法解析,为避免覆盖用户手改内容,本次跳过保存")
|
||||
continue
|
||||
payload = build_envelope(site, config, self._meta.get(site) or self._default_meta())
|
||||
tmp = path.with_name(path.name + ".tmp")
|
||||
with open(tmp, "w", encoding="utf-8", newline="\n") as handle:
|
||||
handle.write(_dump(payload))
|
||||
handle.flush()
|
||||
os.fsync(handle.fileno())
|
||||
pending.append((site, path, tmp, payload, text))
|
||||
except Exception: # broad-except: 清理已写出的临时文件后原样重新抛出
|
||||
for _, _, tmp, _, _ in pending:
|
||||
try:
|
||||
tmp.unlink()
|
||||
except OSError:
|
||||
pass
|
||||
raise
|
||||
|
||||
written = []
|
||||
for site, path, tmp, payload, text in pending:
|
||||
os.replace(tmp, path)
|
||||
self._files[site] = path
|
||||
self._on_disk[site] = text
|
||||
self._meta[site] = {key: value for key, value in payload.items() if key != "config"}
|
||||
written.append(site)
|
||||
|
||||
for site in [s for s in self._files if s not in full_config]:
|
||||
path = self._files.pop(site)
|
||||
self._on_disk.pop(site, None)
|
||||
self._meta.pop(site, None)
|
||||
if path in self._broken or not path.is_file():
|
||||
continue
|
||||
trash = self.sites_dir / TRASH_DIR_NAME
|
||||
trash.mkdir(exist_ok=True)
|
||||
target = trash / f"{path.stem}.{_timestamp()}.json"
|
||||
os.replace(path, target)
|
||||
logger.info(f"站点 {site} 已删除,配置文件移入回收站: {target}")
|
||||
return written
|
||||
|
||||
def _default_meta(self) -> Dict[str, Any]:
|
||||
return {"min_app_version": DEFAULT_MIN_APP_VERSION}
|
||||
|
||||
def _unique_path(self, site: str, taken: set) -> Path:
|
||||
base = self.sites_dir / site_filename(site)
|
||||
path, counter = base, 2
|
||||
while path in taken or (path.exists() and path not in self._files.values()):
|
||||
path = base.with_name(f"{base.stem}~{counter}.json")
|
||||
counter += 1
|
||||
return path
|
||||
|
||||
# ------------------------------------------------------------------ 迁移
|
||||
def migrate_legacy(self) -> Optional[Path]:
|
||||
"""把旧的 config/sites.json 迁移进目录,返回备份文件路径;旧文件无法解析时保持原样并返回 None。"""
|
||||
try:
|
||||
raw = self.legacy_file.read_text(encoding="utf-8-sig").strip()
|
||||
legacy = json.loads(raw) if raw else {}
|
||||
if not isinstance(legacy, dict):
|
||||
raise ValueError("顶层不是对象")
|
||||
except (OSError, ValueError) as exc:
|
||||
logger.error(f"旧版站点配置 {self.legacy_file} 无法解析,暂不迁移(原文件保持不变): {exc}")
|
||||
return None
|
||||
|
||||
from updater import merge_site_records # 与更新器合并 sites.json 的语义保持一致
|
||||
|
||||
self.sites_dir.mkdir(parents=True, exist_ok=True)
|
||||
existing: Dict[str, Tuple[Path, Dict[str, Any]]] = {}
|
||||
for path in self.site_files():
|
||||
try:
|
||||
payload = self.read_envelope(path)
|
||||
except (OSError, ValueError):
|
||||
continue
|
||||
existing[payload["site"]] = (path, payload)
|
||||
|
||||
taken = {path for path, _ in existing.values()}
|
||||
merged_count = created_count = 0
|
||||
for site, config in legacy.items():
|
||||
if not isinstance(config, dict) or (site.startswith("_") and site != "_global"):
|
||||
continue
|
||||
if site in existing:
|
||||
path, payload = existing[site]
|
||||
payload = dict(payload)
|
||||
merged_config = merge_site_records(config, payload["config"])
|
||||
if (
|
||||
site == "_global"
|
||||
and config.get("selector_definitions") == []
|
||||
and payload["config"].get("selector_definitions")
|
||||
):
|
||||
merged_config["selector_definitions"] = payload["config"]["selector_definitions"]
|
||||
payload["config"] = merged_config
|
||||
merged_count += 1
|
||||
else:
|
||||
path = self._unique_path(site, taken)
|
||||
taken.add(path)
|
||||
payload = build_envelope(site, config, self._default_meta())
|
||||
created_count += 1
|
||||
_write_text_atomic(path, _dump(payload))
|
||||
|
||||
backup = self.legacy_file.with_name(f"{self.legacy_file.name}.migrated-{_timestamp()}.bak")
|
||||
os.replace(self.legacy_file, backup)
|
||||
logger.warning(
|
||||
f"已把旧版站点配置迁移到 {self.sites_dir}:新建 {created_count} 个、与发布版合并 {merged_count} 个;"
|
||||
f"原文件已备份为 {backup.name}"
|
||||
)
|
||||
return backup
|
||||
|
||||
|
||||
def load_sites_dict(sites_dir: os.PathLike | str) -> Dict[str, Any]:
|
||||
"""只读地拼出目录里的全部站点(不迁移、不告警),供测试、脚本与导出使用。"""
|
||||
store = SiteStore(sites_dir, Path(sites_dir) / "__no_legacy__.json")
|
||||
return store.load() or {}
|
||||
Reference in new issue
Block a user