Files
universal-web-api/app/services/config/engine.py
T
2026-11-02 22:54:14 +08:00

299 lines
12 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""
app/services/config/engine.py - 配置引擎主类
职责:
- 配置文件读写
- 站点配置管理
- 配置缓存与热重载
- 图片配置、提取器管理
"""
import json
import os
import copy
import re
import threading
import time
from app.core.config import get_logger
from typing import Dict, Optional, List, Any, Set, Union, Callable
from urllib.parse import urljoin
from app.core.parsers import ParserRegistry
from app.models.schemas import (
SiteConfig,
WorkflowStep,
SelectorDefinition,
ADVANCED_FIELDS,
PRESET_ADVANCED_FIELDS,
SITE_ADVANCED_FIELDS,
get_default_image_extraction_config,
get_default_file_paste_config,
get_default_prompt_padding_config,
get_default_site_advanced_config,
get_default_attachment_monitor_config,
get_default_send_confirmation_config,
get_modality_policy,
get_enabled_modalities,
is_modality_enabled,
normalize_modalities_config,
normalize_modality_policy,
)
from app.services.extractor_manager import extractor_manager
from app.services.parser_manager import parser_manager
from app.utils.site_rules import derive_site_card_id, get_site_rule
from app.utils.site_discovery import automatic_discovery_allowed, specific_chat_selectors, selectors_match_chat_html
from app.core.request_transport import (
get_default_request_transport_config,
normalize_request_transport_config,
)
from .cache import ConfigCache
from .site_store import SiteStore
from .managers import GlobalConfigManager, ImagePresetsManager
from .processors import HTMLCleaner, SelectorValidator, AIAnalyzer
# R2-3:模块级定义迁到 engine_common,这里重新导出以保持兼容(外部与测试都从 engine 导入这些名字)
from .engine_common import ( # noqa: E402,F401
ConfigConstants,
DEFAULT_PRESET_NAME,
DEFAULT_WORKFLOW,
PRESET_FIELDS,
_MISSING,
_SITE_STARTUP_JS_PATTERNS,
_coerce_bool,
_merge_config_patch,
get_default_network_config,
get_default_stream_config,
logger,
)
from .engine_local_overrides import ConfigLocalOverridesMixin # noqa: E402
from .engine_migrations import ConfigMigrationsMixin # noqa: E402
from .engine_presets import ConfigPresetsMixin # noqa: E402
from .engine_sites import ConfigSitesMixin # noqa: E402
from .engine_media import ConfigMediaMixin # noqa: E402
from .engine_stream import ConfigStreamMixin # noqa: E402
# ================= 配置引擎主类 =================
class ConfigEngine(
ConfigLocalOverridesMixin,
ConfigMigrationsMixin,
ConfigPresetsMixin,
ConfigSitesMixin,
ConfigMediaMixin,
ConfigStreamMixin,
):
"""配置引擎主类"""
def __init__(self):
self.config_file = ConfigConstants.CONFIG_FILE # 旧版单文件,仅用于自动迁移
self.sites_dir = ConfigConstants.SITES_DIR
self.site_store = SiteStore(self.sites_dir, self.config_file)
self.local_sites_file = ConfigConstants.SITES_LOCAL_FILE
self._io_lock = threading.RLock()
self.last_mtime = 0.0
self.last_signature: tuple = ()
self.last_local_mtime = 0.0
self.sites: Dict[str, SiteConfig] = {}
self._global_default_presets: Dict[str, str] = {}
self._local_default_presets: Dict[str, str] = {}
self._local_sites_payload: Dict[str, Any] = {}
self._cache = ConfigCache(ttl=5.0)
# 子管理器
self.global_config = GlobalConfigManager()
self.image_presets = ImagePresetsManager(ConfigConstants.IMAGE_PRESETS_FILE)
# 加载配置
self._load_config()
self._migrate_global_commands()
# 处理器
self.html_cleaner = HTMLCleaner()
self.validator = SelectorValidator(self.global_config.get_fallback_selectors())
self.ai_analyzer = AIAnalyzer(self.global_config)
# 迁移旧配置(顺序重要:先转预设格式,再补缺失字段,最后清理残留)
self._migrate_loaded_config()
self._apply_local_site_overrides()
logger.debug(f"配置引擎已初始化,已加载 {len(self.sites)} 个站点配置")
def _load_config(self):
with self._io_lock:
return self._load_config_locked()
def _load_config_locked(self):
"""初始化加载站点配置(config/sites/ 目录;发现旧版 sites.json 时自动迁移)"""
try:
data = self.site_store.load()
self.last_signature = self.site_store.signature()
if data is None:
logger.info(f"站点配置目录 {self.sites_dir} 为空,将在首次保存时创建")
self._apply_local_site_overrides()
return
self.last_mtime = time.time()
# 提取并加载 _global;缺失时重置为默认全局配置。
global_section = data.pop("_global", {})
self.global_config.load(global_section if isinstance(global_section, dict) else {})
# 过滤内部键
self.sites = {
k: v for k, v in data.items()
if not k.startswith('_')
}
self._refresh_global_default_presets_from_sites()
self._apply_local_site_overrides()
logger.debug(f"已加载站点配置: {self.sites_dir}({len(self.sites)} 个站点)")
except Exception as e:
logger.error(f"加载配置失败: {e}")
def refresh_if_changed(self):
"""检查文件是否变化,如果变化则重载"""
if not self.site_store.exists() and not os.path.exists(self.local_sites_file):
return
try:
current_signature = self.site_store.signature()
current_local_mtime = os.path.getmtime(self.local_sites_file) if os.path.exists(self.local_sites_file) else 0.0
if current_signature != self.last_signature or current_local_mtime != self.last_local_mtime:
logger.debug("⚡ 检测到站点配置变化,热重载")
self.reload_config()
except Exception as e:
logger.error(f"检查文件变化失败: {e}")
def reload_config(self):
with self._io_lock:
return self._reload_config_locked()
def _reload_config_locked(self):
"""重新加载配置(Hot Reload)"""
self._cache.invalidate()
if not self.site_store.exists():
logger.warning("重载失败:站点配置目录为空")
return
try:
previous = {"_global": self.global_config.to_dict(), **self.sites}
data = self.site_store.load(previous=previous)
if data is None:
logger.warning("重载失败:站点配置目录为空")
return
signature = self.site_store.signature()
# 提取并加载 _global;缺失时重置为默认全局配置。
global_section = data.pop("_global", {})
self.global_config.load(global_section if isinstance(global_section, dict) else {})
self.validator.fallback_selectors = self.global_config.get_fallback_selectors()
# 过滤内部键
self.sites = {
k: v for k, v in data.items()
if not k.startswith('_')
}
self.last_signature = signature
self.last_mtime = time.time()
self._refresh_global_default_presets_from_sites()
self._migrate_loaded_config()
self._apply_local_site_overrides()
logger.debug(f"✅ 配置已热重载 (Sites: {len(self.sites)})")
except json.JSONDecodeError as e:
logger.error(f"❌ 重载配置失败(JSON格式错误),保留旧配置: {e}")
except Exception as e:
logger.error(f"❌ 重载配置失败: {e}")
def save_config(self):
"""公开的保存方法(供 API 调用)"""
return self._save_config()
def _save_config(self) -> bool:
with self._io_lock:
return self._save_config_locked()
def _save_config_locked(self) -> bool:
"""保存站点配置(只重写有变化的站点文件,每个文件原子替换)"""
local_snapshot: Optional[tuple[bool, bytes, Dict[str, Any], Dict[str, str]]] = None
local_overrides_written = False
default_maps_snapshot = (
copy.deepcopy(self._global_default_presets),
copy.deepcopy(self._local_default_presets),
)
try:
self._prune_default_preset_maps()
persisted_sites = {}
for domain, site in self.sites.items():
if domain.startswith('_') or not isinstance(site, dict):
continue
site_copy = copy.deepcopy(site)
from app.utils.site_url import route_domain_matches
if route_domain_matches("arena.ai", domain):
try:
from app.services.arena_model_catalog import load_arena_model_catalog_data
cat_data = load_arena_model_catalog_data()
d_key = str(domain).strip().lower().strip(".")
domain_cat = cat_data.get(d_key, {})
except Exception:
domain_cat = None
presets_dict = site_copy.get("presets")
if isinstance(presets_dict, dict):
for p_key, p_val in presets_dict.items():
if isinstance(p_val, dict) and "model_catalog" in p_val:
if domain_cat is not None and str(p_key).strip() in domain_cat:
p_val.pop("model_catalog", None)
# 同步清理内存中的残留
mem_presets = site.get("presets")
if isinstance(mem_presets, dict):
for p_key, p_val in mem_presets.items():
if isinstance(p_val, dict) and "model_catalog" in p_val:
if domain_cat is not None and str(p_key).strip() in domain_cat:
p_val.pop("model_catalog", None)
persisted_default = self._get_persisted_default_preset(domain, site_copy)
if persisted_default:
site_copy["default_preset"] = persisted_default
else:
site_copy.pop("default_preset", None)
persisted_sites[domain] = site_copy
# 构建完整配置(包含 _global)
full_config = {
"_global": self.global_config.to_dict(),
**persisted_sites
}
local_snapshot = self._snapshot_local_site_overrides_locked()
if local_snapshot is None:
self._global_default_presets, self._local_default_presets = default_maps_snapshot
return False
if not self._save_local_site_overrides():
self._restore_local_site_overrides_locked(local_snapshot)
self._global_default_presets, self._local_default_presets = default_maps_snapshot
return False
local_overrides_written = True
written = self.site_store.save(full_config)
self.last_signature = self.site_store.signature()
self.last_mtime = time.time()
self._cache.invalidate()
logger.info(f"配置已保存: {self.sites_dir}(重写 {len(written)} 个站点文件)")
return True
except Exception as e:
logger.error(f"保存配置失败: {e}")
if local_overrides_written:
self._restore_local_site_overrides_locked(local_snapshot)
self._global_default_presets, self._local_default_presets = default_maps_snapshot
return False
def _migrate_loaded_config(self):
"""Run migrations that apply to freshly loaded site config data."""
self._migrate_to_presets()
self.migrate_site_configs()
self._migrate_site_advanced_to_presets()
self._cleanup_preset_residuals()
try:
from app.services.arena_model_catalog import migrate_and_cleanup_sites_model_catalog
migrate_and_cleanup_sites_model_catalog(self)
except Exception as exc:
logger.warning(f"Arena 模型目录自动迁移失败: {exc}")
__all__ = ['ConfigEngine', 'ConfigConstants', 'DEFAULT_WORKFLOW', 'DEFAULT_PRESET_NAME']