Initial commit

This commit is contained in:
Alex committed 2026-11-02 22:54:14 +08:00
commit 57fc3525eb
7311 files changed
+1291398

No files matched your search

+298
View File
@@ -0,0 +1,298 @@
"""
app/services/config/engine.py - 配置引擎主类
职责:
- 配置文件读写
- 站点配置管理
- 配置缓存与热重载
- 图片配置、提取器管理
"""
import json
import os
import copy
import re
import threading
import time
from app.core.config import get_logger
from typing import Dict, Optional, List, Any, Set, Union, Callable
from urllib.parse import urljoin
from app.core.parsers import ParserRegistry
from app.models.schemas import (
SiteConfig,
WorkflowStep,
SelectorDefinition,
ADVANCED_FIELDS,
PRESET_ADVANCED_FIELDS,
SITE_ADVANCED_FIELDS,
get_default_image_extraction_config,
get_default_file_paste_config,
get_default_prompt_padding_config,
get_default_site_advanced_config,
get_default_attachment_monitor_config,
get_default_send_confirmation_config,
get_modality_policy,
get_enabled_modalities,
is_modality_enabled,
normalize_modalities_config,
normalize_modality_policy,
)
from app.services.extractor_manager import extractor_manager
from app.services.parser_manager import parser_manager
from app.utils.site_rules import derive_site_card_id, get_site_rule
from app.utils.site_discovery import automatic_discovery_allowed, specific_chat_selectors, selectors_match_chat_html
from app.core.request_transport import (
get_default_request_transport_config,
normalize_request_transport_config,
)
from .cache import ConfigCache
from .site_store import SiteStore
from .managers import GlobalConfigManager, ImagePresetsManager
from .processors import HTMLCleaner, SelectorValidator, AIAnalyzer
# R2-3:模块级定义迁到 engine_common,这里重新导出以保持兼容(外部与测试都从 engine 导入这些名字)
from .engine_common import ( # noqa: E402,F401
ConfigConstants,
DEFAULT_PRESET_NAME,
DEFAULT_WORKFLOW,
PRESET_FIELDS,
_MISSING,
_SITE_STARTUP_JS_PATTERNS,
_coerce_bool,
_merge_config_patch,
get_default_network_config,
get_default_stream_config,
logger,
)
from .engine_local_overrides import ConfigLocalOverridesMixin # noqa: E402
from .engine_migrations import ConfigMigrationsMixin # noqa: E402
from .engine_presets import ConfigPresetsMixin # noqa: E402
from .engine_sites import ConfigSitesMixin # noqa: E402
from .engine_media import ConfigMediaMixin # noqa: E402
from .engine_stream import ConfigStreamMixin # noqa: E402
# ================= 配置引擎主类 =================
class ConfigEngine(
ConfigLocalOverridesMixin,
ConfigMigrationsMixin,
ConfigPresetsMixin,
ConfigSitesMixin,
ConfigMediaMixin,
ConfigStreamMixin,
):
"""配置引擎主类"""
def __init__(self):
self.config_file = ConfigConstants.CONFIG_FILE # 旧版单文件,仅用于自动迁移
self.sites_dir = ConfigConstants.SITES_DIR
self.site_store = SiteStore(self.sites_dir, self.config_file)
self.local_sites_file = ConfigConstants.SITES_LOCAL_FILE
self._io_lock = threading.RLock()
self.last_mtime = 0.0
self.last_signature: tuple = ()
self.last_local_mtime = 0.0
self.sites: Dict[str, SiteConfig] = {}
self._global_default_presets: Dict[str, str] = {}
self._local_default_presets: Dict[str, str] = {}
self._local_sites_payload: Dict[str, Any] = {}
self._cache = ConfigCache(ttl=5.0)
# 子管理器
self.global_config = GlobalConfigManager()
self.image_presets = ImagePresetsManager(ConfigConstants.IMAGE_PRESETS_FILE)
# 加载配置
self._load_config()
self._migrate_global_commands()
# 处理器
self.html_cleaner = HTMLCleaner()
self.validator = SelectorValidator(self.global_config.get_fallback_selectors())
self.ai_analyzer = AIAnalyzer(self.global_config)
# 迁移旧配置(顺序重要:先转预设格式,再补缺失字段,最后清理残留)
self._migrate_loaded_config()
self._apply_local_site_overrides()
logger.debug(f"配置引擎已初始化,已加载 {len(self.sites)} 个站点配置")
def _load_config(self):
with self._io_lock:
return self._load_config_locked()
def _load_config_locked(self):
"""初始化加载站点配置(config/sites/ 目录;发现旧版 sites.json 时自动迁移)"""
try:
data = self.site_store.load()
self.last_signature = self.site_store.signature()
if data is None:
logger.info(f"站点配置目录 {self.sites_dir} 为空,将在首次保存时创建")
self._apply_local_site_overrides()
return
self.last_mtime = time.time()
# 提取并加载 _global;缺失时重置为默认全局配置。
global_section = data.pop("_global", {})
self.global_config.load(global_section if isinstance(global_section, dict) else {})
# 过滤内部键
self.sites = {
k: v for k, v in data.items()
if not k.startswith('_')
}
self._refresh_global_default_presets_from_sites()
self._apply_local_site_overrides()
logger.debug(f"已加载站点配置: {self.sites_dir}({len(self.sites)} 个站点)")
except Exception as e:
logger.error(f"加载配置失败: {e}")
def refresh_if_changed(self):
"""检查文件是否变化,如果变化则重载"""
if not self.site_store.exists() and not os.path.exists(self.local_sites_file):
return
try:
current_signature = self.site_store.signature()
current_local_mtime = os.path.getmtime(self.local_sites_file) if os.path.exists(self.local_sites_file) else 0.0
if current_signature != self.last_signature or current_local_mtime != self.last_local_mtime:
logger.debug("⚡ 检测到站点配置变化,热重载")
self.reload_config()
except Exception as e:
logger.error(f"检查文件变化失败: {e}")
def reload_config(self):
with self._io_lock:
return self._reload_config_locked()
def _reload_config_locked(self):
"""重新加载配置(Hot Reload)"""
self._cache.invalidate()
if not self.site_store.exists():
logger.warning("重载失败:站点配置目录为空")
return
try:
previous = {"_global": self.global_config.to_dict(), **self.sites}
data = self.site_store.load(previous=previous)
if data is None:
logger.warning("重载失败:站点配置目录为空")
return
signature = self.site_store.signature()
# 提取并加载 _global;缺失时重置为默认全局配置。
global_section = data.pop("_global", {})
self.global_config.load(global_section if isinstance(global_section, dict) else {})
self.validator.fallback_selectors = self.global_config.get_fallback_selectors()
# 过滤内部键
self.sites = {
k: v for k, v in data.items()
if not k.startswith('_')
}
self.last_signature = signature
self.last_mtime = time.time()
self._refresh_global_default_presets_from_sites()
self._migrate_loaded_config()
self._apply_local_site_overrides()
logger.debug(f"✅ 配置已热重载 (Sites: {len(self.sites)})")
except json.JSONDecodeError as e:
logger.error(f"❌ 重载配置失败(JSON格式错误),保留旧配置: {e}")
except Exception as e:
logger.error(f"❌ 重载配置失败: {e}")
def save_config(self):
"""公开的保存方法(供 API 调用)"""
return self._save_config()
def _save_config(self) -> bool:
with self._io_lock:
return self._save_config_locked()
def _save_config_locked(self) -> bool:
"""保存站点配置(只重写有变化的站点文件,每个文件原子替换)"""
local_snapshot: Optional[tuple[bool, bytes, Dict[str, Any], Dict[str, str]]] = None
local_overrides_written = False
default_maps_snapshot = (
copy.deepcopy(self._global_default_presets),
copy.deepcopy(self._local_default_presets),
)
try:
self._prune_default_preset_maps()
persisted_sites = {}
for domain, site in self.sites.items():
if domain.startswith('_') or not isinstance(site, dict):
continue
site_copy = copy.deepcopy(site)
from app.utils.site_url import route_domain_matches
if route_domain_matches("arena.ai", domain):
try:
from app.services.arena_model_catalog import load_arena_model_catalog_data
cat_data = load_arena_model_catalog_data()
d_key = str(domain).strip().lower().strip(".")
domain_cat = cat_data.get(d_key, {})
except Exception:
domain_cat = None
presets_dict = site_copy.get("presets")
if isinstance(presets_dict, dict):
for p_key, p_val in presets_dict.items():
if isinstance(p_val, dict) and "model_catalog" in p_val:
if domain_cat is not None and str(p_key).strip() in domain_cat:
p_val.pop("model_catalog", None)
# 同步清理内存中的残留
mem_presets = site.get("presets")
if isinstance(mem_presets, dict):
for p_key, p_val in mem_presets.items():
if isinstance(p_val, dict) and "model_catalog" in p_val:
if domain_cat is not None and str(p_key).strip() in domain_cat:
p_val.pop("model_catalog", None)
persisted_default = self._get_persisted_default_preset(domain, site_copy)
if persisted_default:
site_copy["default_preset"] = persisted_default
else:
site_copy.pop("default_preset", None)
persisted_sites[domain] = site_copy
# 构建完整配置(包含 _global)
full_config = {
"_global": self.global_config.to_dict(),
**persisted_sites
}
local_snapshot = self._snapshot_local_site_overrides_locked()
if local_snapshot is None:
self._global_default_presets, self._local_default_presets = default_maps_snapshot
return False
if not self._save_local_site_overrides():
self._restore_local_site_overrides_locked(local_snapshot)
self._global_default_presets, self._local_default_presets = default_maps_snapshot
return False
local_overrides_written = True
written = self.site_store.save(full_config)
self.last_signature = self.site_store.signature()
self.last_mtime = time.time()
self._cache.invalidate()
logger.info(f"配置已保存: {self.sites_dir}(重写 {len(written)} 个站点文件)")
return True
except Exception as e:
logger.error(f"保存配置失败: {e}")
if local_overrides_written:
self._restore_local_site_overrides_locked(local_snapshot)
self._global_default_presets, self._local_default_presets = default_maps_snapshot
return False
def _migrate_loaded_config(self):
"""Run migrations that apply to freshly loaded site config data."""
self._migrate_to_presets()
self.migrate_site_configs()
self._migrate_site_advanced_to_presets()
self._cleanup_preset_residuals()
try:
from app.services.arena_model_catalog import migrate_and_cleanup_sites_model_catalog
migrate_and_cleanup_sites_model_catalog(self)
except Exception as exc:
logger.warning(f"Arena 模型目录自动迁移失败: {exc}")
__all__ = ['ConfigEngine', 'ConfigConstants', 'DEFAULT_WORKFLOW', 'DEFAULT_PRESET_NAME']