Initial commit

This commit is contained in:
Alex committed 2026-11-02 22:54:14 +08:00
commit 57fc3525eb
7311 files changed
+1291398

No files matched your search

+13
View File
@@ -0,0 +1,13 @@
"""
app/services/config - 配置管理模块
职责:
- 导出配置引擎单例(保持向后兼容)
"""
from .engine import ConfigEngine, ConfigConstants, DEFAULT_WORKFLOW
# 创建单例
config_engine = ConfigEngine()
__all__ = ['config_engine', 'ConfigEngine', 'ConfigConstants', 'DEFAULT_WORKFLOW']
+48
View File
@@ -0,0 +1,48 @@
"""
app/services/config/cache.py - lightweight in-memory cache for config reads.
"""
import threading
import time
from typing import Any, Dict, Tuple
_CACHE_MISS = object()
class ConfigCache:
"""Small TTL cache used only for derived config read results."""
def __init__(self, ttl: float = 5.0):
try:
ttl_value = float(ttl)
except (TypeError, ValueError):
ttl_value = 5.0
self._ttl = max(0.0, ttl_value)
self._cache: Dict[str, Tuple[Any, float]] = {}
self._lock = threading.RLock()
def get(self, key: str, default: Any = _CACHE_MISS) -> Any:
now = time.monotonic()
with self._lock:
entry = self._cache.get(key)
if entry is None:
return default
value, created_at = entry
if self._ttl and now - created_at > self._ttl:
self._cache.pop(key, None)
return default
return value
def set(self, key: str, value: Any) -> None:
with self._lock:
self._cache[key] = (value, time.monotonic())
def invalidate(self, key: str = None) -> None:
with self._lock:
if key is None:
self._cache.clear()
else:
self._cache.pop(key, None)
+298
View File
@@ -0,0 +1,298 @@
"""
app/services/config/engine.py - 配置引擎主类
职责:
- 配置文件读写
- 站点配置管理
- 配置缓存与热重载
- 图片配置、提取器管理
"""
import json
import os
import copy
import re
import threading
import time
from app.core.config import get_logger
from typing import Dict, Optional, List, Any, Set, Union, Callable
from urllib.parse import urljoin
from app.core.parsers import ParserRegistry
from app.models.schemas import (
SiteConfig,
WorkflowStep,
SelectorDefinition,
ADVANCED_FIELDS,
PRESET_ADVANCED_FIELDS,
SITE_ADVANCED_FIELDS,
get_default_image_extraction_config,
get_default_file_paste_config,
get_default_prompt_padding_config,
get_default_site_advanced_config,
get_default_attachment_monitor_config,
get_default_send_confirmation_config,
get_modality_policy,
get_enabled_modalities,
is_modality_enabled,
normalize_modalities_config,
normalize_modality_policy,
)
from app.services.extractor_manager import extractor_manager
from app.services.parser_manager import parser_manager
from app.utils.site_rules import derive_site_card_id, get_site_rule
from app.utils.site_discovery import automatic_discovery_allowed, specific_chat_selectors, selectors_match_chat_html
from app.core.request_transport import (
get_default_request_transport_config,
normalize_request_transport_config,
)
from .cache import ConfigCache
from .site_store import SiteStore
from .managers import GlobalConfigManager, ImagePresetsManager
from .processors import HTMLCleaner, SelectorValidator, AIAnalyzer
# R2-3:模块级定义迁到 engine_common,这里重新导出以保持兼容(外部与测试都从 engine 导入这些名字)
from .engine_common import ( # noqa: E402,F401
ConfigConstants,
DEFAULT_PRESET_NAME,
DEFAULT_WORKFLOW,
PRESET_FIELDS,
_MISSING,
_SITE_STARTUP_JS_PATTERNS,
_coerce_bool,
_merge_config_patch,
get_default_network_config,
get_default_stream_config,
logger,
)
from .engine_local_overrides import ConfigLocalOverridesMixin # noqa: E402
from .engine_migrations import ConfigMigrationsMixin # noqa: E402
from .engine_presets import ConfigPresetsMixin # noqa: E402
from .engine_sites import ConfigSitesMixin # noqa: E402
from .engine_media import ConfigMediaMixin # noqa: E402
from .engine_stream import ConfigStreamMixin # noqa: E402
# ================= 配置引擎主类 =================
class ConfigEngine(
ConfigLocalOverridesMixin,
ConfigMigrationsMixin,
ConfigPresetsMixin,
ConfigSitesMixin,
ConfigMediaMixin,
ConfigStreamMixin,
):
"""配置引擎主类"""
def __init__(self):
self.config_file = ConfigConstants.CONFIG_FILE # 旧版单文件,仅用于自动迁移
self.sites_dir = ConfigConstants.SITES_DIR
self.site_store = SiteStore(self.sites_dir, self.config_file)
self.local_sites_file = ConfigConstants.SITES_LOCAL_FILE
self._io_lock = threading.RLock()
self.last_mtime = 0.0
self.last_signature: tuple = ()
self.last_local_mtime = 0.0
self.sites: Dict[str, SiteConfig] = {}
self._global_default_presets: Dict[str, str] = {}
self._local_default_presets: Dict[str, str] = {}
self._local_sites_payload: Dict[str, Any] = {}
self._cache = ConfigCache(ttl=5.0)
# 子管理器
self.global_config = GlobalConfigManager()
self.image_presets = ImagePresetsManager(ConfigConstants.IMAGE_PRESETS_FILE)
# 加载配置
self._load_config()
self._migrate_global_commands()
# 处理器
self.html_cleaner = HTMLCleaner()
self.validator = SelectorValidator(self.global_config.get_fallback_selectors())
self.ai_analyzer = AIAnalyzer(self.global_config)
# 迁移旧配置(顺序重要:先转预设格式,再补缺失字段,最后清理残留)
self._migrate_loaded_config()
self._apply_local_site_overrides()
logger.debug(f"配置引擎已初始化,已加载 {len(self.sites)} 个站点配置")
def _load_config(self):
with self._io_lock:
return self._load_config_locked()
def _load_config_locked(self):
"""初始化加载站点配置(config/sites/ 目录;发现旧版 sites.json 时自动迁移)"""
try:
data = self.site_store.load()
self.last_signature = self.site_store.signature()
if data is None:
logger.info(f"站点配置目录 {self.sites_dir} 为空,将在首次保存时创建")
self._apply_local_site_overrides()
return
self.last_mtime = time.time()
# 提取并加载 _global;缺失时重置为默认全局配置。
global_section = data.pop("_global", {})
self.global_config.load(global_section if isinstance(global_section, dict) else {})
# 过滤内部键
self.sites = {
k: v for k, v in data.items()
if not k.startswith('_')
}
self._refresh_global_default_presets_from_sites()
self._apply_local_site_overrides()
logger.debug(f"已加载站点配置: {self.sites_dir}({len(self.sites)} 个站点)")
except Exception as e:
logger.error(f"加载配置失败: {e}")
def refresh_if_changed(self):
"""检查文件是否变化,如果变化则重载"""
if not self.site_store.exists() and not os.path.exists(self.local_sites_file):
return
try:
current_signature = self.site_store.signature()
current_local_mtime = os.path.getmtime(self.local_sites_file) if os.path.exists(self.local_sites_file) else 0.0
if current_signature != self.last_signature or current_local_mtime != self.last_local_mtime:
logger.debug("⚡ 检测到站点配置变化,热重载")
self.reload_config()
except Exception as e:
logger.error(f"检查文件变化失败: {e}")
def reload_config(self):
with self._io_lock:
return self._reload_config_locked()
def _reload_config_locked(self):
"""重新加载配置(Hot Reload)"""
self._cache.invalidate()
if not self.site_store.exists():
logger.warning("重载失败:站点配置目录为空")
return
try:
previous = {"_global": self.global_config.to_dict(), **self.sites}
data = self.site_store.load(previous=previous)
if data is None:
logger.warning("重载失败:站点配置目录为空")
return
signature = self.site_store.signature()
# 提取并加载 _global;缺失时重置为默认全局配置。
global_section = data.pop("_global", {})
self.global_config.load(global_section if isinstance(global_section, dict) else {})
self.validator.fallback_selectors = self.global_config.get_fallback_selectors()
# 过滤内部键
self.sites = {
k: v for k, v in data.items()
if not k.startswith('_')
}
self.last_signature = signature
self.last_mtime = time.time()
self._refresh_global_default_presets_from_sites()
self._migrate_loaded_config()
self._apply_local_site_overrides()
logger.debug(f"✅ 配置已热重载 (Sites: {len(self.sites)})")
except json.JSONDecodeError as e:
logger.error(f"❌ 重载配置失败(JSON格式错误),保留旧配置: {e}")
except Exception as e:
logger.error(f"❌ 重载配置失败: {e}")
def save_config(self):
"""公开的保存方法(供 API 调用)"""
return self._save_config()
def _save_config(self) -> bool:
with self._io_lock:
return self._save_config_locked()
def _save_config_locked(self) -> bool:
"""保存站点配置(只重写有变化的站点文件,每个文件原子替换)"""
local_snapshot: Optional[tuple[bool, bytes, Dict[str, Any], Dict[str, str]]] = None
local_overrides_written = False
default_maps_snapshot = (
copy.deepcopy(self._global_default_presets),
copy.deepcopy(self._local_default_presets),
)
try:
self._prune_default_preset_maps()
persisted_sites = {}
for domain, site in self.sites.items():
if domain.startswith('_') or not isinstance(site, dict):
continue
site_copy = copy.deepcopy(site)
from app.utils.site_url import route_domain_matches
if route_domain_matches("arena.ai", domain):
try:
from app.services.arena_model_catalog import load_arena_model_catalog_data
cat_data = load_arena_model_catalog_data()
d_key = str(domain).strip().lower().strip(".")
domain_cat = cat_data.get(d_key, {})
except Exception:
domain_cat = None
presets_dict = site_copy.get("presets")
if isinstance(presets_dict, dict):
for p_key, p_val in presets_dict.items():
if isinstance(p_val, dict) and "model_catalog" in p_val:
if domain_cat is not None and str(p_key).strip() in domain_cat:
p_val.pop("model_catalog", None)
# 同步清理内存中的残留
mem_presets = site.get("presets")
if isinstance(mem_presets, dict):
for p_key, p_val in mem_presets.items():
if isinstance(p_val, dict) and "model_catalog" in p_val:
if domain_cat is not None and str(p_key).strip() in domain_cat:
p_val.pop("model_catalog", None)
persisted_default = self._get_persisted_default_preset(domain, site_copy)
if persisted_default:
site_copy["default_preset"] = persisted_default
else:
site_copy.pop("default_preset", None)
persisted_sites[domain] = site_copy
# 构建完整配置(包含 _global)
full_config = {
"_global": self.global_config.to_dict(),
**persisted_sites
}
local_snapshot = self._snapshot_local_site_overrides_locked()
if local_snapshot is None:
self._global_default_presets, self._local_default_presets = default_maps_snapshot
return False
if not self._save_local_site_overrides():
self._restore_local_site_overrides_locked(local_snapshot)
self._global_default_presets, self._local_default_presets = default_maps_snapshot
return False
local_overrides_written = True
written = self.site_store.save(full_config)
self.last_signature = self.site_store.signature()
self.last_mtime = time.time()
self._cache.invalidate()
logger.info(f"配置已保存: {self.sites_dir}(重写 {len(written)} 个站点文件)")
return True
except Exception as e:
logger.error(f"保存配置失败: {e}")
if local_overrides_written:
self._restore_local_site_overrides_locked(local_snapshot)
self._global_default_presets, self._local_default_presets = default_maps_snapshot
return False
def _migrate_loaded_config(self):
"""Run migrations that apply to freshly loaded site config data."""
self._migrate_to_presets()
self.migrate_site_configs()
self._migrate_site_advanced_to_presets()
self._cleanup_preset_residuals()
try:
from app.services.arena_model_catalog import migrate_and_cleanup_sites_model_catalog
migrate_and_cleanup_sites_model_catalog(self)
except Exception as exc:
logger.warning(f"Arena 模型目录自动迁移失败: {exc}")
__all__ = ['ConfigEngine', 'ConfigConstants', 'DEFAULT_WORKFLOW', 'DEFAULT_PRESET_NAME']
+128
View File
@@ -0,0 +1,128 @@
"""ConfigEngine 及其 mixin 共用的模块级定义(R2-3 拆分时从 engine.py 原样迁出,engine.py 仍重新导出)。"""
import os
import copy
import re
from app.core.config import get_logger
from typing import Dict, List, Any
from app.models.schemas import (
WorkflowStep,
get_default_attachment_monitor_config,
get_default_send_confirmation_config,
)
from app.core.request_transport import get_default_request_transport_config
logger = get_logger("CFG_ENG")
# ================= 常量配置 =================
class ConfigConstants:
"""配置引擎常量"""
_PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
# R1-2:站点配置的事实来源是 config/sites/ 目录(每站点一个文件);CONFIG_FILE 只作为旧版单文件的迁移来源
CONFIG_FILE = os.getenv("SITES_CONFIG_FILE", os.path.join(_PROJECT_ROOT, "config", "sites.json"))
SITES_DIR = os.getenv("SITES_CONFIG_DIR", os.path.join(_PROJECT_ROOT, "config", "sites"))
SITES_LOCAL_FILE = os.getenv("SITES_LOCAL_FILE", os.path.join(_PROJECT_ROOT, "config", "sites.local.json"))
COMMANDS_FILE = os.getenv("COMMANDS_CONFIG_FILE", os.path.join(_PROJECT_ROOT, "config", "commands.json"))
COMMANDS_LOCAL_FILE = os.getenv("COMMANDS_LOCAL_FILE", os.path.join(_PROJECT_ROOT, "config", "commands.local.json"))
IMAGE_PRESETS_FILE = os.path.join(_PROJECT_ROOT, "config", "image_presets.json")
MAX_HTML_CHARS = int(os.getenv("MAX_HTML_CHARS", "120000"))
TEXT_TRUNCATE_LENGTH = 80
AI_MAX_RETRIES = 3
AI_RETRY_BASE_DELAY = 1.0
AI_RETRY_MAX_DELAY = 10.0
AI_REQUEST_TIMEOUT = 120
_MISSING = object()
def _coerce_bool(value: Any, default: bool = False) -> bool:
"""Parse bool-like config values without treating every non-empty string as true."""
if isinstance(value, bool):
return value
if value is None:
return default
if isinstance(value, str):
lowered = value.strip().lower()
if lowered in {"1", "true", "yes", "y", "on"}:
return True
if lowered in {"0", "false", "no", "n", "off"}:
return False
return default
if isinstance(value, (int, float)):
return value != 0
return bool(value)
# ================= 预设常量 =================
DEFAULT_PRESET_NAME = "主预设"
# 预设内包含的配置字段(用于迁移和校验)
PRESET_FIELDS = [
"selectors", "workflow", "stream_config",
"image_extraction", "file_paste", "prompt_padding", "stealth",
"extractor_id", "extractor_verified"
]
# 默认工作流
DEFAULT_WORKFLOW: List[WorkflowStep] = [
{"action": "CLICK", "target": "new_chat_btn", "optional": True, "value": None},
{"action": "WAIT", "target": "", "optional": False, "value": "0.5"},
{"action": "FILL_INPUT", "target": "input_box", "optional": False, "value": None},
{"action": "CLICK", "target": "send_btn", "optional": True, "value": None},
{"action": "KEY_PRESS", "target": "Enter", "optional": True, "value": None},
{"action": "STREAM_WAIT", "target": "result_container", "optional": False, "value": None}
]
def get_default_stream_config() -> Dict[str, Any]:
"""获取默认流式配置"""
return {
"mode": "dom", # dom / network
"request_transport": get_default_request_transport_config(),
"hard_timeout": 300, # 全局硬超时(秒)
"send_confirmation": get_default_send_confirmation_config(),
"attachment_monitor": get_default_attachment_monitor_config(),
# 网络监听配置(可选)
"network": None
}
def get_default_network_config() -> Dict[str, Any]:
"""获取默认网络监听配置"""
return {
"listen_pattern": "", # URL 匹配模式(必填)
"stream_match_mode": "keyword", # 流目标匹配模式(keyword / regex)
"stream_match_pattern": "", # 流目标匹配表达式(为空时回退到 listen_pattern)
"parser": "", # 解析器 ID(必填)
"first_response_timeout": 300.0, # 等待首个匹配目标流量的最大时间
"silence_threshold": 3.0, # 静默超时(秒)
"response_interval": 0.5 # 轮询间隔(秒)
}
def _merge_config_patch(base: Any, patch: Any) -> Any:
"""Deep-merge object patches while letting scalars/lists replace existing values."""
if not isinstance(base, dict) or not isinstance(patch, dict):
return copy.deepcopy(patch)
merged = dict(base)
for key, value in patch.items():
existing = base.get(key)
if isinstance(value, dict) and isinstance(existing, dict):
merged[key] = _merge_config_patch(existing, value)
else:
merged[key] = copy.deepcopy(value)
return merged
_SITE_STARTUP_JS_PATTERNS = (
re.compile(r"""location\.(?:assign|replace)\(\s*['"]([^'"]+)['"]\s*\)""", re.IGNORECASE),
re.compile(r"""location\.href\s*=\s*['"]([^'"]+)['"]""", re.IGNORECASE),
)
@@ -0,0 +1,226 @@
"""ConfigEngine 的一部分(R2-3 拆分):本地站点覆盖(sites.local.json)与默认预设状态。
方法原样从 engine.py 迁出,行为不变;通过 mixin 组合回 ConfigEngine,
实例状态都在 ConfigEngine.__init__(engine.py)里初始化。
"""
import json
import os
import copy
from typing import Dict, Optional, Any
from .engine_common import logger
class ConfigLocalOverridesMixin:
"""本地站点覆盖(sites.local.json)与默认预设状态"""
def _load_local_site_overrides(self) -> Dict[str, str]:
with self._io_lock:
return self._load_local_site_overrides_locked()
def _load_local_site_overrides_locked(self) -> Dict[str, str]:
"""加载本地站点覆盖配置,并保留未识别字段。"""
if not os.path.exists(self.local_sites_file):
self.last_local_mtime = 0.0
self._local_sites_payload = {}
return {}
try:
with open(self.local_sites_file, "r", encoding="utf-8-sig") as f:
data = json.load(f)
self.last_local_mtime = os.path.getmtime(self.local_sites_file)
self._local_sites_payload = data if isinstance(data, dict) else {}
defaults = self._local_sites_payload.get("default_presets", {})
if not isinstance(defaults, dict):
return {}
return {
str(domain).strip(): str(preset).strip()
for domain, preset in defaults.items()
if str(domain).strip() and str(preset).strip()
}
except json.JSONDecodeError as e:
logger.error(f"本地站点覆盖配置格式错误: {e}")
return copy.deepcopy(self._local_default_presets or {})
except Exception as e:
logger.error(f"加载本地站点覆盖配置失败: {e}")
return copy.deepcopy(self._local_default_presets or {})
def _prune_default_preset_maps(self) -> None:
active_domains = {
domain for domain, site in self.sites.items()
if not domain.startswith('_') and isinstance(site, dict)
}
self._global_default_presets = {
domain: preset
for domain, preset in self._global_default_presets.items()
if domain in active_domains and str(preset or "").strip()
}
self._local_default_presets = {
domain: preset
for domain, preset in self._local_default_presets.items()
if domain in active_domains and str(preset or "").strip()
}
def _refresh_global_default_presets_from_sites(self) -> None:
next_defaults: Dict[str, str] = {}
for domain, site in self.sites.items():
if domain.startswith('_') or not isinstance(site, dict):
continue
resolved = self._resolve_default_preset_name(site)
if resolved:
next_defaults[domain] = resolved
self._global_default_presets = next_defaults
def _get_persisted_default_preset(self, domain: str, site: Dict[str, Any]) -> Optional[str]:
presets = site.get("presets", {})
if not isinstance(presets, dict) or not presets:
self._global_default_presets.pop(domain, None)
return None
candidate = str(self._global_default_presets.get(domain, "") or "").strip()
if candidate and candidate in presets:
return candidate
resolved = self._resolve_default_preset_name(site)
if resolved:
self._global_default_presets[domain] = resolved
return resolved
self._global_default_presets.pop(domain, None)
return None
def _sync_site_default_preset_state(self, domain: str, site: Dict[str, Any]) -> bool:
presets = site.get("presets", {})
if not isinstance(presets, dict) or not presets:
self._global_default_presets.pop(domain, None)
self._local_default_presets.pop(domain, None)
if "default_preset" in site:
del site["default_preset"]
return True
return False
persisted_default = self._get_persisted_default_preset(domain, site)
local_default = str(self._local_default_presets.get(domain, "") or "").strip()
if local_default and local_default not in presets:
self._local_default_presets.pop(domain, None)
local_default = ""
if local_default and persisted_default and local_default == persisted_default:
self._local_default_presets.pop(domain, None)
local_default = ""
effective_default = local_default or persisted_default
if not effective_default:
effective_default = self._resolve_default_preset_name(site)
if effective_default and site.get("default_preset") != effective_default:
site["default_preset"] = effective_default
return True
if not effective_default and "default_preset" in site:
del site["default_preset"]
return True
return False
def _apply_local_site_overrides(self):
"""将本地默认预设选择覆盖到当前站点配置。"""
self._local_default_presets = self._load_local_site_overrides()
applied = 0
for domain, site in self.sites.items():
if not isinstance(site, dict):
continue
self._sync_site_default_preset_state(domain, site)
local_default = self._local_default_presets.get(domain)
if local_default and site.get("default_preset") == local_default:
applied += 1
if applied > 0:
logger.debug(f"已应用 {applied} 个本地默认预设覆盖")
def _save_local_site_overrides(self) -> bool:
with self._io_lock:
return self._save_local_site_overrides_locked()
def _save_local_site_overrides_locked(self) -> bool:
"""保存本地站点覆盖配置。"""
tmp_file = self.local_sites_file + ".tmp"
self._prune_default_preset_maps()
defaults = {}
for domain, preset_name in self._local_default_presets.items():
site = self.sites.get(domain)
if not isinstance(site, dict):
continue
presets = site.get("presets", {})
preset_value = str(preset_name or "").strip()
if isinstance(presets, dict) and preset_value and preset_value in presets:
defaults[domain] = preset_value
payload = dict(self._local_sites_payload or {})
if os.path.exists(self.local_sites_file):
try:
with open(self.local_sites_file, "r", encoding="utf-8-sig") as f:
latest_payload = json.load(f)
if isinstance(latest_payload, dict):
payload.update(latest_payload)
except Exception:
pass
payload["default_presets"] = defaults
try:
os.makedirs(os.path.dirname(self.local_sites_file), exist_ok=True)
with open(tmp_file, "w", encoding="utf-8", newline="\n") as f:
json.dump(payload, f, indent=2, ensure_ascii=False)
f.flush()
os.fsync(f.fileno())
os.replace(tmp_file, self.local_sites_file)
try:
self.last_local_mtime = os.path.getmtime(self.local_sites_file)
except Exception as e:
logger.warning(f"本地站点覆盖已保存但更新时间戳失败: {e}")
self._local_sites_payload = payload
return True
except Exception as e:
logger.error(f"保存本地站点覆盖配置失败: {e}")
try:
if os.path.exists(tmp_file):
os.remove(tmp_file)
except Exception:
pass
return False
def _snapshot_local_site_overrides_locked(self) -> Optional[tuple[bool, bytes, Dict[str, Any], Dict[str, str]]]:
try:
payload_snapshot = copy.deepcopy(self._local_sites_payload or {})
defaults_snapshot = copy.deepcopy(self._local_default_presets or {})
if not os.path.exists(self.local_sites_file):
return (False, b"", payload_snapshot, defaults_snapshot)
with open(self.local_sites_file, "rb") as f:
return (True, f.read(), payload_snapshot, defaults_snapshot)
except Exception as e:
logger.error(f"读取本地站点覆盖快照失败: {e}")
return None
def _restore_local_site_overrides_locked(self, snapshot: Optional[tuple[bool, bytes, Dict[str, Any], Dict[str, str]]]) -> None:
if snapshot is None:
return
tmp_file = self.local_sites_file + ".restore.tmp"
existed, payload, payload_snapshot, defaults_snapshot = snapshot
try:
if existed:
os.makedirs(os.path.dirname(self.local_sites_file), exist_ok=True)
with open(tmp_file, "wb") as f:
f.write(payload)
f.flush()
os.fsync(f.fileno())
os.replace(tmp_file, self.local_sites_file)
elif os.path.exists(self.local_sites_file):
os.remove(self.local_sites_file)
try:
self.last_local_mtime = os.path.getmtime(self.local_sites_file) if os.path.exists(self.local_sites_file) else 0.0
except Exception as e:
logger.warning(f"本地站点覆盖已恢复但更新时间戳失败: {e}")
self._local_sites_payload = copy.deepcopy(payload_snapshot)
self._local_default_presets = copy.deepcopy(defaults_snapshot)
except Exception as e:
logger.error(f"恢复本地站点覆盖失败: {e}")
try:
if os.path.exists(tmp_file):
os.remove(tmp_file)
except Exception:
pass
+685
View File
@@ -0,0 +1,685 @@
"""ConfigEngine 的一部分(R2-3 拆分):图片提取、文件粘贴、提示词填充配置与图片预设。
方法原样从 engine.py 迁出,行为不变;通过 mixin 组合回 ConfigEngine,
实例状态都在 ConfigEngine.__init__(engine.py)里初始化。
"""
import copy
from typing import Dict, Optional, Any
from app.models.schemas import (
get_default_image_extraction_config,
get_default_file_paste_config,
get_default_prompt_padding_config,
get_default_attachment_monitor_config,
get_default_send_confirmation_config,
get_enabled_modalities,
is_modality_enabled,
normalize_modalities_config,
normalize_modality_policy,
)
from .engine_common import DEFAULT_PRESET_NAME, _coerce_bool, _merge_config_patch, logger
class ConfigMediaMixin:
"""图片提取、文件粘贴、提示词填充配置与图片预设"""
def get_site_image_config(self, domain: str, preset_name: str = None) -> Dict:
"""获取站点的图片提取配置"""
self.refresh_if_changed()
default_config = get_default_image_extraction_config()
data = self._get_site_data(domain, preset_name)
if data is None:
return copy.deepcopy(default_config)
image_config = data.get("image_extraction", {})
return self._validate_image_config(image_config)
def set_site_image_config(self, domain: str, config: Dict,
preset_name: str = None) -> bool:
"""设置站点的图片提取配置"""
self.refresh_if_changed()
data = self._get_site_data(domain, preset_name)
if data is None:
logger.warning(f"站点或预设不存在: {domain}/{preset_name}")
return False
current_config = self._validate_image_config(data.get("image_extraction", {}))
merged_config = _merge_config_patch(current_config, config if isinstance(config, dict) else {})
validated = self._validate_image_config(merged_config)
if not self._assign_preset_field_and_save(data, "image_extraction", validated):
return False
logger.info(f"站点 {domain} [{preset_name or DEFAULT_PRESET_NAME}] 多模态提取配置已更新")
return True
def _validate_image_config(self, config: Dict) -> Dict:
"""验证并规范化多模态提取配置"""
default = get_default_image_extraction_config()
result = copy.deepcopy(default)
if not config:
return result
raw_modalities = config.get("modalities")
if isinstance(raw_modalities, dict):
result["modalities"] = normalize_modalities_config(raw_modalities)
legacy_enabled = None
if "enabled" in config:
legacy_enabled = _coerce_bool(config.get("enabled"), False)
result["enabled"] = legacy_enabled
if (
legacy_enabled is not None
and not isinstance(raw_modalities, dict)
):
result["modalities"]["image"] = normalize_modality_policy("image", legacy_enabled)
elif legacy_enabled is not None and isinstance(raw_modalities, dict):
if not legacy_enabled:
for key in ("image", "audio", "video"):
result["modalities"][key] = normalize_modality_policy(key, False)
elif not get_enabled_modalities(result.get("modalities")):
result["modalities"]["image"] = normalize_modality_policy("image", True)
if "selector" in config and config["selector"]:
result["selector"] = str(config["selector"]).strip()
if not result["selector"]:
result["selector"] = "img"
if "audio_selector" in config and config["audio_selector"]:
result["audio_selector"] = str(config["audio_selector"]).strip()
if not result["audio_selector"]:
result["audio_selector"] = default["audio_selector"]
if "video_selector" in config and config["video_selector"]:
result["video_selector"] = str(config["video_selector"]).strip()
if not result["video_selector"]:
result["video_selector"] = default["video_selector"]
if "container_selector" in config:
val = config["container_selector"]
result["container_selector"] = str(val).strip() if val else None
if "final_target_strategy" in config:
val = str(config["final_target_strategy"] or "").strip().lower()
if val in ("container", "latest_reply", "latest_visual_reply"):
result["final_target_strategy"] = val
if "latest_visual_column" in config:
val = str(config["latest_visual_column"] or "").strip().lower()
if val in ("left", "right"):
result["latest_visual_column"] = val
if "allow_container_fallback" in config:
result["allow_container_fallback"] = _coerce_bool(config.get("allow_container_fallback"), False)
if "force_postprocess" in config:
result["force_postprocess"] = _coerce_bool(config.get("force_postprocess"), False)
if "arena_image_generation" in config:
result["arena_image_generation"] = _coerce_bool(
config.get("arena_image_generation"), False
)
if "direct_postprocess_modalities" in config:
raw_direct_modalities = config.get("direct_postprocess_modalities")
if isinstance(raw_direct_modalities, (list, tuple, set)):
allowed_direct_modalities = []
for item in raw_direct_modalities:
media_type = str(item or "").strip().lower()
if (
media_type in {"image", "audio", "video"}
and is_modality_enabled(result.get("modalities"), media_type)
and media_type not in allowed_direct_modalities
):
allowed_direct_modalities.append(media_type)
if allowed_direct_modalities:
result["direct_postprocess_modalities"] = allowed_direct_modalities
if "debounce_seconds" in config:
try:
val = float(config["debounce_seconds"])
result["debounce_seconds"] = max(0, min(val, 30))
except (ValueError, TypeError):
pass
if "wait_for_load" in config:
result["wait_for_load"] = _coerce_bool(config.get("wait_for_load"), True)
if "load_timeout_seconds" in config:
try:
val = float(config["load_timeout_seconds"])
result["load_timeout_seconds"] = max(1, min(val, 60))
except (ValueError, TypeError):
pass
if "download_blobs" in config:
result["download_blobs"] = _coerce_bool(config.get("download_blobs"), False)
if "max_size_mb" in config:
try:
val = int(config["max_size_mb"])
result["max_size_mb"] = max(1, min(val, 100))
except (ValueError, TypeError):
pass
if "canvas_export_mime" in config:
val = str(config["canvas_export_mime"] or "").strip().lower()
if val in {"image/jpeg", "image/webp", "image/png"}:
result["canvas_export_mime"] = val
if "canvas_export_quality" in config:
try:
val = float(config["canvas_export_quality"])
result["canvas_export_quality"] = max(0.1, min(val, 1.0))
except (ValueError, TypeError):
pass
if "src_allow_patterns" in config:
raw_patterns = config.get("src_allow_patterns")
if isinstance(raw_patterns, str):
raw_patterns = raw_patterns.replace("\r\n", "\n").replace(";", "\n").split("\n")
if isinstance(raw_patterns, (list, tuple, set)):
patterns = []
seen = set()
for item in raw_patterns:
text = str(item or "").strip()
if not text or text in seen:
continue
seen.add(text)
patterns.append(text)
result["src_allow_patterns"] = patterns
if "mode" in config:
val = str(config["mode"]).lower()
if val in ("all", "first", "last"):
result["mode"] = val
if "audio_capture_enabled" in config:
result["audio_capture_enabled"] = _coerce_bool(config.get("audio_capture_enabled"), False)
if "audio_capture_mute_playback" in config:
result["audio_capture_mute_playback"] = _coerce_bool(config.get("audio_capture_mute_playback"), False)
if "audio_capture_preload_enabled" in config:
result["audio_capture_preload_enabled"] = _coerce_bool(config.get("audio_capture_preload_enabled"), False)
if "audio_capture_reload_before_workflow" in config:
result["audio_capture_reload_before_workflow"] = _coerce_bool(config.get("audio_capture_reload_before_workflow"), False)
if "audio_capture_preserve_graph" in config:
result["audio_capture_preserve_graph"] = _coerce_bool(config.get("audio_capture_preserve_graph"), False)
if "audio_capture_terminal_settle_seconds" in config:
try:
val = float(config["audio_capture_terminal_settle_seconds"])
result["audio_capture_terminal_settle_seconds"] = max(0.0, min(val, 5.0))
except (ValueError, TypeError):
pass
if "audio_trigger_selector" in config:
result["audio_trigger_selector"] = str(config["audio_trigger_selector"] or "").strip()
if "audio_trigger_labels" in config:
raw_labels = config.get("audio_trigger_labels")
if isinstance(raw_labels, (list, tuple)):
labels = [
str(item).strip()
for item in raw_labels
if str(item).strip()
]
if labels:
result["audio_trigger_labels"] = labels
if "audio_capture_max_wait_seconds" in config:
try:
val = float(config["audio_capture_max_wait_seconds"])
result["audio_capture_max_wait_seconds"] = max(1.0, min(val, 120.0))
except (ValueError, TypeError):
pass
if "audio_capture_min_wait_seconds" in config:
try:
val = float(config["audio_capture_min_wait_seconds"])
result["audio_capture_min_wait_seconds"] = max(0.2, min(val, 30.0))
except (ValueError, TypeError):
pass
if "audio_capture_hard_max_wait_seconds" in config:
try:
val = float(config["audio_capture_hard_max_wait_seconds"])
result["audio_capture_hard_max_wait_seconds"] = max(1.0, min(val, 180.0))
except (ValueError, TypeError):
pass
if "audio_capture_estimated_chars_per_second" in config:
try:
val = float(config["audio_capture_estimated_chars_per_second"])
result["audio_capture_estimated_chars_per_second"] = max(1.0, min(val, 20.0))
except (ValueError, TypeError):
pass
if "audio_capture_wait_padding_seconds" in config:
try:
val = float(config["audio_capture_wait_padding_seconds"])
result["audio_capture_wait_padding_seconds"] = max(0.0, min(val, 10.0))
except (ValueError, TypeError):
pass
network_capture = dict(result.get("audio_network_capture") or {})
raw_network_capture = config.get("audio_network_capture")
if isinstance(raw_network_capture, dict):
if "enabled" in raw_network_capture:
network_capture["enabled"] = _coerce_bool(raw_network_capture.get("enabled"), False)
if "timeout_seconds" in raw_network_capture:
try:
val = float(raw_network_capture["timeout_seconds"])
network_capture["timeout_seconds"] = max(0.1, min(val, 15.0))
except (ValueError, TypeError):
pass
if "transport" in raw_network_capture:
val = str(raw_network_capture["transport"] or "").strip()
if val in {"page_websocket_probe"}:
network_capture["transport"] = val
if "extractor" in raw_network_capture:
val = str(raw_network_capture["extractor"] or "").strip()
if val in {"voicegenie_ogg_pages", "voicegenie_binary_stream"}:
network_capture["extractor"] = val
if "settle_seconds" in raw_network_capture:
try:
val = float(raw_network_capture["settle_seconds"])
network_capture["settle_seconds"] = max(0.05, min(val, 5.0))
except (ValueError, TypeError):
pass
if "url_patterns" in raw_network_capture:
raw_patterns = raw_network_capture.get("url_patterns")
if isinstance(raw_patterns, (list, tuple)):
patterns = [
str(item).strip()
for item in raw_patterns
if str(item).strip()
]
if patterns:
network_capture["url_patterns"] = patterns
if "max_payload_bytes" in raw_network_capture:
# 修复:此前漏处理该键,导致前端设置的单条载荷上限被静默重置为默认 10MB
try:
val = int(raw_network_capture["max_payload_bytes"])
network_capture["max_payload_bytes"] = max(65536, min(val, 104857600))
except (ValueError, TypeError):
pass
# 兼容旧平铺字段,最终统一收口到新对象
if "audio_network_capture_enabled" in config:
network_capture["enabled"] = _coerce_bool(config.get("audio_network_capture_enabled"), False)
if "audio_network_capture_timeout_seconds" in config:
try:
val = float(config["audio_network_capture_timeout_seconds"])
network_capture["timeout_seconds"] = max(0.1, min(val, 15.0))
except (ValueError, TypeError):
pass
if "audio_network_url_patterns" in config:
raw_patterns = config.get("audio_network_url_patterns")
if isinstance(raw_patterns, (list, tuple)):
patterns = [
str(item).strip()
for item in raw_patterns
if str(item).strip()
]
if patterns:
network_capture["url_patterns"] = patterns
result["audio_network_capture"] = network_capture
browser_tts_fallback = dict(result.get("audio_browser_tts_fallback") or {})
raw_browser_tts_fallback = config.get("audio_browser_tts_fallback")
if isinstance(raw_browser_tts_fallback, dict):
if "enabled" in raw_browser_tts_fallback:
browser_tts_fallback["enabled"] = _coerce_bool(raw_browser_tts_fallback.get("enabled"), False)
if "provider" in raw_browser_tts_fallback:
val = str(raw_browser_tts_fallback["provider"] or "").strip()
if val in {"doubao_samantha"}:
browser_tts_fallback["provider"] = val
if "speaker" in raw_browser_tts_fallback:
val = str(raw_browser_tts_fallback["speaker"] or "").strip()
if val:
browser_tts_fallback["speaker"] = val
if "speech_rate" in raw_browser_tts_fallback:
try:
val = int(raw_browser_tts_fallback["speech_rate"])
browser_tts_fallback["speech_rate"] = max(-100, min(val, 100))
except (ValueError, TypeError):
pass
if "pitch" in raw_browser_tts_fallback:
try:
val = int(raw_browser_tts_fallback["pitch"])
browser_tts_fallback["pitch"] = max(-100, min(val, 100))
except (ValueError, TypeError):
pass
if "format" in raw_browser_tts_fallback:
val = str(raw_browser_tts_fallback["format"] or "").strip().lower()
if val in {"aac"}:
browser_tts_fallback["format"] = val
if "timeout_seconds" in raw_browser_tts_fallback:
try:
val = float(raw_browser_tts_fallback["timeout_seconds"])
browser_tts_fallback["timeout_seconds"] = max(3.0, min(val, 120.0))
except (ValueError, TypeError):
pass
for key in (
"pc_version",
"aid",
"real_aid",
"language",
"device_platform",
"pkg_type",
"region",
"sys_region",
"use_olympus_account",
"samantha_web",
):
if key in raw_browser_tts_fallback:
val = str(raw_browser_tts_fallback[key] or "").strip()
if val:
browser_tts_fallback[key] = val
result["audio_browser_tts_fallback"] = browser_tts_fallback
if "audio_capture_poll_seconds" in config:
try:
val = float(config["audio_capture_poll_seconds"])
result["audio_capture_poll_seconds"] = max(0.05, min(val, 5.0))
except (ValueError, TypeError):
pass
if "audio_capture_silence_seconds" in config:
try:
val = float(config["audio_capture_silence_seconds"])
result["audio_capture_silence_seconds"] = max(0.2, min(val, 30.0))
except (ValueError, TypeError):
pass
if "audio_capture_activity_threshold" in config:
try:
val = float(config["audio_capture_activity_threshold"])
result["audio_capture_activity_threshold"] = max(0.0001, min(val, 0.2))
except (ValueError, TypeError):
pass
if "audio_capture_activity_silence_seconds" in config:
try:
val = float(config["audio_capture_activity_silence_seconds"])
result["audio_capture_activity_silence_seconds"] = max(0.2, min(val, 10.0))
except (ValueError, TypeError):
pass
result["enabled"] = bool(get_enabled_modalities(result.get("modalities")))
return result
def get_site_file_paste_config(self, domain: str, preset_name: str = None) -> dict:
"""获取站点的文件粘贴配置"""
self.refresh_if_changed()
default_config = get_default_file_paste_config()
data = self._get_site_data(domain, preset_name)
if data is None:
result = copy.deepcopy(default_config)
result["send_confirmation"] = get_default_send_confirmation_config()
result["attachment_monitor"] = get_default_attachment_monitor_config()
return result
return self._validate_file_paste_config(
data.get("file_paste", {}),
legacy_stream_config=data.get("stream_config"),
include_attachment_defaults=True,
)
def set_site_file_paste_config(self, domain: str, config: dict,
preset_name: str = None) -> bool:
"""设置站点的文件粘贴配置"""
self.refresh_if_changed()
data = self._get_site_data(domain, preset_name)
if data is None:
logger.warning(f"站点或预设不存在: {domain}/{preset_name}")
return False
current_config = self._validate_file_paste_config(
data.get("file_paste", {}),
legacy_stream_config=data.get("stream_config"),
)
merged_config = _merge_config_patch(current_config, config if isinstance(config, dict) else {})
validated = self._validate_file_paste_config(
merged_config,
legacy_stream_config=data.get("stream_config"),
)
if not self._assign_preset_field_and_save(data, "file_paste", validated):
return False
logger.info(f"站点 {domain} [{preset_name or DEFAULT_PRESET_NAME}] 文件粘贴配置已更新")
return True
def get_all_file_paste_configs(self) -> dict:
"""获取所有站点的文件粘贴配置(使用各站点当前默认预设)"""
self.refresh_if_changed()
result = {}
for domain in self.sites:
if domain.startswith('_'):
continue
data = self._get_site_data(domain)
if data is None:
continue
result[domain] = self._validate_file_paste_config(
data.get("file_paste", {}),
legacy_stream_config=data.get("stream_config"),
include_attachment_defaults=True,
)
return result
def get_site_prompt_padding_config(self, domain: str, preset_name: str = None) -> Dict[str, Any]:
"""获取站点的提示词开头注入配置"""
self.refresh_if_changed()
data = self._get_site_data(domain, preset_name)
if data is None:
return copy.deepcopy(get_default_prompt_padding_config())
return self._validate_prompt_padding_config(data.get("prompt_padding", {}))
def set_site_prompt_padding_config(
self,
domain: str,
config: Dict[str, Any],
preset_name: str = None,
) -> bool:
"""设置站点的提示词开头注入配置"""
self.refresh_if_changed()
data = self._get_site_data(domain, preset_name)
if data is None:
logger.warning(f"站点或预设不存在: {domain}/{preset_name}")
return False
current_config = self._validate_prompt_padding_config(data.get("prompt_padding", {}))
merged_config = _merge_config_patch(current_config, config if isinstance(config, dict) else {})
validated = self._validate_prompt_padding_config(merged_config)
if not self._assign_preset_field_and_save(data, "prompt_padding", validated):
return False
logger.info(f"站点 {domain} [{preset_name or DEFAULT_PRESET_NAME}] 提示词开头注入配置已更新")
return True
def _validate_file_paste_config(
self,
config: dict,
*,
legacy_stream_config: Optional[Dict[str, Any]] = None,
include_attachment_defaults: bool = False,
) -> dict:
"""验证并规范化文件粘贴配置"""
default = get_default_file_paste_config()
result = copy.deepcopy(default)
if not isinstance(config, dict):
config = {}
if "enabled" in config:
result["enabled"] = _coerce_bool(config.get("enabled"), False)
if "threshold" in config:
try:
val = int(config["threshold"])
result["threshold"] = max(1000, min(val, 10000000))
except (ValueError, TypeError):
pass
if "temp_file_type" in config:
val = str(config.get("temp_file_type") or "").strip().lower().lstrip(".")
if val in {"txt", "pdf", "chunk", "error"}:
result["temp_file_type"] = val
if "hint_text" in config:
val = str(config["hint_text"]).strip()
# 限制长度,避免过长的引导文本
hint_val = val[:500] if val else ""
result["hint_text"] = hint_val
# 智能向后兼容:结合老配置的策略类型,防止新字段被无关的旧数据污染
old_temp_type = config.get("temp_file_type", "txt")
if "txt_hint_text" not in config:
result["txt_hint_text"] = hint_val if old_temp_type == "txt" else "完全专注于文件内容"
if "pdf_hint_text" not in config:
result["pdf_hint_text"] = hint_val if old_temp_type == "pdf" else "完全专注于文件内容"
if "error_hint_text" not in config:
result["error_hint_text"] = hint_val if old_temp_type == "error" else "输入文本长度超过限制,已中止发送"
if "txt_hint_text" in config:
val = str(config["txt_hint_text"]).strip()
result["txt_hint_text"] = val[:500] if val else ""
if "pdf_hint_text" in config:
val = str(config["pdf_hint_text"]).strip()
result["pdf_hint_text"] = val[:500] if val else ""
if "error_hint_text" in config:
val = str(config["error_hint_text"]).strip()
result["error_hint_text"] = val[:500] if val else ""
if "reacquire_input_after_upload" in config:
result["reacquire_input_after_upload"] = _coerce_bool(config.get("reacquire_input_after_upload"), False)
if "post_upload_input_selector" in config:
val = str(config["post_upload_input_selector"] or "").strip()
result["post_upload_input_selector"] = val[:500] if val else ""
if "post_upload_settle" in config:
try:
val = float(config["post_upload_settle"])
result["post_upload_settle"] = max(0.0, min(val, 30.0))
except (ValueError, TypeError):
pass
if "upload_signal_timeout" in config:
try:
val = float(config["upload_signal_timeout"])
result["upload_signal_timeout"] = max(0.5, min(val, 120.0))
except (ValueError, TypeError):
pass
if "upload_signal_grace" in config:
try:
val = float(config["upload_signal_grace"])
result["upload_signal_grace"] = max(0.0, min(val, 120.0))
except (ValueError, TypeError):
pass
default_state_probe = copy.deepcopy(default.get("state_probe") or {})
raw_state_probe = config.get("state_probe")
state_probe = copy.deepcopy(default_state_probe)
if isinstance(raw_state_probe, dict):
if "enabled" in raw_state_probe:
state_probe["enabled"] = _coerce_bool(raw_state_probe.get("enabled"), False)
if "code" in raw_state_probe:
code = str(raw_state_probe["code"] or "").strip()
state_probe["code"] = code[:20000] if code else ""
if state_probe:
result["state_probe"] = state_probe
legacy_send_confirmation = {}
if isinstance(legacy_stream_config, dict) and isinstance(legacy_stream_config.get("send_confirmation"), dict):
legacy_send_confirmation.update(legacy_stream_config.get("send_confirmation") or {})
raw_send_confirmation = config.get("send_confirmation")
if isinstance(raw_send_confirmation, dict):
legacy_send_confirmation.update(raw_send_confirmation)
if legacy_send_confirmation or include_attachment_defaults:
result["send_confirmation"] = self._validate_send_confirmation_config(legacy_send_confirmation)
legacy_attachment_monitor = {}
if isinstance(legacy_stream_config, dict) and isinstance(legacy_stream_config.get("attachment_monitor"), dict):
legacy_attachment_monitor.update(legacy_stream_config.get("attachment_monitor") or {})
raw_attachment_monitor = config.get("attachment_monitor")
if isinstance(raw_attachment_monitor, dict):
legacy_attachment_monitor.update(raw_attachment_monitor)
if legacy_attachment_monitor or include_attachment_defaults:
result["attachment_monitor"] = self._validate_attachment_monitor_config(legacy_attachment_monitor)
from app.utils.attachments import attachment_config
if "attachments" in config or include_attachment_defaults:
result["attachments"] = attachment_config(config.get("attachments"))
return result
def _validate_prompt_padding_config(self, config: Dict[str, Any]) -> Dict[str, Any]:
"""验证并规范化提示词开头注入配置。"""
result = copy.deepcopy(get_default_prompt_padding_config())
if not isinstance(config, dict):
return result
if "enabled" in config:
result["enabled"] = _coerce_bool(config.get("enabled"), False)
if "marker_text" in config:
marker_text = str(config.get("marker_text") or "").strip()
result["marker_text"] = marker_text
if "segments_per_side" in config:
try:
segments_per_side = int(config.get("segments_per_side"))
except (TypeError, ValueError):
segments_per_side = int(result["segments_per_side"])
result["segments_per_side"] = max(0, min(segments_per_side, 64))
if "random_insert_enabled" in config:
result["random_insert_enabled"] = _coerce_bool(
config.get("random_insert_enabled"), False
)
if "random_insert_chars" in config:
result["random_insert_chars"] = str(
config.get("random_insert_chars") or ""
)
return result
def list_image_presets(self):
"""列出所有可用的图片配置预设"""
return self.image_presets.list_presets()
def get_image_preset(self, domain: str):
"""获取指定站点的预设信息"""
return self.image_presets.get_preset_for_display(domain)
def apply_image_preset(self, domain: str, preset_domain: str):
"""将预设配置应用到站点"""
preset_config = self.image_presets.get_preset(preset_domain)
if not preset_config:
raise ValueError(f"找不到预设: {preset_domain}")
return self.set_site_image_config(domain, preset_config)
def reload_presets(self):
"""重新加载图片预设"""
self.image_presets.reload()
+374
View File
@@ -0,0 +1,374 @@
"""ConfigEngine 的一部分(R2-3 拆分):历史配置迁移与命令文件读写。
方法原样从 engine.py 迁出,行为不变;通过 mixin 组合回 ConfigEngine,
实例状态都在 ConfigEngine.__init__(engine.py)里初始化。
"""
import json
import os
import copy
from typing import Dict, List, Any
from app.models.schemas import (
PRESET_ADVANCED_FIELDS,
SITE_ADVANCED_FIELDS,
get_default_image_extraction_config,
get_default_file_paste_config,
get_default_prompt_padding_config,
)
from app.utils.site_rules import get_site_rule
from .engine_common import ConfigConstants, DEFAULT_PRESET_NAME, PRESET_FIELDS, logger
class ConfigMigrationsMixin:
"""历史配置迁移与命令文件读写"""
def _load_commands_file(self) -> List[Dict[str, Any]]:
"""加载独立命令配置文件"""
commands_file = ConfigConstants.COMMANDS_FILE
if not os.path.exists(commands_file):
return []
try:
with open(commands_file, "r", encoding="utf-8") as f:
data = json.load(f)
if isinstance(data, dict):
data = data.get("commands", [])
if isinstance(data, list):
return data
logger.warning(f"命令配置文件格式无效: {commands_file}")
return []
except json.JSONDecodeError as e:
logger.error(f"命令配置文件格式错误: {e}")
return []
except Exception as e:
logger.error(f"加载命令配置失败: {e}")
return []
def _save_commands_file(self, commands: List[Dict[str, Any]]) -> bool:
"""保存独立命令配置文件"""
commands_file = ConfigConstants.COMMANDS_FILE
tmp_file = commands_file + ".tmp"
try:
os.makedirs(os.path.dirname(commands_file), exist_ok=True)
payload = {"commands": commands}
with open(tmp_file, "w", encoding="utf-8", newline="\n") as f:
json.dump(payload, f, indent=2, ensure_ascii=False)
f.flush()
os.fsync(f.fileno())
os.replace(tmp_file, commands_file)
logger.info(f"命令配置已保存: {commands_file}")
return True
except Exception as e:
logger.error(f"保存命令配置失败: {e}")
try:
if os.path.exists(tmp_file):
os.remove(tmp_file)
except Exception:
pass
return False
@staticmethod
def _merge_commands(existing: List[Dict[str, Any]], incoming: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
"""保留已有命令,仅追加不存在的命令"""
merged = list(existing or [])
seen_ids = {
str(cmd.get("id", "")).strip()
for cmd in merged
if isinstance(cmd, dict) and str(cmd.get("id", "")).strip()
}
seen_names = {
str(cmd.get("name", "")).strip()
for cmd in merged
if isinstance(cmd, dict) and str(cmd.get("name", "")).strip()
}
for cmd in incoming or []:
if not isinstance(cmd, dict):
continue
command_id = str(cmd.get("id", "")).strip()
command_name = str(cmd.get("name", "")).strip()
if command_id and command_id in seen_ids:
continue
if command_name and command_name in seen_names:
continue
merged.append(cmd)
if command_id:
seen_ids.add(command_id)
if command_name:
seen_names.add(command_name)
return merged
def _migrate_global_commands(self):
"""将旧版 _global.commands 迁移到独立文件"""
legacy_commands = self.global_config.get("commands", [])
existing_commands = self._load_commands_file()
if not isinstance(legacy_commands, list):
legacy_commands = []
merged_commands = self._merge_commands(existing_commands, legacy_commands)
if legacy_commands or not os.path.exists(ConfigConstants.COMMANDS_FILE):
self._save_commands_file(merged_commands)
if self.global_config.remove("commands"):
self._save_config()
def _migrate_to_presets(self):
"""
将旧格式(扁平)站点配置迁移为预设格式
旧格式: { "selectors": {...}, "workflow": [...], ... }
新格式: { "presets": { "主预设": { "selectors": {...}, ... } } }
"""
migrated_count = 0
for domain in list(self.sites.keys()):
if domain.startswith('_'):
continue
site_config = self.sites[domain]
# 已经是预设格式,跳过
if "presets" in site_config:
continue
# 将所有已知配置字段提取到主预设中
preset_data = {}
remaining = {}
for key, value in site_config.items():
if key in PRESET_FIELDS:
preset_data[key] = value
elif key == "advanced":
remaining[key] = value
else:
# 未知字段也放入预设(保留用户自定义数据)
preset_data[key] = value
# 构建新格式
self.sites[domain] = {
"default_preset": DEFAULT_PRESET_NAME,
"presets": {
DEFAULT_PRESET_NAME: preset_data
},
**remaining,
}
migrated_count += 1
logger.debug(f"迁移站点配置: {domain} → 预设格式")
if migrated_count > 0:
self._save_config()
logger.info(f"✅ 已迁移 {migrated_count} 个站点配置为预设格式")
def _cleanup_preset_residuals(self):
"""
清理站点配置中预设外的残留字段
当站点已有 presets 结构时,顶层不应再有 selectors/workflow/file_paste 等字段。
这些残留通常由旧版 bug 或手动编辑产生。
"""
cleaned_count = 0
default_fixed_count = 0
for domain in list(self.sites.keys()):
if domain.startswith('_'):
continue
site_config = self.sites[domain]
# 只处理已有 presets 结构的站点
if "presets" not in site_config:
continue
# 找出预设外的残留字段
residual_keys = []
for key in list(site_config.keys()):
if key == "presets":
continue
if key in PRESET_FIELDS:
residual_keys.append(key)
# 删除残留
for key in residual_keys:
del site_config[key]
cleaned_count += 1
logger.debug(f"清理残留: {domain}.{key}")
if self._normalize_site_default_preset(domain, site_config):
default_fixed_count += 1
logger.debug(f"修正默认预设: {domain} -> {site_config.get('default_preset')}")
if cleaned_count > 0 or default_fixed_count > 0:
self._save_config()
logger.info(
f"✅ 已清理 {cleaned_count} 个预设外残留字段,"
f"修正 {default_fixed_count} 个站点默认预设"
)
def _migrate_site_advanced_to_presets(self):
"""
规范化已有预设级 advanced,避免启动时改写站点级 advanced 语义。
站点根级 advanced 仍作为所有预设的共享基线;预设级 advanced 只保存
覆盖项。因此这里不再把站点级时序字段复制到所有预设,也不从站点级
删除这些字段,只清理明显放错位置的 Cookie 字段并规范化已有覆盖项。
"""
cleaned_count = 0
changed = False
for domain, site_config in self.sites.items():
if domain.startswith("_") or not isinstance(site_config, dict):
continue
presets = site_config.get("presets", {})
if not isinstance(presets, dict) or not presets:
continue
for preset_name, preset_data in presets.items():
if not isinstance(preset_data, dict):
continue
preset_advanced = preset_data.get("advanced")
if not isinstance(preset_advanced, dict):
continue
for key in list(SITE_ADVANCED_FIELDS):
if key in preset_advanced:
del preset_advanced[key]
cleaned_count += 1
changed = True
normalized_preset_advanced = self._normalize_site_advanced_config(
preset_advanced
)
for key in list(preset_advanced.keys()):
if key in PRESET_ADVANCED_FIELDS:
value = normalized_preset_advanced[key]
if preset_advanced.get(key) != value:
preset_advanced[key] = value
cleaned_count += 1
changed = True
if preset_advanced != preset_data.get("advanced"):
logger.debug(f"规范化预设高级配置: {domain}/{preset_name}")
if not preset_advanced:
del preset_data["advanced"]
cleaned_count += 1
changed = True
if changed:
self._save_config()
logger.info(
f"✅ 已规范化预设高级配置,清理 {cleaned_count} 个残留项"
)
def _guess_stealth(self, domain: str) -> bool:
"""Guess whether stealth mode should default to enabled."""
rule = get_site_rule(domain)
if "stealth_default" in rule:
enabled = bool(rule.get("stealth_default"))
if enabled:
logger.info(f"检测到默认启用低熵模式的域名: {domain}")
return enabled
return False
def migrate_site_configs(self):
"""迁移旧版站点配置,补充各预设中缺失的字段"""
migrated_count = 0
default_image_config = get_default_image_extraction_config()
default_file_paste = get_default_file_paste_config()
default_prompt_padding = get_default_prompt_padding_config()
obsolete_stream_keys = {
"silence_threshold",
"initial_wait",
"enable_wrapper_search",
"rerender_wait",
"content_shrink_tolerance",
}
obsolete_network_keys = set()
for domain, site_config in self.sites.items():
if domain.startswith("_"):
continue
presets = site_config.get("presets", {})
for preset_name, preset_data in presets.items():
if "image_extraction" not in preset_data:
preset_data["image_extraction"] = copy.deepcopy(default_image_config)
migrated_count += 1
logger.debug(f"迁移: {domain}/{preset_name} (添加 image_extraction)")
else:
normalized_image_config = self._validate_image_config(preset_data.get("image_extraction", {}))
if normalized_image_config != preset_data.get("image_extraction"):
preset_data["image_extraction"] = normalized_image_config
migrated_count += 1
logger.debug(f"迁移: {domain}/{preset_name} (规范化 image_extraction)")
if "file_paste" not in preset_data:
preset_data["file_paste"] = copy.deepcopy(default_file_paste)
migrated_count += 1
logger.debug(f"迁移: {domain}/{preset_name} (添加 file_paste)")
if "prompt_padding" not in preset_data:
preset_data["prompt_padding"] = copy.deepcopy(default_prompt_padding)
migrated_count += 1
logger.debug(f"迁移: {domain}/{preset_name} (添加 prompt_padding)")
else:
normalized_prompt_padding = self._validate_prompt_padding_config(
preset_data.get("prompt_padding", {})
)
if normalized_prompt_padding != preset_data.get("prompt_padding"):
preset_data["prompt_padding"] = normalized_prompt_padding
migrated_count += 1
logger.debug(f"迁移: {domain}/{preset_name} (规范化 prompt_padding)")
stream_config = preset_data.get("stream_config")
moved_attachment_rules = False
if not isinstance(stream_config, dict):
stream_config = {}
preset_data["stream_config"] = stream_config
normalized_file_paste = self._validate_file_paste_config(
preset_data.get("file_paste", {}),
legacy_stream_config=stream_config,
)
if normalized_file_paste != preset_data.get("file_paste"):
preset_data["file_paste"] = normalized_file_paste
migrated_count += 1
logger.debug(f"迁移: {domain}/{preset_name} (规范化 file_paste)")
for legacy_key in ("send_confirmation", "attachment_monitor"):
if legacy_key in stream_config:
del stream_config[legacy_key]
moved_attachment_rules = True
if isinstance(stream_config, dict):
removed_stream = False
for key in list(obsolete_stream_keys):
if key in stream_config:
del stream_config[key]
removed_stream = True
network_config = stream_config.get("network")
removed_network = False
if isinstance(network_config, dict):
for key in list(obsolete_network_keys):
if key in network_config:
del network_config[key]
removed_network = True
if removed_stream or removed_network or moved_attachment_rules:
migrated_count += 1
logger.debug(f"迁移: {domain}/{preset_name} (清理废弃的流式配置字段)")
if migrated_count > 0:
self._save_config()
logger.info(f"已迁移 {migrated_count} 个预设配置")
return migrated_count
+667
View File
@@ -0,0 +1,667 @@
"""ConfigEngine 的一部分(R2-3 拆分):预设的解析、创建、改名、删除及引用同步。
方法原样从 engine.py 迁出,行为不变;通过 mixin 组合回 ConfigEngine,
实例状态都在 ConfigEngine.__init__(engine.py)里初始化。
"""
import json
import copy
from typing import Dict, Optional, List, Any
from .engine_common import DEFAULT_PRESET_NAME, DEFAULT_WORKFLOW, _MISSING, logger
class ConfigPresetsMixin:
"""预设的解析、创建、改名、删除及引用同步"""
def _resolve_default_preset_name(self, site: Dict[str, Any]) -> Optional[str]:
"""解析站点有效默认预设名(不修改原对象)"""
presets = site.get("presets", {})
if not presets:
return None
configured_default = site.get("default_preset")
if isinstance(configured_default, str):
configured_default = configured_default.strip()
else:
configured_default = None
if configured_default and configured_default in presets:
return configured_default
if DEFAULT_PRESET_NAME in presets:
return DEFAULT_PRESET_NAME
return next(iter(presets))
def _normalize_site_default_preset(self, domain: str, site: Dict[str, Any]) -> bool:
"""
规范化站点 default_preset 字段
Returns:
是否发生修改
"""
return self._sync_site_default_preset_state(domain, site)
def _get_site_data(self, domain: str, preset_name: str = None) -> Optional[Dict]:
"""
获取指定站点的预设配置数据(可变引用)
查找顺序:
- preset_name 显式提供时:只匹配该预设(含别名)
- preset_name 为空时:按站点默认预设 / 主预设 / 第一个可用预设回退
Args:
domain: 站点域名
preset_name: 预设名称,None 则使用默认
Returns:
预设配置字典的引用(可直接修改),或 None
"""
if domain not in self.sites:
return None
site = self.sites[domain]
presets = site.get("presets", {})
if not presets:
return None
requested_preset = str(preset_name or "").strip()
if requested_preset:
resolved_target = self._resolve_preset_alias_key(requested_preset, presets)
if resolved_target != requested_preset:
logger.debug(f"预设别名命中: '{requested_preset}' -> '{resolved_target}'")
if resolved_target in presets:
return presets[resolved_target]
return None
resolved_default = self._resolve_default_preset_name(site)
if resolved_default and resolved_default in presets:
return presets[resolved_default]
if DEFAULT_PRESET_NAME in presets:
return presets[DEFAULT_PRESET_NAME]
first_key = next(iter(presets))
logger.warning(f"默认预设不存在,使用第一个预设: '{first_key}'")
return presets[first_key]
@staticmethod
def _resolve_preset_alias_key(target: Any, presets: Dict[str, Any]) -> str:
"""兼容历史命名:允许不带“预设_”前缀的名字命中真实预设键。"""
normalized = str(target or "").strip()
if not normalized or not isinstance(presets, dict):
return normalized
if normalized in presets:
return normalized
candidates = []
if normalized.startswith("预设_"):
stripped = normalized[len("预设_"):].strip()
if stripped:
candidates.append(stripped)
else:
candidates.append(f"预设_{normalized}")
for candidate in candidates:
if candidate in presets:
return candidate
return normalized
def _get_site_data_readonly(self, domain: str, preset_name: str = None) -> Optional[Dict]:
"""获取预设配置的深拷贝(只读用途)"""
with self._io_lock:
data = self._get_site_data(domain, preset_name)
if data is None:
return None
return copy.deepcopy(data)
def _get_preset_data_exact(self, domain: str, preset_name: str = None) -> Optional[Dict]:
"""获取预设配置引用;显式预设不存在时不回退到默认预设。"""
site = self.sites.get(domain)
if not isinstance(site, dict):
return None
presets = site.get("presets", {})
if not isinstance(presets, dict) or not presets:
return None
if preset_name is None or not str(preset_name).strip():
target = self._resolve_default_preset_name(site)
else:
target = str(preset_name).strip()
resolved_target = self._resolve_preset_alias_key(target, presets)
if resolved_target and resolved_target in presets:
return presets[resolved_target]
return None
def list_presets(self, domain: str) -> List[str]:
"""获取指定站点的所有预设名称"""
self.refresh_if_changed()
if domain not in self.sites:
return []
site = self.sites[domain]
presets = site.get("presets", {})
return list(presets.keys())
def get_default_preset(self, domain: str) -> Optional[str]:
"""获取指定站点的默认预设名称(已解析回退)"""
self.refresh_if_changed()
cache_key = f"default_preset:{domain}"
cached = self._cache.get(cache_key, _MISSING)
if cached is not _MISSING:
return cached
site = self.sites.get(domain)
if not site:
self._cache.set(cache_key, None)
return None
result = self._resolve_default_preset_name(site)
self._cache.set(cache_key, result)
return result
def _assign_preset_field_and_save(
self,
preset_data: Dict[str, Any],
field_name: str,
value: Any,
) -> bool:
previous_value = preset_data.get(field_name, _MISSING)
if previous_value is not _MISSING:
previous_value = copy.deepcopy(previous_value)
preset_data[field_name] = value
if self._save_config():
return True
if previous_value is _MISSING:
preset_data.pop(field_name, None)
else:
preset_data[field_name] = previous_value
return False
def set_default_preset(self, domain: str, preset_name: str) -> bool:
"""设置指定站点的默认预设"""
self.refresh_if_changed()
site = self.sites.get(domain)
if not site:
return False
presets = site.get("presets", {})
if preset_name not in presets:
logger.warning(f"默认预设设置失败,预设不存在: {domain}/{preset_name}")
return False
previous_default = site.get("default_preset", _MISSING)
if previous_default is not _MISSING:
previous_default = copy.deepcopy(previous_default)
previous_default_maps = (
copy.deepcopy(getattr(self, "_global_default_presets", {})),
copy.deepcopy(getattr(self, "_local_default_presets", {})),
)
persisted_default = self._get_persisted_default_preset(domain, site)
if persisted_default == preset_name:
self._local_default_presets.pop(domain, None)
else:
self._local_default_presets[domain] = preset_name
site["default_preset"] = preset_name
if not self._save_local_site_overrides():
if previous_default is _MISSING:
site.pop("default_preset", None)
else:
site["default_preset"] = previous_default
self._global_default_presets, self._local_default_presets = previous_default_maps
return False
logger.info(f"✅ 站点 {domain} 默认预设已设置为: '{preset_name}'(仅本地覆盖)")
return True
def create_preset(self, domain: str, new_name: str,
source_name: str = None) -> bool:
"""
创建新预设(克隆自现有预设)
Args:
domain: 站点域名
new_name: 新预设名称
source_name: 要克隆的源预设名称,None 则克隆主预设
Returns:
是否成功
"""
self.refresh_if_changed()
if domain not in self.sites:
logger.warning(f"站点不存在: {domain}")
return False
site = self.sites[domain]
presets = site.get("presets", {})
if new_name in presets:
logger.warning(f"预设已存在: {new_name}")
return False
# 获取源预设
if source_name is not None and str(source_name or "").strip():
requested_source = str(source_name or "").strip()
source = self._resolve_preset_alias_key(requested_source, presets)
source_data = presets.get(source)
if not source_data:
logger.warning(f"源预设不存在: {requested_source}")
return False
else:
source = DEFAULT_PRESET_NAME
source_data = presets.get(source)
if not source_data:
# 未显式指定源时才尝试第一个可用预设
if presets:
source = next(iter(presets))
source_data = presets[source]
else:
logger.warning(f"没有可克隆的源预设")
return False
previous_default = site.get("default_preset", _MISSING)
if previous_default is not _MISSING:
previous_default = copy.deepcopy(previous_default)
previous_presets = copy.deepcopy(presets)
previous_global_default = copy.deepcopy(self._global_default_presets)
previous_local_default = copy.deepcopy(self._local_default_presets)
# 深拷贝创建新预设
presets[new_name] = copy.deepcopy(source_data)
self._normalize_site_default_preset(domain, site)
if not self._save_config():
site["presets"] = previous_presets
if previous_default is _MISSING:
site.pop("default_preset", None)
else:
site["default_preset"] = previous_default
self._global_default_presets = previous_global_default
self._local_default_presets = previous_local_default
return False
logger.info(f"✅ 站点 {domain} 创建预设: '{new_name}' (克隆自 '{source}')")
return True
@staticmethod
def _build_preset_rename_map(old_name: Any, new_name: Any) -> Dict[str, str]:
old_raw = str(old_name or "").strip()
new_raw = str(new_name or "").strip()
if not old_raw or not new_raw:
return {}
def _strip_prefix(value: str) -> str:
if value.startswith("预设_"):
return value[len("预设_"):].strip()
return value
def _add_prefix(value: str) -> str:
return value if value.startswith("预设_") else f"预设_{value}"
mapping: Dict[str, str] = {}
old_plain = _strip_prefix(old_raw)
new_plain = _strip_prefix(new_raw)
old_prefixed = _add_prefix(old_raw)
new_prefixed = _add_prefix(new_raw)
for src, dst in (
(old_raw, new_raw),
(old_plain, new_plain),
(old_prefixed, new_prefixed),
):
src_text = str(src or "").strip()
dst_text = str(dst or "").strip()
if src_text and dst_text:
mapping[src_text] = dst_text
return mapping
@staticmethod
def _command_targets_domain(command: Dict[str, Any], domain: str) -> bool:
normalized_domain = str(domain or "").strip().lower()
if not normalized_domain or not isinstance(command, dict):
return False
trigger = command.get("trigger", {}) or {}
command_domain = str(trigger.get("domain", "") or "").strip().lower()
if not command_domain:
return False
return (
command_domain == normalized_domain
or command_domain.endswith(f".{normalized_domain}")
or normalized_domain.endswith(f".{command_domain}")
)
def _rename_preset_references_in_commands(
self,
domain: str,
rename_map: Dict[str, str],
) -> int:
if not rename_map:
return 0
commands = self._load_commands_file()
if not commands:
return 0
updated = 0
for command in commands:
if not self._command_targets_domain(command, domain):
continue
actions = command.get("actions", [])
if not isinstance(actions, list):
continue
for action in actions:
if not isinstance(action, dict):
continue
current_preset = str(action.get("preset_name", "") or "").strip()
replacement = rename_map.get(current_preset)
if not replacement or replacement == current_preset:
continue
action["preset_name"] = replacement
updated += 1
if updated > 0:
self._save_commands_file(commands)
return updated
def _rename_preset_references_in_active_tabs(
self,
domain: str,
rename_map: Dict[str, str],
) -> int:
if not rename_map:
return 0
try:
from app.core import browser as browser_module
instance = getattr(browser_module, "_browser_instance", None)
if instance is None:
return 0
pool = getattr(instance, "_tab_pool", None)
if pool is None or not hasattr(pool, "get_sessions_snapshot"):
return 0
updated = 0
for session in pool.get_sessions_snapshot():
session_domain = str(getattr(session, "current_domain", "") or "").strip().lower()
if session_domain != str(domain or "").strip().lower():
continue
current_preset = str(getattr(session, "preset_name", "") or "").strip()
replacement = rename_map.get(current_preset)
if not replacement or replacement == current_preset:
continue
if hasattr(session, "set_preset"):
session.set_preset(replacement, source=getattr(session, "preset_override_source", None))
else:
session.preset_name = replacement
updated += 1
return updated
except Exception as e:
logger.debug(f"同步活动标签页预设引用失败(忽略): {e}")
return 0
def _clear_preset_references_in_active_tabs(self, domain: str, preset_name: str) -> int:
"""预设被删除后,把在用标签页里指向它的 preset_name 置为 None(回落站点默认预设)。"""
target = str(preset_name or "").strip()
if not target:
return 0
# 兼容历史命名:带/不带“预设_”前缀都视为命中同一个预设
targets = {target}
if target.startswith("预设_"):
stripped = target[len("预设_"):].strip()
if stripped:
targets.add(stripped)
else:
targets.add(f"预设_{target}")
try:
from app.core import browser as browser_module
instance = getattr(browser_module, "_browser_instance", None)
if instance is None:
return 0
pool = getattr(instance, "_tab_pool", None)
if pool is None or not hasattr(pool, "get_sessions_snapshot"):
return 0
cleared = 0
for session in pool.get_sessions_snapshot():
session_domain = str(getattr(session, "current_domain", "") or "").strip().lower()
if session_domain != str(domain or "").strip().lower():
continue
current_preset = str(getattr(session, "preset_name", "") or "").strip()
if current_preset not in targets:
continue
if hasattr(session, "set_preset"):
session.set_preset(None, source=None)
else:
session.preset_name = None
cleared += 1
return cleared
except Exception as e:
logger.debug(f"清理活动标签页预设引用失败(忽略): {e}")
return 0
def _sync_preset_overrides_file(self, domain: str, old_preset: str, new_preset: Optional[str] = None) -> int:
"""当预设重命名或删除时,同步清理/更新 preset_overrides.local.json"""
target = str(old_preset or "").strip()
if not target:
return 0
from pathlib import Path
config_path = Path("config/preset_overrides.local.json")
if not config_path.exists():
return 0
from app.utils.site_url import extract_remote_site_domain
import json
try:
with open(config_path, "r", encoding="utf-8-sig") as f:
payload = json.load(f)
if not isinstance(payload, dict):
return 0
urls = payload.get("urls", {})
if not isinstance(urls, dict):
return 0
domain_norm = str(domain or "").strip().lower()
changed = 0
keys_to_delete = []
for url_key, p_name in list(urls.items()):
url_domain = str(extract_remote_site_domain(url_key) or "").strip().lower()
if domain_norm and url_domain != domain_norm:
continue
if p_name == target or p_name == f"预设_{target}" or (target.startswith("预设_") and p_name == target[3:]):
if new_preset:
urls[url_key] = new_preset
changed += 1
else:
keys_to_delete.append(url_key)
changed += 1
for k in keys_to_delete:
urls.pop(k, None)
if changed > 0:
from app.core.config import atomic_write_json
atomic_write_json(config_path, payload)
try:
from app.core import browser as browser_module
instance = getattr(browser_module, "_browser_instance", None)
pool = getattr(instance, "_tab_pool", None)
if pool is not None and hasattr(pool, "apply_runtime_config"):
pool.apply_runtime_config(preset_overrides=payload)
except Exception:
pass
return changed
except Exception as e:
logger.debug(f"同步预设本地配置失败: {e}")
return 0
def delete_preset(self, domain: str, preset_name: str) -> bool:
"""
删除预设(不允许删除最后一个预设)
Args:
domain: 站点域名
preset_name: 要删除的预设名称
Returns:
是否成功
"""
self.refresh_if_changed()
if domain not in self.sites:
return False
site = self.sites[domain]
presets = site.get("presets", {})
resolved_preset_name = self._resolve_preset_alias_key(preset_name, presets)
if resolved_preset_name not in presets:
logger.warning(f"预设不存在: {preset_name}")
return False
if len(presets) <= 1:
logger.warning(f"不能删除最后一个预设")
return False
previous_default = site.get("default_preset", _MISSING)
if previous_default is not _MISSING:
previous_default = copy.deepcopy(previous_default)
removed_preset = copy.deepcopy(presets[resolved_preset_name])
previous_global_default = copy.deepcopy(self._global_default_presets)
previous_local_default = copy.deepcopy(self._local_default_presets)
del presets[resolved_preset_name]
self._normalize_site_default_preset(domain, site)
if not self._save_config():
presets[resolved_preset_name] = removed_preset
if previous_default is _MISSING:
site.pop("default_preset", None)
else:
site["default_preset"] = previous_default
self._global_default_presets = previous_global_default
self._local_default_presets = previous_local_default
return False
# 预设已删除并落盘:清理在用标签页对它的残留引用,否则 session.preset_name 会指向不存在的预设
cleared_tab_refs = 0
try:
cleared_tab_refs = self._clear_preset_references_in_active_tabs(domain, resolved_preset_name)
except Exception as e:
logger.warning(f"清理活动标签页预设引用失败(忽略): {e}")
# 同步清理持久化 preset_overrides.local.json
try:
self._sync_preset_overrides_file(domain, resolved_preset_name, new_preset=None)
except Exception as e:
logger.warning(f"清理本地预设记忆文件失败(忽略): {e}")
logger.info(
f"✅ 站点 {domain} 删除预设: '{resolved_preset_name}' "
f"(活动标签页同步 {cleared_tab_refs} 处)"
)
return True
def rename_preset(self, domain: str, old_name: str, new_name: str) -> bool:
"""重命名预设"""
self.refresh_if_changed()
if domain not in self.sites:
return False
site = self.sites[domain]
presets = site.get("presets", {})
resolved_old_name = self._resolve_preset_alias_key(old_name, presets)
if resolved_old_name not in presets:
return False
if new_name in presets:
logger.warning(f"预设名已存在: {new_name}")
return False
rename_map = self._build_preset_rename_map(resolved_old_name, new_name)
default_preset = site.get("default_preset")
previous_default = site.get("default_preset", _MISSING)
if previous_default is not _MISSING:
previous_default = copy.deepcopy(previous_default)
previous_presets = copy.deepcopy(presets)
previous_global_default = copy.deepcopy(self._global_default_presets)
previous_local_default = copy.deepcopy(self._local_default_presets)
local_default = str(self._local_default_presets.get(domain, "") or "").strip()
# 保持顺序:创建有序副本
new_presets = {}
for key, value in presets.items():
if key == resolved_old_name:
new_presets[new_name] = value
else:
new_presets[key] = value
site["presets"] = new_presets
if default_preset == resolved_old_name:
site["default_preset"] = new_name
if local_default:
local_replacement = rename_map.get(local_default)
if local_replacement:
self._local_default_presets[domain] = local_replacement
self._normalize_site_default_preset(domain, site)
if not self._save_config():
site["presets"] = previous_presets
if previous_default is _MISSING:
site.pop("default_preset", None)
else:
site["default_preset"] = previous_default
self._global_default_presets = previous_global_default
self._local_default_presets = previous_local_default
return False
updated_command_refs = self._rename_preset_references_in_commands(domain, rename_map)
updated_tab_refs = self._rename_preset_references_in_active_tabs(domain, rename_map)
try:
self._sync_preset_overrides_file(domain, resolved_old_name, new_preset=new_name)
except Exception as e:
logger.warning(f"同步本地预设记忆文件失败(忽略): {e}")
logger.info(
f"站点 {domain} 重命名预设: '{resolved_old_name}' → '{new_name}' "
f"(命令引用同步 {updated_command_refs} 处, 活动标签页同步 {updated_tab_refs} 处)"
)
return True
def get_preset_selectors(self, domain: str, preset_name: str = None) -> Dict:
"""获取指定预设的选择器配置"""
data = self._get_site_data_readonly(domain, preset_name)
return data.get("selectors", {}) if data else {}
def set_preset_selectors(self, domain: str, selectors: Dict,
preset_name: str = None) -> bool:
"""设置指定预设的选择器配置"""
self.refresh_if_changed()
data = self._get_site_data(domain, preset_name)
if data is None:
return False
if not self._assign_preset_field_and_save(data, "selectors", selectors):
return False
logger.info(f"站点 {domain} [{preset_name or DEFAULT_PRESET_NAME}] 选择器已更新")
return True
def get_preset_workflow(self, domain: str, preset_name: str = None) -> List:
"""获取指定预设的工作流配置"""
data = self._get_site_data_readonly(domain, preset_name)
return data.get("workflow", DEFAULT_WORKFLOW) if data else DEFAULT_WORKFLOW
def set_preset_workflow(self, domain: str, workflow: List,
preset_name: str = None) -> bool:
"""设置指定预设的工作流配置"""
from app.core.workflow.flow_runtime import has_control_flow, validate_workflow
if has_control_flow(workflow):
validate_workflow(workflow)
self.refresh_if_changed()
data = self._get_site_data(domain, preset_name)
if data is None:
return False
if not self._assign_preset_field_and_save(data, "workflow", workflow):
return False
logger.info(f"站点 {domain} [{preset_name or DEFAULT_PRESET_NAME}] 工作流已更新")
return True
+535
View File
@@ -0,0 +1,535 @@
"""ConfigEngine 的一部分(R2-3 拆分):站点目录、起始页推断、站点高级配置与选择器定义。
方法原样从 engine.py 迁出,行为不变;通过 mixin 组合回 ConfigEngine,
实例状态都在 ConfigEngine.__init__(engine.py)里初始化。
"""
import copy
from typing import Dict, Optional, List, Any, Set, Union, Callable
from urllib.parse import urljoin
from app.models.schemas import (
SiteConfig,
SelectorDefinition,
ADVANCED_FIELDS,
PRESET_ADVANCED_FIELDS,
SITE_ADVANCED_FIELDS,
get_default_image_extraction_config,
get_default_file_paste_config,
get_default_prompt_padding_config,
get_default_site_advanced_config,
)
from app.utils.site_rules import derive_site_card_id, get_site_rule
from app.utils.site_discovery import (
automatic_discovery_allowed,
specific_chat_selectors,
selectors_match_chat_html,
)
from .engine_common import (
DEFAULT_PRESET_NAME,
DEFAULT_WORKFLOW,
_MISSING,
_SITE_STARTUP_JS_PATTERNS,
_coerce_bool,
get_default_stream_config,
logger,
)
class ConfigSitesMixin:
"""站点目录、起始页推断、站点高级配置与选择器定义"""
def _extract_startup_url_from_script(self, script: str) -> str:
text = str(script or "").strip()
if not text:
return ""
for pattern in _SITE_STARTUP_JS_PATTERNS:
match = pattern.search(text)
if match:
return str(match.group(1) or "").strip()
return ""
def _normalize_startup_url(self, domain: str, raw_url: str) -> str:
normalized_domain = str(domain or "").strip().lower().strip(".")
text = str(raw_url or "").strip()
if not normalized_domain:
return text
if not text:
return f"https://{normalized_domain}"
if text.startswith(("http://", "https://")):
return text
return urljoin(f"https://{normalized_domain}", text)
def _infer_site_startup_url(self, domain: str, site: Dict[str, Any]) -> str:
preset_name = self._resolve_default_preset_name(site)
preset_data = self._get_site_data_readonly(domain, preset_name)
workflow = preset_data.get("workflow", []) if isinstance(preset_data, dict) else []
if isinstance(workflow, list):
for step in workflow:
if str((step or {}).get("action") or "").strip().upper() != "JS_EXEC":
continue
startup_url = self._extract_startup_url_from_script((step or {}).get("value"))
if startup_url:
return self._normalize_startup_url(domain, startup_url)
return self._normalize_startup_url(domain, "")
def get_site_catalog_entry(self, domain: str, fallback_order: int = 0) -> Optional[Dict[str, Any]]:
self.refresh_if_changed()
site = self.sites.get(domain)
if not isinstance(site, dict) or str(domain or "").startswith("_"):
return None
rule = get_site_rule(domain)
startup_url = rule.get("startup_url") or self._infer_site_startup_url(domain, site)
display_name = str(rule.get("display_name") or domain).strip() or str(domain)
card_id = str(rule.get("card_id") or derive_site_card_id(domain)).strip() or derive_site_card_id(domain)
guide_priority = rule.get("guide_priority")
if not isinstance(guide_priority, int):
guide_priority = int(fallback_order)
entry = {
"domain": str(domain),
"display_name": display_name,
"url": self._normalize_startup_url(domain, startup_url),
"card_id": card_id,
"guide_priority": guide_priority,
}
if isinstance(rule.get("route_aliases"), list):
entry["route_aliases"] = [str(item) for item in rule.get("route_aliases", [])]
if "stealth_default" in rule:
entry["stealth_default"] = bool(rule.get("stealth_default"))
return entry
def list_site_catalog(self) -> List[Dict[str, Any]]:
self.refresh_if_changed()
ordered_domains = [
domain for domain, site in self.sites.items()
if not domain.startswith("_") and isinstance(site, dict)
]
entries: List[Dict[str, Any]] = []
for index, domain in enumerate(ordered_domains):
entry = self.get_site_catalog_entry(domain, fallback_order=index)
if entry:
entries.append(entry)
entries.sort(key=lambda item: (int(item.get("guide_priority", 0)), str(item.get("domain") or "")))
return entries
def _normalize_site_advanced_config(
self,
raw_config: Optional[Dict[str, Any]] = None,
*,
base_config: Optional[Dict[str, Any]] = None,
) -> Dict[str, Any]:
"""合并并规范化高级配置。"""
normalized = {
**get_default_site_advanced_config(),
}
if isinstance(base_config, dict):
normalized.update(copy.deepcopy(base_config))
if isinstance(raw_config, dict):
normalized.update(copy.deepcopy(raw_config))
normalized["independent_cookies"] = _coerce_bool(normalized.get("independent_cookies"), False)
normalized["independent_cookies_auto_takeover"] = _coerce_bool(
normalized.get("independent_cookies_auto_takeover"), False
)
normalized["input_box_stability_wait_enabled"] = _coerce_bool(
normalized.get("input_box_stability_wait_enabled"), False
)
normalized["input_box_stability_wait_after_new_chat_only"] = _coerce_bool(
normalized.get("input_box_stability_wait_after_new_chat_only"), True
)
try:
timeout_value = float(normalized.get("input_box_stability_wait_timeout", 1.5))
except Exception:
timeout_value = 1.5
normalized["input_box_stability_wait_timeout"] = max(0.1, min(timeout_value, 10.0))
normalized["url_transition_wait_on_new_chat"] = _coerce_bool(
normalized.get("url_transition_wait_on_new_chat"), False
)
raw_patterns = normalized.get("url_transition_wait_patterns") or []
if isinstance(raw_patterns, str):
raw_patterns = raw_patterns.replace("\n", ",").replace(";", ",").split(",")
if not isinstance(raw_patterns, (list, tuple, set)):
raw_patterns = []
normalized["url_transition_wait_patterns"] = [
str(pattern or "").strip()
for pattern in raw_patterns
if str(pattern or "").strip()
]
normalized["send_confirmation_check_enabled"] = _coerce_bool(
normalized.get("send_confirmation_check_enabled"), False
)
try:
send_timeout_value = float(normalized.get("send_confirmation_check_timeout", 1.5))
except Exception:
send_timeout_value = 1.5
normalized["send_confirmation_check_timeout"] = max(0.1, min(send_timeout_value, 10.0))
normalized["skip_new_chat_on_retry"] = _coerce_bool(
normalized.get("skip_new_chat_on_retry"), False
)
return normalized
def get_site_advanced_config(self, domain: str, preset_name: str = None) -> Dict[str, Any]:
"""获取站点高级配置;传入 preset_name 时叠加对应预设的覆盖项。"""
self.refresh_if_changed()
site = self.sites.get(domain)
if not site:
return self._normalize_site_advanced_config()
raw_config = site.get("advanced") if isinstance(site, dict) else None
normalized = self._normalize_site_advanced_config(
raw_config if isinstance(raw_config, dict) else {}
)
requested_preset = str(preset_name or "").strip()
if requested_preset:
preset_data = self._get_preset_data_exact(domain, requested_preset)
if preset_data is None:
logger.warning(f"站点高级配置预设不存在: {domain}/{requested_preset}")
return normalized
preset_advanced = (
preset_data.get("advanced")
if isinstance(preset_data, dict)
else None
)
if isinstance(preset_advanced, dict):
preset_advanced = {
key: value
for key, value in preset_advanced.items()
if key in PRESET_ADVANCED_FIELDS
}
normalized = self._normalize_site_advanced_config(
preset_advanced,
base_config=normalized,
)
return normalized
def set_site_advanced_config(self, domain: str, config: Dict[str, Any]) -> bool:
"""设置站点级高级配置。"""
with self._io_lock:
self.refresh_if_changed()
site = self.sites.get(domain)
if not site:
logger.warning(f"站点不存在: {domain}")
return False
previous_advanced = site.get("advanced", _MISSING)
if previous_advanced is not _MISSING:
previous_advanced = copy.deepcopy(previous_advanced)
existing = site.get("advanced") if isinstance(site.get("advanced"), dict) else {}
stored = copy.deepcopy(existing)
normalized = self._normalize_site_advanced_config(config or {}, base_config=existing)
provided_keys = {
key
for key in ((config or {}).keys() if isinstance(config, dict) else set())
if key in ADVANCED_FIELDS
}
if not provided_keys and previous_advanced is _MISSING:
return True
for key in ADVANCED_FIELDS:
if key in provided_keys:
stored[key] = normalized[key]
site["advanced"] = stored
if self._save_config_locked():
return True
if previous_advanced is _MISSING:
site.pop("advanced", None)
else:
site["advanced"] = previous_advanced
return False
def set_preset_advanced_config(
self,
domain: str,
config: Dict[str, Any],
preset_name: str = None,
*,
prune_inherited_fields: Optional[Set[str]] = None,
) -> bool:
"""设置当前预设的高级配置;拒绝混入站点级字段。"""
with self._io_lock:
self.refresh_if_changed()
site = self.sites.get(domain)
if not isinstance(site, dict):
logger.warning(f"站点不存在: {domain}")
return False
requested_preset = str(preset_name or "").strip()
if not requested_preset:
logger.warning(f"预设级高级配置缺少 preset_name: {domain}")
return False
data = self._get_preset_data_exact(domain, requested_preset)
if data is None:
logger.warning(f"站点或预设不存在: {domain}/{requested_preset}")
return False
raw_config = config if isinstance(config, dict) else {}
invalid_site_keys = {
key
for key in raw_config.keys()
if key in SITE_ADVANCED_FIELDS
}
if invalid_site_keys:
joined = ", ".join(sorted(invalid_site_keys))
logger.warning(f"预设级高级配置不能包含站点级字段: {domain}/{requested_preset} ({joined})")
return False
previous_advanced = data.get("advanced", _MISSING)
if previous_advanced is not _MISSING:
previous_advanced = copy.deepcopy(previous_advanced)
stored = copy.deepcopy(data.get("advanced") or {})
if not isinstance(stored, dict):
stored = {}
for key in list(SITE_ADVANCED_FIELDS):
stored.pop(key, None)
site_advanced = site.get("advanced") if isinstance(site.get("advanced"), dict) else {}
inherited = self._normalize_site_advanced_config(site_advanced)
normalized = self._normalize_site_advanced_config(
raw_config,
base_config=stored,
)
provided_keys = {
key
for key in raw_config.keys()
if key in PRESET_ADVANCED_FIELDS
}
if not provided_keys and previous_advanced is _MISSING:
return True
prune_inherited_keys = set(prune_inherited_fields or set())
for key in PRESET_ADVANCED_FIELDS:
if key in provided_keys:
if (
key in prune_inherited_keys
and key not in stored
and normalized[key] == inherited.get(key)
):
stored.pop(key, None)
else:
stored[key] = normalized[key]
if not stored and previous_advanced is _MISSING:
return True
data["advanced"] = stored
if self._save_config_locked():
return True
if previous_advanced is _MISSING:
data.pop("advanced", None)
else:
data["advanced"] = previous_advanced
return False
def list_sites(self) -> Dict[str, Any]:
"""获取所有站点配置(过滤内部键)"""
self.refresh_if_changed()
with self._io_lock:
return copy.deepcopy({
domain: config
for domain, config in self.sites.items()
if not domain.startswith('_')
})
def get_site_config(self, domain: str, html_content: Optional[Union[str, Callable[[], str]]] = None,
preset_name: str = None) -> Optional[SiteConfig]:
"""
获取站点配置(缓存 + AI 分析)
Args:
domain: 站点域名
html_content: 页面 HTML 或用于延迟获取 HTML 的可调用对象(用于 AI 分析未知站点)
preset_name: 预设名称,None 则使用默认预设
"""
self.refresh_if_changed()
if domain in self.sites:
cache_key = f"site_config:{domain}:{preset_name or ''}"
cached = self._cache.get(cache_key, _MISSING)
if cached is not _MISSING:
return copy.deepcopy(cached)
site = self.sites.get(domain, {})
config = self._get_site_data(domain, preset_name)
if config is None:
logger.warning(f"站点 {domain} 无可用预设")
return None
used_preset = preset_name or self._resolve_default_preset_name(site) or DEFAULT_PRESET_NAME
previous_config = None
# 补充缺失字段
changed = False
if "workflow" not in config:
if previous_config is None:
previous_config = copy.deepcopy(config)
config["workflow"] = DEFAULT_WORKFLOW
changed = True
if "image_extraction" not in config:
if previous_config is None:
previous_config = copy.deepcopy(config)
config["image_extraction"] = get_default_image_extraction_config()
changed = True
if "file_paste" not in config:
if previous_config is None:
previous_config = copy.deepcopy(config)
config["file_paste"] = get_default_file_paste_config()
changed = True
else:
normalized_file_paste = self._validate_file_paste_config(
config.get("file_paste", {}),
legacy_stream_config=config.get("stream_config"),
)
if normalized_file_paste != config.get("file_paste"):
if previous_config is None:
previous_config = copy.deepcopy(config)
config["file_paste"] = normalized_file_paste
changed = True
if "prompt_padding" not in config:
if previous_config is None:
previous_config = copy.deepcopy(config)
config["prompt_padding"] = get_default_prompt_padding_config()
changed = True
else:
normalized_prompt_padding = self._validate_prompt_padding_config(
config.get("prompt_padding", {})
)
if normalized_prompt_padding != config.get("prompt_padding"):
if previous_config is None:
previous_config = copy.deepcopy(config)
config["prompt_padding"] = normalized_prompt_padding
changed = True
if changed:
completed_config = copy.deepcopy(config)
if not self._save_config():
config.clear()
config.update(previous_config)
logger.warning(
f"站点 {domain} [{used_preset}] 配置自动补全保存失败,"
"已仅对当前请求返回补全配置"
)
return completed_config
logger.debug(f"使用缓存配置: {domain} [预设: {used_preset}]")
result = copy.deepcopy(config)
self._cache.set(cache_key, result)
return copy.deepcopy(result)
if not automatic_discovery_allowed(domain):
logger.info(f"跳过非聊天站点的自动收录: {domain}(已有手动配置不受影响)")
return None
resolved_html = ""
if callable(html_content):
try:
resolved_html = str(html_content() or "")
except Exception as e:
logger.warning(f"获取页面 HTML 失败: {e}")
resolved_html = ""
elif html_content is not None:
resolved_html = str(html_content or "")
if resolved_html.strip():
logger.info(f"🔍 未知域名 {domain},启动 AI 识别...")
clean_html = self.html_cleaner.clean(resolved_html)
selectors = self.ai_analyzer.analyze(clean_html)
if specific_chat_selectors(selectors):
selectors = self.validator.validate(selectors)
if selectors_match_chat_html(selectors, clean_html):
new_preset: SiteConfig = {
"selectors": selectors,
"workflow": DEFAULT_WORKFLOW,
"stealth": self._guess_stealth(domain),
"stream_config": copy.deepcopy(get_default_stream_config()),
"image_extraction": get_default_image_extraction_config(),
"file_paste": get_default_file_paste_config(),
"prompt_padding": get_default_prompt_padding_config(),
}
self.sites[domain] = {
"default_preset": DEFAULT_PRESET_NAME,
"presets": {
DEFAULT_PRESET_NAME: new_preset
}
}
if self._save_config():
logger.info(f"✅ 配置已生成并保存: {domain}")
else:
logger.warning(f"⚠️ 配置已生成但保存失败,仅保留在当前运行内存: {domain}")
return copy.deepcopy(new_preset)
logger.warning(f"⚠️ AI 分析失败,使用通用回退配置: {domain}")
else:
logger.warning(f"⚠️ 未知域名 {domain} 且未提供网页 HTML,使用通用回退配置")
fallback_selectors = self.global_config.get_fallback_selectors()
fallback_preset: SiteConfig = {
"selectors": fallback_selectors,
"workflow": DEFAULT_WORKFLOW,
"stealth": False,
"stream_config": copy.deepcopy(get_default_stream_config()),
"image_extraction": get_default_image_extraction_config(),
"file_paste": get_default_file_paste_config(),
"prompt_padding": get_default_prompt_padding_config(),
}
# A generic fallback is only for this caller. It is not proof that this
# domain hosts an AI chat product, and must never enter the site catalog.
logger.info(f"通用回退仅用于当前请求,未自动添加站点: {domain}")
return copy.deepcopy(fallback_preset)
def delete_site_config(self, domain: str) -> bool:
"""删除指定站点配置"""
self.refresh_if_changed()
if domain in self.sites:
removed_site = copy.deepcopy(self.sites[domain])
previous_global_default = copy.deepcopy(self._global_default_presets)
previous_local_default = copy.deepcopy(self._local_default_presets)
del self.sites[domain]
if not self._save_config():
self.sites[domain] = removed_site
self._global_default_presets = previous_global_default
self._local_default_presets = previous_local_default
return False
logger.info(f"已删除配置: {domain}")
return True
return False
def get_selector_definitions(self) -> List[SelectorDefinition]:
"""获取元素定义列表"""
return self.global_config.get_selector_definitions()
def set_selector_definitions(self, definitions: List[SelectorDefinition]) -> bool:
"""设置元素定义列表并保存"""
previous_definitions = self.global_config.get_selector_definitions()
previous_fallback_selectors = copy.deepcopy(self.validator.fallback_selectors)
self.global_config.set_selector_definitions(definitions)
# 更新验证器的回退选择器
self.validator.fallback_selectors = self.global_config.get_fallback_selectors()
# 保存配置
if not self._save_config():
self.global_config.set_selector_definitions(previous_definitions)
self.validator.fallback_selectors = previous_fallback_selectors
return False
logger.info(f"元素定义已更新: {len(definitions)} 个")
return True
+485
View File
@@ -0,0 +1,485 @@
"""ConfigEngine 的一部分(R2-3 拆分):提取器、流式/网络监听配置及其校验、解析器列表。
方法原样从 engine.py 迁出,行为不变;通过 mixin 组合回 ConfigEngine,
实例状态都在 ConfigEngine.__init__(engine.py)里初始化。
"""
import copy
from typing import Dict, Optional, List, Any
from app.core.parsers import ParserRegistry
from app.models.schemas import get_default_attachment_monitor_config, get_default_send_confirmation_config
from app.services.extractor_manager import extractor_manager
from app.services.parser_manager import parser_manager
from app.core.request_transport import normalize_request_transport_config
from .engine_common import (
DEFAULT_PRESET_NAME,
_MISSING,
_coerce_bool,
_merge_config_patch,
get_default_network_config,
get_default_stream_config,
logger,
)
class ConfigStreamMixin:
"""提取器、流式/网络监听配置及其校验、解析器列表"""
def get_site_extractor(self, domain: str, preset_name: str = None):
"""获取站点的提取器实例"""
self.refresh_if_changed()
data = self._get_site_data(domain, preset_name)
if data is not None:
return extractor_manager.get_extractor_for_site(data)
return extractor_manager.get_extractor()
def set_site_extractor(self, domain: str, extractor_id: str,
preset_name: str = None) -> bool:
"""为站点设置提取器"""
self.refresh_if_changed()
data = self._get_site_data(domain, preset_name)
if data is None:
logger.warning(f"站点或预设不存在: {domain}/{preset_name}")
return False
from app.core.extractors import ExtractorRegistry
if not ExtractorRegistry.exists(extractor_id):
logger.error(f"提取器不存在: {extractor_id}")
return False
previous_extractor_id = data.get("extractor_id", _MISSING)
if previous_extractor_id is not _MISSING:
previous_extractor_id = copy.deepcopy(previous_extractor_id)
previous_verified = data.get("extractor_verified", _MISSING)
if previous_verified is not _MISSING:
previous_verified = copy.deepcopy(previous_verified)
data["extractor_id"] = extractor_id
data["extractor_verified"] = False
if not self._save_config():
if previous_extractor_id is _MISSING:
data.pop("extractor_id", None)
else:
data["extractor_id"] = previous_extractor_id
if previous_verified is _MISSING:
data.pop("extractor_verified", None)
else:
data["extractor_verified"] = previous_verified
return False
logger.info(f"站点 {domain} [{preset_name or DEFAULT_PRESET_NAME}] 已绑定提取器: {extractor_id}")
return True
def set_site_extractor_verified(self, domain: str, verified: bool = True,
preset_name: str = None) -> bool:
"""设置站点提取器验证状态"""
self.refresh_if_changed()
data = self._get_site_data(domain, preset_name)
if data is None:
return False
if not self._assign_preset_field_and_save(data, "extractor_verified", verified):
return False
return True
def get_site_stream_config(self, domain: str, preset_name: str = None) -> Dict[str, Any]:
"""
获取站点的流式配置
Args:
domain: 站点域名
preset_name: 预设名称
Returns:
完整的流式配置(包含默认值)
"""
self.refresh_if_changed()
cache_key = f"stream_config:{domain}:{preset_name or ''}"
cached = self._cache.get(cache_key, _MISSING)
if cached is not _MISSING:
return copy.deepcopy(cached)
default_config = get_default_stream_config()
data = self._get_site_data(domain, preset_name)
if data is None:
result = copy.deepcopy(default_config)
self._cache.set(cache_key, result)
return copy.deepcopy(result)
stream_config = data.get("stream_config", {})
# 合并默认值
result = copy.deepcopy(default_config)
# 更新顶层字段
for key in ["mode", "hard_timeout"]:
if key in stream_config:
result[key] = stream_config[key]
if isinstance(stream_config.get("request_transport"), dict):
result["request_transport"] = normalize_request_transport_config(
stream_config.get("request_transport")
)
# 处理 send_confirmation 配置
if isinstance(stream_config.get("send_confirmation"), dict):
result["send_confirmation"].update(stream_config["send_confirmation"])
# 处理 attachment_monitor 配置
if isinstance(stream_config.get("attachment_monitor"), dict):
result["attachment_monitor"].update(stream_config["attachment_monitor"])
file_paste_config = data.get("file_paste", {})
if isinstance(file_paste_config, dict):
if isinstance(file_paste_config.get("send_confirmation"), dict):
result["send_confirmation"].update(file_paste_config["send_confirmation"])
if isinstance(file_paste_config.get("attachment_monitor"), dict):
result["attachment_monitor"].update(file_paste_config["attachment_monitor"])
# 处理 network 配置
if stream_config.get("network"):
network_default = get_default_network_config()
network_config = stream_config["network"]
result["network"] = network_default.copy()
for key in [
"listen_pattern",
"stream_match_mode",
"stream_match_pattern",
"parser",
"hard_timeout",
"first_response_timeout",
"first_content_timeout",
"initial_target_body_wait",
"silence_threshold",
"response_interval",
]:
if key in network_config:
result["network"][key] = network_config[key]
self._cache.set(cache_key, copy.deepcopy(result))
return result
def set_site_stream_config(self, domain: str, config: Dict[str, Any],
preset_name: str = None) -> bool:
"""
设置站点的流式配置
Args:
domain: 站点域名
config: 流式配置(部分或完整)
preset_name: 预设名称
Returns:
是否成功
"""
self.refresh_if_changed()
data = self._get_site_data(domain, preset_name)
if data is None:
logger.warning(f"站点或预设不存在: {domain}/{preset_name}")
return False
previous_file_paste = data.get("file_paste", _MISSING)
if previous_file_paste is not _MISSING:
previous_file_paste = copy.deepcopy(previous_file_paste)
previous_stream = data.get("stream_config", _MISSING)
if previous_stream is not _MISSING:
previous_stream = copy.deepcopy(previous_stream)
legacy_file_paste_updates: Dict[str, Any] = {}
if isinstance(config.get("send_confirmation"), dict):
legacy_file_paste_updates["send_confirmation"] = config.get("send_confirmation") or {}
if isinstance(config.get("attachment_monitor"), dict):
legacy_file_paste_updates["attachment_monitor"] = config.get("attachment_monitor") or {}
# 验证并规范化配置
current_config = self.get_site_stream_config(domain, preset_name)
merged_config = _merge_config_patch(current_config, config if isinstance(config, dict) else {})
validated = self._validate_stream_config(merged_config)
validated.pop("send_confirmation", None)
validated.pop("attachment_monitor", None)
if legacy_file_paste_updates:
existing_file_paste = self._validate_file_paste_config(
data.get("file_paste", {}),
legacy_stream_config=data.get("stream_config"),
)
merged_file_paste = _merge_config_patch(existing_file_paste, legacy_file_paste_updates)
data["file_paste"] = self._validate_file_paste_config(
merged_file_paste,
legacy_stream_config=data.get("stream_config"),
)
data["stream_config"] = validated
if not self._save_config():
if previous_stream is _MISSING:
data.pop("stream_config", None)
else:
data["stream_config"] = previous_stream
if previous_file_paste is _MISSING:
data.pop("file_paste", None)
else:
data["file_paste"] = previous_file_paste
return False
logger.info(f"站点 {domain} [{preset_name or DEFAULT_PRESET_NAME}] 流式配置已更新 (mode={validated.get('mode')})")
return True
def _validate_stream_config(self, config: Dict[str, Any]) -> Dict[str, Any]:
"""
验证并规范化流式配置
Args:
config: 原始配置
Returns:
规范化后的配置
"""
result = get_default_stream_config()
if not config:
return result
mode_explicitly_set = False
# 验证 mode
if "mode" in config:
mode = str(config["mode"]).lower()
if mode in ("dom", "network"):
result["mode"] = mode
mode_explicitly_set = True
# 验证数值字段
for key in ["hard_timeout"]:
if key in config:
try:
val = float(config[key])
if key == "hard_timeout":
result[key] = max(10, min(val, 600))
except (ValueError, TypeError):
pass
# 验证 send_confirmation 配置
if isinstance(config.get("send_confirmation"), dict):
result["send_confirmation"] = self._validate_send_confirmation_config(
config["send_confirmation"]
)
if isinstance(config.get("request_transport"), dict):
result["request_transport"] = normalize_request_transport_config(
config["request_transport"]
)
# 验证 attachment_monitor 配置
if isinstance(config.get("attachment_monitor"), dict):
result["attachment_monitor"] = self._validate_attachment_monitor_config(
config["attachment_monitor"]
)
# 验证 network 配置
if config.get("network"):
network_config = self._validate_network_config(config["network"])
if network_config:
result["network"] = network_config
# 仅在调用方没有显式指定 mode 时,才根据 network 配置自动切到 network。
# 这样切回 DOM 时可以保留 parser/listen_pattern,而不会被后端强制改回 network。
if (
not mode_explicitly_set
and network_config.get("parser")
and network_config.get("listen_pattern")
):
result["mode"] = "network"
return result
def _validate_send_confirmation_config(self, config: Dict[str, Any]) -> Dict[str, Any]:
"""验证发送成功判定配置。"""
result = get_default_send_confirmation_config()
if not isinstance(config, dict):
return result
numeric_ranges = {
"post_click_observe_window": (0.0, 15.0),
"pre_retry_probe_window": (0.0, 5.0),
"retry_observe_window": (0.0, 15.0),
"attachment_observe_window": (0.0, 30.0),
"retry_interval": (0.0, 30.0),
"retry_cooldown_window": (0.0, 30.0),
}
for key, (minimum, maximum) in numeric_ranges.items():
if key not in config:
continue
try:
value = float(config[key])
except (TypeError, ValueError):
continue
result[key] = max(minimum, min(value, maximum))
if "max_retry_count" in config:
try:
value = int(config["max_retry_count"])
except (TypeError, ValueError):
value = None
if value is not None:
result["max_retry_count"] = max(0, min(value, 10))
retry_action = str(config.get("retry_action") or "").strip().lower()
if retry_action in {"click_send_btn", "key_press"}:
result["retry_action"] = retry_action
if "retry_key_combo" in config:
retry_key_combo = str(config.get("retry_key_combo") or "").strip()
if retry_key_combo:
result["retry_key_combo"] = retry_key_combo[:64]
bool_fields = [
"retry_on_unconfirmed_send",
"accept_attachment_change",
"accept_attachment_disappear",
"accept_probe_confirmation",
"retry_block_on_stop_button",
"retry_block_if_generating",
"trust_network_activity",
"trust_generating_indicator",
"trust_send_disabled_with_input_shrink",
]
for key in bool_fields:
if key not in config:
continue
result[key] = _coerce_bool(config.get(key), bool(result.get(key, False)))
sensitivity = str(config.get("attachment_sensitivity") or "").strip().lower()
if sensitivity in {"low", "medium", "high"}:
result["attachment_sensitivity"] = sensitivity
return result
def _validate_attachment_monitor_config(self, config: Dict[str, Any]) -> Dict[str, Any]:
"""验证附件上传/发送前判定规则。"""
result = get_default_attachment_monitor_config()
if not isinstance(config, dict):
return result
list_fields = [
"root_selectors",
"attachment_selectors",
"pending_selectors",
"status_selectors",
"content_exclusion_selectors",
"busy_text_markers",
"ignored_busy_text_markers",
"send_button_disabled_markers",
]
for key in list_fields:
raw_value = config.get(key)
if raw_value is None:
continue
if not isinstance(raw_value, list):
continue
cleaned = []
for item in raw_value:
value = str(item or "").strip()
if value and value not in cleaned:
cleaned.append(value)
result[key] = cleaned
numeric_ranges = {
"idle_timeout": (0.5, 60.0),
"hard_max_wait": (1.0, 300.0),
}
for key, (minimum, maximum) in numeric_ranges.items():
if key not in config:
continue
try:
value = float(config[key])
except (TypeError, ValueError):
continue
result[key] = max(minimum, min(value, maximum))
bool_fields = [
"require_attachment_present",
"require_upload_signal_before_ready",
"continue_once_on_unconfirmed_send",
]
for key in bool_fields:
if key not in config:
continue
result[key] = _coerce_bool(config.get(key), bool(result.get(key, False)))
return result
def _validate_network_config(self, config: Dict[str, Any]) -> Optional[Dict[str, Any]]:
"""
验证网络监听配置
Args:
config: 原始网络配置
Returns:
规范化后的配置,无效则返回 None
"""
if not config:
return None
result = get_default_network_config()
# listen_pattern(必填)
if "listen_pattern" in config:
pattern = str(config["listen_pattern"]).strip()
if pattern:
result["listen_pattern"] = pattern
if "stream_match_mode" in config:
match_mode = str(config["stream_match_mode"]).strip().lower()
if match_mode in {"keyword", "regex"}:
result["stream_match_mode"] = match_mode
if "stream_match_pattern" in config:
result["stream_match_pattern"] = str(config["stream_match_pattern"]).strip()
# parser(必填,需验证存在性)
if "parser" in config:
parser_id = str(config["parser"]).strip()
if parser_id:
# 验证解析器是否存在
if ParserRegistry.exists(parser_id):
result["parser"] = parser_id
else:
logger.warning(f"解析器不存在: {parser_id}")
# 仍然保存,允许后续添加解析器
result["parser"] = parser_id
# 验证数值字段
for key in ["first_response_timeout", "silence_threshold", "response_interval"]:
if key in config:
try:
val = float(config[key])
if key == "first_response_timeout":
result[key] = max(0.5, min(val, 300))
elif key == "silence_threshold":
result[key] = max(0.5, min(val, 30))
elif key == "response_interval":
result[key] = max(0.1, min(val, 5))
except (ValueError, TypeError):
pass
# 检查是否有有效配置
if not result["listen_pattern"] or not result["parser"]:
return None
return result
def list_available_parsers(self) -> List[Dict[str, str]]:
"""
列出所有可用的响应解析器
Returns:
解析器信息列表
"""
return parser_manager.list_parsers()
def get_extractor_manager(self):
"""获取提取器管理器实例"""
return extractor_manager
def get_parser_manager(self):
"""获取解析器管理器实例"""
return parser_manager
+440
View File
@@ -0,0 +1,440 @@
"""
app/services/config/managers.py - 配置子管理器
职责:
- 全局配置管理(元素定义)
- 图片预设管理
"""
import json
import os
import copy
from app.core.config import get_logger
from typing import Dict, List, Any, Optional
from app.models.schemas import (
SelectorDefinition,
get_default_selector_definitions,
get_default_image_extraction_config
)
logger = get_logger("CFG_MGR")
# ================= 全局配置管理器 =================
class GlobalConfigManager:
"""
全局配置管理器
管理 _global 节点中的配置,包括:
- selector_definitions: 元素定义列表
- 以及任意扩展字段(通过 get/set 访问)
"""
def __init__(self):
self._selector_definitions: List[SelectorDefinition] = get_default_selector_definitions()
self._extra: Dict[str, Any] = {} # 通用扩展存储
def load(self, global_section: Dict[str, Any]):
"""从 _global 节点加载配置"""
self._extra = {}
self._selector_definitions = get_default_selector_definitions()
if not global_section:
return
if "selector_definitions" in global_section:
defs = global_section["selector_definitions"]
if isinstance(defs, list):
self._selector_definitions = self._merge_selector_definitions(defs)
logger.debug(f"已加载 {len(defs)} 个元素定义")
# 加载所有非 selector_definitions 的字段到通用存储
for key, value in global_section.items():
if key != "selector_definitions":
self._extra[key] = copy.deepcopy(value)
if "commands" in self._extra:
cmd_count = len(self._extra["commands"]) if isinstance(self._extra["commands"], list) else 0
logger.debug(f"已加载 {cmd_count} 个自动化命令")
def get_selector_definitions(self) -> List[SelectorDefinition]:
"""获取元素定义列表"""
return copy.deepcopy(self._selector_definitions)
def set_selector_definitions(self, definitions: List[SelectorDefinition]):
"""设置元素定义列表"""
self._selector_definitions = self._merge_selector_definitions(definitions)
@staticmethod
def _normalize_selector_definition(raw_item: Any) -> Optional[SelectorDefinition]:
if not isinstance(raw_item, dict):
return None
key = raw_item.get("key")
if not isinstance(key, str):
return None
key = key.strip()
if not key:
return None
description = raw_item.get("description", "")
if not isinstance(description, str):
description = str(description) if description is not None else ""
return {
"key": key,
"description": description,
"enabled": bool(raw_item.get("enabled", True)),
"required": bool(raw_item.get("required", False)),
}
def _merge_selector_definitions(self, definitions: List[SelectorDefinition]) -> List[SelectorDefinition]:
"""合并内置默认字段,确保升级后新增的选择器能自动出现。"""
merged = []
existing_keys = set()
for item in definitions or []:
normalized = self._normalize_selector_definition(item)
if not normalized:
continue
key = normalized["key"]
if key in existing_keys:
continue
merged.append(normalized)
existing_keys.add(key)
for default_def in get_default_selector_definitions():
normalized_default = self._normalize_selector_definition(default_def)
if not normalized_default:
continue
key = normalized_default.get("key")
if key not in existing_keys:
merged.append(normalized_default)
existing_keys.add(key)
return merged
def get_enabled_definitions(self) -> List[SelectorDefinition]:
"""获取启用的元素定义"""
return [
copy.deepcopy(d)
for d in self._selector_definitions
if isinstance(d, dict) and d.get("enabled", True)
]
def get_fallback_selectors(self) -> Dict[str, Optional[str]]:
"""
生成回退选择器字典
基于元素定义生成,用于 SelectorValidator
"""
fallback_map = {
"input_box": "textarea",
"send_btn": 'button[type="submit"]',
"result_container": "div",
"new_chat_btn": None,
"retry_send_btn": None,
"message_wrapper": None,
"generating_indicator": None,
"upload_btn": None,
"file_input": 'input[type="file"]',
"drop_zone": None,
}
result = {}
for d in self._selector_definitions:
if not isinstance(d, dict):
continue
key = d.get("key")
if not isinstance(key, str) or not key:
continue
result[key] = fallback_map.get(key, None)
return result
def build_prompt_selector_list(self) -> str:
"""
生成 AI 提示词中的元素查找列表
只包含 enabled=True 的元素
"""
lines = []
for d in self._selector_definitions:
if not isinstance(d, dict):
continue
if not d.get("enabled", True):
continue
key = str(d.get("key") or "").strip()
if not key:
continue
desc = str(d.get("description") or "")
required = d.get("required", False)
if required:
tag = "[REQUIRED]"
else:
tag = "[OPTIONAL, return null if not found]"
lines.append(f"- `{key}`: {desc} {tag}")
return "\n".join(lines)
def build_prompt_json_keys(self) -> str:
"""
生成 AI 提示词中的 JSON 输出格式说明
"""
lines = []
for d in self._selector_definitions:
if not isinstance(d, dict):
continue
if not d.get("enabled", True):
continue
key = str(d.get("key") or "").strip()
if not key:
continue
desc = str(d.get("description") or "")
lines.append(f'- `{key}`: {desc}')
return "\n".join(lines)
def get(self, key: str, default: Any = None) -> Any:
"""获取通用配置值"""
return copy.deepcopy(self._extra.get(key, default))
def set(self, key: str, value: Any):
"""设置通用配置值"""
self._extra[key] = copy.deepcopy(value)
def remove(self, key: str) -> bool:
"""删除通用配置值"""
if key not in self._extra:
return False
del self._extra[key]
return True
def to_dict(self) -> Dict[str, Any]:
"""导出为字典(用于保存)"""
result = {
"selector_definitions": self._selector_definitions
}
# 合并通用存储中的所有字段
result.update(self._extra)
return copy.deepcopy(result)
# ================= 图片预设管理器 =================
class ImagePresetsManager:
"""
图片提取预设管理器
职责:
- 加载预设配置文件
- 提供站点预设查询
- 应用预设到站点配置
"""
def __init__(self, presets_file: str):
self.presets_file = presets_file
self.presets: Dict[str, Any] = {}
self._load_presets()
@staticmethod
def _default_presets_payload() -> Dict[str, Any]:
return {
"_default": {
"name": "Default",
"description": "Built-in fallback image extraction preset",
"image_extraction": get_default_image_extraction_config(),
}
}
@staticmethod
def _get_preset_image_config(preset_data: Any) -> Optional[Dict[str, Any]]:
if not isinstance(preset_data, dict):
return None
config = preset_data.get("image_extraction")
return copy.deepcopy(config) if isinstance(config, dict) else None
@staticmethod
def _get_preset_text(preset_data: Dict[str, Any], key: str, default: str = "") -> str:
value = preset_data.get(key, default)
if value is None:
return default
if isinstance(value, (str, int, float, bool)):
return str(value)
return default
def _find_matching_preset(self, domain: str) -> Optional[tuple[str, Dict[str, Any]]]:
domain = str(domain or "").strip().lower().rstrip(".")
if not domain:
return None
normalized_presets = []
for preset_domain, preset_data in self.presets.items():
if preset_domain.startswith("_") or not isinstance(preset_data, dict):
continue
normalized_preset_domain = str(preset_domain).strip().lower().rstrip(".")
if not normalized_preset_domain:
continue
normalized_presets.append((preset_domain, preset_data, normalized_preset_domain))
for preset_domain, preset_data, normalized_preset_domain in normalized_presets:
if domain == normalized_preset_domain:
return preset_domain, preset_data
suffix_matches = [
item
for item in normalized_presets
if domain.endswith(f".{item[2]}")
]
if suffix_matches:
preset_domain, preset_data, _ = max(
suffix_matches,
key=lambda item: len(item[2]),
)
logger.debug(f"使用模糊匹配预设: {domain} -> {preset_domain}")
return preset_domain, preset_data
return None
def _use_default_presets_or_keep_existing(self, keep_existing: bool) -> None:
if keep_existing and self.presets:
logger.warning("图片预设加载失败,保留当前内存中的预设")
return
self.presets = self._default_presets_payload()
def _load_presets(self, keep_existing_on_error: bool = False):
"""加载预设配置文件"""
if not os.path.exists(self.presets_file):
logger.warning(f"图片预设文件不存在: {self.presets_file}")
self._use_default_presets_or_keep_existing(keep_existing_on_error)
return
try:
with open(self.presets_file, 'r', encoding='utf-8') as f:
data = json.load(f)
if not isinstance(data, dict):
logger.error("预设文件格式错误: 顶层必须是对象")
self._use_default_presets_or_keep_existing(keep_existing_on_error)
return
# 移除元数据
loaded_presets = {
str(k): v
for k, v in data.items()
if k != "_meta" and isinstance(v, dict)
}
skipped_count = len([k for k, v in data.items() if k != "_meta" and not isinstance(v, dict)])
if skipped_count:
logger.warning(f"已跳过 {skipped_count} 个格式无效的图片预设")
if not loaded_presets:
self._use_default_presets_or_keep_existing(keep_existing_on_error)
logger.debug(f"已加载 {len(self.presets)} 个图片预设")
return
self.presets = loaded_presets
logger.debug(f"已加载 {len(self.presets)} 个图片预设")
except json.JSONDecodeError as e:
logger.error(f"预设文件格式错误: {e}")
self._use_default_presets_or_keep_existing(keep_existing_on_error)
except Exception as e:
logger.error(f"加载预设失败: {e}")
self._use_default_presets_or_keep_existing(keep_existing_on_error)
def get_preset(self, domain: str) -> Optional[Dict]:
"""
获取站点的预设配置
Args:
domain: 站点域名
Returns:
预设配置字典,不存在返回 None
"""
match = self._find_matching_preset(domain)
if match is None:
return None
return self._get_preset_image_config(match[1])
def list_presets(self) -> List[Dict[str, Any]]:
"""
列出所有可用预设
Returns:
预设列表,每项包含 domain、name、description、enabled
"""
result = []
for domain, data in self.presets.items():
if domain == "_meta":
continue
if not isinstance(data, dict):
continue
config = self._get_preset_image_config(data) or {}
item = {
"domain": domain,
"name": self._get_preset_text(data, "name", domain),
"description": self._get_preset_text(data, "description", ""),
"enabled": config.get("enabled", False),
"notes": self._get_preset_text(data, "notes", ""),
"config": config
}
result.append(item)
result.sort(key=lambda x: x["domain"])
return result
def get_preset_for_display(self, domain: str) -> Dict[str, Any]:
"""
获取用于显示的预设信息
Args:
domain: 站点域名
Returns:
包含 available、preset_domain、config 的字典
"""
match = self._find_matching_preset(domain)
if match is None:
preset_config = None
else:
preset_config = self._get_preset_image_config(match[1])
if preset_config:
matched_domain, matched_data = match
return {
"available": True,
"preset_domain": matched_domain,
"name": self._get_preset_text(
matched_data,
"name",
"",
),
"config": preset_config
}
return {
"available": False,
"preset_domain": None,
"name": None,
"config": None
}
def reload(self):
"""重新加载预设文件"""
self._load_presets(keep_existing_on_error=True)
__all__ = ['GlobalConfigManager', 'ImagePresetsManager']
+661
View File
@@ -0,0 +1,661 @@
"""
app/services/config/processors.py - 配置处理器
职责:
- HTML 清理
- 选择器验证
- AI 页面分析
"""
import json
import importlib.util
import os
import re
import time
from app.core.config import get_logger
from app.utils.site_discovery import admitted_selectors
from typing import Dict, Optional
from urllib import request, error
import bs4
from bs4 import BeautifulSoup
from app.core.config import AppConfig
from .managers import GlobalConfigManager
logger = get_logger("CFG_PRC")
_BS4_PARSER = "lxml" if importlib.util.find_spec("lxml") is not None else "html.parser"
# ================= 常量 =================
# HTML 清理配置
MAX_HTML_CHARS = int(os.getenv("MAX_HTML_CHARS", "120000"))
TEXT_TRUNCATE_LENGTH = 80
# AI 请求配置
AI_MAX_RETRIES = 3
AI_RETRY_BASE_DELAY = 1.0
AI_RETRY_MAX_DELAY = 10.0
AI_REQUEST_TIMEOUT = 120
# 无效选择器语法模式
INVALID_SYNTAX_PATTERNS = [
(r'~\s*\.\.', '~ .. 无效语法'),
(r'\.\.\s*$', '结尾 .. 无效'),
(r'>>\s', '>> 无效语法'),
(r':has\(', ':has() 兼容性差'),
(r'\s~\s*$', '结尾 ~ 无效'),
]
# ================= HTML 清理器 =================
class HTMLCleaner:
"""HTML 清理器"""
TAGS_TO_REMOVE = [
'script', 'style', 'meta', 'link', 'noscript',
'img', 'video', 'audio', 'iframe', 'canvas',
'path', 'rect', 'circle', 'polygon', 'defs', 'clipPath',
'header', 'footer', 'nav', 'aside',
]
ALLOWED_ATTRS = [
'id', 'class', 'name', 'placeholder', 'aria-label', 'role',
'data-testid', 'type', 'disabled', 'value', 'title', 'tabindex',
'contenteditable', 'href'
]
INTERACTIVE_TAGS = ['input', 'textarea', 'button', 'form', 'a']
CORE_AREA_SELECTORS = [
'[role="main"]',
'main',
'#app',
'#root',
'.chat',
'.conversation',
'.message',
]
def __init__(self, max_chars: int = None, text_truncate: int = None):
self.max_chars = max_chars or MAX_HTML_CHARS
self.text_truncate = text_truncate or TEXT_TRUNCATE_LENGTH
def clean(self, html: str) -> str:
"""深度清理 HTML"""
logger.debug("开始 HTML 清理...")
original_length = len(html)
soup = BeautifulSoup(html, _BS4_PARSER)
try:
interactive_elements = self._extract_interactive_elements(soup)
for tag in soup(self.TAGS_TO_REMOVE):
tag.decompose()
for element in soup(text=lambda t: isinstance(t, bs4.element.Comment)):
element.extract()
for tag in soup.find_all(True):
if tag.string and len(tag.string) > self.text_truncate:
tag.string = tag.string[:self.text_truncate] + "..."
attrs = dict(tag.attrs)
for attr in attrs:
if attr not in self.ALLOWED_ATTRS:
del tag.attrs[attr]
if 'class' in tag.attrs and isinstance(tag.attrs['class'], list):
tag.attrs['class'] = " ".join(tag.attrs['class'])
clean_html = str(soup.body) if soup.body else str(soup)
clean_html = re.sub(r'\s+', ' ', clean_html).strip()
if len(clean_html) > self.max_chars:
logger.warning(f"HTML 过长 ({len(clean_html)}),执行智能截断...")
clean_html = self._smart_truncate(clean_html, interactive_elements)
final_length = len(clean_html)
reduction = 100 - (final_length / original_length * 100) if original_length > 0 else 0
logger.info(f"HTML 清理完成: {original_length} → {final_length} 字符 (减少 {reduction:.1f}%)")
return clean_html
finally:
soup.decompose()
def _extract_interactive_elements(self, soup: BeautifulSoup) -> str:
elements = []
for tag_name in self.INTERACTIVE_TAGS:
for element in soup.find_all(tag_name):
context = self._get_element_with_context(element, levels=2)
if context:
elements.append(context)
unique_elements = list(dict.fromkeys(elements))
return "\n".join(unique_elements)
def _get_element_with_context(self, element, levels: int = 2) -> str:
try:
current = element
for _ in range(levels):
if current.parent and current.parent.name not in ['body', 'html', '[document]']:
current = current.parent
else:
break
html_str = str(current)
if len(html_str) > 2000:
html_str = html_str[:2000] + "..."
return html_str
except Exception:
return str(element)
def _smart_truncate(self, html: str, interactive_html: str) -> str:
interactive_budget = int(self.max_chars * 0.3)
if len(interactive_html) > interactive_budget:
interactive_html = interactive_html[:interactive_budget]
remaining_budget = self.max_chars - len(interactive_html) - 100
if remaining_budget <= 0:
logger.warning("交互元素已占满预算")
return interactive_html
core_html = self._extract_core_area(html)
if core_html and len(core_html) <= remaining_budget:
result = core_html + "\n<!-- INTERACTIVE ELEMENTS -->\n" + interactive_html
return result[:self.max_chars]
head_budget = remaining_budget // 3
tail_budget = remaining_budget // 3
head_part = self._truncate_at_tag_boundary(html[:head_budget * 2], head_budget, from_end=False)
tail_part = self._truncate_at_tag_boundary(html[-tail_budget * 2:], tail_budget, from_end=True)
result = (
head_part +
"\n<!-- TRUNCATED: MIDDLE SECTION -->\n" +
"<!-- INTERACTIVE ELEMENTS START -->\n" +
interactive_html +
"\n<!-- INTERACTIVE ELEMENTS END -->\n" +
tail_part
)
if len(result) > self.max_chars:
result = result[:self.max_chars]
return result
def _extract_core_area(self, html: str) -> Optional[str]:
soup = None
try:
soup = BeautifulSoup(html, _BS4_PARSER)
for selector in self.CORE_AREA_SELECTORS:
try:
element = soup.select_one(selector)
if element:
core_html = str(element)
logger.debug(f"找到核心区域: {selector} ({len(core_html)} chars)")
return core_html
except Exception:
continue
return None
except Exception as e:
logger.debug(f"提取核心区域失败: {e}")
return None
finally:
if soup is not None:
soup.decompose()
def _truncate_at_tag_boundary(self, html: str, max_len: int, from_end: bool = False) -> str:
if len(html) <= max_len:
return html
if from_end:
start_pos = len(html) - max_len
tag_start = html.find('<', start_pos)
if tag_start != -1 and tag_start < len(html) - 100:
return html[tag_start:]
return html[-max_len:]
else:
tag_end = html.rfind('>', 0, max_len)
if tag_end != -1 and tag_end > 100:
return html[:tag_end + 1]
return html[:max_len]
# ================= 选择器验证器 =================
class SelectorValidator:
"""选择器验证器"""
def __init__(self, fallback_selectors: Dict[str, Optional[str]] = None):
self.fallback_selectors = fallback_selectors or {}
def validate(self, selectors: Dict[str, Optional[str]]) -> Dict[str, Optional[str]]:
"""验证并修复选择器"""
fixed = {}
for key, selector in selectors.items():
if selector is None:
fixed[key] = self.fallback_selectors.get(key)
continue
is_invalid = False
invalid_reason = ""
for pattern, reason in INVALID_SYNTAX_PATTERNS:
if re.search(pattern, selector):
is_invalid = True
invalid_reason = reason
break
if is_invalid:
logger.warning(f"❌ 无效选择器 [{key}]: {selector}")
logger.warning(f" 原因: {invalid_reason}")
repaired = self._try_repair(selector)
if repaired:
logger.info(f" ✅ 修复为: {repaired}")
fixed[key] = repaired
else:
fallback = self.fallback_selectors.get(key)
logger.info(f" 🔄 回退为: {fallback}")
fixed[key] = fallback
else:
if re.search(r'\._[a-f0-9]{5,}|^\.[a-f0-9]{6,}', selector):
logger.info(f"ℹ️ 哈希类名 [{key}]: {selector} (可能不稳定,但保留)")
fixed[key] = selector
return fixed
def _try_repair(self, selector: str) -> Optional[str]:
tag_match = re.match(r'^(\w+)', selector)
if not tag_match:
return None
tag = tag_match.group(1)
attr_patterns = [
r'(\[name=["\']?\w+["\']?\])',
r'(\[type=["\']?\w+["\']?\])',
r'(\[role=["\']?\w+["\']?\])',
r'(#[\w-]+)',
]
for pattern in attr_patterns:
match = re.search(pattern, selector)
if match:
return tag + match.group(1)
return tag
# ================= AI 分析器 =================
class AIAnalyzer:
"""AI 页面分析器(支持 OpenAI 兼容 / Gemini 官方 / Claude 官方)"""
# ===== Provider 注册表 =====
# 每个 provider 定义如何构建请求和解析响应
# 新增 provider 只需在此处注册,无需修改其他逻辑
PROVIDERS = {
"openai": {
"build_url": "_build_url_openai",
"build_headers": "_build_headers_openai",
"build_body": "_build_body_openai",
"parse_response": "_parse_response_openai",
},
"gemini": {
"build_url": "_build_url_gemini",
"build_headers": "_build_headers_gemini",
"build_body": "_build_body_gemini",
"parse_response": "_parse_response_gemini",
},
"claude": {
"build_url": "_build_url_claude",
"build_headers": "_build_headers_claude",
"build_body": "_build_body_claude",
"parse_response": "_parse_response_claude",
},
}
# URL 关键词 → provider 类型的映射(用于自动检测)
URL_PROVIDER_HINTS = [
("generativelanguage.googleapis.com", "gemini"),
("api.anthropic.com", "claude"),
("openai.azure.com", "azure"), # 未来可扩展
]
def __init__(self, global_config: GlobalConfigManager = None):
self.api_key = AppConfig.get_helper_api_key()
base_url = AppConfig.get_helper_base_url()
self.base_url = base_url.rstrip('/') if base_url else "http://127.0.0.1:5104/v1"
self.model = AppConfig.get_helper_model()
self.global_config = global_config
# 确定 provider 类型
configured_provider = AppConfig.get_helper_api_provider()
if configured_provider == "auto" or configured_provider not in self.PROVIDERS:
self.provider = self._detect_provider(self.base_url)
else:
self.provider = configured_provider
if not self.api_key:
logger.warning("⚠️ 未配置 HELPER_API_KEY,AI 分析功能将不可用")
else:
logger.debug(f"AI 分析器已配置: Provider={self.provider}, URL={self.base_url}, Model={self.model}")
# ================= 公共接口 =================
def analyze(self, html: str) -> Optional[Dict[str, str]]:
"""分析 HTML 并返回选择器"""
if not self.api_key:
logger.error("API Key 未配置")
return None
prompt = self._build_prompt(html)
for attempt in range(AI_MAX_RETRIES):
try:
logger.info(f"正在请求 AI 分析(尝试 {attempt + 1}/{AI_MAX_RETRIES},Provider: {self.provider})...")
response = self._request_ai(prompt)
if response:
analysis = self._extract_json(response)
if analysis is not None:
selectors = admitted_selectors(analysis)
if selectors:
logger.info("✅ 已确认 AI 聊天页并识别到专用定位器")
return selectors
logger.info("未确认 AI 聊天界面或缺少专用回复容器,不自动收录")
return None
logger.warning(f"第 {attempt + 1} 次分析失败")
except Exception as e:
logger.error(f"AI 请求异常: {e}")
if attempt < AI_MAX_RETRIES - 1:
delay = min(
AI_RETRY_BASE_DELAY * (2 ** attempt),
AI_RETRY_MAX_DELAY
)
jitter = delay * 0.1 * (0.5 - os.urandom(1)[0] / 255)
sleep_time = delay + jitter
logger.info(f"等待 {sleep_time:.2f}s 后重试...")
time.sleep(sleep_time)
logger.error("❌ AI 分析失败(已达最大重试次数)")
return None
# ================= Provider 自动检测 =================
def _detect_provider(self, base_url: str) -> str:
"""根据 URL 自动推断 API 类型"""
if not base_url:
return "openai"
url_lower = base_url.lower()
for keyword, provider in self.URL_PROVIDER_HINTS:
if keyword in url_lower:
logger.info(f"自动检测到 API 类型: {provider}(基于 URL 包含 '{keyword}')")
return provider
# 默认回退到 OpenAI 兼容格式
return "openai"
# ================= 统一请求调度 =================
def _request_ai(self, prompt: str) -> Optional[str]:
"""向 AI API 发送请求(根据 provider 自动适配格式)"""
provider_config = self.PROVIDERS.get(self.provider)
if not provider_config:
logger.error(f"未知的 Provider: {self.provider},回退到 openai")
provider_config = self.PROVIDERS["openai"]
# 通过注册表动态调用对应的构建/解析方法
url = getattr(self, provider_config["build_url"])(prompt)
headers = getattr(self, provider_config["build_headers"])()
body = getattr(self, provider_config["build_body"])(prompt)
parse_fn = getattr(self, provider_config["parse_response"])
try:
req = request.Request(
url,
data=json.dumps(body).encode('utf-8'),
headers=headers
)
with request.urlopen(req, timeout=AI_REQUEST_TIMEOUT) as response:
response_text = response.read().decode('utf-8')
try:
json_resp = json.loads(response_text)
return parse_fn(json_resp)
except json.JSONDecodeError:
logger.error(f"AI 响应 JSON 解析失败,原始响应前 500 字符: {response_text[:500]}")
return None
except error.HTTPError as e:
error_body = ""
try:
error_body = e.read().decode('utf-8')[:500]
except Exception:
pass
logger.error(f"HTTP 错误 {e.code}: {e.reason}" + (f",响应: {error_body}" if error_body else ""))
return None
except error.URLError as e:
logger.error(f"网络错误: {e.reason}")
return None
except Exception as e:
# 兜底:捕获 socket.timeout 等未被上层覆盖的异常
logger.error(f"请求异常: {type(e).__name__}: {e}")
return None
# ================= OpenAI 兼容格式 =================
def _build_url_openai(self, prompt: str) -> str:
return f"{self.base_url}/chat/completions"
def _build_headers_openai(self) -> Dict[str, str]:
return {
"Content-Type": "application/json",
"Authorization": f"Bearer {self.api_key}",
}
def _build_body_openai(self, prompt: str) -> Dict:
return {
"model": self.model,
"messages": [{"role": "user", "content": prompt}],
"temperature": 0.1,
}
def _parse_response_openai(self, json_resp: Dict) -> Optional[str]:
try:
if "choices" in json_resp and len(json_resp["choices"]) > 0:
return json_resp["choices"][0]["message"]["content"]
except (KeyError, IndexError, TypeError) as e:
logger.error(f"OpenAI 响应解析失败: {e}")
return None
# ================= Gemini 官方格式 =================
def _build_url_gemini(self, prompt: str) -> str:
# Gemini 官方 API: base_url 应为 https://generativelanguage.googleapis.com
# 完整端点: /v1beta/models/{model}:generateContent?key={api_key}
base = self.base_url.rstrip('/')
# 如果用户配置的 base_url 已经包含 /v1beta 或 /v1,不再追加
if "/v1beta" not in base and "/v1/" not in base:
base = f"{base}/v1beta"
return f"{base}/models/{self.model}:generateContent?key={self.api_key}"
def _build_headers_gemini(self) -> Dict[str, str]:
# Gemini 官方通过 URL 参数传递 key,不需要 Authorization 头
return {
"Content-Type": "application/json",
}
def _build_body_gemini(self, prompt: str) -> Dict:
return {
"contents": [
{
"parts": [
{"text": prompt}
]
}
],
"generationConfig": {
"temperature": 0.1,
},
}
def _parse_response_gemini(self, json_resp: Dict) -> Optional[str]:
try:
candidates = json_resp.get("candidates", [])
if candidates:
parts = candidates[0].get("content", {}).get("parts", [])
if parts:
return parts[0].get("text")
except (KeyError, IndexError, TypeError) as e:
logger.error(f"Gemini 响应解析失败: {e}")
# 检查是否有错误信息
if "error" in json_resp:
err = json_resp["error"]
logger.error(f"Gemini API 错误: [{err.get('code')}] {err.get('message', '')}")
return None
# ================= Claude 官方格式 =================
def _build_url_claude(self, prompt: str) -> str:
base = self.base_url.rstrip('/')
# Claude 官方端点: /v1/messages
if not base.endswith("/v1"):
base = f"{base}/v1"
return f"{base}/messages"
def _build_headers_claude(self) -> Dict[str, str]:
return {
"Content-Type": "application/json",
"x-api-key": self.api_key,
"anthropic-version": "2023-06-01",
}
def _build_body_claude(self, prompt: str) -> Dict:
return {
"model": self.model,
"max_tokens": 4096,
"messages": [{"role": "user", "content": prompt}],
"temperature": 0.1,
}
def _parse_response_claude(self, json_resp: Dict) -> Optional[str]:
try:
content = json_resp.get("content", [])
if content:
# Claude 响应的 content 是列表,每项有 type 和 text
text_parts = [block["text"] for block in content if block.get("type") == "text"]
if text_parts:
return "\n".join(text_parts)
except (KeyError, IndexError, TypeError) as e:
logger.error(f"Claude 响应解析失败: {e}")
# 检查是否有错误信息
if "error" in json_resp:
err = json_resp["error"]
logger.error(f"Claude API 错误: [{err.get('type')}] {err.get('message', '')}")
return None
# ================= Prompt 构建(不变)=================
def _build_prompt(self, clean_html: str) -> str:
"""构建 AI 提示词(动态版)"""
if self.global_config:
selector_list = self.global_config.build_prompt_selector_list()
json_keys = self.global_config.build_prompt_json_keys()
else:
selector_list = self._build_default_selector_list()
json_keys = self._build_default_json_keys()
lines = [
"You are a web interface classifier. First determine whether this is an AI CONVERSATION interface; do not assume every page is a chat app.",
"Return {\"is_chat_site\": false} for search engines, search results, login/consent pages, articles, ordinary forms, and pages with insufficient evidence.",
"A textarea, submit button, generic div, or the word AI alone is NOT evidence of an AI conversation interface.",
"Only for a genuine AI chat page with a composer and a specific assistant-reply container, return is_chat_site: true and the selector keys below. Do not invent selectors for missing elements.",
"",
"## CRITICAL RULES:",
"1. **Uniqueness is Key**: Ensure selectors matches ONLY the intended element.",
"2. **Distinguish AI vs User**: For `result_container`, specify the selector to target the **AI's response text** only. It MUST exclude user prompts, sidebars, or chat history.",
"3. **Use Hierarchy**: If a class like `.prose` or `.markdown` is used for both User and AI, you MUST find a unique parent class to differentiate (e.g., `.bot-msg .prose`).",
"4. **Syntax**: Use standard CSS selectors. Spaces for descendants (e.g., `div.bot p`) are encouraged for precision.",
"5. **No Invalid Syntax**: Do NOT use `xpath`, `~`, `:has()`, or `text()`.",
"",
"## PREFERENCE ORDER:",
"1. `id`, `name`, `data-testid` (Most preferred)",
"2. `button[type=\"submit\"]`",
"3. Unique parent class + target class (e.g., `.response-area .content`)",
"4. Hashed classes (only if no other option exists)",
"",
"## ELEMENTS TO FIND:",
selector_list,
"",
"## REQUIRED OUTPUT (JSON ONLY):",
"Return a JSON object with boolean is_chat_site plus these keys ONLY when is_chat_site is true:",
json_keys,
"",
"## HTML:",
clean_html
]
return "\n".join(lines)
def _build_default_selector_list(self) -> str:
"""默认的元素列表(回退用)"""
return """- `input_box`: The text input area (textarea/input). [REQUIRED]
- `send_btn`: The button that sends the message. [REQUIRED]
- `result_container`: The container for the AI's generated text response. [REQUIRED]
- `new_chat_btn`: Button to start a fresh conversation. [OPTIONAL, return null if not found]"""
def _build_default_json_keys(self) -> str:
"""默认的 JSON 键说明(回退用)"""
return """- `input_box`: The text input area
- `send_btn`: The send button
- `result_container`: AI response container
- `new_chat_btn`: New chat button (or null)"""
# ================= JSON 提取(不变)=================
def _extract_json(self, text: str) -> Optional[Dict]:
"""从 AI 响应中提取 JSON"""
try:
match = re.search(r'```(?:json)?\s*(\{[\s\S]*?\})\s*```', text)
if match:
return json.loads(match.group(1))
match = re.search(r'(\{[\s\S]*\})', text)
if match:
return json.loads(match.group(1))
return json.loads(text)
except json.JSONDecodeError as e:
logger.error(f"JSON 解析失败: {e}")
return None
__all__ = ['HTMLCleaner', 'SelectorValidator', 'AIAnalyzer']
+154
View File
@@ -0,0 +1,154 @@
"""R1-1:站点适配器 Schema v1。
只校验“形状”:类型、必填项、嵌套关系,以及 default_preset 必须指向已有预设。
语义校验(工作流 DSL、选择器能否命中等)仍由 ``flow_runtime.validate_workflow`` 和
ConfigEngine 的各个 ``_validate_*`` 负责。为了不误伤现有配置和用户自建预设,未知字段一律放行,
预设内部也不设必填项。
单站点文件(R1-2,``config/sites/<站点>.json``)的外层信封::
{"schema_version": 1, "site": "chat.deepseek.com", "adapter_version": "2026.09.26",
"min_app_version": "3.0.0", "last_verified": "2026-09-26", "config": {...}}
``config`` 与旧版 ``config/sites.json`` 中对应节点的内容完全相同。
"""
from __future__ import annotations
from typing import Any, Dict, Iterable, List
from jsonschema import Draft202012Validator
SITE_FILE_SCHEMA_VERSION = 1
GLOBAL_KEY = "_global"
_OBJECT = {"type": "object"}
WORKFLOW_STEP_SCHEMA: Dict[str, Any] = {
"type": "object",
"required": ["action"],
"properties": {
"action": {"type": "string", "minLength": 1},
"target": {"type": ["string", "null"]},
"optional": {"type": "boolean"},
"label": {"type": "string"},
"selector": {"type": "string"},
"execution": _OBJECT,
"flow_version": {"type": "integer"},
"retry_safe": {"type": "boolean"},
},
}
PRESET_SCHEMA: Dict[str, Any] = {
"type": "object",
"properties": {
"selectors": {"type": "object", "additionalProperties": {"type": ["string", "null"]}},
"workflow": {"type": "array", "items": WORKFLOW_STEP_SCHEMA},
"stealth": {"type": "boolean"},
"stream_config": _OBJECT,
"image_extraction": _OBJECT,
"file_paste": _OBJECT,
"prompt_padding": _OBJECT,
"advanced": _OBJECT,
},
}
SITE_CONFIG_SCHEMA: Dict[str, Any] = {
"$schema": "https://json-schema.org/draft/2020-12/schema",
"type": "object",
"required": ["presets"],
"properties": {
"presets": {"type": "object", "minProperties": 1, "additionalProperties": PRESET_SCHEMA},
"default_preset": {"type": "string", "minLength": 1},
"advanced": _OBJECT,
},
}
GLOBAL_CONFIG_SCHEMA: Dict[str, Any] = {
"$schema": "https://json-schema.org/draft/2020-12/schema",
"type": "object",
"properties": {
"selector_definitions": {
"type": "array",
"items": {
"type": "object",
"required": ["key"],
"properties": {
"key": {"type": "string", "minLength": 1},
"description": {"type": "string"},
"enabled": {"type": "boolean"},
"required": {"type": "boolean"},
},
},
},
},
}
SITE_FILE_SCHEMA: Dict[str, Any] = {
"$schema": "https://json-schema.org/draft/2020-12/schema",
"type": "object",
"required": ["schema_version", "site", "config"],
"properties": {
"schema_version": {"const": SITE_FILE_SCHEMA_VERSION},
"site": {"type": "string", "minLength": 1},
"adapter_version": {"type": "string"},
"min_app_version": {"type": "string", "pattern": r"^\d+(\.\d+){0,3}$"},
"last_verified": {"type": "string", "pattern": r"^\d{4}-\d{2}-\d{2}$"},
"config": _OBJECT,
},
}
_VALIDATORS = {
name: Draft202012Validator(schema)
for name, schema in (
("site", SITE_CONFIG_SCHEMA),
("global", GLOBAL_CONFIG_SCHEMA),
("file", SITE_FILE_SCHEMA),
)
}
def _format_path(prefix: str, parts: Iterable[Any]) -> str:
path = prefix
for part in parts:
path += f"[{part}]" if isinstance(part, int) else f".{part}"
return path
def _schema_errors(kind: str, payload: Any, prefix: str) -> List[str]:
errors = sorted(_VALIDATORS[kind].iter_errors(payload), key=lambda e: list(map(str, e.absolute_path)))
return [f"{_format_path(prefix, error.absolute_path)}: {error.message}" for error in errors]
def site_config_errors(site: str, config: Any) -> List[str]:
"""校验单个站点(或 ``_global``)的配置,返回可读的路径级错误列表;空列表表示通过。"""
prefix = f"sites[{site}]"
if site == GLOBAL_KEY:
return _schema_errors("global", config, prefix)
errors = _schema_errors("site", config, prefix)
if not errors and isinstance(config, dict):
default = config.get("default_preset")
presets = config.get("presets") or {}
if default and default not in presets:
errors.append(f"{prefix}.default_preset: '{default}' 不在 presets 中(可选:{', '.join(presets)})")
return errors
def site_file_errors(payload: Any) -> List[str]:
"""校验单站点文件(信封 + config)。"""
errors = _schema_errors("file", payload, "file")
if errors:
return errors
return site_config_errors(payload["site"], payload["config"])
def sites_errors(sites: Any) -> Dict[str, List[str]]:
"""校验整份站点配置字典(旧 sites.json 结构),返回 {站点: [错误...]},只包含有错误的站点。"""
if not isinstance(sites, dict):
return {"*": [f"sites: 应为对象,实际为 {type(sites).__name__}"]}
result: Dict[str, List[str]] = {}
for site, config in sites.items():
errors = site_config_errors(str(site), config)
if errors:
result[str(site)] = errors
return result
+312
View File
@@ -0,0 +1,312 @@
"""R1-2:站点配置目录存储(``config/sites/<站点>.json``)。
旧版把全部站点放在一个 271KB 的 ``config/sites.json`` 里;现在每个站点(以及 ``_global``)一个文件,
外层是 R1-1 定义的信封(见 ``site_schema.py``)。本模块负责:
- 读取目录并拼回与旧 ``sites.json`` **完全相同**的字典(``{"_global": {...}, "<域名>": {...}}``),
所以 ConfigEngine 以外的代码无需改动;
- 只重写内容有变化的站点文件(两阶段:先写全部临时文件,再逐个 ``os.replace``),保留信封元数据;
被删除的站点移入 ``.trash/``,不直接删除;
- 自动迁移旧的 ``config/sites.json``:目录为空时直接拆分;目录里已有发布版默认文件时(通过更新器升级的场景),
按更新器 ``merge_site_records`` 的语义合并(同一站点优先保留本地值、补入发布版新增字段);
迁移后旧文件改名为 ``sites.json.migrated-<时间>.bak``;
- 读不懂的文件(JSON 损坏或信封不完整)只告警跳过,并且**永远不会被保存逻辑覆盖或移入回收站**;
热重载时这类站点沿用上一次成功加载的配置。
"""
from __future__ import annotations
import json
import os
import time
from pathlib import Path
from typing import Any, Dict, List, Optional, Tuple
from app.core.config import get_logger
from app.services.config.site_schema import SITE_FILE_SCHEMA_VERSION, site_file_errors
logger = get_logger("SITE_STORE")
INDEX_FILE_NAME = "index.json"
TRASH_DIR_NAME = ".trash"
DEFAULT_MIN_APP_VERSION = "3.0.0"
_ENVELOPE_ORDER = ("schema_version", "site", "adapter_version", "min_app_version", "last_verified")
_WINDOWS_RESERVED = {"con", "prn", "aux", "nul", *(f"com{i}" for i in range(1, 10)), *(f"lpt{i}" for i in range(1, 10))}
def canonical_config_text(config: Any) -> str:
"""与格式、键顺序无关的规范化文本,用于变更检测与摘要(R1-3 的 config_sha256 也基于它)。"""
return json.dumps(config, ensure_ascii=False, sort_keys=True, separators=(",", ":"))
def site_filename(site: str) -> str:
"""站点键 -> 文件名。保留 ``[A-Za-z0-9._-]``,其余字节百分号编码;避开 Windows 保留名与 index.json。"""
encoded = "".join(
ch if (ch.isascii() and (ch.isalnum() or ch in "._-")) else "".join(f"%{b:02X}" for b in ch.encode("utf-8"))
for ch in str(site)
)
if not encoded or encoded.startswith(".") or encoded.split(".")[0].lower() in _WINDOWS_RESERVED | {"index"}:
encoded = "%5F" + encoded
return f"{encoded}.json"
def build_envelope(site: str, config: Dict[str, Any], meta: Optional[Dict[str, Any]] = None) -> Dict[str, Any]:
meta = dict(meta or {})
payload: Dict[str, Any] = {"schema_version": SITE_FILE_SCHEMA_VERSION, "site": site}
for key in _ENVELOPE_ORDER[2:]:
if meta.get(key) not in (None, ""):
payload[key] = meta[key]
for key, value in meta.items(): # 其余元数据(例如 R1-3 的 origin)原样保留
if key not in payload and key != "config":
payload[key] = value
payload["config"] = config
return payload
def _dump(payload: Dict[str, Any]) -> str:
return json.dumps(payload, indent=2, ensure_ascii=False) + "\n"
def _write_text_atomic(path: Path, text: str) -> None:
tmp = path.with_name(path.name + ".tmp")
with open(tmp, "w", encoding="utf-8", newline="\n") as handle:
handle.write(text)
handle.flush()
os.fsync(handle.fileno())
os.replace(tmp, path)
def _timestamp() -> str:
return time.strftime("%Y%m%d-%H%M%S")
class SiteStore:
"""``config/sites/`` 目录的读写与旧版迁移。线程安全由调用方(ConfigEngine 的 ``_io_lock``)保证。"""
def __init__(self, sites_dir: os.PathLike | str, legacy_file: os.PathLike | str):
self.sites_dir = Path(sites_dir)
self.legacy_file = Path(legacy_file)
self._files: Dict[str, Path] = {}
self._meta: Dict[str, Dict[str, Any]] = {}
self._on_disk: Dict[str, str] = {}
self._broken: Dict[Path, str] = {}
self._broken_sites: Dict[Path, str] = {} # 损坏文件 -> 上次成功加载时对应的站点
# ------------------------------------------------------------------ 查询
def site_files(self) -> List[Path]:
if not self.sites_dir.is_dir():
return []
return sorted(
path for path in self.sites_dir.glob("*.json")
if path.is_file() and path.name != INDEX_FILE_NAME and not path.name.startswith(".")
)
def exists(self) -> bool:
return bool(self.site_files()) or self.legacy_file.is_file()
def signature(self) -> Tuple:
"""目录内容签名(文件名、mtime、大小),用于热重载的变更检测。"""
entries = []
for path in self.site_files():
try:
stat = path.stat()
except OSError:
continue
entries.append((path.name, stat.st_mtime_ns, stat.st_size))
if self.legacy_file.is_file():
stat = self.legacy_file.stat()
entries.append(("<legacy>", stat.st_mtime_ns, stat.st_size))
return tuple(entries)
def metadata(self, site: str) -> Dict[str, Any]:
return dict(self._meta.get(site) or {})
def broken_files(self) -> Dict[str, str]:
return {str(path): reason for path, reason in self._broken.items()}
def file_for(self, site: str) -> Path:
return self._files.get(site) or (self.sites_dir / site_filename(site))
# ------------------------------------------------------------------ 读取
def read_envelope(self, path: Path) -> Dict[str, Any]:
payload = json.loads(path.read_text(encoding="utf-8-sig"))
if not isinstance(payload, dict) or not isinstance(payload.get("site"), str) or not isinstance(payload.get("config"), dict):
raise ValueError("不是有效的站点文件(缺少 site/config 字段)")
return payload
def load(self, previous: Optional[Dict[str, Any]] = None) -> Optional[Dict[str, Any]]:
"""读取全部站点,返回旧 sites.json 结构的字典;目录与旧文件都不存在时返回 None。
``previous`` 是上一次加载的结果:热重载时,暂时读不懂的文件沿用其中对应站点的配置。
"""
if self.legacy_file.is_file():
self.migrate_legacy()
files = self.site_files()
if not files:
return None
data: Dict[str, Any] = {}
files_map: Dict[str, Path] = {}
meta: Dict[str, Dict[str, Any]] = {}
on_disk: Dict[str, str] = {}
broken: Dict[Path, str] = {}
previous_sites_by_path = {path: site for site, path in self._files.items()}
previous_sites_by_path.update(self._broken_sites)
broken_sites: Dict[Path, str] = {}
for path in files:
try:
payload = self.read_envelope(path)
except (OSError, ValueError) as exc: # JSON 损坏、编码错误(均为 ValueError)、信封不完整、读文件失败
broken[path] = str(exc)
logger.error(f"站点配置文件无法读取,已跳过且不会被覆盖: {path.name}: {exc}")
old_site = previous_sites_by_path.get(path)
if old_site:
broken_sites[path] = old_site
if previous and old_site in previous:
data[old_site] = previous[old_site]
logger.warning(f"站点 {old_site} 暂时沿用上一次成功加载的配置")
continue
site = payload["site"]
if site in files_map:
broken[path] = f"与 {files_map[site].name} 重复声明站点 {site}"
logger.error(f"站点配置文件重复声明 {site},已忽略: {path.name}")
continue
for error in site_file_errors(payload)[:5]:
logger.warning(f"站点配置 Schema 告警({path.name}): {error}")
data[site] = payload["config"]
files_map[site] = path
meta[site] = {key: value for key, value in payload.items() if key != "config"}
on_disk[site] = canonical_config_text(payload["config"])
self._files, self._meta, self._on_disk = files_map, meta, on_disk
self._broken, self._broken_sites = broken, broken_sites
return data
# ------------------------------------------------------------------ 保存
def save(self, full_config: Dict[str, Any]) -> List[str]:
"""保存旧 sites.json 结构的字典,返回实际重写的站点列表。"""
self.sites_dir.mkdir(parents=True, exist_ok=True)
pending: List[Tuple[str, Path, Path, Dict[str, Any], str]] = []
taken = {path for path in self._files.values()} | set(self._broken)
try:
for site, config in full_config.items():
if not isinstance(config, dict):
continue
text = canonical_config_text(config)
path = self._files.get(site)
if path is not None and self._on_disk.get(site) == text and path.is_file():
continue
if path is None:
path = self._unique_path(site, taken)
taken.add(path)
if path in self._broken:
logger.error(f"站点 {site} 对应的文件 {path.name} 无法解析,为避免覆盖用户手改内容,本次跳过保存")
continue
payload = build_envelope(site, config, self._meta.get(site) or self._default_meta())
tmp = path.with_name(path.name + ".tmp")
with open(tmp, "w", encoding="utf-8", newline="\n") as handle:
handle.write(_dump(payload))
handle.flush()
os.fsync(handle.fileno())
pending.append((site, path, tmp, payload, text))
except Exception: # broad-except: 清理已写出的临时文件后原样重新抛出
for _, _, tmp, _, _ in pending:
try:
tmp.unlink()
except OSError:
pass
raise
written = []
for site, path, tmp, payload, text in pending:
os.replace(tmp, path)
self._files[site] = path
self._on_disk[site] = text
self._meta[site] = {key: value for key, value in payload.items() if key != "config"}
written.append(site)
for site in [s for s in self._files if s not in full_config]:
path = self._files.pop(site)
self._on_disk.pop(site, None)
self._meta.pop(site, None)
if path in self._broken or not path.is_file():
continue
trash = self.sites_dir / TRASH_DIR_NAME
trash.mkdir(exist_ok=True)
target = trash / f"{path.stem}.{_timestamp()}.json"
os.replace(path, target)
logger.info(f"站点 {site} 已删除,配置文件移入回收站: {target}")
return written
def _default_meta(self) -> Dict[str, Any]:
return {"min_app_version": DEFAULT_MIN_APP_VERSION}
def _unique_path(self, site: str, taken: set) -> Path:
base = self.sites_dir / site_filename(site)
path, counter = base, 2
while path in taken or (path.exists() and path not in self._files.values()):
path = base.with_name(f"{base.stem}~{counter}.json")
counter += 1
return path
# ------------------------------------------------------------------ 迁移
def migrate_legacy(self) -> Optional[Path]:
"""把旧的 config/sites.json 迁移进目录,返回备份文件路径;旧文件无法解析时保持原样并返回 None。"""
try:
raw = self.legacy_file.read_text(encoding="utf-8-sig").strip()
legacy = json.loads(raw) if raw else {}
if not isinstance(legacy, dict):
raise ValueError("顶层不是对象")
except (OSError, ValueError) as exc:
logger.error(f"旧版站点配置 {self.legacy_file} 无法解析,暂不迁移(原文件保持不变): {exc}")
return None
from updater import merge_site_records # 与更新器合并 sites.json 的语义保持一致
self.sites_dir.mkdir(parents=True, exist_ok=True)
existing: Dict[str, Tuple[Path, Dict[str, Any]]] = {}
for path in self.site_files():
try:
payload = self.read_envelope(path)
except (OSError, ValueError):
continue
existing[payload["site"]] = (path, payload)
taken = {path for path, _ in existing.values()}
merged_count = created_count = 0
for site, config in legacy.items():
if not isinstance(config, dict) or (site.startswith("_") and site != "_global"):
continue
if site in existing:
path, payload = existing[site]
payload = dict(payload)
merged_config = merge_site_records(config, payload["config"])
if (
site == "_global"
and config.get("selector_definitions") == []
and payload["config"].get("selector_definitions")
):
merged_config["selector_definitions"] = payload["config"]["selector_definitions"]
payload["config"] = merged_config
merged_count += 1
else:
path = self._unique_path(site, taken)
taken.add(path)
payload = build_envelope(site, config, self._default_meta())
created_count += 1
_write_text_atomic(path, _dump(payload))
backup = self.legacy_file.with_name(f"{self.legacy_file.name}.migrated-{_timestamp()}.bak")
os.replace(self.legacy_file, backup)
logger.warning(
f"已把旧版站点配置迁移到 {self.sites_dir}:新建 {created_count} 个、与发布版合并 {merged_count} 个;"
f"原文件已备份为 {backup.name}"
)
return backup
def load_sites_dict(sites_dir: os.PathLike | str) -> Dict[str, Any]:
"""只读地拼出目录里的全部站点(不迁移、不告警),供测试、脚本与导出使用。"""
store = SiteStore(sites_dir, Path(sites_dir) / "__no_legacy__.json")
return store.load() or {}