Initial commit

This commit is contained in:
Alex committed 2026-11-02 22:54:14 +08:00
commit 57fc3525eb
7311 files changed
+1291398

No files matched your search

+535
View File
@@ -0,0 +1,535 @@
"""ConfigEngine 的一部分(R2-3 拆分):站点目录、起始页推断、站点高级配置与选择器定义。
方法原样从 engine.py 迁出,行为不变;通过 mixin 组合回 ConfigEngine,
实例状态都在 ConfigEngine.__init__(engine.py)里初始化。
"""
import copy
from typing import Dict, Optional, List, Any, Set, Union, Callable
from urllib.parse import urljoin
from app.models.schemas import (
SiteConfig,
SelectorDefinition,
ADVANCED_FIELDS,
PRESET_ADVANCED_FIELDS,
SITE_ADVANCED_FIELDS,
get_default_image_extraction_config,
get_default_file_paste_config,
get_default_prompt_padding_config,
get_default_site_advanced_config,
)
from app.utils.site_rules import derive_site_card_id, get_site_rule
from app.utils.site_discovery import (
automatic_discovery_allowed,
specific_chat_selectors,
selectors_match_chat_html,
)
from .engine_common import (
DEFAULT_PRESET_NAME,
DEFAULT_WORKFLOW,
_MISSING,
_SITE_STARTUP_JS_PATTERNS,
_coerce_bool,
get_default_stream_config,
logger,
)
class ConfigSitesMixin:
"""站点目录、起始页推断、站点高级配置与选择器定义"""
def _extract_startup_url_from_script(self, script: str) -> str:
text = str(script or "").strip()
if not text:
return ""
for pattern in _SITE_STARTUP_JS_PATTERNS:
match = pattern.search(text)
if match:
return str(match.group(1) or "").strip()
return ""
def _normalize_startup_url(self, domain: str, raw_url: str) -> str:
normalized_domain = str(domain or "").strip().lower().strip(".")
text = str(raw_url or "").strip()
if not normalized_domain:
return text
if not text:
return f"https://{normalized_domain}"
if text.startswith(("http://", "https://")):
return text
return urljoin(f"https://{normalized_domain}", text)
def _infer_site_startup_url(self, domain: str, site: Dict[str, Any]) -> str:
preset_name = self._resolve_default_preset_name(site)
preset_data = self._get_site_data_readonly(domain, preset_name)
workflow = preset_data.get("workflow", []) if isinstance(preset_data, dict) else []
if isinstance(workflow, list):
for step in workflow:
if str((step or {}).get("action") or "").strip().upper() != "JS_EXEC":
continue
startup_url = self._extract_startup_url_from_script((step or {}).get("value"))
if startup_url:
return self._normalize_startup_url(domain, startup_url)
return self._normalize_startup_url(domain, "")
def get_site_catalog_entry(self, domain: str, fallback_order: int = 0) -> Optional[Dict[str, Any]]:
self.refresh_if_changed()
site = self.sites.get(domain)
if not isinstance(site, dict) or str(domain or "").startswith("_"):
return None
rule = get_site_rule(domain)
startup_url = rule.get("startup_url") or self._infer_site_startup_url(domain, site)
display_name = str(rule.get("display_name") or domain).strip() or str(domain)
card_id = str(rule.get("card_id") or derive_site_card_id(domain)).strip() or derive_site_card_id(domain)
guide_priority = rule.get("guide_priority")
if not isinstance(guide_priority, int):
guide_priority = int(fallback_order)
entry = {
"domain": str(domain),
"display_name": display_name,
"url": self._normalize_startup_url(domain, startup_url),
"card_id": card_id,
"guide_priority": guide_priority,
}
if isinstance(rule.get("route_aliases"), list):
entry["route_aliases"] = [str(item) for item in rule.get("route_aliases", [])]
if "stealth_default" in rule:
entry["stealth_default"] = bool(rule.get("stealth_default"))
return entry
def list_site_catalog(self) -> List[Dict[str, Any]]:
self.refresh_if_changed()
ordered_domains = [
domain for domain, site in self.sites.items()
if not domain.startswith("_") and isinstance(site, dict)
]
entries: List[Dict[str, Any]] = []
for index, domain in enumerate(ordered_domains):
entry = self.get_site_catalog_entry(domain, fallback_order=index)
if entry:
entries.append(entry)
entries.sort(key=lambda item: (int(item.get("guide_priority", 0)), str(item.get("domain") or "")))
return entries
def _normalize_site_advanced_config(
self,
raw_config: Optional[Dict[str, Any]] = None,
*,
base_config: Optional[Dict[str, Any]] = None,
) -> Dict[str, Any]:
"""合并并规范化高级配置。"""
normalized = {
**get_default_site_advanced_config(),
}
if isinstance(base_config, dict):
normalized.update(copy.deepcopy(base_config))
if isinstance(raw_config, dict):
normalized.update(copy.deepcopy(raw_config))
normalized["independent_cookies"] = _coerce_bool(normalized.get("independent_cookies"), False)
normalized["independent_cookies_auto_takeover"] = _coerce_bool(
normalized.get("independent_cookies_auto_takeover"), False
)
normalized["input_box_stability_wait_enabled"] = _coerce_bool(
normalized.get("input_box_stability_wait_enabled"), False
)
normalized["input_box_stability_wait_after_new_chat_only"] = _coerce_bool(
normalized.get("input_box_stability_wait_after_new_chat_only"), True
)
try:
timeout_value = float(normalized.get("input_box_stability_wait_timeout", 1.5))
except Exception:
timeout_value = 1.5
normalized["input_box_stability_wait_timeout"] = max(0.1, min(timeout_value, 10.0))
normalized["url_transition_wait_on_new_chat"] = _coerce_bool(
normalized.get("url_transition_wait_on_new_chat"), False
)
raw_patterns = normalized.get("url_transition_wait_patterns") or []
if isinstance(raw_patterns, str):
raw_patterns = raw_patterns.replace("\n", ",").replace(";", ",").split(",")
if not isinstance(raw_patterns, (list, tuple, set)):
raw_patterns = []
normalized["url_transition_wait_patterns"] = [
str(pattern or "").strip()
for pattern in raw_patterns
if str(pattern or "").strip()
]
normalized["send_confirmation_check_enabled"] = _coerce_bool(
normalized.get("send_confirmation_check_enabled"), False
)
try:
send_timeout_value = float(normalized.get("send_confirmation_check_timeout", 1.5))
except Exception:
send_timeout_value = 1.5
normalized["send_confirmation_check_timeout"] = max(0.1, min(send_timeout_value, 10.0))
normalized["skip_new_chat_on_retry"] = _coerce_bool(
normalized.get("skip_new_chat_on_retry"), False
)
return normalized
def get_site_advanced_config(self, domain: str, preset_name: str = None) -> Dict[str, Any]:
"""获取站点高级配置;传入 preset_name 时叠加对应预设的覆盖项。"""
self.refresh_if_changed()
site = self.sites.get(domain)
if not site:
return self._normalize_site_advanced_config()
raw_config = site.get("advanced") if isinstance(site, dict) else None
normalized = self._normalize_site_advanced_config(
raw_config if isinstance(raw_config, dict) else {}
)
requested_preset = str(preset_name or "").strip()
if requested_preset:
preset_data = self._get_preset_data_exact(domain, requested_preset)
if preset_data is None:
logger.warning(f"站点高级配置预设不存在: {domain}/{requested_preset}")
return normalized
preset_advanced = (
preset_data.get("advanced")
if isinstance(preset_data, dict)
else None
)
if isinstance(preset_advanced, dict):
preset_advanced = {
key: value
for key, value in preset_advanced.items()
if key in PRESET_ADVANCED_FIELDS
}
normalized = self._normalize_site_advanced_config(
preset_advanced,
base_config=normalized,
)
return normalized
def set_site_advanced_config(self, domain: str, config: Dict[str, Any]) -> bool:
"""设置站点级高级配置。"""
with self._io_lock:
self.refresh_if_changed()
site = self.sites.get(domain)
if not site:
logger.warning(f"站点不存在: {domain}")
return False
previous_advanced = site.get("advanced", _MISSING)
if previous_advanced is not _MISSING:
previous_advanced = copy.deepcopy(previous_advanced)
existing = site.get("advanced") if isinstance(site.get("advanced"), dict) else {}
stored = copy.deepcopy(existing)
normalized = self._normalize_site_advanced_config(config or {}, base_config=existing)
provided_keys = {
key
for key in ((config or {}).keys() if isinstance(config, dict) else set())
if key in ADVANCED_FIELDS
}
if not provided_keys and previous_advanced is _MISSING:
return True
for key in ADVANCED_FIELDS:
if key in provided_keys:
stored[key] = normalized[key]
site["advanced"] = stored
if self._save_config_locked():
return True
if previous_advanced is _MISSING:
site.pop("advanced", None)
else:
site["advanced"] = previous_advanced
return False
def set_preset_advanced_config(
self,
domain: str,
config: Dict[str, Any],
preset_name: str = None,
*,
prune_inherited_fields: Optional[Set[str]] = None,
) -> bool:
"""设置当前预设的高级配置;拒绝混入站点级字段。"""
with self._io_lock:
self.refresh_if_changed()
site = self.sites.get(domain)
if not isinstance(site, dict):
logger.warning(f"站点不存在: {domain}")
return False
requested_preset = str(preset_name or "").strip()
if not requested_preset:
logger.warning(f"预设级高级配置缺少 preset_name: {domain}")
return False
data = self._get_preset_data_exact(domain, requested_preset)
if data is None:
logger.warning(f"站点或预设不存在: {domain}/{requested_preset}")
return False
raw_config = config if isinstance(config, dict) else {}
invalid_site_keys = {
key
for key in raw_config.keys()
if key in SITE_ADVANCED_FIELDS
}
if invalid_site_keys:
joined = ", ".join(sorted(invalid_site_keys))
logger.warning(f"预设级高级配置不能包含站点级字段: {domain}/{requested_preset} ({joined})")
return False
previous_advanced = data.get("advanced", _MISSING)
if previous_advanced is not _MISSING:
previous_advanced = copy.deepcopy(previous_advanced)
stored = copy.deepcopy(data.get("advanced") or {})
if not isinstance(stored, dict):
stored = {}
for key in list(SITE_ADVANCED_FIELDS):
stored.pop(key, None)
site_advanced = site.get("advanced") if isinstance(site.get("advanced"), dict) else {}
inherited = self._normalize_site_advanced_config(site_advanced)
normalized = self._normalize_site_advanced_config(
raw_config,
base_config=stored,
)
provided_keys = {
key
for key in raw_config.keys()
if key in PRESET_ADVANCED_FIELDS
}
if not provided_keys and previous_advanced is _MISSING:
return True
prune_inherited_keys = set(prune_inherited_fields or set())
for key in PRESET_ADVANCED_FIELDS:
if key in provided_keys:
if (
key in prune_inherited_keys
and key not in stored
and normalized[key] == inherited.get(key)
):
stored.pop(key, None)
else:
stored[key] = normalized[key]
if not stored and previous_advanced is _MISSING:
return True
data["advanced"] = stored
if self._save_config_locked():
return True
if previous_advanced is _MISSING:
data.pop("advanced", None)
else:
data["advanced"] = previous_advanced
return False
def list_sites(self) -> Dict[str, Any]:
"""获取所有站点配置(过滤内部键)"""
self.refresh_if_changed()
with self._io_lock:
return copy.deepcopy({
domain: config
for domain, config in self.sites.items()
if not domain.startswith('_')
})
def get_site_config(self, domain: str, html_content: Optional[Union[str, Callable[[], str]]] = None,
preset_name: str = None) -> Optional[SiteConfig]:
"""
获取站点配置(缓存 + AI 分析)
Args:
domain: 站点域名
html_content: 页面 HTML 或用于延迟获取 HTML 的可调用对象(用于 AI 分析未知站点)
preset_name: 预设名称,None 则使用默认预设
"""
self.refresh_if_changed()
if domain in self.sites:
cache_key = f"site_config:{domain}:{preset_name or ''}"
cached = self._cache.get(cache_key, _MISSING)
if cached is not _MISSING:
return copy.deepcopy(cached)
site = self.sites.get(domain, {})
config = self._get_site_data(domain, preset_name)
if config is None:
logger.warning(f"站点 {domain} 无可用预设")
return None
used_preset = preset_name or self._resolve_default_preset_name(site) or DEFAULT_PRESET_NAME
previous_config = None
# 补充缺失字段
changed = False
if "workflow" not in config:
if previous_config is None:
previous_config = copy.deepcopy(config)
config["workflow"] = DEFAULT_WORKFLOW
changed = True
if "image_extraction" not in config:
if previous_config is None:
previous_config = copy.deepcopy(config)
config["image_extraction"] = get_default_image_extraction_config()
changed = True
if "file_paste" not in config:
if previous_config is None:
previous_config = copy.deepcopy(config)
config["file_paste"] = get_default_file_paste_config()
changed = True
else:
normalized_file_paste = self._validate_file_paste_config(
config.get("file_paste", {}),
legacy_stream_config=config.get("stream_config"),
)
if normalized_file_paste != config.get("file_paste"):
if previous_config is None:
previous_config = copy.deepcopy(config)
config["file_paste"] = normalized_file_paste
changed = True
if "prompt_padding" not in config:
if previous_config is None:
previous_config = copy.deepcopy(config)
config["prompt_padding"] = get_default_prompt_padding_config()
changed = True
else:
normalized_prompt_padding = self._validate_prompt_padding_config(
config.get("prompt_padding", {})
)
if normalized_prompt_padding != config.get("prompt_padding"):
if previous_config is None:
previous_config = copy.deepcopy(config)
config["prompt_padding"] = normalized_prompt_padding
changed = True
if changed:
completed_config = copy.deepcopy(config)
if not self._save_config():
config.clear()
config.update(previous_config)
logger.warning(
f"站点 {domain} [{used_preset}] 配置自动补全保存失败,"
"已仅对当前请求返回补全配置"
)
return completed_config
logger.debug(f"使用缓存配置: {domain} [预设: {used_preset}]")
result = copy.deepcopy(config)
self._cache.set(cache_key, result)
return copy.deepcopy(result)
if not automatic_discovery_allowed(domain):
logger.info(f"跳过非聊天站点的自动收录: {domain}(已有手动配置不受影响)")
return None
resolved_html = ""
if callable(html_content):
try:
resolved_html = str(html_content() or "")
except Exception as e:
logger.warning(f"获取页面 HTML 失败: {e}")
resolved_html = ""
elif html_content is not None:
resolved_html = str(html_content or "")
if resolved_html.strip():
logger.info(f"🔍 未知域名 {domain},启动 AI 识别...")
clean_html = self.html_cleaner.clean(resolved_html)
selectors = self.ai_analyzer.analyze(clean_html)
if specific_chat_selectors(selectors):
selectors = self.validator.validate(selectors)
if selectors_match_chat_html(selectors, clean_html):
new_preset: SiteConfig = {
"selectors": selectors,
"workflow": DEFAULT_WORKFLOW,
"stealth": self._guess_stealth(domain),
"stream_config": copy.deepcopy(get_default_stream_config()),
"image_extraction": get_default_image_extraction_config(),
"file_paste": get_default_file_paste_config(),
"prompt_padding": get_default_prompt_padding_config(),
}
self.sites[domain] = {
"default_preset": DEFAULT_PRESET_NAME,
"presets": {
DEFAULT_PRESET_NAME: new_preset
}
}
if self._save_config():
logger.info(f"✅ 配置已生成并保存: {domain}")
else:
logger.warning(f"⚠️ 配置已生成但保存失败,仅保留在当前运行内存: {domain}")
return copy.deepcopy(new_preset)
logger.warning(f"⚠️ AI 分析失败,使用通用回退配置: {domain}")
else:
logger.warning(f"⚠️ 未知域名 {domain} 且未提供网页 HTML,使用通用回退配置")
fallback_selectors = self.global_config.get_fallback_selectors()
fallback_preset: SiteConfig = {
"selectors": fallback_selectors,
"workflow": DEFAULT_WORKFLOW,
"stealth": False,
"stream_config": copy.deepcopy(get_default_stream_config()),
"image_extraction": get_default_image_extraction_config(),
"file_paste": get_default_file_paste_config(),
"prompt_padding": get_default_prompt_padding_config(),
}
# A generic fallback is only for this caller. It is not proof that this
# domain hosts an AI chat product, and must never enter the site catalog.
logger.info(f"通用回退仅用于当前请求,未自动添加站点: {domain}")
return copy.deepcopy(fallback_preset)
def delete_site_config(self, domain: str) -> bool:
"""删除指定站点配置"""
self.refresh_if_changed()
if domain in self.sites:
removed_site = copy.deepcopy(self.sites[domain])
previous_global_default = copy.deepcopy(self._global_default_presets)
previous_local_default = copy.deepcopy(self._local_default_presets)
del self.sites[domain]
if not self._save_config():
self.sites[domain] = removed_site
self._global_default_presets = previous_global_default
self._local_default_presets = previous_local_default
return False
logger.info(f"已删除配置: {domain}")
return True
return False
def get_selector_definitions(self) -> List[SelectorDefinition]:
"""获取元素定义列表"""
return self.global_config.get_selector_definitions()
def set_selector_definitions(self, definitions: List[SelectorDefinition]) -> bool:
"""设置元素定义列表并保存"""
previous_definitions = self.global_config.get_selector_definitions()
previous_fallback_selectors = copy.deepcopy(self.validator.fallback_selectors)
self.global_config.set_selector_definitions(definitions)
# 更新验证器的回退选择器
self.validator.fallback_selectors = self.global_config.get_fallback_selectors()
# 保存配置
if not self._save_config():
self.global_config.set_selector_definitions(previous_definitions)
self.validator.fallback_selectors = previous_fallback_selectors
return False
logger.info(f"元素定义已更新: {len(definitions)} 个")
return True