Files
universal-web-api/app/services/config/engine_sites.py
T
2026-11-02 22:54:14 +08:00

536 lines
22 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""ConfigEngine 的一部分(R2-3 拆分):站点目录、起始页推断、站点高级配置与选择器定义。
方法原样从 engine.py 迁出,行为不变;通过 mixin 组合回 ConfigEngine,
实例状态都在 ConfigEngine.__init__(engine.py)里初始化。
"""
import copy
from typing import Dict, Optional, List, Any, Set, Union, Callable
from urllib.parse import urljoin
from app.models.schemas import (
SiteConfig,
SelectorDefinition,
ADVANCED_FIELDS,
PRESET_ADVANCED_FIELDS,
SITE_ADVANCED_FIELDS,
get_default_image_extraction_config,
get_default_file_paste_config,
get_default_prompt_padding_config,
get_default_site_advanced_config,
)
from app.utils.site_rules import derive_site_card_id, get_site_rule
from app.utils.site_discovery import (
automatic_discovery_allowed,
specific_chat_selectors,
selectors_match_chat_html,
)
from .engine_common import (
DEFAULT_PRESET_NAME,
DEFAULT_WORKFLOW,
_MISSING,
_SITE_STARTUP_JS_PATTERNS,
_coerce_bool,
get_default_stream_config,
logger,
)
class ConfigSitesMixin:
"""站点目录、起始页推断、站点高级配置与选择器定义"""
def _extract_startup_url_from_script(self, script: str) -> str:
text = str(script or "").strip()
if not text:
return ""
for pattern in _SITE_STARTUP_JS_PATTERNS:
match = pattern.search(text)
if match:
return str(match.group(1) or "").strip()
return ""
def _normalize_startup_url(self, domain: str, raw_url: str) -> str:
normalized_domain = str(domain or "").strip().lower().strip(".")
text = str(raw_url or "").strip()
if not normalized_domain:
return text
if not text:
return f"https://{normalized_domain}"
if text.startswith(("http://", "https://")):
return text
return urljoin(f"https://{normalized_domain}", text)
def _infer_site_startup_url(self, domain: str, site: Dict[str, Any]) -> str:
preset_name = self._resolve_default_preset_name(site)
preset_data = self._get_site_data_readonly(domain, preset_name)
workflow = preset_data.get("workflow", []) if isinstance(preset_data, dict) else []
if isinstance(workflow, list):
for step in workflow:
if str((step or {}).get("action") or "").strip().upper() != "JS_EXEC":
continue
startup_url = self._extract_startup_url_from_script((step or {}).get("value"))
if startup_url:
return self._normalize_startup_url(domain, startup_url)
return self._normalize_startup_url(domain, "")
def get_site_catalog_entry(self, domain: str, fallback_order: int = 0) -> Optional[Dict[str, Any]]:
self.refresh_if_changed()
site = self.sites.get(domain)
if not isinstance(site, dict) or str(domain or "").startswith("_"):
return None
rule = get_site_rule(domain)
startup_url = rule.get("startup_url") or self._infer_site_startup_url(domain, site)
display_name = str(rule.get("display_name") or domain).strip() or str(domain)
card_id = str(rule.get("card_id") or derive_site_card_id(domain)).strip() or derive_site_card_id(domain)
guide_priority = rule.get("guide_priority")
if not isinstance(guide_priority, int):
guide_priority = int(fallback_order)
entry = {
"domain": str(domain),
"display_name": display_name,
"url": self._normalize_startup_url(domain, startup_url),
"card_id": card_id,
"guide_priority": guide_priority,
}
if isinstance(rule.get("route_aliases"), list):
entry["route_aliases"] = [str(item) for item in rule.get("route_aliases", [])]
if "stealth_default" in rule:
entry["stealth_default"] = bool(rule.get("stealth_default"))
return entry
def list_site_catalog(self) -> List[Dict[str, Any]]:
self.refresh_if_changed()
ordered_domains = [
domain for domain, site in self.sites.items()
if not domain.startswith("_") and isinstance(site, dict)
]
entries: List[Dict[str, Any]] = []
for index, domain in enumerate(ordered_domains):
entry = self.get_site_catalog_entry(domain, fallback_order=index)
if entry:
entries.append(entry)
entries.sort(key=lambda item: (int(item.get("guide_priority", 0)), str(item.get("domain") or "")))
return entries
def _normalize_site_advanced_config(
self,
raw_config: Optional[Dict[str, Any]] = None,
*,
base_config: Optional[Dict[str, Any]] = None,
) -> Dict[str, Any]:
"""合并并规范化高级配置。"""
normalized = {
**get_default_site_advanced_config(),
}
if isinstance(base_config, dict):
normalized.update(copy.deepcopy(base_config))
if isinstance(raw_config, dict):
normalized.update(copy.deepcopy(raw_config))
normalized["independent_cookies"] = _coerce_bool(normalized.get("independent_cookies"), False)
normalized["independent_cookies_auto_takeover"] = _coerce_bool(
normalized.get("independent_cookies_auto_takeover"), False
)
normalized["input_box_stability_wait_enabled"] = _coerce_bool(
normalized.get("input_box_stability_wait_enabled"), False
)
normalized["input_box_stability_wait_after_new_chat_only"] = _coerce_bool(
normalized.get("input_box_stability_wait_after_new_chat_only"), True
)
try:
timeout_value = float(normalized.get("input_box_stability_wait_timeout", 1.5))
except Exception:
timeout_value = 1.5
normalized["input_box_stability_wait_timeout"] = max(0.1, min(timeout_value, 10.0))
normalized["url_transition_wait_on_new_chat"] = _coerce_bool(
normalized.get("url_transition_wait_on_new_chat"), False
)
raw_patterns = normalized.get("url_transition_wait_patterns") or []
if isinstance(raw_patterns, str):
raw_patterns = raw_patterns.replace("\n", ",").replace(";", ",").split(",")
if not isinstance(raw_patterns, (list, tuple, set)):
raw_patterns = []
normalized["url_transition_wait_patterns"] = [
str(pattern or "").strip()
for pattern in raw_patterns
if str(pattern or "").strip()
]
normalized["send_confirmation_check_enabled"] = _coerce_bool(
normalized.get("send_confirmation_check_enabled"), False
)
try:
send_timeout_value = float(normalized.get("send_confirmation_check_timeout", 1.5))
except Exception:
send_timeout_value = 1.5
normalized["send_confirmation_check_timeout"] = max(0.1, min(send_timeout_value, 10.0))
normalized["skip_new_chat_on_retry"] = _coerce_bool(
normalized.get("skip_new_chat_on_retry"), False
)
return normalized
def get_site_advanced_config(self, domain: str, preset_name: str = None) -> Dict[str, Any]:
"""获取站点高级配置;传入 preset_name 时叠加对应预设的覆盖项。"""
self.refresh_if_changed()
site = self.sites.get(domain)
if not site:
return self._normalize_site_advanced_config()
raw_config = site.get("advanced") if isinstance(site, dict) else None
normalized = self._normalize_site_advanced_config(
raw_config if isinstance(raw_config, dict) else {}
)
requested_preset = str(preset_name or "").strip()
if requested_preset:
preset_data = self._get_preset_data_exact(domain, requested_preset)
if preset_data is None:
logger.warning(f"站点高级配置预设不存在: {domain}/{requested_preset}")
return normalized
preset_advanced = (
preset_data.get("advanced")
if isinstance(preset_data, dict)
else None
)
if isinstance(preset_advanced, dict):
preset_advanced = {
key: value
for key, value in preset_advanced.items()
if key in PRESET_ADVANCED_FIELDS
}
normalized = self._normalize_site_advanced_config(
preset_advanced,
base_config=normalized,
)
return normalized
def set_site_advanced_config(self, domain: str, config: Dict[str, Any]) -> bool:
"""设置站点级高级配置。"""
with self._io_lock:
self.refresh_if_changed()
site = self.sites.get(domain)
if not site:
logger.warning(f"站点不存在: {domain}")
return False
previous_advanced = site.get("advanced", _MISSING)
if previous_advanced is not _MISSING:
previous_advanced = copy.deepcopy(previous_advanced)
existing = site.get("advanced") if isinstance(site.get("advanced"), dict) else {}
stored = copy.deepcopy(existing)
normalized = self._normalize_site_advanced_config(config or {}, base_config=existing)
provided_keys = {
key
for key in ((config or {}).keys() if isinstance(config, dict) else set())
if key in ADVANCED_FIELDS
}
if not provided_keys and previous_advanced is _MISSING:
return True
for key in ADVANCED_FIELDS:
if key in provided_keys:
stored[key] = normalized[key]
site["advanced"] = stored
if self._save_config_locked():
return True
if previous_advanced is _MISSING:
site.pop("advanced", None)
else:
site["advanced"] = previous_advanced
return False
def set_preset_advanced_config(
self,
domain: str,
config: Dict[str, Any],
preset_name: str = None,
*,
prune_inherited_fields: Optional[Set[str]] = None,
) -> bool:
"""设置当前预设的高级配置;拒绝混入站点级字段。"""
with self._io_lock:
self.refresh_if_changed()
site = self.sites.get(domain)
if not isinstance(site, dict):
logger.warning(f"站点不存在: {domain}")
return False
requested_preset = str(preset_name or "").strip()
if not requested_preset:
logger.warning(f"预设级高级配置缺少 preset_name: {domain}")
return False
data = self._get_preset_data_exact(domain, requested_preset)
if data is None:
logger.warning(f"站点或预设不存在: {domain}/{requested_preset}")
return False
raw_config = config if isinstance(config, dict) else {}
invalid_site_keys = {
key
for key in raw_config.keys()
if key in SITE_ADVANCED_FIELDS
}
if invalid_site_keys:
joined = ", ".join(sorted(invalid_site_keys))
logger.warning(f"预设级高级配置不能包含站点级字段: {domain}/{requested_preset} ({joined})")
return False
previous_advanced = data.get("advanced", _MISSING)
if previous_advanced is not _MISSING:
previous_advanced = copy.deepcopy(previous_advanced)
stored = copy.deepcopy(data.get("advanced") or {})
if not isinstance(stored, dict):
stored = {}
for key in list(SITE_ADVANCED_FIELDS):
stored.pop(key, None)
site_advanced = site.get("advanced") if isinstance(site.get("advanced"), dict) else {}
inherited = self._normalize_site_advanced_config(site_advanced)
normalized = self._normalize_site_advanced_config(
raw_config,
base_config=stored,
)
provided_keys = {
key
for key in raw_config.keys()
if key in PRESET_ADVANCED_FIELDS
}
if not provided_keys and previous_advanced is _MISSING:
return True
prune_inherited_keys = set(prune_inherited_fields or set())
for key in PRESET_ADVANCED_FIELDS:
if key in provided_keys:
if (
key in prune_inherited_keys
and key not in stored
and normalized[key] == inherited.get(key)
):
stored.pop(key, None)
else:
stored[key] = normalized[key]
if not stored and previous_advanced is _MISSING:
return True
data["advanced"] = stored
if self._save_config_locked():
return True
if previous_advanced is _MISSING:
data.pop("advanced", None)
else:
data["advanced"] = previous_advanced
return False
def list_sites(self) -> Dict[str, Any]:
"""获取所有站点配置(过滤内部键)"""
self.refresh_if_changed()
with self._io_lock:
return copy.deepcopy({
domain: config
for domain, config in self.sites.items()
if not domain.startswith('_')
})
def get_site_config(self, domain: str, html_content: Optional[Union[str, Callable[[], str]]] = None,
preset_name: str = None) -> Optional[SiteConfig]:
"""
获取站点配置(缓存 + AI 分析)
Args:
domain: 站点域名
html_content: 页面 HTML 或用于延迟获取 HTML 的可调用对象(用于 AI 分析未知站点)
preset_name: 预设名称,None 则使用默认预设
"""
self.refresh_if_changed()
if domain in self.sites:
cache_key = f"site_config:{domain}:{preset_name or ''}"
cached = self._cache.get(cache_key, _MISSING)
if cached is not _MISSING:
return copy.deepcopy(cached)
site = self.sites.get(domain, {})
config = self._get_site_data(domain, preset_name)
if config is None:
logger.warning(f"站点 {domain} 无可用预设")
return None
used_preset = preset_name or self._resolve_default_preset_name(site) or DEFAULT_PRESET_NAME
previous_config = None
# 补充缺失字段
changed = False
if "workflow" not in config:
if previous_config is None:
previous_config = copy.deepcopy(config)
config["workflow"] = DEFAULT_WORKFLOW
changed = True
if "image_extraction" not in config:
if previous_config is None:
previous_config = copy.deepcopy(config)
config["image_extraction"] = get_default_image_extraction_config()
changed = True
if "file_paste" not in config:
if previous_config is None:
previous_config = copy.deepcopy(config)
config["file_paste"] = get_default_file_paste_config()
changed = True
else:
normalized_file_paste = self._validate_file_paste_config(
config.get("file_paste", {}),
legacy_stream_config=config.get("stream_config"),
)
if normalized_file_paste != config.get("file_paste"):
if previous_config is None:
previous_config = copy.deepcopy(config)
config["file_paste"] = normalized_file_paste
changed = True
if "prompt_padding" not in config:
if previous_config is None:
previous_config = copy.deepcopy(config)
config["prompt_padding"] = get_default_prompt_padding_config()
changed = True
else:
normalized_prompt_padding = self._validate_prompt_padding_config(
config.get("prompt_padding", {})
)
if normalized_prompt_padding != config.get("prompt_padding"):
if previous_config is None:
previous_config = copy.deepcopy(config)
config["prompt_padding"] = normalized_prompt_padding
changed = True
if changed:
completed_config = copy.deepcopy(config)
if not self._save_config():
config.clear()
config.update(previous_config)
logger.warning(
f"站点 {domain} [{used_preset}] 配置自动补全保存失败,"
"已仅对当前请求返回补全配置"
)
return completed_config
logger.debug(f"使用缓存配置: {domain} [预设: {used_preset}]")
result = copy.deepcopy(config)
self._cache.set(cache_key, result)
return copy.deepcopy(result)
if not automatic_discovery_allowed(domain):
logger.info(f"跳过非聊天站点的自动收录: {domain}(已有手动配置不受影响)")
return None
resolved_html = ""
if callable(html_content):
try:
resolved_html = str(html_content() or "")
except Exception as e:
logger.warning(f"获取页面 HTML 失败: {e}")
resolved_html = ""
elif html_content is not None:
resolved_html = str(html_content or "")
if resolved_html.strip():
logger.info(f"🔍 未知域名 {domain},启动 AI 识别...")
clean_html = self.html_cleaner.clean(resolved_html)
selectors = self.ai_analyzer.analyze(clean_html)
if specific_chat_selectors(selectors):
selectors = self.validator.validate(selectors)
if selectors_match_chat_html(selectors, clean_html):
new_preset: SiteConfig = {
"selectors": selectors,
"workflow": DEFAULT_WORKFLOW,
"stealth": self._guess_stealth(domain),
"stream_config": copy.deepcopy(get_default_stream_config()),
"image_extraction": get_default_image_extraction_config(),
"file_paste": get_default_file_paste_config(),
"prompt_padding": get_default_prompt_padding_config(),
}
self.sites[domain] = {
"default_preset": DEFAULT_PRESET_NAME,
"presets": {
DEFAULT_PRESET_NAME: new_preset
}
}
if self._save_config():
logger.info(f"✅ 配置已生成并保存: {domain}")
else:
logger.warning(f"⚠️ 配置已生成但保存失败,仅保留在当前运行内存: {domain}")
return copy.deepcopy(new_preset)
logger.warning(f"⚠️ AI 分析失败,使用通用回退配置: {domain}")
else:
logger.warning(f"⚠️ 未知域名 {domain} 且未提供网页 HTML,使用通用回退配置")
fallback_selectors = self.global_config.get_fallback_selectors()
fallback_preset: SiteConfig = {
"selectors": fallback_selectors,
"workflow": DEFAULT_WORKFLOW,
"stealth": False,
"stream_config": copy.deepcopy(get_default_stream_config()),
"image_extraction": get_default_image_extraction_config(),
"file_paste": get_default_file_paste_config(),
"prompt_padding": get_default_prompt_padding_config(),
}
# A generic fallback is only for this caller. It is not proof that this
# domain hosts an AI chat product, and must never enter the site catalog.
logger.info(f"通用回退仅用于当前请求,未自动添加站点: {domain}")
return copy.deepcopy(fallback_preset)
def delete_site_config(self, domain: str) -> bool:
"""删除指定站点配置"""
self.refresh_if_changed()
if domain in self.sites:
removed_site = copy.deepcopy(self.sites[domain])
previous_global_default = copy.deepcopy(self._global_default_presets)
previous_local_default = copy.deepcopy(self._local_default_presets)
del self.sites[domain]
if not self._save_config():
self.sites[domain] = removed_site
self._global_default_presets = previous_global_default
self._local_default_presets = previous_local_default
return False
logger.info(f"已删除配置: {domain}")
return True
return False
def get_selector_definitions(self) -> List[SelectorDefinition]:
"""获取元素定义列表"""
return self.global_config.get_selector_definitions()
def set_selector_definitions(self, definitions: List[SelectorDefinition]) -> bool:
"""设置元素定义列表并保存"""
previous_definitions = self.global_config.get_selector_definitions()
previous_fallback_selectors = copy.deepcopy(self.validator.fallback_selectors)
self.global_config.set_selector_definitions(definitions)
# 更新验证器的回退选择器
self.validator.fallback_selectors = self.global_config.get_fallback_selectors()
# 保存配置
if not self._save_config():
self.global_config.set_selector_definitions(previous_definitions)
self.validator.fallback_selectors = previous_fallback_selectors
return False
logger.info(f"元素定义已更新: {len(definitions)} 个")
return True