"""ConfigEngine 的一部分(R2-3 拆分):站点目录、起始页推断、站点高级配置与选择器定义。 方法原样从 engine.py 迁出,行为不变;通过 mixin 组合回 ConfigEngine, 实例状态都在 ConfigEngine.__init__(engine.py)里初始化。 """ import copy from typing import Dict, Optional, List, Any, Set, Union, Callable from urllib.parse import urljoin from app.models.schemas import ( SiteConfig, SelectorDefinition, ADVANCED_FIELDS, PRESET_ADVANCED_FIELDS, SITE_ADVANCED_FIELDS, get_default_image_extraction_config, get_default_file_paste_config, get_default_prompt_padding_config, get_default_site_advanced_config, ) from app.utils.site_rules import derive_site_card_id, get_site_rule from app.utils.site_discovery import ( automatic_discovery_allowed, specific_chat_selectors, selectors_match_chat_html, ) from .engine_common import ( DEFAULT_PRESET_NAME, DEFAULT_WORKFLOW, _MISSING, _SITE_STARTUP_JS_PATTERNS, _coerce_bool, get_default_stream_config, logger, ) class ConfigSitesMixin: """站点目录、起始页推断、站点高级配置与选择器定义""" def _extract_startup_url_from_script(self, script: str) -> str: text = str(script or "").strip() if not text: return "" for pattern in _SITE_STARTUP_JS_PATTERNS: match = pattern.search(text) if match: return str(match.group(1) or "").strip() return "" def _normalize_startup_url(self, domain: str, raw_url: str) -> str: normalized_domain = str(domain or "").strip().lower().strip(".") text = str(raw_url or "").strip() if not normalized_domain: return text if not text: return f"https://{normalized_domain}" if text.startswith(("http://", "https://")): return text return urljoin(f"https://{normalized_domain}", text) def _infer_site_startup_url(self, domain: str, site: Dict[str, Any]) -> str: preset_name = self._resolve_default_preset_name(site) preset_data = self._get_site_data_readonly(domain, preset_name) workflow = preset_data.get("workflow", []) if isinstance(preset_data, dict) else [] if isinstance(workflow, list): for step in workflow: if str((step or {}).get("action") or "").strip().upper() != "JS_EXEC": continue startup_url = self._extract_startup_url_from_script((step or {}).get("value")) if startup_url: return self._normalize_startup_url(domain, startup_url) return self._normalize_startup_url(domain, "") def get_site_catalog_entry(self, domain: str, fallback_order: int = 0) -> Optional[Dict[str, Any]]: self.refresh_if_changed() site = self.sites.get(domain) if not isinstance(site, dict) or str(domain or "").startswith("_"): return None rule = get_site_rule(domain) startup_url = rule.get("startup_url") or self._infer_site_startup_url(domain, site) display_name = str(rule.get("display_name") or domain).strip() or str(domain) card_id = str(rule.get("card_id") or derive_site_card_id(domain)).strip() or derive_site_card_id(domain) guide_priority = rule.get("guide_priority") if not isinstance(guide_priority, int): guide_priority = int(fallback_order) entry = { "domain": str(domain), "display_name": display_name, "url": self._normalize_startup_url(domain, startup_url), "card_id": card_id, "guide_priority": guide_priority, } if isinstance(rule.get("route_aliases"), list): entry["route_aliases"] = [str(item) for item in rule.get("route_aliases", [])] if "stealth_default" in rule: entry["stealth_default"] = bool(rule.get("stealth_default")) return entry def list_site_catalog(self) -> List[Dict[str, Any]]: self.refresh_if_changed() ordered_domains = [ domain for domain, site in self.sites.items() if not domain.startswith("_") and isinstance(site, dict) ] entries: List[Dict[str, Any]] = [] for index, domain in enumerate(ordered_domains): entry = self.get_site_catalog_entry(domain, fallback_order=index) if entry: entries.append(entry) entries.sort(key=lambda item: (int(item.get("guide_priority", 0)), str(item.get("domain") or ""))) return entries def _normalize_site_advanced_config( self, raw_config: Optional[Dict[str, Any]] = None, *, base_config: Optional[Dict[str, Any]] = None, ) -> Dict[str, Any]: """合并并规范化高级配置。""" normalized = { **get_default_site_advanced_config(), } if isinstance(base_config, dict): normalized.update(copy.deepcopy(base_config)) if isinstance(raw_config, dict): normalized.update(copy.deepcopy(raw_config)) normalized["independent_cookies"] = _coerce_bool(normalized.get("independent_cookies"), False) normalized["independent_cookies_auto_takeover"] = _coerce_bool( normalized.get("independent_cookies_auto_takeover"), False ) normalized["input_box_stability_wait_enabled"] = _coerce_bool( normalized.get("input_box_stability_wait_enabled"), False ) normalized["input_box_stability_wait_after_new_chat_only"] = _coerce_bool( normalized.get("input_box_stability_wait_after_new_chat_only"), True ) try: timeout_value = float(normalized.get("input_box_stability_wait_timeout", 1.5)) except Exception: timeout_value = 1.5 normalized["input_box_stability_wait_timeout"] = max(0.1, min(timeout_value, 10.0)) normalized["url_transition_wait_on_new_chat"] = _coerce_bool( normalized.get("url_transition_wait_on_new_chat"), False ) raw_patterns = normalized.get("url_transition_wait_patterns") or [] if isinstance(raw_patterns, str): raw_patterns = raw_patterns.replace("\n", ",").replace(";", ",").split(",") if not isinstance(raw_patterns, (list, tuple, set)): raw_patterns = [] normalized["url_transition_wait_patterns"] = [ str(pattern or "").strip() for pattern in raw_patterns if str(pattern or "").strip() ] normalized["send_confirmation_check_enabled"] = _coerce_bool( normalized.get("send_confirmation_check_enabled"), False ) try: send_timeout_value = float(normalized.get("send_confirmation_check_timeout", 1.5)) except Exception: send_timeout_value = 1.5 normalized["send_confirmation_check_timeout"] = max(0.1, min(send_timeout_value, 10.0)) normalized["skip_new_chat_on_retry"] = _coerce_bool( normalized.get("skip_new_chat_on_retry"), False ) return normalized def get_site_advanced_config(self, domain: str, preset_name: str = None) -> Dict[str, Any]: """获取站点高级配置;传入 preset_name 时叠加对应预设的覆盖项。""" self.refresh_if_changed() site = self.sites.get(domain) if not site: return self._normalize_site_advanced_config() raw_config = site.get("advanced") if isinstance(site, dict) else None normalized = self._normalize_site_advanced_config( raw_config if isinstance(raw_config, dict) else {} ) requested_preset = str(preset_name or "").strip() if requested_preset: preset_data = self._get_preset_data_exact(domain, requested_preset) if preset_data is None: logger.warning(f"站点高级配置预设不存在: {domain}/{requested_preset}") return normalized preset_advanced = ( preset_data.get("advanced") if isinstance(preset_data, dict) else None ) if isinstance(preset_advanced, dict): preset_advanced = { key: value for key, value in preset_advanced.items() if key in PRESET_ADVANCED_FIELDS } normalized = self._normalize_site_advanced_config( preset_advanced, base_config=normalized, ) return normalized def set_site_advanced_config(self, domain: str, config: Dict[str, Any]) -> bool: """设置站点级高级配置。""" with self._io_lock: self.refresh_if_changed() site = self.sites.get(domain) if not site: logger.warning(f"站点不存在: {domain}") return False previous_advanced = site.get("advanced", _MISSING) if previous_advanced is not _MISSING: previous_advanced = copy.deepcopy(previous_advanced) existing = site.get("advanced") if isinstance(site.get("advanced"), dict) else {} stored = copy.deepcopy(existing) normalized = self._normalize_site_advanced_config(config or {}, base_config=existing) provided_keys = { key for key in ((config or {}).keys() if isinstance(config, dict) else set()) if key in ADVANCED_FIELDS } if not provided_keys and previous_advanced is _MISSING: return True for key in ADVANCED_FIELDS: if key in provided_keys: stored[key] = normalized[key] site["advanced"] = stored if self._save_config_locked(): return True if previous_advanced is _MISSING: site.pop("advanced", None) else: site["advanced"] = previous_advanced return False def set_preset_advanced_config( self, domain: str, config: Dict[str, Any], preset_name: str = None, *, prune_inherited_fields: Optional[Set[str]] = None, ) -> bool: """设置当前预设的高级配置;拒绝混入站点级字段。""" with self._io_lock: self.refresh_if_changed() site = self.sites.get(domain) if not isinstance(site, dict): logger.warning(f"站点不存在: {domain}") return False requested_preset = str(preset_name or "").strip() if not requested_preset: logger.warning(f"预设级高级配置缺少 preset_name: {domain}") return False data = self._get_preset_data_exact(domain, requested_preset) if data is None: logger.warning(f"站点或预设不存在: {domain}/{requested_preset}") return False raw_config = config if isinstance(config, dict) else {} invalid_site_keys = { key for key in raw_config.keys() if key in SITE_ADVANCED_FIELDS } if invalid_site_keys: joined = ", ".join(sorted(invalid_site_keys)) logger.warning(f"预设级高级配置不能包含站点级字段: {domain}/{requested_preset} ({joined})") return False previous_advanced = data.get("advanced", _MISSING) if previous_advanced is not _MISSING: previous_advanced = copy.deepcopy(previous_advanced) stored = copy.deepcopy(data.get("advanced") or {}) if not isinstance(stored, dict): stored = {} for key in list(SITE_ADVANCED_FIELDS): stored.pop(key, None) site_advanced = site.get("advanced") if isinstance(site.get("advanced"), dict) else {} inherited = self._normalize_site_advanced_config(site_advanced) normalized = self._normalize_site_advanced_config( raw_config, base_config=stored, ) provided_keys = { key for key in raw_config.keys() if key in PRESET_ADVANCED_FIELDS } if not provided_keys and previous_advanced is _MISSING: return True prune_inherited_keys = set(prune_inherited_fields or set()) for key in PRESET_ADVANCED_FIELDS: if key in provided_keys: if ( key in prune_inherited_keys and key not in stored and normalized[key] == inherited.get(key) ): stored.pop(key, None) else: stored[key] = normalized[key] if not stored and previous_advanced is _MISSING: return True data["advanced"] = stored if self._save_config_locked(): return True if previous_advanced is _MISSING: data.pop("advanced", None) else: data["advanced"] = previous_advanced return False def list_sites(self) -> Dict[str, Any]: """获取所有站点配置(过滤内部键)""" self.refresh_if_changed() with self._io_lock: return copy.deepcopy({ domain: config for domain, config in self.sites.items() if not domain.startswith('_') }) def get_site_config(self, domain: str, html_content: Optional[Union[str, Callable[[], str]]] = None, preset_name: str = None) -> Optional[SiteConfig]: """ 获取站点配置(缓存 + AI 分析) Args: domain: 站点域名 html_content: 页面 HTML 或用于延迟获取 HTML 的可调用对象(用于 AI 分析未知站点) preset_name: 预设名称,None 则使用默认预设 """ self.refresh_if_changed() if domain in self.sites: cache_key = f"site_config:{domain}:{preset_name or ''}" cached = self._cache.get(cache_key, _MISSING) if cached is not _MISSING: return copy.deepcopy(cached) site = self.sites.get(domain, {}) config = self._get_site_data(domain, preset_name) if config is None: logger.warning(f"站点 {domain} 无可用预设") return None used_preset = preset_name or self._resolve_default_preset_name(site) or DEFAULT_PRESET_NAME previous_config = None # 补充缺失字段 changed = False if "workflow" not in config: if previous_config is None: previous_config = copy.deepcopy(config) config["workflow"] = DEFAULT_WORKFLOW changed = True if "image_extraction" not in config: if previous_config is None: previous_config = copy.deepcopy(config) config["image_extraction"] = get_default_image_extraction_config() changed = True if "file_paste" not in config: if previous_config is None: previous_config = copy.deepcopy(config) config["file_paste"] = get_default_file_paste_config() changed = True else: normalized_file_paste = self._validate_file_paste_config( config.get("file_paste", {}), legacy_stream_config=config.get("stream_config"), ) if normalized_file_paste != config.get("file_paste"): if previous_config is None: previous_config = copy.deepcopy(config) config["file_paste"] = normalized_file_paste changed = True if "prompt_padding" not in config: if previous_config is None: previous_config = copy.deepcopy(config) config["prompt_padding"] = get_default_prompt_padding_config() changed = True else: normalized_prompt_padding = self._validate_prompt_padding_config( config.get("prompt_padding", {}) ) if normalized_prompt_padding != config.get("prompt_padding"): if previous_config is None: previous_config = copy.deepcopy(config) config["prompt_padding"] = normalized_prompt_padding changed = True if changed: completed_config = copy.deepcopy(config) if not self._save_config(): config.clear() config.update(previous_config) logger.warning( f"站点 {domain} [{used_preset}] 配置自动补全保存失败," "已仅对当前请求返回补全配置" ) return completed_config logger.debug(f"使用缓存配置: {domain} [预设: {used_preset}]") result = copy.deepcopy(config) self._cache.set(cache_key, result) return copy.deepcopy(result) if not automatic_discovery_allowed(domain): logger.info(f"跳过非聊天站点的自动收录: {domain}(已有手动配置不受影响)") return None resolved_html = "" if callable(html_content): try: resolved_html = str(html_content() or "") except Exception as e: logger.warning(f"获取页面 HTML 失败: {e}") resolved_html = "" elif html_content is not None: resolved_html = str(html_content or "") if resolved_html.strip(): logger.info(f"🔍 未知域名 {domain},启动 AI 识别...") clean_html = self.html_cleaner.clean(resolved_html) selectors = self.ai_analyzer.analyze(clean_html) if specific_chat_selectors(selectors): selectors = self.validator.validate(selectors) if selectors_match_chat_html(selectors, clean_html): new_preset: SiteConfig = { "selectors": selectors, "workflow": DEFAULT_WORKFLOW, "stealth": self._guess_stealth(domain), "stream_config": copy.deepcopy(get_default_stream_config()), "image_extraction": get_default_image_extraction_config(), "file_paste": get_default_file_paste_config(), "prompt_padding": get_default_prompt_padding_config(), } self.sites[domain] = { "default_preset": DEFAULT_PRESET_NAME, "presets": { DEFAULT_PRESET_NAME: new_preset } } if self._save_config(): logger.info(f"✅ 配置已生成并保存: {domain}") else: logger.warning(f"⚠️ 配置已生成但保存失败,仅保留在当前运行内存: {domain}") return copy.deepcopy(new_preset) logger.warning(f"⚠️ AI 分析失败,使用通用回退配置: {domain}") else: logger.warning(f"⚠️ 未知域名 {domain} 且未提供网页 HTML,使用通用回退配置") fallback_selectors = self.global_config.get_fallback_selectors() fallback_preset: SiteConfig = { "selectors": fallback_selectors, "workflow": DEFAULT_WORKFLOW, "stealth": False, "stream_config": copy.deepcopy(get_default_stream_config()), "image_extraction": get_default_image_extraction_config(), "file_paste": get_default_file_paste_config(), "prompt_padding": get_default_prompt_padding_config(), } # A generic fallback is only for this caller. It is not proof that this # domain hosts an AI chat product, and must never enter the site catalog. logger.info(f"通用回退仅用于当前请求,未自动添加站点: {domain}") return copy.deepcopy(fallback_preset) def delete_site_config(self, domain: str) -> bool: """删除指定站点配置""" self.refresh_if_changed() if domain in self.sites: removed_site = copy.deepcopy(self.sites[domain]) previous_global_default = copy.deepcopy(self._global_default_presets) previous_local_default = copy.deepcopy(self._local_default_presets) del self.sites[domain] if not self._save_config(): self.sites[domain] = removed_site self._global_default_presets = previous_global_default self._local_default_presets = previous_local_default return False logger.info(f"已删除配置: {domain}") return True return False def get_selector_definitions(self) -> List[SelectorDefinition]: """获取元素定义列表""" return self.global_config.get_selector_definitions() def set_selector_definitions(self, definitions: List[SelectorDefinition]) -> bool: """设置元素定义列表并保存""" previous_definitions = self.global_config.get_selector_definitions() previous_fallback_selectors = copy.deepcopy(self.validator.fallback_selectors) self.global_config.set_selector_definitions(definitions) # 更新验证器的回退选择器 self.validator.fallback_selectors = self.global_config.get_fallback_selectors() # 保存配置 if not self._save_config(): self.global_config.set_selector_definitions(previous_definitions) self.validator.fallback_selectors = previous_fallback_selectors return False logger.info(f"元素定义已更新: {len(definitions)} 个") return True