""" app/services/config/processors.py - 配置处理器 职责: - HTML 清理 - 选择器验证 - AI 页面分析 """ import json import importlib.util import os import re import time from app.core.config import get_logger from app.utils.site_discovery import admitted_selectors from typing import Dict, Optional from urllib import request, error import bs4 from bs4 import BeautifulSoup from app.core.config import AppConfig from .managers import GlobalConfigManager logger = get_logger("CFG_PRC") _BS4_PARSER = "lxml" if importlib.util.find_spec("lxml") is not None else "html.parser" # ================= 常量 ================= # HTML 清理配置 MAX_HTML_CHARS = int(os.getenv("MAX_HTML_CHARS", "120000")) TEXT_TRUNCATE_LENGTH = 80 # AI 请求配置 AI_MAX_RETRIES = 3 AI_RETRY_BASE_DELAY = 1.0 AI_RETRY_MAX_DELAY = 10.0 AI_REQUEST_TIMEOUT = 120 # 无效选择器语法模式 INVALID_SYNTAX_PATTERNS = [ (r'~\s*\.\.', '~ .. 无效语法'), (r'\.\.\s*$', '结尾 .. 无效'), (r'>>\s', '>> 无效语法'), (r':has\(', ':has() 兼容性差'), (r'\s~\s*$', '结尾 ~ 无效'), ] # ================= HTML 清理器 ================= class HTMLCleaner: """HTML 清理器""" TAGS_TO_REMOVE = [ 'script', 'style', 'meta', 'link', 'noscript', 'img', 'video', 'audio', 'iframe', 'canvas', 'path', 'rect', 'circle', 'polygon', 'defs', 'clipPath', 'header', 'footer', 'nav', 'aside', ] ALLOWED_ATTRS = [ 'id', 'class', 'name', 'placeholder', 'aria-label', 'role', 'data-testid', 'type', 'disabled', 'value', 'title', 'tabindex', 'contenteditable', 'href' ] INTERACTIVE_TAGS = ['input', 'textarea', 'button', 'form', 'a'] CORE_AREA_SELECTORS = [ '[role="main"]', 'main', '#app', '#root', '.chat', '.conversation', '.message', ] def __init__(self, max_chars: int = None, text_truncate: int = None): self.max_chars = max_chars or MAX_HTML_CHARS self.text_truncate = text_truncate or TEXT_TRUNCATE_LENGTH def clean(self, html: str) -> str: """深度清理 HTML""" logger.debug("开始 HTML 清理...") original_length = len(html) soup = BeautifulSoup(html, _BS4_PARSER) try: interactive_elements = self._extract_interactive_elements(soup) for tag in soup(self.TAGS_TO_REMOVE): tag.decompose() for element in soup(text=lambda t: isinstance(t, bs4.element.Comment)): element.extract() for tag in soup.find_all(True): if tag.string and len(tag.string) > self.text_truncate: tag.string = tag.string[:self.text_truncate] + "..." attrs = dict(tag.attrs) for attr in attrs: if attr not in self.ALLOWED_ATTRS: del tag.attrs[attr] if 'class' in tag.attrs and isinstance(tag.attrs['class'], list): tag.attrs['class'] = " ".join(tag.attrs['class']) clean_html = str(soup.body) if soup.body else str(soup) clean_html = re.sub(r'\s+', ' ', clean_html).strip() if len(clean_html) > self.max_chars: logger.warning(f"HTML 过长 ({len(clean_html)}),执行智能截断...") clean_html = self._smart_truncate(clean_html, interactive_elements) final_length = len(clean_html) reduction = 100 - (final_length / original_length * 100) if original_length > 0 else 0 logger.info(f"HTML 清理完成: {original_length} → {final_length} 字符 (减少 {reduction:.1f}%)") return clean_html finally: soup.decompose() def _extract_interactive_elements(self, soup: BeautifulSoup) -> str: elements = [] for tag_name in self.INTERACTIVE_TAGS: for element in soup.find_all(tag_name): context = self._get_element_with_context(element, levels=2) if context: elements.append(context) unique_elements = list(dict.fromkeys(elements)) return "\n".join(unique_elements) def _get_element_with_context(self, element, levels: int = 2) -> str: try: current = element for _ in range(levels): if current.parent and current.parent.name not in ['body', 'html', '[document]']: current = current.parent else: break html_str = str(current) if len(html_str) > 2000: html_str = html_str[:2000] + "..." return html_str except Exception: return str(element) def _smart_truncate(self, html: str, interactive_html: str) -> str: interactive_budget = int(self.max_chars * 0.3) if len(interactive_html) > interactive_budget: interactive_html = interactive_html[:interactive_budget] remaining_budget = self.max_chars - len(interactive_html) - 100 if remaining_budget <= 0: logger.warning("交互元素已占满预算") return interactive_html core_html = self._extract_core_area(html) if core_html and len(core_html) <= remaining_budget: result = core_html + "\n\n" + interactive_html return result[:self.max_chars] head_budget = remaining_budget // 3 tail_budget = remaining_budget // 3 head_part = self._truncate_at_tag_boundary(html[:head_budget * 2], head_budget, from_end=False) tail_part = self._truncate_at_tag_boundary(html[-tail_budget * 2:], tail_budget, from_end=True) result = ( head_part + "\n\n" + "\n" + interactive_html + "\n\n" + tail_part ) if len(result) > self.max_chars: result = result[:self.max_chars] return result def _extract_core_area(self, html: str) -> Optional[str]: soup = None try: soup = BeautifulSoup(html, _BS4_PARSER) for selector in self.CORE_AREA_SELECTORS: try: element = soup.select_one(selector) if element: core_html = str(element) logger.debug(f"找到核心区域: {selector} ({len(core_html)} chars)") return core_html except Exception: continue return None except Exception as e: logger.debug(f"提取核心区域失败: {e}") return None finally: if soup is not None: soup.decompose() def _truncate_at_tag_boundary(self, html: str, max_len: int, from_end: bool = False) -> str: if len(html) <= max_len: return html if from_end: start_pos = len(html) - max_len tag_start = html.find('<', start_pos) if tag_start != -1 and tag_start < len(html) - 100: return html[tag_start:] return html[-max_len:] else: tag_end = html.rfind('>', 0, max_len) if tag_end != -1 and tag_end > 100: return html[:tag_end + 1] return html[:max_len] # ================= 选择器验证器 ================= class SelectorValidator: """选择器验证器""" def __init__(self, fallback_selectors: Dict[str, Optional[str]] = None): self.fallback_selectors = fallback_selectors or {} def validate(self, selectors: Dict[str, Optional[str]]) -> Dict[str, Optional[str]]: """验证并修复选择器""" fixed = {} for key, selector in selectors.items(): if selector is None: fixed[key] = self.fallback_selectors.get(key) continue is_invalid = False invalid_reason = "" for pattern, reason in INVALID_SYNTAX_PATTERNS: if re.search(pattern, selector): is_invalid = True invalid_reason = reason break if is_invalid: logger.warning(f"❌ 无效选择器 [{key}]: {selector}") logger.warning(f" 原因: {invalid_reason}") repaired = self._try_repair(selector) if repaired: logger.info(f" ✅ 修复为: {repaired}") fixed[key] = repaired else: fallback = self.fallback_selectors.get(key) logger.info(f" 🔄 回退为: {fallback}") fixed[key] = fallback else: if re.search(r'\._[a-f0-9]{5,}|^\.[a-f0-9]{6,}', selector): logger.info(f"ℹ️ 哈希类名 [{key}]: {selector} (可能不稳定,但保留)") fixed[key] = selector return fixed def _try_repair(self, selector: str) -> Optional[str]: tag_match = re.match(r'^(\w+)', selector) if not tag_match: return None tag = tag_match.group(1) attr_patterns = [ r'(\[name=["\']?\w+["\']?\])', r'(\[type=["\']?\w+["\']?\])', r'(\[role=["\']?\w+["\']?\])', r'(#[\w-]+)', ] for pattern in attr_patterns: match = re.search(pattern, selector) if match: return tag + match.group(1) return tag # ================= AI 分析器 ================= class AIAnalyzer: """AI 页面分析器(支持 OpenAI 兼容 / Gemini 官方 / Claude 官方)""" # ===== Provider 注册表 ===== # 每个 provider 定义如何构建请求和解析响应 # 新增 provider 只需在此处注册,无需修改其他逻辑 PROVIDERS = { "openai": { "build_url": "_build_url_openai", "build_headers": "_build_headers_openai", "build_body": "_build_body_openai", "parse_response": "_parse_response_openai", }, "gemini": { "build_url": "_build_url_gemini", "build_headers": "_build_headers_gemini", "build_body": "_build_body_gemini", "parse_response": "_parse_response_gemini", }, "claude": { "build_url": "_build_url_claude", "build_headers": "_build_headers_claude", "build_body": "_build_body_claude", "parse_response": "_parse_response_claude", }, } # URL 关键词 → provider 类型的映射(用于自动检测) URL_PROVIDER_HINTS = [ ("generativelanguage.googleapis.com", "gemini"), ("api.anthropic.com", "claude"), ("openai.azure.com", "azure"), # 未来可扩展 ] def __init__(self, global_config: GlobalConfigManager = None): self.api_key = AppConfig.get_helper_api_key() base_url = AppConfig.get_helper_base_url() self.base_url = base_url.rstrip('/') if base_url else "http://127.0.0.1:5104/v1" self.model = AppConfig.get_helper_model() self.global_config = global_config # 确定 provider 类型 configured_provider = AppConfig.get_helper_api_provider() if configured_provider == "auto" or configured_provider not in self.PROVIDERS: self.provider = self._detect_provider(self.base_url) else: self.provider = configured_provider if not self.api_key: logger.warning("⚠️ 未配置 HELPER_API_KEY,AI 分析功能将不可用") else: logger.debug(f"AI 分析器已配置: Provider={self.provider}, URL={self.base_url}, Model={self.model}") # ================= 公共接口 ================= def analyze(self, html: str) -> Optional[Dict[str, str]]: """分析 HTML 并返回选择器""" if not self.api_key: logger.error("API Key 未配置") return None prompt = self._build_prompt(html) for attempt in range(AI_MAX_RETRIES): try: logger.info(f"正在请求 AI 分析(尝试 {attempt + 1}/{AI_MAX_RETRIES},Provider: {self.provider})...") response = self._request_ai(prompt) if response: analysis = self._extract_json(response) if analysis is not None: selectors = admitted_selectors(analysis) if selectors: logger.info("✅ 已确认 AI 聊天页并识别到专用定位器") return selectors logger.info("未确认 AI 聊天界面或缺少专用回复容器,不自动收录") return None logger.warning(f"第 {attempt + 1} 次分析失败") except Exception as e: logger.error(f"AI 请求异常: {e}") if attempt < AI_MAX_RETRIES - 1: delay = min( AI_RETRY_BASE_DELAY * (2 ** attempt), AI_RETRY_MAX_DELAY ) jitter = delay * 0.1 * (0.5 - os.urandom(1)[0] / 255) sleep_time = delay + jitter logger.info(f"等待 {sleep_time:.2f}s 后重试...") time.sleep(sleep_time) logger.error("❌ AI 分析失败(已达最大重试次数)") return None # ================= Provider 自动检测 ================= def _detect_provider(self, base_url: str) -> str: """根据 URL 自动推断 API 类型""" if not base_url: return "openai" url_lower = base_url.lower() for keyword, provider in self.URL_PROVIDER_HINTS: if keyword in url_lower: logger.info(f"自动检测到 API 类型: {provider}(基于 URL 包含 '{keyword}')") return provider # 默认回退到 OpenAI 兼容格式 return "openai" # ================= 统一请求调度 ================= def _request_ai(self, prompt: str) -> Optional[str]: """向 AI API 发送请求(根据 provider 自动适配格式)""" provider_config = self.PROVIDERS.get(self.provider) if not provider_config: logger.error(f"未知的 Provider: {self.provider},回退到 openai") provider_config = self.PROVIDERS["openai"] # 通过注册表动态调用对应的构建/解析方法 url = getattr(self, provider_config["build_url"])(prompt) headers = getattr(self, provider_config["build_headers"])() body = getattr(self, provider_config["build_body"])(prompt) parse_fn = getattr(self, provider_config["parse_response"]) try: req = request.Request( url, data=json.dumps(body).encode('utf-8'), headers=headers ) with request.urlopen(req, timeout=AI_REQUEST_TIMEOUT) as response: response_text = response.read().decode('utf-8') try: json_resp = json.loads(response_text) return parse_fn(json_resp) except json.JSONDecodeError: logger.error(f"AI 响应 JSON 解析失败,原始响应前 500 字符: {response_text[:500]}") return None except error.HTTPError as e: error_body = "" try: error_body = e.read().decode('utf-8')[:500] except Exception: pass logger.error(f"HTTP 错误 {e.code}: {e.reason}" + (f",响应: {error_body}" if error_body else "")) return None except error.URLError as e: logger.error(f"网络错误: {e.reason}") return None except Exception as e: # 兜底:捕获 socket.timeout 等未被上层覆盖的异常 logger.error(f"请求异常: {type(e).__name__}: {e}") return None # ================= OpenAI 兼容格式 ================= def _build_url_openai(self, prompt: str) -> str: return f"{self.base_url}/chat/completions" def _build_headers_openai(self) -> Dict[str, str]: return { "Content-Type": "application/json", "Authorization": f"Bearer {self.api_key}", } def _build_body_openai(self, prompt: str) -> Dict: return { "model": self.model, "messages": [{"role": "user", "content": prompt}], "temperature": 0.1, } def _parse_response_openai(self, json_resp: Dict) -> Optional[str]: try: if "choices" in json_resp and len(json_resp["choices"]) > 0: return json_resp["choices"][0]["message"]["content"] except (KeyError, IndexError, TypeError) as e: logger.error(f"OpenAI 响应解析失败: {e}") return None # ================= Gemini 官方格式 ================= def _build_url_gemini(self, prompt: str) -> str: # Gemini 官方 API: base_url 应为 https://generativelanguage.googleapis.com # 完整端点: /v1beta/models/{model}:generateContent?key={api_key} base = self.base_url.rstrip('/') # 如果用户配置的 base_url 已经包含 /v1beta 或 /v1,不再追加 if "/v1beta" not in base and "/v1/" not in base: base = f"{base}/v1beta" return f"{base}/models/{self.model}:generateContent?key={self.api_key}" def _build_headers_gemini(self) -> Dict[str, str]: # Gemini 官方通过 URL 参数传递 key,不需要 Authorization 头 return { "Content-Type": "application/json", } def _build_body_gemini(self, prompt: str) -> Dict: return { "contents": [ { "parts": [ {"text": prompt} ] } ], "generationConfig": { "temperature": 0.1, }, } def _parse_response_gemini(self, json_resp: Dict) -> Optional[str]: try: candidates = json_resp.get("candidates", []) if candidates: parts = candidates[0].get("content", {}).get("parts", []) if parts: return parts[0].get("text") except (KeyError, IndexError, TypeError) as e: logger.error(f"Gemini 响应解析失败: {e}") # 检查是否有错误信息 if "error" in json_resp: err = json_resp["error"] logger.error(f"Gemini API 错误: [{err.get('code')}] {err.get('message', '')}") return None # ================= Claude 官方格式 ================= def _build_url_claude(self, prompt: str) -> str: base = self.base_url.rstrip('/') # Claude 官方端点: /v1/messages if not base.endswith("/v1"): base = f"{base}/v1" return f"{base}/messages" def _build_headers_claude(self) -> Dict[str, str]: return { "Content-Type": "application/json", "x-api-key": self.api_key, "anthropic-version": "2023-06-01", } def _build_body_claude(self, prompt: str) -> Dict: return { "model": self.model, "max_tokens": 4096, "messages": [{"role": "user", "content": prompt}], "temperature": 0.1, } def _parse_response_claude(self, json_resp: Dict) -> Optional[str]: try: content = json_resp.get("content", []) if content: # Claude 响应的 content 是列表,每项有 type 和 text text_parts = [block["text"] for block in content if block.get("type") == "text"] if text_parts: return "\n".join(text_parts) except (KeyError, IndexError, TypeError) as e: logger.error(f"Claude 响应解析失败: {e}") # 检查是否有错误信息 if "error" in json_resp: err = json_resp["error"] logger.error(f"Claude API 错误: [{err.get('type')}] {err.get('message', '')}") return None # ================= Prompt 构建(不变)================= def _build_prompt(self, clean_html: str) -> str: """构建 AI 提示词(动态版)""" if self.global_config: selector_list = self.global_config.build_prompt_selector_list() json_keys = self.global_config.build_prompt_json_keys() else: selector_list = self._build_default_selector_list() json_keys = self._build_default_json_keys() lines = [ "You are a web interface classifier. First determine whether this is an AI CONVERSATION interface; do not assume every page is a chat app.", "Return {\"is_chat_site\": false} for search engines, search results, login/consent pages, articles, ordinary forms, and pages with insufficient evidence.", "A textarea, submit button, generic div, or the word AI alone is NOT evidence of an AI conversation interface.", "Only for a genuine AI chat page with a composer and a specific assistant-reply container, return is_chat_site: true and the selector keys below. Do not invent selectors for missing elements.", "", "## CRITICAL RULES:", "1. **Uniqueness is Key**: Ensure selectors matches ONLY the intended element.", "2. **Distinguish AI vs User**: For `result_container`, specify the selector to target the **AI's response text** only. It MUST exclude user prompts, sidebars, or chat history.", "3. **Use Hierarchy**: If a class like `.prose` or `.markdown` is used for both User and AI, you MUST find a unique parent class to differentiate (e.g., `.bot-msg .prose`).", "4. **Syntax**: Use standard CSS selectors. Spaces for descendants (e.g., `div.bot p`) are encouraged for precision.", "5. **No Invalid Syntax**: Do NOT use `xpath`, `~`, `:has()`, or `text()`.", "", "## PREFERENCE ORDER:", "1. `id`, `name`, `data-testid` (Most preferred)", "2. `button[type=\"submit\"]`", "3. Unique parent class + target class (e.g., `.response-area .content`)", "4. Hashed classes (only if no other option exists)", "", "## ELEMENTS TO FIND:", selector_list, "", "## REQUIRED OUTPUT (JSON ONLY):", "Return a JSON object with boolean is_chat_site plus these keys ONLY when is_chat_site is true:", json_keys, "", "## HTML:", clean_html ] return "\n".join(lines) def _build_default_selector_list(self) -> str: """默认的元素列表(回退用)""" return """- `input_box`: The text input area (textarea/input). [REQUIRED] - `send_btn`: The button that sends the message. [REQUIRED] - `result_container`: The container for the AI's generated text response. [REQUIRED] - `new_chat_btn`: Button to start a fresh conversation. [OPTIONAL, return null if not found]""" def _build_default_json_keys(self) -> str: """默认的 JSON 键说明(回退用)""" return """- `input_box`: The text input area - `send_btn`: The send button - `result_container`: AI response container - `new_chat_btn`: New chat button (or null)""" # ================= JSON 提取(不变)================= def _extract_json(self, text: str) -> Optional[Dict]: """从 AI 响应中提取 JSON""" try: match = re.search(r'```(?:json)?\s*(\{[\s\S]*?\})\s*```', text) if match: return json.loads(match.group(1)) match = re.search(r'(\{[\s\S]*\})', text) if match: return json.loads(match.group(1)) return json.loads(text) except json.JSONDecodeError as e: logger.error(f"JSON 解析失败: {e}") return None __all__ = ['HTMLCleaner', 'SelectorValidator', 'AIAnalyzer']