""" app/core/extractors/hybrid_mode.py - 混合智能提取器 结合 DOM 直接提取和深度 JS 注入的优点: - 先快速检测页面特征(是否有代码块、公式等) - 简单内容 → 轻量提取(性能优先) - 复杂内容 → 深度提取(准确优先) """ from typing import Any from app.core.extractors.base import BaseExtractor from app.core.driver import as_element class HybridExtractor(BaseExtractor): """ 混合智能提取器 策略: 1. 快速检测元素是否包含复杂内容(代码块、LaTeX、Shadow DOM) 2. 简单内容:直接读取 textContent(快速,低开销) 3. 复杂内容:JS 深度遍历提取(准确,支持格式化) 优势: - 兼顾性能与准确性 - 自动适应不同类型的页面 - 失败时自动降级 """ # ============ 元数据 ============ @classmethod def get_id(cls) -> str: return "hybrid_mode" @classmethod def get_name(cls) -> str: return "混合智能模式" @classmethod def get_description(cls) -> str: return "自动检测内容复杂度,智能选择最优提取策略(推荐)" # ============ 检测脚本 ============ # 轻量级复杂度检测 JS DETECT_COMPLEXITY_JS = """ return (function() { var result = { hasCode: false, hasLatex: false, hasShadow: false, hasTable: false, textLength: 0 }; try { // 检测代码块 result.hasCode = !!( this.querySelector('pre') || this.querySelector('code') || this.querySelector('ms-code-block') ); // 检测 LaTeX 公式 result.hasLatex = !!( this.querySelector('.katex') || this.querySelector('ms-katex') || this.querySelector('.MathJax') || this.querySelector('math') ); // 检测表格 result.hasTable = !!(this.querySelector('table')); // 检测 Shadow DOM(需要深度遍历) var checkShadow = function(el, depth) { if (depth > 5) return false; if (el.shadowRoot) return true; var children = el.children || []; for (var i = 0; i < Math.min(children.length, 20); i++) { if (checkShadow(children[i], depth + 1)) return true; } return false; }; result.hasShadow = checkShadow(this, 0); // 获取文本长度(用于判断是否值得深度提取) result.textLength = (this.textContent || '').length; } catch(e) { result.error = e.message; } // 综合判断 result.isComplex = result.hasCode || result.hasLatex || result.hasShadow || result.hasTable; result.needsDeep = result.isComplex && result.textLength > 50; return result; }).call(this); """ # 深度提取 JS(来自 deep_mode.py,此处简化展示关键部分) DEEP_EXTRACT_JS = r""" return (function () { function normNewline(s){ return (s||'').replace(/\r\n/g,'\n').replace(/\r/g,'\n'); } function isEl(n){ return n && n.nodeType === 1; } function hasAncestorTag(el, tagNameLower){ var p = el; while (p) { if (p.nodeType === 1 && (p.tagName||'').toLowerCase() === tagNameLower) return true; p = p.parentElement; } return false; } function rstripNewlines(s){ s = normNewline(s); return s.replace(/\n+$/g, ''); } function ignorableEl(el){ if (!isEl(el)) return false; var tag = (el.tagName||'').toLowerCase(); if (tag === 'button' || tag === 'svg') return true; if (tag === 'mat-expansion-panel-header') return true; var cls = (el.className && typeof el.className === 'string') ? el.className : ''; if (cls.indexOf('actions-container') >= 0) return true; if (cls.indexOf('turn-footer') >= 0) return true; if (cls.indexOf('material-symbols') >= 0) return true; if (cls.indexOf('material-icons') >= 0) return true; if (cls.indexOf('mat-icon') >= 0) return true; var aria = (el.getAttribute && (el.getAttribute('aria-label')||'')) || ''; if (/download|copy|expand|collapse|edit/i.test(aria)) return true; return false; } function extractKatex(el){ try { var ann = el.querySelector && el.querySelector('annotation[encoding="application/x-tex"]'); if (ann && ann.textContent) return ann.textContent; } catch(e) {} return ''; } function walk(node, out){ if (!node) return; if (isEl(node) && node.shadowRoot) walk(node.shadowRoot, out); if (node.nodeType === 11) { var kidsF = node.childNodes ? Array.prototype.slice.call(node.childNodes) : []; for (var iF = 0; iF < kidsF.length; iF++) walk(kidsF[iF], out); return; } if (node.nodeType === 3) { out.push(node.nodeValue || ''); return; } if (!isEl(node)) return; var el = node; if (ignorableEl(el)) return; var tag = (el.tagName||'').toLowerCase(); var cls = (el.className && typeof el.className === 'string') ? el.className : ''; // KaTeX 处理 if (tag === 'ms-katex' || cls.indexOf('katex') >= 0) { var tex = extractKatex(el); if (tex) { var inline = (cls.indexOf('inline') >= 0); out.push(inline ? (' $' + tex + '$ ') : ('\n$$\n' + tex + '\n$$\n')); } return; } // 代码块处理 if ((tag === 'pre' || tag === 'code') && !hasAncestorTag(el, 'ms-katex')) { var tcode = normNewline(el.textContent || ''); if (tcode && tcode.replace(/\s+/g,'').length > 0) { out.push('\n```\n' + rstripNewlines(tcode) + '\n```\n'); } return; } // 递归子节点 var kids = el.childNodes ? Array.prototype.slice.call(el.childNodes) : []; if (kids.length) { var isBlock = 'p div section li ul ol table tr h1 h2 h3 h4 h5 h6 br'.indexOf(tag) >= 0; if (isBlock) out.push('\n'); for (var k = 0; k < kids.length; k++) walk(kids[k], out); if (isBlock) out.push('\n'); } } try { var out = []; walk(this, out); var s = normNewline(out.join('')); s = s.replace(/[ \t]+\n/g, '\n').replace(/\n{3,}/g, '\n\n'); return s.trim(); } catch (e) { return normNewline(this.textContent || '').trim(); } }).call(this); """ # 内容子节点选择器 CONTENT_SELECTORS = [ 'css:.markdown', 'css:.prose', 'css:.response-content-markdown', 'css:.ds-markdown', 'css:.message-content', 'css:[class*="markdown"]', 'css:[class*="content"]', ] # ============ 核心方法 ============ def extract_text(self, element) -> str: """ 智能提取文本 流程: 1. 定位内容节点 2. 检测复杂度 3. 选择提取策略 4. 失败时自动降级 """ if not element: return "" # 1. 定位内容子节点 target = self.find_content_node(element) # 2. 检测复杂度 complexity = self._detect_complexity(target) # 3. 根据复杂度选择策略 if complexity.get('needsDeep', False): # 复杂内容:使用深度提取 text = self._deep_extract(target) if text: return text # 简单内容或深度提取失败:使用轻量提取 return self._simple_extract(target) def get_anchor(self, element) -> str: """ 获取元素锚点 复用 deep_mode 的稳定锚点策略 """ if not element: return "" try: # 1. 优先使用稳定的 ID 类属性 stable_attrs = ['data-message-id', 'data-turn-id', 'data-testid', 'id'] for attr in stable_attrs: try: val = element.attr(attr) if val: return f"{attr}={val}" except Exception: pass # 2. 回退:tag + class + DOM 位置 tag = element.tag if hasattr(element, 'tag') else 'unknown' cls = "" try: cls = element.attr('class') or "" except Exception: pass classes = cls.split()[:3] class_part = f"|cls={'.'.join(classes)}" if classes else "" # 3. DOM 位置 index_part = "" try: index = as_element(element).run_js(""" const parent = this.parentElement; if (!parent) return -1; return Array.from(parent.children).indexOf(this); """) if index is not None and index >= 0: index_part = f"|idx={index}" except Exception: pass return f"tag:{tag}{class_part}{index_part}" except Exception: return "" def find_content_node(self, element) -> Any: """ 定位内容子节点 尝试找到实际包含内容的子元素,避免读取杂项 """ if not element: return element for selector in self.CONTENT_SELECTORS: try: child = as_element(element).find(selector, timeout=0.05) if child: # 验证有实际内容 text = as_element(child).run_js("return (this.textContent || '').trim()") if text and len(str(text)) > 10: return child except Exception: pass return element # ============ 私有方法 ============ def _detect_complexity(self, element) -> dict: """ 检测元素复杂度 Returns: { 'hasCode': bool, 'hasLatex': bool, 'hasShadow': bool, 'hasTable': bool, 'isComplex': bool, 'needsDeep': bool } """ try: result = as_element(element).run_js(self.DETECT_COMPLEXITY_JS) if isinstance(result, dict): return result except Exception: pass # 检测失败时默认简单模式 return {'isComplex': False, 'needsDeep': False} def _simple_extract(self, element) -> str: """ 简单提取(高性能) 直接读取 textContent,不做复杂处理 """ try: # 优先使用 .text 属性 if hasattr(element, 'text') and element.text: return self._normalize(element.text) # 回退:JS 读取 text = as_element(element).run_js("return this.textContent || this.innerText || ''") return self._normalize(str(text)) if text else "" except Exception: return "" def _deep_extract(self, element) -> str: """ 深度提取(高准确度) 使用 JS 注入遍历 DOM 树 """ try: text = as_element(element).run_js(self.DEEP_EXTRACT_JS) if text and str(text).strip(): return self._normalize(str(text)) except Exception: pass return "" def _normalize(self, text: str) -> str: """标准化文本""" if not text: return "" text = text.replace('\r\n', '\n').replace('\r', '\n') return text.strip() __all__ = ['HybridExtractor']