Files
universal-web-api/app/core/extractors/hybrid_mode.py
T
2026-11-02 22:54:14 +08:00

396 lines
12 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""
app/core/extractors/hybrid_mode.py - 混合智能提取器
结合 DOM 直接提取和深度 JS 注入的优点:
- 先快速检测页面特征(是否有代码块、公式等)
- 简单内容 → 轻量提取(性能优先)
- 复杂内容 → 深度提取(准确优先)
"""
from typing import Any
from app.core.extractors.base import BaseExtractor
from app.core.driver import as_element
class HybridExtractor(BaseExtractor):
"""
混合智能提取器
策略:
1. 快速检测元素是否包含复杂内容(代码块、LaTeX、Shadow DOM)
2. 简单内容:直接读取 textContent(快速,低开销)
3. 复杂内容:JS 深度遍历提取(准确,支持格式化)
优势:
- 兼顾性能与准确性
- 自动适应不同类型的页面
- 失败时自动降级
"""
# ============ 元数据 ============
@classmethod
def get_id(cls) -> str:
return "hybrid_mode"
@classmethod
def get_name(cls) -> str:
return "混合智能模式"
@classmethod
def get_description(cls) -> str:
return "自动检测内容复杂度,智能选择最优提取策略(推荐)"
# ============ 检测脚本 ============
# 轻量级复杂度检测 JS
DETECT_COMPLEXITY_JS = """
return (function() {
var result = {
hasCode: false,
hasLatex: false,
hasShadow: false,
hasTable: false,
textLength: 0
};
try {
// 检测代码块
result.hasCode = !!(
this.querySelector('pre') ||
this.querySelector('code') ||
this.querySelector('ms-code-block')
);
// 检测 LaTeX 公式
result.hasLatex = !!(
this.querySelector('.katex') ||
this.querySelector('ms-katex') ||
this.querySelector('.MathJax') ||
this.querySelector('math')
);
// 检测表格
result.hasTable = !!(this.querySelector('table'));
// 检测 Shadow DOM(需要深度遍历)
var checkShadow = function(el, depth) {
if (depth > 5) return false;
if (el.shadowRoot) return true;
var children = el.children || [];
for (var i = 0; i < Math.min(children.length, 20); i++) {
if (checkShadow(children[i], depth + 1)) return true;
}
return false;
};
result.hasShadow = checkShadow(this, 0);
// 获取文本长度(用于判断是否值得深度提取)
result.textLength = (this.textContent || '').length;
} catch(e) {
result.error = e.message;
}
// 综合判断
result.isComplex = result.hasCode || result.hasLatex || result.hasShadow || result.hasTable;
result.needsDeep = result.isComplex && result.textLength > 50;
return result;
}).call(this);
"""
# 深度提取 JS(来自 deep_mode.py,此处简化展示关键部分)
DEEP_EXTRACT_JS = r"""
return (function () {
function normNewline(s){ return (s||'').replace(/\r\n/g,'\n').replace(/\r/g,'\n'); }
function isEl(n){ return n && n.nodeType === 1; }
function hasAncestorTag(el, tagNameLower){
var p = el;
while (p) {
if (p.nodeType === 1 && (p.tagName||'').toLowerCase() === tagNameLower) return true;
p = p.parentElement;
}
return false;
}
function rstripNewlines(s){
s = normNewline(s);
return s.replace(/\n+$/g, '');
}
function ignorableEl(el){
if (!isEl(el)) return false;
var tag = (el.tagName||'').toLowerCase();
if (tag === 'button' || tag === 'svg') return true;
if (tag === 'mat-expansion-panel-header') return true;
var cls = (el.className && typeof el.className === 'string') ? el.className : '';
if (cls.indexOf('actions-container') >= 0) return true;
if (cls.indexOf('turn-footer') >= 0) return true;
if (cls.indexOf('material-symbols') >= 0) return true;
if (cls.indexOf('material-icons') >= 0) return true;
if (cls.indexOf('mat-icon') >= 0) return true;
var aria = (el.getAttribute && (el.getAttribute('aria-label')||'')) || '';
if (/download|copy|expand|collapse|edit/i.test(aria)) return true;
return false;
}
function extractKatex(el){
try {
var ann = el.querySelector && el.querySelector('annotation[encoding="application/x-tex"]');
if (ann && ann.textContent) return ann.textContent;
} catch(e) {}
return '';
}
function walk(node, out){
if (!node) return;
if (isEl(node) && node.shadowRoot) walk(node.shadowRoot, out);
if (node.nodeType === 11) {
var kidsF = node.childNodes ? Array.prototype.slice.call(node.childNodes) : [];
for (var iF = 0; iF < kidsF.length; iF++) walk(kidsF[iF], out);
return;
}
if (node.nodeType === 3) {
out.push(node.nodeValue || '');
return;
}
if (!isEl(node)) return;
var el = node;
if (ignorableEl(el)) return;
var tag = (el.tagName||'').toLowerCase();
var cls = (el.className && typeof el.className === 'string') ? el.className : '';
// KaTeX 处理
if (tag === 'ms-katex' || cls.indexOf('katex') >= 0) {
var tex = extractKatex(el);
if (tex) {
var inline = (cls.indexOf('inline') >= 0);
out.push(inline ? (' $' + tex + '$ ') : ('\n$$\n' + tex + '\n$$\n'));
}
return;
}
// 代码块处理
if ((tag === 'pre' || tag === 'code') && !hasAncestorTag(el, 'ms-katex')) {
var tcode = normNewline(el.textContent || '');
if (tcode && tcode.replace(/\s+/g,'').length > 0) {
out.push('\n```\n' + rstripNewlines(tcode) + '\n```\n');
}
return;
}
// 递归子节点
var kids = el.childNodes ? Array.prototype.slice.call(el.childNodes) : [];
if (kids.length) {
var isBlock = 'p div section li ul ol table tr h1 h2 h3 h4 h5 h6 br'.indexOf(tag) >= 0;
if (isBlock) out.push('\n');
for (var k = 0; k < kids.length; k++) walk(kids[k], out);
if (isBlock) out.push('\n');
}
}
try {
var out = [];
walk(this, out);
var s = normNewline(out.join(''));
s = s.replace(/[ \t]+\n/g, '\n').replace(/\n{3,}/g, '\n\n');
return s.trim();
} catch (e) {
return normNewline(this.textContent || '').trim();
}
}).call(this);
"""
# 内容子节点选择器
CONTENT_SELECTORS = [
'css:.markdown',
'css:.prose',
'css:.response-content-markdown',
'css:.ds-markdown',
'css:.message-content',
'css:[class*="markdown"]',
'css:[class*="content"]',
]
# ============ 核心方法 ============
def extract_text(self, element) -> str:
"""
智能提取文本
流程:
1. 定位内容节点
2. 检测复杂度
3. 选择提取策略
4. 失败时自动降级
"""
if not element:
return ""
# 1. 定位内容子节点
target = self.find_content_node(element)
# 2. 检测复杂度
complexity = self._detect_complexity(target)
# 3. 根据复杂度选择策略
if complexity.get('needsDeep', False):
# 复杂内容:使用深度提取
text = self._deep_extract(target)
if text:
return text
# 简单内容或深度提取失败:使用轻量提取
return self._simple_extract(target)
def get_anchor(self, element) -> str:
"""
获取元素锚点
复用 deep_mode 的稳定锚点策略
"""
if not element:
return ""
try:
# 1. 优先使用稳定的 ID 类属性
stable_attrs = ['data-message-id', 'data-turn-id', 'data-testid', 'id']
for attr in stable_attrs:
try:
val = element.attr(attr)
if val:
return f"{attr}={val}"
except Exception:
pass
# 2. 回退:tag + class + DOM 位置
tag = element.tag if hasattr(element, 'tag') else 'unknown'
cls = ""
try:
cls = element.attr('class') or ""
except Exception:
pass
classes = cls.split()[:3]
class_part = f"|cls={'.'.join(classes)}" if classes else ""
# 3. DOM 位置
index_part = ""
try:
index = as_element(element).run_js("""
const parent = this.parentElement;
if (!parent) return -1;
return Array.from(parent.children).indexOf(this);
""")
if index is not None and index >= 0:
index_part = f"|idx={index}"
except Exception:
pass
return f"tag:{tag}{class_part}{index_part}"
except Exception:
return ""
def find_content_node(self, element) -> Any:
"""
定位内容子节点
尝试找到实际包含内容的子元素,避免读取杂项
"""
if not element:
return element
for selector in self.CONTENT_SELECTORS:
try:
child = as_element(element).find(selector, timeout=0.05)
if child:
# 验证有实际内容
text = as_element(child).run_js("return (this.textContent || '').trim()")
if text and len(str(text)) > 10:
return child
except Exception:
pass
return element
# ============ 私有方法 ============
def _detect_complexity(self, element) -> dict:
"""
检测元素复杂度
Returns:
{
'hasCode': bool,
'hasLatex': bool,
'hasShadow': bool,
'hasTable': bool,
'isComplex': bool,
'needsDeep': bool
}
"""
try:
result = as_element(element).run_js(self.DETECT_COMPLEXITY_JS)
if isinstance(result, dict):
return result
except Exception:
pass
# 检测失败时默认简单模式
return {'isComplex': False, 'needsDeep': False}
def _simple_extract(self, element) -> str:
"""
简单提取(高性能)
直接读取 textContent,不做复杂处理
"""
try:
# 优先使用 .text 属性
if hasattr(element, 'text') and element.text:
return self._normalize(element.text)
# 回退:JS 读取
text = as_element(element).run_js("return this.textContent || this.innerText || ''")
return self._normalize(str(text)) if text else ""
except Exception:
return ""
def _deep_extract(self, element) -> str:
"""
深度提取(高准确度)
使用 JS 注入遍历 DOM 树
"""
try:
text = as_element(element).run_js(self.DEEP_EXTRACT_JS)
if text and str(text).strip():
return self._normalize(str(text))
except Exception:
pass
return ""
def _normalize(self, text: str) -> str:
"""标准化文本"""
if not text:
return ""
text = text.replace('\r\n', '\n').replace('\r', '\n')
return text.strip()
__all__ = ['HybridExtractor']