Initial commit
This commit is contained in:
commit
57fc3525eb
7311 files changed
+1291398
No files matched your search
@@ -0,0 +1,396 @@
|
||||
"""
|
||||
app/core/extractors/hybrid_mode.py - 混合智能提取器
|
||||
|
||||
结合 DOM 直接提取和深度 JS 注入的优点:
|
||||
- 先快速检测页面特征(是否有代码块、公式等)
|
||||
- 简单内容 → 轻量提取(性能优先)
|
||||
- 复杂内容 → 深度提取(准确优先)
|
||||
"""
|
||||
|
||||
from typing import Any
|
||||
|
||||
from app.core.extractors.base import BaseExtractor
|
||||
from app.core.driver import as_element
|
||||
|
||||
|
||||
class HybridExtractor(BaseExtractor):
|
||||
"""
|
||||
混合智能提取器
|
||||
|
||||
策略:
|
||||
1. 快速检测元素是否包含复杂内容(代码块、LaTeX、Shadow DOM)
|
||||
2. 简单内容:直接读取 textContent(快速,低开销)
|
||||
3. 复杂内容:JS 深度遍历提取(准确,支持格式化)
|
||||
|
||||
优势:
|
||||
- 兼顾性能与准确性
|
||||
- 自动适应不同类型的页面
|
||||
- 失败时自动降级
|
||||
"""
|
||||
|
||||
# ============ 元数据 ============
|
||||
|
||||
@classmethod
|
||||
def get_id(cls) -> str:
|
||||
return "hybrid_mode"
|
||||
|
||||
@classmethod
|
||||
def get_name(cls) -> str:
|
||||
return "混合智能模式"
|
||||
|
||||
@classmethod
|
||||
def get_description(cls) -> str:
|
||||
return "自动检测内容复杂度,智能选择最优提取策略(推荐)"
|
||||
|
||||
# ============ 检测脚本 ============
|
||||
|
||||
# 轻量级复杂度检测 JS
|
||||
DETECT_COMPLEXITY_JS = """
|
||||
return (function() {
|
||||
var result = {
|
||||
hasCode: false,
|
||||
hasLatex: false,
|
||||
hasShadow: false,
|
||||
hasTable: false,
|
||||
textLength: 0
|
||||
};
|
||||
|
||||
try {
|
||||
// 检测代码块
|
||||
result.hasCode = !!(
|
||||
this.querySelector('pre') ||
|
||||
this.querySelector('code') ||
|
||||
this.querySelector('ms-code-block')
|
||||
);
|
||||
|
||||
// 检测 LaTeX 公式
|
||||
result.hasLatex = !!(
|
||||
this.querySelector('.katex') ||
|
||||
this.querySelector('ms-katex') ||
|
||||
this.querySelector('.MathJax') ||
|
||||
this.querySelector('math')
|
||||
);
|
||||
|
||||
// 检测表格
|
||||
result.hasTable = !!(this.querySelector('table'));
|
||||
|
||||
// 检测 Shadow DOM(需要深度遍历)
|
||||
var checkShadow = function(el, depth) {
|
||||
if (depth > 5) return false;
|
||||
if (el.shadowRoot) return true;
|
||||
var children = el.children || [];
|
||||
for (var i = 0; i < Math.min(children.length, 20); i++) {
|
||||
if (checkShadow(children[i], depth + 1)) return true;
|
||||
}
|
||||
return false;
|
||||
};
|
||||
result.hasShadow = checkShadow(this, 0);
|
||||
|
||||
// 获取文本长度(用于判断是否值得深度提取)
|
||||
result.textLength = (this.textContent || '').length;
|
||||
|
||||
} catch(e) {
|
||||
result.error = e.message;
|
||||
}
|
||||
|
||||
// 综合判断
|
||||
result.isComplex = result.hasCode || result.hasLatex || result.hasShadow || result.hasTable;
|
||||
result.needsDeep = result.isComplex && result.textLength > 50;
|
||||
|
||||
return result;
|
||||
}).call(this);
|
||||
"""
|
||||
|
||||
# 深度提取 JS(来自 deep_mode.py,此处简化展示关键部分)
|
||||
DEEP_EXTRACT_JS = r"""
|
||||
return (function () {
|
||||
function normNewline(s){ return (s||'').replace(/\r\n/g,'\n').replace(/\r/g,'\n'); }
|
||||
function isEl(n){ return n && n.nodeType === 1; }
|
||||
|
||||
function hasAncestorTag(el, tagNameLower){
|
||||
var p = el;
|
||||
while (p) {
|
||||
if (p.nodeType === 1 && (p.tagName||'').toLowerCase() === tagNameLower) return true;
|
||||
p = p.parentElement;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
function rstripNewlines(s){
|
||||
s = normNewline(s);
|
||||
return s.replace(/\n+$/g, '');
|
||||
}
|
||||
|
||||
function ignorableEl(el){
|
||||
if (!isEl(el)) return false;
|
||||
var tag = (el.tagName||'').toLowerCase();
|
||||
if (tag === 'button' || tag === 'svg') return true;
|
||||
if (tag === 'mat-expansion-panel-header') return true;
|
||||
|
||||
var cls = (el.className && typeof el.className === 'string') ? el.className : '';
|
||||
if (cls.indexOf('actions-container') >= 0) return true;
|
||||
if (cls.indexOf('turn-footer') >= 0) return true;
|
||||
if (cls.indexOf('material-symbols') >= 0) return true;
|
||||
if (cls.indexOf('material-icons') >= 0) return true;
|
||||
if (cls.indexOf('mat-icon') >= 0) return true;
|
||||
|
||||
var aria = (el.getAttribute && (el.getAttribute('aria-label')||'')) || '';
|
||||
if (/download|copy|expand|collapse|edit/i.test(aria)) return true;
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
function extractKatex(el){
|
||||
try {
|
||||
var ann = el.querySelector && el.querySelector('annotation[encoding="application/x-tex"]');
|
||||
if (ann && ann.textContent) return ann.textContent;
|
||||
} catch(e) {}
|
||||
return '';
|
||||
}
|
||||
|
||||
function walk(node, out){
|
||||
if (!node) return;
|
||||
if (isEl(node) && node.shadowRoot) walk(node.shadowRoot, out);
|
||||
|
||||
if (node.nodeType === 11) {
|
||||
var kidsF = node.childNodes ? Array.prototype.slice.call(node.childNodes) : [];
|
||||
for (var iF = 0; iF < kidsF.length; iF++) walk(kidsF[iF], out);
|
||||
return;
|
||||
}
|
||||
|
||||
if (node.nodeType === 3) {
|
||||
out.push(node.nodeValue || '');
|
||||
return;
|
||||
}
|
||||
|
||||
if (!isEl(node)) return;
|
||||
var el = node;
|
||||
if (ignorableEl(el)) return;
|
||||
|
||||
var tag = (el.tagName||'').toLowerCase();
|
||||
var cls = (el.className && typeof el.className === 'string') ? el.className : '';
|
||||
|
||||
// KaTeX 处理
|
||||
if (tag === 'ms-katex' || cls.indexOf('katex') >= 0) {
|
||||
var tex = extractKatex(el);
|
||||
if (tex) {
|
||||
var inline = (cls.indexOf('inline') >= 0);
|
||||
out.push(inline ? (' $' + tex + '$ ') : ('\n$$\n' + tex + '\n$$\n'));
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
// 代码块处理
|
||||
if ((tag === 'pre' || tag === 'code') && !hasAncestorTag(el, 'ms-katex')) {
|
||||
var tcode = normNewline(el.textContent || '');
|
||||
if (tcode && tcode.replace(/\s+/g,'').length > 0) {
|
||||
out.push('\n```\n' + rstripNewlines(tcode) + '\n```\n');
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
// 递归子节点
|
||||
var kids = el.childNodes ? Array.prototype.slice.call(el.childNodes) : [];
|
||||
if (kids.length) {
|
||||
var isBlock = 'p div section li ul ol table tr h1 h2 h3 h4 h5 h6 br'.indexOf(tag) >= 0;
|
||||
if (isBlock) out.push('\n');
|
||||
for (var k = 0; k < kids.length; k++) walk(kids[k], out);
|
||||
if (isBlock) out.push('\n');
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
var out = [];
|
||||
walk(this, out);
|
||||
var s = normNewline(out.join(''));
|
||||
s = s.replace(/[ \t]+\n/g, '\n').replace(/\n{3,}/g, '\n\n');
|
||||
return s.trim();
|
||||
} catch (e) {
|
||||
return normNewline(this.textContent || '').trim();
|
||||
}
|
||||
}).call(this);
|
||||
"""
|
||||
|
||||
# 内容子节点选择器
|
||||
CONTENT_SELECTORS = [
|
||||
'css:.markdown',
|
||||
'css:.prose',
|
||||
'css:.response-content-markdown',
|
||||
'css:.ds-markdown',
|
||||
'css:.message-content',
|
||||
'css:[class*="markdown"]',
|
||||
'css:[class*="content"]',
|
||||
]
|
||||
|
||||
# ============ 核心方法 ============
|
||||
|
||||
def extract_text(self, element) -> str:
|
||||
"""
|
||||
智能提取文本
|
||||
|
||||
流程:
|
||||
1. 定位内容节点
|
||||
2. 检测复杂度
|
||||
3. 选择提取策略
|
||||
4. 失败时自动降级
|
||||
"""
|
||||
if not element:
|
||||
return ""
|
||||
|
||||
# 1. 定位内容子节点
|
||||
target = self.find_content_node(element)
|
||||
|
||||
# 2. 检测复杂度
|
||||
complexity = self._detect_complexity(target)
|
||||
|
||||
# 3. 根据复杂度选择策略
|
||||
if complexity.get('needsDeep', False):
|
||||
# 复杂内容:使用深度提取
|
||||
text = self._deep_extract(target)
|
||||
if text:
|
||||
return text
|
||||
|
||||
# 简单内容或深度提取失败:使用轻量提取
|
||||
return self._simple_extract(target)
|
||||
|
||||
def get_anchor(self, element) -> str:
|
||||
"""
|
||||
获取元素锚点
|
||||
|
||||
复用 deep_mode 的稳定锚点策略
|
||||
"""
|
||||
if not element:
|
||||
return ""
|
||||
|
||||
try:
|
||||
# 1. 优先使用稳定的 ID 类属性
|
||||
stable_attrs = ['data-message-id', 'data-turn-id', 'data-testid', 'id']
|
||||
for attr in stable_attrs:
|
||||
try:
|
||||
val = element.attr(attr)
|
||||
if val:
|
||||
return f"{attr}={val}"
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# 2. 回退:tag + class + DOM 位置
|
||||
tag = element.tag if hasattr(element, 'tag') else 'unknown'
|
||||
|
||||
cls = ""
|
||||
try:
|
||||
cls = element.attr('class') or ""
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
classes = cls.split()[:3]
|
||||
class_part = f"|cls={'.'.join(classes)}" if classes else ""
|
||||
|
||||
# 3. DOM 位置
|
||||
index_part = ""
|
||||
try:
|
||||
index = as_element(element).run_js("""
|
||||
const parent = this.parentElement;
|
||||
if (!parent) return -1;
|
||||
return Array.from(parent.children).indexOf(this);
|
||||
""")
|
||||
if index is not None and index >= 0:
|
||||
index_part = f"|idx={index}"
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return f"tag:{tag}{class_part}{index_part}"
|
||||
|
||||
except Exception:
|
||||
return ""
|
||||
|
||||
def find_content_node(self, element) -> Any:
|
||||
"""
|
||||
定位内容子节点
|
||||
|
||||
尝试找到实际包含内容的子元素,避免读取杂项
|
||||
"""
|
||||
if not element:
|
||||
return element
|
||||
|
||||
for selector in self.CONTENT_SELECTORS:
|
||||
try:
|
||||
child = as_element(element).find(selector, timeout=0.05)
|
||||
if child:
|
||||
# 验证有实际内容
|
||||
text = as_element(child).run_js("return (this.textContent || '').trim()")
|
||||
if text and len(str(text)) > 10:
|
||||
return child
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return element
|
||||
|
||||
# ============ 私有方法 ============
|
||||
|
||||
def _detect_complexity(self, element) -> dict:
|
||||
"""
|
||||
检测元素复杂度
|
||||
|
||||
Returns:
|
||||
{
|
||||
'hasCode': bool,
|
||||
'hasLatex': bool,
|
||||
'hasShadow': bool,
|
||||
'hasTable': bool,
|
||||
'isComplex': bool,
|
||||
'needsDeep': bool
|
||||
}
|
||||
"""
|
||||
try:
|
||||
result = as_element(element).run_js(self.DETECT_COMPLEXITY_JS)
|
||||
if isinstance(result, dict):
|
||||
return result
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# 检测失败时默认简单模式
|
||||
return {'isComplex': False, 'needsDeep': False}
|
||||
|
||||
def _simple_extract(self, element) -> str:
|
||||
"""
|
||||
简单提取(高性能)
|
||||
|
||||
直接读取 textContent,不做复杂处理
|
||||
"""
|
||||
try:
|
||||
# 优先使用 .text 属性
|
||||
if hasattr(element, 'text') and element.text:
|
||||
return self._normalize(element.text)
|
||||
|
||||
# 回退:JS 读取
|
||||
text = as_element(element).run_js("return this.textContent || this.innerText || ''")
|
||||
return self._normalize(str(text)) if text else ""
|
||||
|
||||
except Exception:
|
||||
return ""
|
||||
|
||||
def _deep_extract(self, element) -> str:
|
||||
"""
|
||||
深度提取(高准确度)
|
||||
|
||||
使用 JS 注入遍历 DOM 树
|
||||
"""
|
||||
try:
|
||||
text = as_element(element).run_js(self.DEEP_EXTRACT_JS)
|
||||
if text and str(text).strip():
|
||||
return self._normalize(str(text))
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return ""
|
||||
|
||||
def _normalize(self, text: str) -> str:
|
||||
"""标准化文本"""
|
||||
if not text:
|
||||
return ""
|
||||
|
||||
text = text.replace('\r\n', '\n').replace('\r', '\n')
|
||||
return text.strip()
|
||||
|
||||
|
||||
__all__ = ['HybridExtractor']
|
||||
Reference in new issue
Block a user