Initial commit
This commit is contained in:
commit
57fc3525eb
7311 files changed
+1291398
No files matched your search
@@ -0,0 +1,534 @@
|
||||
"""
|
||||
app/core/extractors/deep_mode.py - 深度内容提取器 (v2.2)
|
||||
|
||||
v2.2 修复:
|
||||
- find_content_node 正确识别容器元素,不再误匹配段落
|
||||
- 支持 DeepSeek ds-markdown 结构
|
||||
"""
|
||||
|
||||
from typing import Any, Optional, Dict, List
|
||||
|
||||
from app.core.extractors.base import BaseExtractor
|
||||
from app.core.extractors.image_extractor import image_extractor
|
||||
from app.core.driver import as_element
|
||||
|
||||
|
||||
class DeepBrowserExtractor(BaseExtractor):
|
||||
"""深度浏览器内容提取器"""
|
||||
|
||||
@classmethod
|
||||
def get_id(cls) -> str:
|
||||
return "deep_mode_v1"
|
||||
|
||||
@classmethod
|
||||
def get_name(cls) -> str:
|
||||
return "深度模式 (JS注入)"
|
||||
|
||||
@classmethod
|
||||
def get_description(cls) -> str:
|
||||
return "通过 JavaScript 注入深度提取内容,支持 LaTeX、代码块、Shadow DOM"
|
||||
|
||||
DEEP_EXTRACT_JS = r"""
|
||||
return (function () {
|
||||
function normNewline(s){ return (s||'').replace(/\r\n/g,'\n').replace(/\r/g,'\n'); }
|
||||
function isEl(n){ return n && n.nodeType === 1; }
|
||||
function rstripNewlines(s){ return normNewline(s).replace(/\n+$/g, ''); }
|
||||
|
||||
function hasAncestorTag(el, tagNameLower){
|
||||
var p = el;
|
||||
while (p) {
|
||||
if (p.nodeType === 1 && (p.tagName||'').toLowerCase() === tagNameLower) return true;
|
||||
p = p.parentElement;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
function hasAncestorClass(el, classPattern){
|
||||
var p = el;
|
||||
while (p) {
|
||||
if (p.nodeType === 1) {
|
||||
var cls = (p.className && typeof p.className === 'string') ? p.className : '';
|
||||
if (cls.indexOf(classPattern) >= 0) return true;
|
||||
}
|
||||
p = p.parentElement;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
function ignorableEl(el){
|
||||
if (!isEl(el)) return false;
|
||||
var tag = (el.tagName||'').toLowerCase();
|
||||
|
||||
if (tag === 'button' || tag === 'svg') return true;
|
||||
if (tag === 'mat-expansion-panel-header') return true;
|
||||
|
||||
var cls = (el.className && typeof el.className === 'string') ? el.className : '';
|
||||
|
||||
// DeepSeek: 代码块横幅区域
|
||||
if (cls.indexOf('md-code-block-banner-wrap') >= 0) return true;
|
||||
if (cls.indexOf('code-info-button-text') >= 0) return true;
|
||||
|
||||
// GPT: 代码块工具栏
|
||||
if (cls.indexOf('sticky') >= 0 && el.querySelector && el.querySelector('button')) return true;
|
||||
if (cls.indexOf('bg-token-sidebar-surface-primary') >= 0 && el.querySelector && el.querySelector('button')) return true;
|
||||
|
||||
// 通用 UI 元素
|
||||
if (cls.indexOf('actions-container') >= 0) return true;
|
||||
if (cls.indexOf('projected-actions-wrapper') >= 0) return true;
|
||||
if (cls.indexOf('turn-footer') >= 0) return true;
|
||||
if (cls.indexOf('mat-expansion-indicator') >= 0) return true;
|
||||
if (cls.indexOf('material-symbols') >= 0) return true;
|
||||
if (cls.indexOf('material-icons') >= 0) return true;
|
||||
if (cls.indexOf('mat-icon') >= 0) return true;
|
||||
|
||||
// Gemini: 代码块按钮区
|
||||
if (cls.indexOf('buttons') >= 0 && el.closest && el.closest('code-block')) return true;
|
||||
|
||||
var aria = (el.getAttribute && (el.getAttribute('aria-label')||'')) || '';
|
||||
if (/download|copy|expand|collapse|edit|rerun|open options|good response|bad response|复制/i.test(aria)) return true;
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
// ========== LaTeX 提取 ==========
|
||||
|
||||
function extractKatexAnnotation(el){
|
||||
try {
|
||||
var ann = el.querySelector && el.querySelector('annotation[encoding="application/x-tex"]');
|
||||
if (ann && ann.textContent) return ann.textContent;
|
||||
} catch(e) {}
|
||||
return '';
|
||||
}
|
||||
|
||||
function extractDataMath(el){
|
||||
try {
|
||||
var dm = el.getAttribute && el.getAttribute('data-math');
|
||||
if (dm) return dm;
|
||||
} catch(e) {}
|
||||
return '';
|
||||
}
|
||||
|
||||
// ========== 代码块提取 ==========
|
||||
|
||||
function extractDeepSeekCodeBlock(el){
|
||||
var lang = '';
|
||||
var code = '';
|
||||
try {
|
||||
var langSpan = el.querySelector('.d813de27');
|
||||
if (langSpan) lang = (langSpan.textContent || '').trim();
|
||||
var pre = el.querySelector('pre');
|
||||
if (pre) code = normNewline(pre.textContent || '');
|
||||
} catch(e) {}
|
||||
return { lang: lang, code: code };
|
||||
}
|
||||
|
||||
function extractGeminiCodeBlock(el){
|
||||
var lang = '';
|
||||
var code = '';
|
||||
try {
|
||||
var langEl = el.querySelector('.code-block-decoration span');
|
||||
if (langEl) lang = (langEl.textContent || '').trim();
|
||||
var codeEl = el.querySelector('code[data-test-id="code-content"]');
|
||||
if (codeEl) code = normNewline(codeEl.textContent || '');
|
||||
} catch(e) {}
|
||||
return { lang: lang, code: code };
|
||||
}
|
||||
|
||||
function extractGPTCodeBlock(el){
|
||||
var lang = '';
|
||||
var code = '';
|
||||
try {
|
||||
var codeEl = el.querySelector('code[class*="language-"]');
|
||||
if (codeEl) {
|
||||
var clsStr = codeEl.className || '';
|
||||
var match = clsStr.match(/language-(\S+)/);
|
||||
if (match) lang = match[1];
|
||||
code = normNewline(codeEl.textContent || '');
|
||||
} else {
|
||||
codeEl = el.querySelector('code');
|
||||
if (codeEl) code = normNewline(codeEl.textContent || '');
|
||||
}
|
||||
if (!lang) {
|
||||
var headerDiv = el.previousElementSibling;
|
||||
if (headerDiv && headerDiv.textContent) {
|
||||
var t = (headerDiv.textContent || '').trim();
|
||||
if (t && t.length < 30 && !/复制|copy/i.test(t)) lang = t;
|
||||
}
|
||||
}
|
||||
} catch(e) {}
|
||||
return { lang: lang, code: code };
|
||||
}
|
||||
|
||||
function extractMsCodeBlock(el){
|
||||
var best = '';
|
||||
try {
|
||||
var nodes = el.querySelectorAll ? el.querySelectorAll('pre code') : [];
|
||||
for (var i = 0; i < nodes.length; i++) {
|
||||
var c = nodes[i];
|
||||
if (!c) continue;
|
||||
if (hasAncestorTag(c, 'ms-katex')) continue;
|
||||
var ccls = (c.className && typeof c.className === 'string') ? c.className : '';
|
||||
if (ccls.indexOf('rendered') >= 0) continue;
|
||||
try {
|
||||
if (c.querySelector && c.querySelector('.katex')) continue;
|
||||
} catch(e2){}
|
||||
var t = normNewline(c.textContent || '');
|
||||
if (t && t.replace(/\s+/g,'').length > best.replace(/\s+/g,'').length) best = t;
|
||||
}
|
||||
} catch(e) {}
|
||||
return best;
|
||||
}
|
||||
|
||||
// ========== 主遍历逻辑 ==========
|
||||
|
||||
function walk(node, out){
|
||||
if (!node) return;
|
||||
|
||||
if (isEl(node) && node.shadowRoot) {
|
||||
walk(node.shadowRoot, out);
|
||||
}
|
||||
|
||||
if (node.nodeType === 11) {
|
||||
var kidsF = node.childNodes ? Array.prototype.slice.call(node.childNodes) : [];
|
||||
for (var iF = 0; iF < kidsF.length; iF++) walk(kidsF[iF], out);
|
||||
return;
|
||||
}
|
||||
|
||||
if (node.nodeType === 3) {
|
||||
out.push(node.nodeValue || '');
|
||||
return;
|
||||
}
|
||||
|
||||
if (!isEl(node)) return;
|
||||
var el = node;
|
||||
|
||||
if (ignorableEl(el)) return;
|
||||
|
||||
var tag = (el.tagName||'').toLowerCase();
|
||||
var cls = (el.className && typeof el.className === 'string') ? el.className : '';
|
||||
|
||||
// ===== DeepSeek 代码块 =====
|
||||
if (cls.indexOf('md-code-block') >= 0 && tag === 'div') {
|
||||
var ds = extractDeepSeekCodeBlock(el);
|
||||
if (ds.code && ds.code.replace(/\s+/g,'').length > 0) {
|
||||
out.push('\n```' + (ds.lang || '') + '\n');
|
||||
out.push(rstripNewlines(ds.code));
|
||||
out.push('\n```\n');
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
// ===== Gemini 代码块 =====
|
||||
if (tag === 'code-block') {
|
||||
var gm = extractGeminiCodeBlock(el);
|
||||
if (gm.code && gm.code.replace(/\s+/g,'').length > 0) {
|
||||
out.push('\n```' + (gm.lang ? gm.lang.toLowerCase() : '') + '\n');
|
||||
out.push(rstripNewlines(gm.code));
|
||||
out.push('\n```\n');
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
// ===== GPT 代码块 =====
|
||||
if (tag === 'pre' && el.querySelector && el.querySelector('code[class*="language-"]')) {
|
||||
var gpt = extractGPTCodeBlock(el);
|
||||
if (gpt.code && gpt.code.replace(/\s+/g,'').length > 0) {
|
||||
out.push('\n```' + (gpt.lang || '') + '\n');
|
||||
out.push(rstripNewlines(gpt.code));
|
||||
out.push('\n```\n');
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
// ===== ms-katex (LaTeX) =====
|
||||
if (tag === 'ms-katex') {
|
||||
var tex = extractKatexAnnotation(el);
|
||||
if (tex) {
|
||||
var inline = (cls.indexOf('inline') >= 0);
|
||||
out.push(inline ? (' $' + tex + '$ ') : ('\n$$\n' + tex + '\n$$\n'));
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
// ===== Gemini LaTeX (span[data-math]) =====
|
||||
var dataMath = extractDataMath(el);
|
||||
if (dataMath) {
|
||||
var isBlockMath = cls.indexOf('math-block') >= 0 || cls.indexOf('math-display') >= 0;
|
||||
out.push(isBlockMath ? ('\n$$\n' + dataMath + '\n$$\n') : (' $' + dataMath + '$ '));
|
||||
return;
|
||||
}
|
||||
|
||||
// ===== GPT/通用 .katex =====
|
||||
if (cls.indexOf('katex') >= 0) {
|
||||
var tex2 = extractKatexAnnotation(el);
|
||||
if (tex2) out.push(' $' + tex2 + '$ ');
|
||||
return;
|
||||
}
|
||||
|
||||
// ===== ms-code-block =====
|
||||
if (tag === 'ms-code-block') {
|
||||
var msCode = extractMsCodeBlock(el);
|
||||
if (msCode && msCode.replace(/\s+/g,'').length > 0) {
|
||||
out.push('\n```\n');
|
||||
out.push(rstripNewlines(msCode));
|
||||
out.push('\n```\n');
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
// ===== 通用 pre/code 回退 =====
|
||||
if (tag === 'pre' && !hasAncestorTag(el, 'ms-katex') && !hasAncestorClass(el, 'md-code-block')) {
|
||||
var tcode = normNewline(el.textContent || '');
|
||||
if (tcode && tcode.replace(/\s+/g,'').length > 0) {
|
||||
out.push('\n```\n');
|
||||
out.push(rstripNewlines(tcode));
|
||||
out.push('\n```\n');
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
// ===== Markdown 标题 h1-h6 =====
|
||||
if (/^h[1-6]$/.test(tag)) {
|
||||
var level = parseInt(tag.charAt(1), 10);
|
||||
var hashes = '';
|
||||
for (var h = 0; h < level; h++) hashes += '#';
|
||||
out.push('\n' + hashes + ' ');
|
||||
var hKids = el.childNodes ? Array.prototype.slice.call(el.childNodes) : [];
|
||||
for (var hk = 0; hk < hKids.length; hk++) walk(hKids[hk], out);
|
||||
out.push('\n');
|
||||
return;
|
||||
}
|
||||
|
||||
// ===== Markdown 加粗 strong/b =====
|
||||
if (tag === 'strong' || tag === 'b') {
|
||||
out.push('**');
|
||||
var bKids = el.childNodes ? Array.prototype.slice.call(el.childNodes) : [];
|
||||
for (var bk = 0; bk < bKids.length; bk++) walk(bKids[bk], out);
|
||||
out.push('**');
|
||||
return;
|
||||
}
|
||||
|
||||
// ===== Markdown 斜体 em/i =====
|
||||
if (tag === 'em' || tag === 'i') {
|
||||
out.push('*');
|
||||
var iKids = el.childNodes ? Array.prototype.slice.call(el.childNodes) : [];
|
||||
for (var ik = 0; ik < iKids.length; ik++) walk(iKids[ik], out);
|
||||
out.push('*');
|
||||
return;
|
||||
}
|
||||
|
||||
// ===== Markdown 水平线 hr =====
|
||||
if (tag === 'hr') {
|
||||
out.push('\n---\n');
|
||||
return;
|
||||
}
|
||||
|
||||
// ===== 通用块级元素 =====
|
||||
var kids = el.childNodes ? Array.prototype.slice.call(el.childNodes) : [];
|
||||
if (kids.length) {
|
||||
var isBlock = ('p div section article li ul ol table tr'.indexOf(tag) >= 0);
|
||||
if (isBlock) out.push('\n');
|
||||
for (var k = 0; k < kids.length; k++) walk(kids[k], out);
|
||||
if (isBlock) out.push('\n');
|
||||
}
|
||||
}
|
||||
|
||||
// ========== 入口 ==========
|
||||
try {
|
||||
var out = [];
|
||||
walk(this, out);
|
||||
|
||||
var s = normNewline(out.join(''));
|
||||
s = s.replace(/[ \t]+\n/g, '\n');
|
||||
s = s.replace(/\n{3,}/g, '\n\n');
|
||||
return s.trim();
|
||||
} catch (e) {
|
||||
try {
|
||||
return normNewline(this.textContent || this.innerText || '').trim();
|
||||
} catch (e2) {
|
||||
return '';
|
||||
}
|
||||
}
|
||||
}).call(this);
|
||||
"""
|
||||
|
||||
# 注意:移除了 'css:[class*="markdown"]' 这种容易误匹配的选择器
|
||||
CONTENT_CHILD_SELECTORS = [
|
||||
'css:.markdown-body', # GitHub style
|
||||
'css:.prose', # Tailwind
|
||||
'css:.turn-content', # Gemini
|
||||
'css:.response-content-markdown',
|
||||
'css:.message-content',
|
||||
]
|
||||
CONTENT_CHILD_SELECTOR_COMBINED = (
|
||||
'css:.markdown-body, .prose, .turn-content, '
|
||||
'.response-content-markdown, .message-content'
|
||||
)
|
||||
|
||||
def find_content_node(self, element) -> Any:
|
||||
"""
|
||||
定位内容子节点
|
||||
|
||||
v2.2 修复:正确识别容器元素,不再误匹配段落
|
||||
"""
|
||||
if not element:
|
||||
return element
|
||||
|
||||
# ===== 关键修复:检查元素本身是否已经是目标容器 =====
|
||||
try:
|
||||
ele_class = element.attr('class') or ""
|
||||
ele_tag = element.tag.lower() if hasattr(element, 'tag') else ""
|
||||
|
||||
# 已知的内容容器类名(直接返回,不再向下查找)
|
||||
container_patterns = [
|
||||
'ds-markdown', # DeepSeek
|
||||
'markdown-body', # GitHub style
|
||||
'prose', # Tailwind
|
||||
'message-content', # 通用
|
||||
'response-content', # 通用
|
||||
]
|
||||
|
||||
for pattern in container_patterns:
|
||||
if pattern in ele_class:
|
||||
# 确保不是段落子元素
|
||||
if 'paragraph' not in ele_class and 'html' not in ele_class:
|
||||
return element
|
||||
|
||||
# 如果是 div/article 且类名包含 markdown/prose,直接返回
|
||||
if ele_tag in ('div', 'article', 'section'):
|
||||
if ('markdown' in ele_class or 'prose' in ele_class) and 'paragraph' not in ele_class:
|
||||
return element
|
||||
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# ===== 向下查找(只在必要时执行)=====
|
||||
try:
|
||||
children = as_element(element).find_all(self.CONTENT_CHILD_SELECTOR_COMBINED, timeout=0.08)
|
||||
if children and not isinstance(children, list):
|
||||
children = [children]
|
||||
except Exception:
|
||||
children = []
|
||||
|
||||
for child in (children or [])[:8]:
|
||||
try:
|
||||
if not child:
|
||||
continue
|
||||
child_class = child.attr('class') or ""
|
||||
if 'paragraph' in child_class:
|
||||
continue
|
||||
child_text = as_element(child).run_js("return this.textContent || this.innerText || ''")
|
||||
if child_text and len(str(child_text).strip()) > 0:
|
||||
return child
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return element
|
||||
|
||||
def extract_text(self, element) -> str:
|
||||
"""从元素中提取纯文本"""
|
||||
if not element:
|
||||
return ""
|
||||
|
||||
target_ele = self.find_content_node(element)
|
||||
|
||||
try:
|
||||
text = as_element(target_ele).run_js(self.DEEP_EXTRACT_JS)
|
||||
if text and str(text).strip():
|
||||
return str(text).replace('\r\n', '\n').replace('\r', '\n')
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
try:
|
||||
t = target_ele.text if hasattr(target_ele, 'text') else ""
|
||||
return (t or "").replace('\r\n', '\n').replace('\r', '\n')
|
||||
except Exception:
|
||||
return ""
|
||||
|
||||
def get_anchor(self, element) -> str:
|
||||
"""获取元素锚点"""
|
||||
if not element:
|
||||
return ""
|
||||
try:
|
||||
stable_attrs = ['data-message-id', 'data-turn-id', 'data-testid', 'id']
|
||||
for attr in stable_attrs:
|
||||
try:
|
||||
val = element.attr(attr)
|
||||
if val:
|
||||
return f"{attr}={val}"
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
tag = element.tag if hasattr(element, 'tag') else 'unknown'
|
||||
|
||||
cls = ""
|
||||
try:
|
||||
cls = element.attr('class') or ""
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
classes = cls.split()[:3]
|
||||
class_part = f"|cls={'.'.join(classes)}" if classes else ""
|
||||
|
||||
index_part = ""
|
||||
try:
|
||||
index = as_element(element).run_js("""
|
||||
const parent = this.parentElement;
|
||||
if (!parent) return -1;
|
||||
const siblings = Array.from(parent.children);
|
||||
return siblings.indexOf(this);
|
||||
""")
|
||||
if index is not None and index >= 0:
|
||||
index_part = f"|idx={index}"
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return f"tag:{tag}{class_part}{index_part}"
|
||||
except Exception:
|
||||
return ""
|
||||
def extract_images(
|
||||
self,
|
||||
element,
|
||||
config: Optional[Dict] = None,
|
||||
container_selector_fallback: Optional[str] = None
|
||||
) -> List[Dict]:
|
||||
"""
|
||||
从元素中提取图片(Phase A 新增)
|
||||
|
||||
Args:
|
||||
element: 页面元素对象
|
||||
config: 图片提取配置(ImageExtractionConfig)
|
||||
container_selector_fallback: 容器选择器回退值
|
||||
|
||||
Returns:
|
||||
图片数据列表,每项符合 ImageData 格式
|
||||
"""
|
||||
if not element:
|
||||
return []
|
||||
|
||||
|
||||
return image_extractor.extract(
|
||||
element,
|
||||
config=config,
|
||||
container_selector_fallback=container_selector_fallback
|
||||
)
|
||||
|
||||
def extract_media(
|
||||
self,
|
||||
element,
|
||||
config: Optional[Dict] = None,
|
||||
container_selector_fallback: Optional[str] = None
|
||||
) -> List[Dict]:
|
||||
"""从元素中提取多模态资源。"""
|
||||
if not element:
|
||||
return []
|
||||
|
||||
from app.core.extractors.media_extractor import media_extractor
|
||||
|
||||
return media_extractor.extract(
|
||||
element,
|
||||
config=config,
|
||||
container_selector_fallback=container_selector_fallback
|
||||
)
|
||||
|
||||
__all__ = ['DeepBrowserExtractor']
|
||||
Reference in new issue
Block a user