""" app/core/extractors/deep_mode.py - 深度内容提取器 (v2.2) v2.2 修复: - find_content_node 正确识别容器元素,不再误匹配段落 - 支持 DeepSeek ds-markdown 结构 """ from typing import Any, Optional, Dict, List from app.core.extractors.base import BaseExtractor from app.core.extractors.image_extractor import image_extractor from app.core.driver import as_element class DeepBrowserExtractor(BaseExtractor): """深度浏览器内容提取器""" @classmethod def get_id(cls) -> str: return "deep_mode_v1" @classmethod def get_name(cls) -> str: return "深度模式 (JS注入)" @classmethod def get_description(cls) -> str: return "通过 JavaScript 注入深度提取内容,支持 LaTeX、代码块、Shadow DOM" DEEP_EXTRACT_JS = r""" return (function () { function normNewline(s){ return (s||'').replace(/\r\n/g,'\n').replace(/\r/g,'\n'); } function isEl(n){ return n && n.nodeType === 1; } function rstripNewlines(s){ return normNewline(s).replace(/\n+$/g, ''); } function hasAncestorTag(el, tagNameLower){ var p = el; while (p) { if (p.nodeType === 1 && (p.tagName||'').toLowerCase() === tagNameLower) return true; p = p.parentElement; } return false; } function hasAncestorClass(el, classPattern){ var p = el; while (p) { if (p.nodeType === 1) { var cls = (p.className && typeof p.className === 'string') ? p.className : ''; if (cls.indexOf(classPattern) >= 0) return true; } p = p.parentElement; } return false; } function ignorableEl(el){ if (!isEl(el)) return false; var tag = (el.tagName||'').toLowerCase(); if (tag === 'button' || tag === 'svg') return true; if (tag === 'mat-expansion-panel-header') return true; var cls = (el.className && typeof el.className === 'string') ? el.className : ''; // DeepSeek: 代码块横幅区域 if (cls.indexOf('md-code-block-banner-wrap') >= 0) return true; if (cls.indexOf('code-info-button-text') >= 0) return true; // GPT: 代码块工具栏 if (cls.indexOf('sticky') >= 0 && el.querySelector && el.querySelector('button')) return true; if (cls.indexOf('bg-token-sidebar-surface-primary') >= 0 && el.querySelector && el.querySelector('button')) return true; // 通用 UI 元素 if (cls.indexOf('actions-container') >= 0) return true; if (cls.indexOf('projected-actions-wrapper') >= 0) return true; if (cls.indexOf('turn-footer') >= 0) return true; if (cls.indexOf('mat-expansion-indicator') >= 0) return true; if (cls.indexOf('material-symbols') >= 0) return true; if (cls.indexOf('material-icons') >= 0) return true; if (cls.indexOf('mat-icon') >= 0) return true; // Gemini: 代码块按钮区 if (cls.indexOf('buttons') >= 0 && el.closest && el.closest('code-block')) return true; var aria = (el.getAttribute && (el.getAttribute('aria-label')||'')) || ''; if (/download|copy|expand|collapse|edit|rerun|open options|good response|bad response|复制/i.test(aria)) return true; return false; } // ========== LaTeX 提取 ========== function extractKatexAnnotation(el){ try { var ann = el.querySelector && el.querySelector('annotation[encoding="application/x-tex"]'); if (ann && ann.textContent) return ann.textContent; } catch(e) {} return ''; } function extractDataMath(el){ try { var dm = el.getAttribute && el.getAttribute('data-math'); if (dm) return dm; } catch(e) {} return ''; } // ========== 代码块提取 ========== function extractDeepSeekCodeBlock(el){ var lang = ''; var code = ''; try { var langSpan = el.querySelector('.d813de27'); if (langSpan) lang = (langSpan.textContent || '').trim(); var pre = el.querySelector('pre'); if (pre) code = normNewline(pre.textContent || ''); } catch(e) {} return { lang: lang, code: code }; } function extractGeminiCodeBlock(el){ var lang = ''; var code = ''; try { var langEl = el.querySelector('.code-block-decoration span'); if (langEl) lang = (langEl.textContent || '').trim(); var codeEl = el.querySelector('code[data-test-id="code-content"]'); if (codeEl) code = normNewline(codeEl.textContent || ''); } catch(e) {} return { lang: lang, code: code }; } function extractGPTCodeBlock(el){ var lang = ''; var code = ''; try { var codeEl = el.querySelector('code[class*="language-"]'); if (codeEl) { var clsStr = codeEl.className || ''; var match = clsStr.match(/language-(\S+)/); if (match) lang = match[1]; code = normNewline(codeEl.textContent || ''); } else { codeEl = el.querySelector('code'); if (codeEl) code = normNewline(codeEl.textContent || ''); } if (!lang) { var headerDiv = el.previousElementSibling; if (headerDiv && headerDiv.textContent) { var t = (headerDiv.textContent || '').trim(); if (t && t.length < 30 && !/复制|copy/i.test(t)) lang = t; } } } catch(e) {} return { lang: lang, code: code }; } function extractMsCodeBlock(el){ var best = ''; try { var nodes = el.querySelectorAll ? el.querySelectorAll('pre code') : []; for (var i = 0; i < nodes.length; i++) { var c = nodes[i]; if (!c) continue; if (hasAncestorTag(c, 'ms-katex')) continue; var ccls = (c.className && typeof c.className === 'string') ? c.className : ''; if (ccls.indexOf('rendered') >= 0) continue; try { if (c.querySelector && c.querySelector('.katex')) continue; } catch(e2){} var t = normNewline(c.textContent || ''); if (t && t.replace(/\s+/g,'').length > best.replace(/\s+/g,'').length) best = t; } } catch(e) {} return best; } // ========== 主遍历逻辑 ========== function walk(node, out){ if (!node) return; if (isEl(node) && node.shadowRoot) { walk(node.shadowRoot, out); } if (node.nodeType === 11) { var kidsF = node.childNodes ? Array.prototype.slice.call(node.childNodes) : []; for (var iF = 0; iF < kidsF.length; iF++) walk(kidsF[iF], out); return; } if (node.nodeType === 3) { out.push(node.nodeValue || ''); return; } if (!isEl(node)) return; var el = node; if (ignorableEl(el)) return; var tag = (el.tagName||'').toLowerCase(); var cls = (el.className && typeof el.className === 'string') ? el.className : ''; // ===== DeepSeek 代码块 ===== if (cls.indexOf('md-code-block') >= 0 && tag === 'div') { var ds = extractDeepSeekCodeBlock(el); if (ds.code && ds.code.replace(/\s+/g,'').length > 0) { out.push('\n```' + (ds.lang || '') + '\n'); out.push(rstripNewlines(ds.code)); out.push('\n```\n'); } return; } // ===== Gemini 代码块 ===== if (tag === 'code-block') { var gm = extractGeminiCodeBlock(el); if (gm.code && gm.code.replace(/\s+/g,'').length > 0) { out.push('\n```' + (gm.lang ? gm.lang.toLowerCase() : '') + '\n'); out.push(rstripNewlines(gm.code)); out.push('\n```\n'); } return; } // ===== GPT 代码块 ===== if (tag === 'pre' && el.querySelector && el.querySelector('code[class*="language-"]')) { var gpt = extractGPTCodeBlock(el); if (gpt.code && gpt.code.replace(/\s+/g,'').length > 0) { out.push('\n```' + (gpt.lang || '') + '\n'); out.push(rstripNewlines(gpt.code)); out.push('\n```\n'); } return; } // ===== ms-katex (LaTeX) ===== if (tag === 'ms-katex') { var tex = extractKatexAnnotation(el); if (tex) { var inline = (cls.indexOf('inline') >= 0); out.push(inline ? (' $' + tex + '$ ') : ('\n$$\n' + tex + '\n$$\n')); } return; } // ===== Gemini LaTeX (span[data-math]) ===== var dataMath = extractDataMath(el); if (dataMath) { var isBlockMath = cls.indexOf('math-block') >= 0 || cls.indexOf('math-display') >= 0; out.push(isBlockMath ? ('\n$$\n' + dataMath + '\n$$\n') : (' $' + dataMath + '$ ')); return; } // ===== GPT/通用 .katex ===== if (cls.indexOf('katex') >= 0) { var tex2 = extractKatexAnnotation(el); if (tex2) out.push(' $' + tex2 + '$ '); return; } // ===== ms-code-block ===== if (tag === 'ms-code-block') { var msCode = extractMsCodeBlock(el); if (msCode && msCode.replace(/\s+/g,'').length > 0) { out.push('\n```\n'); out.push(rstripNewlines(msCode)); out.push('\n```\n'); } return; } // ===== 通用 pre/code 回退 ===== if (tag === 'pre' && !hasAncestorTag(el, 'ms-katex') && !hasAncestorClass(el, 'md-code-block')) { var tcode = normNewline(el.textContent || ''); if (tcode && tcode.replace(/\s+/g,'').length > 0) { out.push('\n```\n'); out.push(rstripNewlines(tcode)); out.push('\n```\n'); } return; } // ===== Markdown 标题 h1-h6 ===== if (/^h[1-6]$/.test(tag)) { var level = parseInt(tag.charAt(1), 10); var hashes = ''; for (var h = 0; h < level; h++) hashes += '#'; out.push('\n' + hashes + ' '); var hKids = el.childNodes ? Array.prototype.slice.call(el.childNodes) : []; for (var hk = 0; hk < hKids.length; hk++) walk(hKids[hk], out); out.push('\n'); return; } // ===== Markdown 加粗 strong/b ===== if (tag === 'strong' || tag === 'b') { out.push('**'); var bKids = el.childNodes ? Array.prototype.slice.call(el.childNodes) : []; for (var bk = 0; bk < bKids.length; bk++) walk(bKids[bk], out); out.push('**'); return; } // ===== Markdown 斜体 em/i ===== if (tag === 'em' || tag === 'i') { out.push('*'); var iKids = el.childNodes ? Array.prototype.slice.call(el.childNodes) : []; for (var ik = 0; ik < iKids.length; ik++) walk(iKids[ik], out); out.push('*'); return; } // ===== Markdown 水平线 hr ===== if (tag === 'hr') { out.push('\n---\n'); return; } // ===== 通用块级元素 ===== var kids = el.childNodes ? Array.prototype.slice.call(el.childNodes) : []; if (kids.length) { var isBlock = ('p div section article li ul ol table tr'.indexOf(tag) >= 0); if (isBlock) out.push('\n'); for (var k = 0; k < kids.length; k++) walk(kids[k], out); if (isBlock) out.push('\n'); } } // ========== 入口 ========== try { var out = []; walk(this, out); var s = normNewline(out.join('')); s = s.replace(/[ \t]+\n/g, '\n'); s = s.replace(/\n{3,}/g, '\n\n'); return s.trim(); } catch (e) { try { return normNewline(this.textContent || this.innerText || '').trim(); } catch (e2) { return ''; } } }).call(this); """ # 注意:移除了 'css:[class*="markdown"]' 这种容易误匹配的选择器 CONTENT_CHILD_SELECTORS = [ 'css:.markdown-body', # GitHub style 'css:.prose', # Tailwind 'css:.turn-content', # Gemini 'css:.response-content-markdown', 'css:.message-content', ] CONTENT_CHILD_SELECTOR_COMBINED = ( 'css:.markdown-body, .prose, .turn-content, ' '.response-content-markdown, .message-content' ) def find_content_node(self, element) -> Any: """ 定位内容子节点 v2.2 修复:正确识别容器元素,不再误匹配段落 """ if not element: return element # ===== 关键修复:检查元素本身是否已经是目标容器 ===== try: ele_class = element.attr('class') or "" ele_tag = element.tag.lower() if hasattr(element, 'tag') else "" # 已知的内容容器类名(直接返回,不再向下查找) container_patterns = [ 'ds-markdown', # DeepSeek 'markdown-body', # GitHub style 'prose', # Tailwind 'message-content', # 通用 'response-content', # 通用 ] for pattern in container_patterns: if pattern in ele_class: # 确保不是段落子元素 if 'paragraph' not in ele_class and 'html' not in ele_class: return element # 如果是 div/article 且类名包含 markdown/prose,直接返回 if ele_tag in ('div', 'article', 'section'): if ('markdown' in ele_class or 'prose' in ele_class) and 'paragraph' not in ele_class: return element except Exception: pass # ===== 向下查找(只在必要时执行)===== try: children = as_element(element).find_all(self.CONTENT_CHILD_SELECTOR_COMBINED, timeout=0.08) if children and not isinstance(children, list): children = [children] except Exception: children = [] for child in (children or [])[:8]: try: if not child: continue child_class = child.attr('class') or "" if 'paragraph' in child_class: continue child_text = as_element(child).run_js("return this.textContent || this.innerText || ''") if child_text and len(str(child_text).strip()) > 0: return child except Exception: pass return element def extract_text(self, element) -> str: """从元素中提取纯文本""" if not element: return "" target_ele = self.find_content_node(element) try: text = as_element(target_ele).run_js(self.DEEP_EXTRACT_JS) if text and str(text).strip(): return str(text).replace('\r\n', '\n').replace('\r', '\n') except Exception: pass try: t = target_ele.text if hasattr(target_ele, 'text') else "" return (t or "").replace('\r\n', '\n').replace('\r', '\n') except Exception: return "" def get_anchor(self, element) -> str: """获取元素锚点""" if not element: return "" try: stable_attrs = ['data-message-id', 'data-turn-id', 'data-testid', 'id'] for attr in stable_attrs: try: val = element.attr(attr) if val: return f"{attr}={val}" except Exception: pass tag = element.tag if hasattr(element, 'tag') else 'unknown' cls = "" try: cls = element.attr('class') or "" except Exception: pass classes = cls.split()[:3] class_part = f"|cls={'.'.join(classes)}" if classes else "" index_part = "" try: index = as_element(element).run_js(""" const parent = this.parentElement; if (!parent) return -1; const siblings = Array.from(parent.children); return siblings.indexOf(this); """) if index is not None and index >= 0: index_part = f"|idx={index}" except Exception: pass return f"tag:{tag}{class_part}{index_part}" except Exception: return "" def extract_images( self, element, config: Optional[Dict] = None, container_selector_fallback: Optional[str] = None ) -> List[Dict]: """ 从元素中提取图片(Phase A 新增) Args: element: 页面元素对象 config: 图片提取配置(ImageExtractionConfig) container_selector_fallback: 容器选择器回退值 Returns: 图片数据列表,每项符合 ImageData 格式 """ if not element: return [] return image_extractor.extract( element, config=config, container_selector_fallback=container_selector_fallback ) def extract_media( self, element, config: Optional[Dict] = None, container_selector_fallback: Optional[str] = None ) -> List[Dict]: """从元素中提取多模态资源。""" if not element: return [] from app.core.extractors.media_extractor import media_extractor return media_extractor.extract( element, config=config, container_selector_fallback=container_selector_fallback ) __all__ = ['DeepBrowserExtractor']