Files
universal-web-api/app/core/extractors/dom_mode.py
T
2026-11-02 22:54:14 +08:00

182 lines
4.9 KiB
Python

"""
app/core/extractors/dom_mode.py - DOM 直接提取器
轻量级提取器,直接读取元素的 textContent/innerText。
适用于结构简单、无复杂格式的页面。
"""
from typing import Any
from app.core.extractors.base import BaseExtractor
from app.core.driver import as_element
class DOMDirectExtractor(BaseExtractor):
"""
DOM 直接提取器
特点:
- 直接读取 textContent/innerText
- 无 JavaScript 注入(性能高)
- 不处理 LaTeX/代码块
- 适合简单页面
"""
# ============ 元数据 ============
@classmethod
def get_id(cls) -> str:
return "dom_direct"
@classmethod
def get_name(cls) -> str:
return "DOM 直接提取"
@classmethod
def get_description(cls) -> str:
return "直接读取 DOM textContent(轻量级,不支持复杂格式)"
CONTENT_CHILD_SELECTOR_COMBINED = 'css:.markdown, .prose, [class*="content"]'
# ============ 提取逻辑 ============
def extract_text(self, element) -> str:
"""
直接提取元素的文本内容
Args:
element: 页面元素对象
Returns:
提取的文本
"""
if not element:
return ""
try:
# 优先使用 .text 属性
if hasattr(element, 'text'):
text = element.text
if text:
return self._normalize_text(text)
# 回退:使用 textContent
text = as_element(element).run_js("return this.textContent || this.innerText || ''")
return self._normalize_text(str(text)) if text else ""
except Exception:
return ""
def get_anchor(self, element) -> str:
"""
获取元素锚点(复用 deep_mode 的逻辑)
Args:
element: 页面元素对象
Returns:
锚点字符串
"""
if not element:
return ""
try:
# 1. 优先使用稳定的 ID 类属性
stable_attrs = ['data-message-id', 'data-turn-id', 'data-testid', 'id']
for attr in stable_attrs:
try:
val = element.attr(attr)
if val:
return f"{attr}={val}"
except Exception:
pass
# 2. 回退:tag + class
tag = element.tag if hasattr(element, 'tag') else 'unknown'
cls = ""
try:
cls = element.attr('class') or ""
except Exception:
pass
classes = cls.split()[:3]
class_part = f"|cls={'.'.join(classes)}" if classes else ""
# 3. 加入 DOM 位置
index_part = ""
try:
index = as_element(element).run_js("""
const parent = this.parentElement;
if (!parent) return -1;
const siblings = Array.from(parent.children);
return siblings.indexOf(this);
""")
if index is not None and index >= 0:
index_part = f"|idx={index}"
except Exception:
pass
return f"tag:{tag}{class_part}{index_part}"
except Exception:
return ""
def find_content_node(self, element) -> Any:
"""
定位内容子节点
简化版:直接使用常见选择器
Args:
element: 父元素
Returns:
内容子节点或原元素
"""
if not element:
return element
try:
children = as_element(element).find_all(self.CONTENT_CHILD_SELECTOR_COMBINED, timeout=0.08)
if children and not isinstance(children, list):
children = [children]
except Exception:
children = []
for child in (children or [])[:8]:
try:
if not child:
continue
text = as_element(child).run_js("return this.textContent || ''")
if text and len(str(text).strip()) > 0:
return child
except Exception:
pass
return element
# ============ 辅助方法 ============
def _normalize_text(self, text: str) -> str:
"""
标准化文本
- 统一换行符
- 去除首尾空白
Args:
text: 原始文本
Returns:
标准化后的文本
"""
if not text:
return ""
text = text.replace('\r\n', '\n').replace('\r', '\n')
return text.strip()
__all__ = ['DOMDirectExtractor']