Initial commit
This commit is contained in:
commit
57fc3525eb
7311 files changed
+1291398
No files matched your search
@@ -0,0 +1,391 @@
|
||||
"""
|
||||
app/services/extractor_manager.py - 提取器配置管理器
|
||||
|
||||
职责:
|
||||
- 加载 config/extractors.json
|
||||
- 管理提取器配置
|
||||
- 根据站点配置返回提取器实例
|
||||
- 支持导入/导出提取器配置
|
||||
"""
|
||||
|
||||
import os
|
||||
import json
|
||||
import copy
|
||||
from app.core.config import atomic_write_json, get_logger
|
||||
from typing import Dict, Optional, List, Any
|
||||
|
||||
from app.core.extractors import ExtractorRegistry, BaseExtractor
|
||||
|
||||
|
||||
logger = get_logger("EXTRACT")
|
||||
|
||||
|
||||
class ExtractorConfigManager:
|
||||
"""
|
||||
提取器配置管理器
|
||||
|
||||
负责:
|
||||
1. 加载和保存 extractors.json
|
||||
2. 根据配置动态加载提取器
|
||||
3. 为站点提供提取器实例
|
||||
"""
|
||||
|
||||
# 配置文件路径
|
||||
_PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
CONFIG_FILE = os.path.join(_PROJECT_ROOT, "config", "extractors.json")
|
||||
|
||||
def __init__(self):
|
||||
self._config: Dict[str, Any] = {}
|
||||
self._extractors_config: Dict[str, Dict] = {}
|
||||
self._default_id: str = "deep_mode_v1"
|
||||
self._last_mtime: float = 0.0
|
||||
|
||||
# 加载配置
|
||||
self._load_config()
|
||||
|
||||
def _load_config(self, *, preserve_on_error: bool = False):
|
||||
"""加载 extractors.json 配置文件"""
|
||||
if not os.path.exists(self.CONFIG_FILE):
|
||||
logger.warning(f"提取器配置文件不存在: {self.CONFIG_FILE},使用默认配置")
|
||||
if not preserve_on_error:
|
||||
self._use_defaults()
|
||||
return
|
||||
|
||||
try:
|
||||
next_mtime = os.path.getmtime(self.CONFIG_FILE)
|
||||
|
||||
with open(self.CONFIG_FILE, "r", encoding="utf-8") as f:
|
||||
loaded_config = json.load(f)
|
||||
|
||||
if not isinstance(loaded_config, dict):
|
||||
raise ValueError("提取器配置必须是对象")
|
||||
extractors_config = loaded_config.get("extractors", {})
|
||||
if not isinstance(extractors_config, dict):
|
||||
raise ValueError("提取器配置 extractors 必须是对象")
|
||||
invalid_entries = [
|
||||
extractor_id
|
||||
for extractor_id, extractor_config in extractors_config.items()
|
||||
if not isinstance(extractor_config, dict)
|
||||
]
|
||||
if invalid_entries:
|
||||
raise ValueError(
|
||||
"提取器配置项必须是对象: "
|
||||
+ ", ".join(str(extractor_id) for extractor_id in invalid_entries)
|
||||
)
|
||||
|
||||
self._config = loaded_config
|
||||
self._extractors_config = extractors_config
|
||||
self._default_id = str(loaded_config.get("default") or "deep_mode_v1").strip() or "deep_mode_v1"
|
||||
self._config["extractors"] = self._extractors_config
|
||||
self._config["default"] = self._default_id
|
||||
self._config.setdefault("version", "1.0")
|
||||
self._last_mtime = next_mtime
|
||||
|
||||
# 动态加载配置中定义的提取器
|
||||
self._register_from_config()
|
||||
|
||||
logger.debug(
|
||||
f"已加载提取器配置: {len(self._extractors_config)} 个提取器,"
|
||||
f"默认: {self._default_id}"
|
||||
)
|
||||
|
||||
except json.JSONDecodeError as e:
|
||||
logger.error(f"提取器配置文件格式错误: {e}")
|
||||
if not preserve_on_error:
|
||||
self._use_defaults()
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"加载提取器配置失败: {e}")
|
||||
if not preserve_on_error:
|
||||
self._use_defaults()
|
||||
|
||||
def _use_defaults(self):
|
||||
"""使用默认配置"""
|
||||
self._extractors_config = {
|
||||
"deep_mode_v1": {
|
||||
"id": "deep_mode_v1",
|
||||
"name": "深度模式 (JS注入)",
|
||||
"description": "通过 JavaScript 注入深度提取内容",
|
||||
"class": "DeepBrowserExtractor",
|
||||
"module": "app.core.extractors.deep_mode",
|
||||
"enabled": True,
|
||||
"config": {}
|
||||
}
|
||||
}
|
||||
self._default_id = "deep_mode_v1"
|
||||
self._config = {
|
||||
"extractors": self._extractors_config,
|
||||
"default": self._default_id,
|
||||
"version": "1.0",
|
||||
}
|
||||
|
||||
def _register_from_config(self):
|
||||
"""根据配置文件动态注册提取器"""
|
||||
for extractor_id, config in self._extractors_config.items():
|
||||
if not isinstance(config, dict):
|
||||
logger.warning(f"跳过无效提取器配置 [{extractor_id}]: 配置项必须是对象")
|
||||
continue
|
||||
if not config.get("enabled", True):
|
||||
logger.debug(f"跳过禁用的提取器: {extractor_id}")
|
||||
continue
|
||||
|
||||
# 检查是否已注册
|
||||
if ExtractorRegistry.exists(extractor_id):
|
||||
logger.debug(f"提取器已注册: {extractor_id}")
|
||||
continue
|
||||
|
||||
# 尝试动态加载
|
||||
module_path = config.get("module")
|
||||
class_name = config.get("class")
|
||||
|
||||
if module_path and class_name:
|
||||
try:
|
||||
ExtractorRegistry.load_from_module(module_path, class_name, extractor_id)
|
||||
except Exception as e:
|
||||
logger.warning(f"动态加载提取器失败 [{extractor_id}]: {e}")
|
||||
|
||||
def refresh_if_changed(self):
|
||||
"""检查配置文件是否变化,如果变化则重载"""
|
||||
if not os.path.exists(self.CONFIG_FILE):
|
||||
return
|
||||
|
||||
try:
|
||||
current_mtime = os.path.getmtime(self.CONFIG_FILE)
|
||||
if current_mtime != self._last_mtime:
|
||||
logger.info("检测到提取器配置变化,重新加载")
|
||||
self._load_config(preserve_on_error=True)
|
||||
except Exception as e:
|
||||
logger.error(f"检查配置变化失败: {e}")
|
||||
|
||||
def reload_config(self) -> None:
|
||||
"""强制重新加载提取器配置,供配置备份恢复后热重载使用。"""
|
||||
self._load_config(preserve_on_error=False)
|
||||
|
||||
# ================= 提取器获取 =================
|
||||
|
||||
def get_extractor(self, extractor_id: Optional[str] = None) -> BaseExtractor:
|
||||
"""
|
||||
获取提取器实例
|
||||
|
||||
Args:
|
||||
extractor_id: 提取器 ID(None 则返回默认)
|
||||
|
||||
Returns:
|
||||
提取器实例
|
||||
"""
|
||||
eid = extractor_id or self._default_id
|
||||
|
||||
try:
|
||||
return ExtractorRegistry.get(eid)
|
||||
except ValueError:
|
||||
logger.warning(f"提取器不存在: {eid},使用默认: {self._default_id}")
|
||||
return ExtractorRegistry.get(self._default_id)
|
||||
|
||||
def get_extractor_for_site(self, site_config: Dict) -> BaseExtractor:
|
||||
"""
|
||||
根据站点配置获取提取器
|
||||
|
||||
Args:
|
||||
site_config: 站点配置字典(包含 extractor_id 字段)
|
||||
|
||||
Returns:
|
||||
提取器实例
|
||||
"""
|
||||
extractor_id = site_config.get("extractor_id")
|
||||
return self.get_extractor(extractor_id)
|
||||
|
||||
def get_default_id(self) -> str:
|
||||
"""获取默认提取器 ID"""
|
||||
return self._default_id
|
||||
|
||||
def set_default(self, extractor_id: str) -> bool:
|
||||
"""
|
||||
设置默认提取器
|
||||
|
||||
Args:
|
||||
extractor_id: 提取器 ID
|
||||
|
||||
Returns:
|
||||
是否成功
|
||||
"""
|
||||
if not ExtractorRegistry.exists(extractor_id):
|
||||
logger.error(f"无法设置默认提取器:{extractor_id} 不存在")
|
||||
return False
|
||||
|
||||
previous_default = self._default_id
|
||||
previous_config = copy.deepcopy(self._config)
|
||||
previous_extractors_config = copy.deepcopy(self._extractors_config)
|
||||
|
||||
try:
|
||||
self._default_id = extractor_id
|
||||
self._config["extractors"] = self._extractors_config
|
||||
self._config["default"] = extractor_id
|
||||
self._save_config()
|
||||
except Exception as exc:
|
||||
self._default_id = previous_default
|
||||
self._config = previous_config
|
||||
self._extractors_config = self._config.get("extractors", previous_extractors_config)
|
||||
logger.error(f"设置默认提取器失败,已回滚: {exc}")
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
# ================= 配置查询 =================
|
||||
|
||||
def list_extractors(self) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
列出所有可用的提取器
|
||||
|
||||
Returns:
|
||||
提取器信息列表
|
||||
"""
|
||||
result = []
|
||||
|
||||
# 从注册表获取(确保包含运行时注册的)
|
||||
for info in ExtractorRegistry.list_all():
|
||||
extractor_id = info["id"]
|
||||
|
||||
# 合并配置文件中的信息
|
||||
config = self._extractors_config.get(extractor_id, {})
|
||||
if not isinstance(config, dict):
|
||||
config = {}
|
||||
|
||||
result.append({
|
||||
"id": extractor_id,
|
||||
"name": config.get("name") or info.get("name", extractor_id),
|
||||
"description": config.get("description") or info.get("description", ""),
|
||||
"enabled": config.get("enabled", True),
|
||||
"is_default": extractor_id == self._default_id
|
||||
})
|
||||
|
||||
return result
|
||||
|
||||
def get_extractor_config(self, extractor_id: str) -> Optional[Dict]:
|
||||
"""获取提取器配置"""
|
||||
config = self._extractors_config.get(extractor_id)
|
||||
return copy.deepcopy(config) if isinstance(config, dict) else None
|
||||
|
||||
def has_extractor(self, extractor_id: str) -> bool:
|
||||
"""检查提取器是否已经注册。"""
|
||||
return ExtractorRegistry.exists(extractor_id)
|
||||
|
||||
# ================= 导入/导出 =================
|
||||
|
||||
def export_config(self) -> Dict[str, Any]:
|
||||
"""
|
||||
导出提取器配置
|
||||
|
||||
Returns:
|
||||
完整配置字典(可保存为 JSON)
|
||||
"""
|
||||
return {
|
||||
"extractors": copy.deepcopy(self._extractors_config),
|
||||
"default": self._default_id,
|
||||
"version": self._config.get("version", "1.0")
|
||||
}
|
||||
|
||||
def import_config(self, config: Dict[str, Any]) -> bool:
|
||||
"""
|
||||
导入提取器配置
|
||||
|
||||
Args:
|
||||
config: 配置字典
|
||||
|
||||
Returns:
|
||||
是否成功
|
||||
"""
|
||||
try:
|
||||
# 验证格式
|
||||
if not isinstance(config, dict):
|
||||
logger.error("导入失败:配置必须是对象")
|
||||
return False
|
||||
extractors = config.get("extractors")
|
||||
if "extractors" not in config:
|
||||
logger.error("导入失败:缺少 extractors 字段")
|
||||
return False
|
||||
if not isinstance(extractors, dict):
|
||||
logger.error("导入失败:extractors 必须是对象")
|
||||
return False
|
||||
invalid_entries = [
|
||||
extractor_id
|
||||
for extractor_id, extractor_config in extractors.items()
|
||||
if not isinstance(extractor_config, dict)
|
||||
]
|
||||
if invalid_entries:
|
||||
logger.error(
|
||||
"导入失败:提取器配置必须是对象: "
|
||||
+ ", ".join(str(extractor_id) for extractor_id in invalid_entries)
|
||||
)
|
||||
return False
|
||||
default_id = str(config.get("default") or "deep_mode_v1").strip() or "deep_mode_v1"
|
||||
default_entry = extractors.get(default_id)
|
||||
if isinstance(default_entry, dict) and not default_entry.get("enabled", True):
|
||||
logger.error(f"导入失败:默认提取器已禁用: {default_id}")
|
||||
return False
|
||||
if default_entry is None and not ExtractorRegistry.exists(default_id):
|
||||
logger.error(f"导入失败:默认提取器不存在: {default_id}")
|
||||
return False
|
||||
|
||||
previous_config = copy.deepcopy(self._config)
|
||||
previous_extractors_config = copy.deepcopy(self._extractors_config)
|
||||
previous_default_id = self._default_id
|
||||
previous_registry = copy.copy(getattr(ExtractorRegistry, "_extractors", {}))
|
||||
|
||||
next_config = copy.deepcopy(config)
|
||||
self._extractors_config = copy.deepcopy(next_config["extractors"])
|
||||
self._default_id = default_id
|
||||
next_config["extractors"] = self._extractors_config
|
||||
next_config["default"] = self._default_id
|
||||
self._config = next_config
|
||||
|
||||
# 保存到文件
|
||||
saved_config = False
|
||||
try:
|
||||
self._save_config()
|
||||
saved_config = True
|
||||
self._register_from_config()
|
||||
if not ExtractorRegistry.exists(self._default_id):
|
||||
raise ValueError(f"默认提取器无法注册: {self._default_id}")
|
||||
except Exception:
|
||||
self._config = previous_config
|
||||
self._extractors_config = self._config.get("extractors", previous_extractors_config)
|
||||
self._default_id = previous_default_id
|
||||
registry = getattr(ExtractorRegistry, "_extractors", None)
|
||||
if isinstance(registry, dict):
|
||||
registry.clear()
|
||||
registry.update(previous_registry)
|
||||
if saved_config:
|
||||
try:
|
||||
self._save_config()
|
||||
except Exception as rollback_error:
|
||||
logger.error(f"导入失败后恢复提取器配置文件失败: {rollback_error}")
|
||||
raise
|
||||
|
||||
logger.info(f"成功导入提取器配置: {len(self._extractors_config)} 个")
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"导入配置失败: {e}")
|
||||
return False
|
||||
|
||||
def _save_config(self):
|
||||
"""保存配置到文件"""
|
||||
try:
|
||||
atomic_write_json(self.CONFIG_FILE, self._config)
|
||||
try:
|
||||
self._last_mtime = os.path.getmtime(self.CONFIG_FILE)
|
||||
except Exception as mtime_error:
|
||||
logger.warning(f"提取器配置已保存但更新时间戳失败: {mtime_error}")
|
||||
logger.info("提取器配置已保存")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"保存配置失败: {e}")
|
||||
raise
|
||||
|
||||
|
||||
# ================= 单例 =================
|
||||
|
||||
extractor_manager = ExtractorConfigManager()
|
||||
|
||||
|
||||
__all__ = ['ExtractorConfigManager', 'extractor_manager']
|
||||
Reference in new issue
Block a user