392 lines
14 KiB
Python
392 lines
14 KiB
Python
"""
|
||
app/services/extractor_manager.py - 提取器配置管理器
|
||
|
||
职责:
|
||
- 加载 config/extractors.json
|
||
- 管理提取器配置
|
||
- 根据站点配置返回提取器实例
|
||
- 支持导入/导出提取器配置
|
||
"""
|
||
|
||
import os
|
||
import json
|
||
import copy
|
||
from app.core.config import atomic_write_json, get_logger
|
||
from typing import Dict, Optional, List, Any
|
||
|
||
from app.core.extractors import ExtractorRegistry, BaseExtractor
|
||
|
||
|
||
logger = get_logger("EXTRACT")
|
||
|
||
|
||
class ExtractorConfigManager:
|
||
"""
|
||
提取器配置管理器
|
||
|
||
负责:
|
||
1. 加载和保存 extractors.json
|
||
2. 根据配置动态加载提取器
|
||
3. 为站点提供提取器实例
|
||
"""
|
||
|
||
# 配置文件路径
|
||
_PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||
CONFIG_FILE = os.path.join(_PROJECT_ROOT, "config", "extractors.json")
|
||
|
||
def __init__(self):
|
||
self._config: Dict[str, Any] = {}
|
||
self._extractors_config: Dict[str, Dict] = {}
|
||
self._default_id: str = "deep_mode_v1"
|
||
self._last_mtime: float = 0.0
|
||
|
||
# 加载配置
|
||
self._load_config()
|
||
|
||
def _load_config(self, *, preserve_on_error: bool = False):
|
||
"""加载 extractors.json 配置文件"""
|
||
if not os.path.exists(self.CONFIG_FILE):
|
||
logger.warning(f"提取器配置文件不存在: {self.CONFIG_FILE},使用默认配置")
|
||
if not preserve_on_error:
|
||
self._use_defaults()
|
||
return
|
||
|
||
try:
|
||
next_mtime = os.path.getmtime(self.CONFIG_FILE)
|
||
|
||
with open(self.CONFIG_FILE, "r", encoding="utf-8") as f:
|
||
loaded_config = json.load(f)
|
||
|
||
if not isinstance(loaded_config, dict):
|
||
raise ValueError("提取器配置必须是对象")
|
||
extractors_config = loaded_config.get("extractors", {})
|
||
if not isinstance(extractors_config, dict):
|
||
raise ValueError("提取器配置 extractors 必须是对象")
|
||
invalid_entries = [
|
||
extractor_id
|
||
for extractor_id, extractor_config in extractors_config.items()
|
||
if not isinstance(extractor_config, dict)
|
||
]
|
||
if invalid_entries:
|
||
raise ValueError(
|
||
"提取器配置项必须是对象: "
|
||
+ ", ".join(str(extractor_id) for extractor_id in invalid_entries)
|
||
)
|
||
|
||
self._config = loaded_config
|
||
self._extractors_config = extractors_config
|
||
self._default_id = str(loaded_config.get("default") or "deep_mode_v1").strip() or "deep_mode_v1"
|
||
self._config["extractors"] = self._extractors_config
|
||
self._config["default"] = self._default_id
|
||
self._config.setdefault("version", "1.0")
|
||
self._last_mtime = next_mtime
|
||
|
||
# 动态加载配置中定义的提取器
|
||
self._register_from_config()
|
||
|
||
logger.debug(
|
||
f"已加载提取器配置: {len(self._extractors_config)} 个提取器,"
|
||
f"默认: {self._default_id}"
|
||
)
|
||
|
||
except json.JSONDecodeError as e:
|
||
logger.error(f"提取器配置文件格式错误: {e}")
|
||
if not preserve_on_error:
|
||
self._use_defaults()
|
||
|
||
except Exception as e:
|
||
logger.error(f"加载提取器配置失败: {e}")
|
||
if not preserve_on_error:
|
||
self._use_defaults()
|
||
|
||
def _use_defaults(self):
|
||
"""使用默认配置"""
|
||
self._extractors_config = {
|
||
"deep_mode_v1": {
|
||
"id": "deep_mode_v1",
|
||
"name": "深度模式 (JS注入)",
|
||
"description": "通过 JavaScript 注入深度提取内容",
|
||
"class": "DeepBrowserExtractor",
|
||
"module": "app.core.extractors.deep_mode",
|
||
"enabled": True,
|
||
"config": {}
|
||
}
|
||
}
|
||
self._default_id = "deep_mode_v1"
|
||
self._config = {
|
||
"extractors": self._extractors_config,
|
||
"default": self._default_id,
|
||
"version": "1.0",
|
||
}
|
||
|
||
def _register_from_config(self):
|
||
"""根据配置文件动态注册提取器"""
|
||
for extractor_id, config in self._extractors_config.items():
|
||
if not isinstance(config, dict):
|
||
logger.warning(f"跳过无效提取器配置 [{extractor_id}]: 配置项必须是对象")
|
||
continue
|
||
if not config.get("enabled", True):
|
||
logger.debug(f"跳过禁用的提取器: {extractor_id}")
|
||
continue
|
||
|
||
# 检查是否已注册
|
||
if ExtractorRegistry.exists(extractor_id):
|
||
logger.debug(f"提取器已注册: {extractor_id}")
|
||
continue
|
||
|
||
# 尝试动态加载
|
||
module_path = config.get("module")
|
||
class_name = config.get("class")
|
||
|
||
if module_path and class_name:
|
||
try:
|
||
ExtractorRegistry.load_from_module(module_path, class_name, extractor_id)
|
||
except Exception as e:
|
||
logger.warning(f"动态加载提取器失败 [{extractor_id}]: {e}")
|
||
|
||
def refresh_if_changed(self):
|
||
"""检查配置文件是否变化,如果变化则重载"""
|
||
if not os.path.exists(self.CONFIG_FILE):
|
||
return
|
||
|
||
try:
|
||
current_mtime = os.path.getmtime(self.CONFIG_FILE)
|
||
if current_mtime != self._last_mtime:
|
||
logger.info("检测到提取器配置变化,重新加载")
|
||
self._load_config(preserve_on_error=True)
|
||
except Exception as e:
|
||
logger.error(f"检查配置变化失败: {e}")
|
||
|
||
def reload_config(self) -> None:
|
||
"""强制重新加载提取器配置,供配置备份恢复后热重载使用。"""
|
||
self._load_config(preserve_on_error=False)
|
||
|
||
# ================= 提取器获取 =================
|
||
|
||
def get_extractor(self, extractor_id: Optional[str] = None) -> BaseExtractor:
|
||
"""
|
||
获取提取器实例
|
||
|
||
Args:
|
||
extractor_id: 提取器 ID(None 则返回默认)
|
||
|
||
Returns:
|
||
提取器实例
|
||
"""
|
||
eid = extractor_id or self._default_id
|
||
|
||
try:
|
||
return ExtractorRegistry.get(eid)
|
||
except ValueError:
|
||
logger.warning(f"提取器不存在: {eid},使用默认: {self._default_id}")
|
||
return ExtractorRegistry.get(self._default_id)
|
||
|
||
def get_extractor_for_site(self, site_config: Dict) -> BaseExtractor:
|
||
"""
|
||
根据站点配置获取提取器
|
||
|
||
Args:
|
||
site_config: 站点配置字典(包含 extractor_id 字段)
|
||
|
||
Returns:
|
||
提取器实例
|
||
"""
|
||
extractor_id = site_config.get("extractor_id")
|
||
return self.get_extractor(extractor_id)
|
||
|
||
def get_default_id(self) -> str:
|
||
"""获取默认提取器 ID"""
|
||
return self._default_id
|
||
|
||
def set_default(self, extractor_id: str) -> bool:
|
||
"""
|
||
设置默认提取器
|
||
|
||
Args:
|
||
extractor_id: 提取器 ID
|
||
|
||
Returns:
|
||
是否成功
|
||
"""
|
||
if not ExtractorRegistry.exists(extractor_id):
|
||
logger.error(f"无法设置默认提取器:{extractor_id} 不存在")
|
||
return False
|
||
|
||
previous_default = self._default_id
|
||
previous_config = copy.deepcopy(self._config)
|
||
previous_extractors_config = copy.deepcopy(self._extractors_config)
|
||
|
||
try:
|
||
self._default_id = extractor_id
|
||
self._config["extractors"] = self._extractors_config
|
||
self._config["default"] = extractor_id
|
||
self._save_config()
|
||
except Exception as exc:
|
||
self._default_id = previous_default
|
||
self._config = previous_config
|
||
self._extractors_config = self._config.get("extractors", previous_extractors_config)
|
||
logger.error(f"设置默认提取器失败,已回滚: {exc}")
|
||
return False
|
||
|
||
return True
|
||
|
||
# ================= 配置查询 =================
|
||
|
||
def list_extractors(self) -> List[Dict[str, Any]]:
|
||
"""
|
||
列出所有可用的提取器
|
||
|
||
Returns:
|
||
提取器信息列表
|
||
"""
|
||
result = []
|
||
|
||
# 从注册表获取(确保包含运行时注册的)
|
||
for info in ExtractorRegistry.list_all():
|
||
extractor_id = info["id"]
|
||
|
||
# 合并配置文件中的信息
|
||
config = self._extractors_config.get(extractor_id, {})
|
||
if not isinstance(config, dict):
|
||
config = {}
|
||
|
||
result.append({
|
||
"id": extractor_id,
|
||
"name": config.get("name") or info.get("name", extractor_id),
|
||
"description": config.get("description") or info.get("description", ""),
|
||
"enabled": config.get("enabled", True),
|
||
"is_default": extractor_id == self._default_id
|
||
})
|
||
|
||
return result
|
||
|
||
def get_extractor_config(self, extractor_id: str) -> Optional[Dict]:
|
||
"""获取提取器配置"""
|
||
config = self._extractors_config.get(extractor_id)
|
||
return copy.deepcopy(config) if isinstance(config, dict) else None
|
||
|
||
def has_extractor(self, extractor_id: str) -> bool:
|
||
"""检查提取器是否已经注册。"""
|
||
return ExtractorRegistry.exists(extractor_id)
|
||
|
||
# ================= 导入/导出 =================
|
||
|
||
def export_config(self) -> Dict[str, Any]:
|
||
"""
|
||
导出提取器配置
|
||
|
||
Returns:
|
||
完整配置字典(可保存为 JSON)
|
||
"""
|
||
return {
|
||
"extractors": copy.deepcopy(self._extractors_config),
|
||
"default": self._default_id,
|
||
"version": self._config.get("version", "1.0")
|
||
}
|
||
|
||
def import_config(self, config: Dict[str, Any]) -> bool:
|
||
"""
|
||
导入提取器配置
|
||
|
||
Args:
|
||
config: 配置字典
|
||
|
||
Returns:
|
||
是否成功
|
||
"""
|
||
try:
|
||
# 验证格式
|
||
if not isinstance(config, dict):
|
||
logger.error("导入失败:配置必须是对象")
|
||
return False
|
||
extractors = config.get("extractors")
|
||
if "extractors" not in config:
|
||
logger.error("导入失败:缺少 extractors 字段")
|
||
return False
|
||
if not isinstance(extractors, dict):
|
||
logger.error("导入失败:extractors 必须是对象")
|
||
return False
|
||
invalid_entries = [
|
||
extractor_id
|
||
for extractor_id, extractor_config in extractors.items()
|
||
if not isinstance(extractor_config, dict)
|
||
]
|
||
if invalid_entries:
|
||
logger.error(
|
||
"导入失败:提取器配置必须是对象: "
|
||
+ ", ".join(str(extractor_id) for extractor_id in invalid_entries)
|
||
)
|
||
return False
|
||
default_id = str(config.get("default") or "deep_mode_v1").strip() or "deep_mode_v1"
|
||
default_entry = extractors.get(default_id)
|
||
if isinstance(default_entry, dict) and not default_entry.get("enabled", True):
|
||
logger.error(f"导入失败:默认提取器已禁用: {default_id}")
|
||
return False
|
||
if default_entry is None and not ExtractorRegistry.exists(default_id):
|
||
logger.error(f"导入失败:默认提取器不存在: {default_id}")
|
||
return False
|
||
|
||
previous_config = copy.deepcopy(self._config)
|
||
previous_extractors_config = copy.deepcopy(self._extractors_config)
|
||
previous_default_id = self._default_id
|
||
previous_registry = copy.copy(getattr(ExtractorRegistry, "_extractors", {}))
|
||
|
||
next_config = copy.deepcopy(config)
|
||
self._extractors_config = copy.deepcopy(next_config["extractors"])
|
||
self._default_id = default_id
|
||
next_config["extractors"] = self._extractors_config
|
||
next_config["default"] = self._default_id
|
||
self._config = next_config
|
||
|
||
# 保存到文件
|
||
saved_config = False
|
||
try:
|
||
self._save_config()
|
||
saved_config = True
|
||
self._register_from_config()
|
||
if not ExtractorRegistry.exists(self._default_id):
|
||
raise ValueError(f"默认提取器无法注册: {self._default_id}")
|
||
except Exception:
|
||
self._config = previous_config
|
||
self._extractors_config = self._config.get("extractors", previous_extractors_config)
|
||
self._default_id = previous_default_id
|
||
registry = getattr(ExtractorRegistry, "_extractors", None)
|
||
if isinstance(registry, dict):
|
||
registry.clear()
|
||
registry.update(previous_registry)
|
||
if saved_config:
|
||
try:
|
||
self._save_config()
|
||
except Exception as rollback_error:
|
||
logger.error(f"导入失败后恢复提取器配置文件失败: {rollback_error}")
|
||
raise
|
||
|
||
logger.info(f"成功导入提取器配置: {len(self._extractors_config)} 个")
|
||
return True
|
||
|
||
except Exception as e:
|
||
logger.error(f"导入配置失败: {e}")
|
||
return False
|
||
|
||
def _save_config(self):
|
||
"""保存配置到文件"""
|
||
try:
|
||
atomic_write_json(self.CONFIG_FILE, self._config)
|
||
try:
|
||
self._last_mtime = os.path.getmtime(self.CONFIG_FILE)
|
||
except Exception as mtime_error:
|
||
logger.warning(f"提取器配置已保存但更新时间戳失败: {mtime_error}")
|
||
logger.info("提取器配置已保存")
|
||
|
||
except Exception as e:
|
||
logger.error(f"保存配置失败: {e}")
|
||
raise
|
||
|
||
|
||
# ================= 单例 =================
|
||
|
||
extractor_manager = ExtractorConfigManager()
|
||
|
||
|
||
__all__ = ['ExtractorConfigManager', 'extractor_manager']
|