"""R1-2:站点配置目录存储(``config/sites/<站点>.json``)。 旧版把全部站点放在一个 271KB 的 ``config/sites.json`` 里;现在每个站点(以及 ``_global``)一个文件, 外层是 R1-1 定义的信封(见 ``site_schema.py``)。本模块负责: - 读取目录并拼回与旧 ``sites.json`` **完全相同**的字典(``{"_global": {...}, "<域名>": {...}}``), 所以 ConfigEngine 以外的代码无需改动; - 只重写内容有变化的站点文件(两阶段:先写全部临时文件,再逐个 ``os.replace``),保留信封元数据; 被删除的站点移入 ``.trash/``,不直接删除; - 自动迁移旧的 ``config/sites.json``:目录为空时直接拆分;目录里已有发布版默认文件时(通过更新器升级的场景), 按更新器 ``merge_site_records`` 的语义合并(同一站点优先保留本地值、补入发布版新增字段); 迁移后旧文件改名为 ``sites.json.migrated-<时间>.bak``; - 读不懂的文件(JSON 损坏或信封不完整)只告警跳过,并且**永远不会被保存逻辑覆盖或移入回收站**; 热重载时这类站点沿用上一次成功加载的配置。 """ from __future__ import annotations import json import os import time from pathlib import Path from typing import Any, Dict, List, Optional, Tuple from app.core.config import get_logger from app.services.config.site_schema import SITE_FILE_SCHEMA_VERSION, site_file_errors logger = get_logger("SITE_STORE") INDEX_FILE_NAME = "index.json" TRASH_DIR_NAME = ".trash" DEFAULT_MIN_APP_VERSION = "3.0.0" _ENVELOPE_ORDER = ("schema_version", "site", "adapter_version", "min_app_version", "last_verified") _WINDOWS_RESERVED = {"con", "prn", "aux", "nul", *(f"com{i}" for i in range(1, 10)), *(f"lpt{i}" for i in range(1, 10))} def canonical_config_text(config: Any) -> str: """与格式、键顺序无关的规范化文本,用于变更检测与摘要(R1-3 的 config_sha256 也基于它)。""" return json.dumps(config, ensure_ascii=False, sort_keys=True, separators=(",", ":")) def site_filename(site: str) -> str: """站点键 -> 文件名。保留 ``[A-Za-z0-9._-]``,其余字节百分号编码;避开 Windows 保留名与 index.json。""" encoded = "".join( ch if (ch.isascii() and (ch.isalnum() or ch in "._-")) else "".join(f"%{b:02X}" for b in ch.encode("utf-8")) for ch in str(site) ) if not encoded or encoded.startswith(".") or encoded.split(".")[0].lower() in _WINDOWS_RESERVED | {"index"}: encoded = "%5F" + encoded return f"{encoded}.json" def build_envelope(site: str, config: Dict[str, Any], meta: Optional[Dict[str, Any]] = None) -> Dict[str, Any]: meta = dict(meta or {}) payload: Dict[str, Any] = {"schema_version": SITE_FILE_SCHEMA_VERSION, "site": site} for key in _ENVELOPE_ORDER[2:]: if meta.get(key) not in (None, ""): payload[key] = meta[key] for key, value in meta.items(): # 其余元数据(例如 R1-3 的 origin)原样保留 if key not in payload and key != "config": payload[key] = value payload["config"] = config return payload def _dump(payload: Dict[str, Any]) -> str: return json.dumps(payload, indent=2, ensure_ascii=False) + "\n" def _write_text_atomic(path: Path, text: str) -> None: tmp = path.with_name(path.name + ".tmp") with open(tmp, "w", encoding="utf-8", newline="\n") as handle: handle.write(text) handle.flush() os.fsync(handle.fileno()) os.replace(tmp, path) def _timestamp() -> str: return time.strftime("%Y%m%d-%H%M%S") class SiteStore: """``config/sites/`` 目录的读写与旧版迁移。线程安全由调用方(ConfigEngine 的 ``_io_lock``)保证。""" def __init__(self, sites_dir: os.PathLike | str, legacy_file: os.PathLike | str): self.sites_dir = Path(sites_dir) self.legacy_file = Path(legacy_file) self._files: Dict[str, Path] = {} self._meta: Dict[str, Dict[str, Any]] = {} self._on_disk: Dict[str, str] = {} self._broken: Dict[Path, str] = {} self._broken_sites: Dict[Path, str] = {} # 损坏文件 -> 上次成功加载时对应的站点 # ------------------------------------------------------------------ 查询 def site_files(self) -> List[Path]: if not self.sites_dir.is_dir(): return [] return sorted( path for path in self.sites_dir.glob("*.json") if path.is_file() and path.name != INDEX_FILE_NAME and not path.name.startswith(".") ) def exists(self) -> bool: return bool(self.site_files()) or self.legacy_file.is_file() def signature(self) -> Tuple: """目录内容签名(文件名、mtime、大小),用于热重载的变更检测。""" entries = [] for path in self.site_files(): try: stat = path.stat() except OSError: continue entries.append((path.name, stat.st_mtime_ns, stat.st_size)) if self.legacy_file.is_file(): stat = self.legacy_file.stat() entries.append(("", stat.st_mtime_ns, stat.st_size)) return tuple(entries) def metadata(self, site: str) -> Dict[str, Any]: return dict(self._meta.get(site) or {}) def broken_files(self) -> Dict[str, str]: return {str(path): reason for path, reason in self._broken.items()} def file_for(self, site: str) -> Path: return self._files.get(site) or (self.sites_dir / site_filename(site)) # ------------------------------------------------------------------ 读取 def read_envelope(self, path: Path) -> Dict[str, Any]: payload = json.loads(path.read_text(encoding="utf-8-sig")) if not isinstance(payload, dict) or not isinstance(payload.get("site"), str) or not isinstance(payload.get("config"), dict): raise ValueError("不是有效的站点文件(缺少 site/config 字段)") return payload def load(self, previous: Optional[Dict[str, Any]] = None) -> Optional[Dict[str, Any]]: """读取全部站点,返回旧 sites.json 结构的字典;目录与旧文件都不存在时返回 None。 ``previous`` 是上一次加载的结果:热重载时,暂时读不懂的文件沿用其中对应站点的配置。 """ if self.legacy_file.is_file(): self.migrate_legacy() files = self.site_files() if not files: return None data: Dict[str, Any] = {} files_map: Dict[str, Path] = {} meta: Dict[str, Dict[str, Any]] = {} on_disk: Dict[str, str] = {} broken: Dict[Path, str] = {} previous_sites_by_path = {path: site for site, path in self._files.items()} previous_sites_by_path.update(self._broken_sites) broken_sites: Dict[Path, str] = {} for path in files: try: payload = self.read_envelope(path) except (OSError, ValueError) as exc: # JSON 损坏、编码错误(均为 ValueError)、信封不完整、读文件失败 broken[path] = str(exc) logger.error(f"站点配置文件无法读取,已跳过且不会被覆盖: {path.name}: {exc}") old_site = previous_sites_by_path.get(path) if old_site: broken_sites[path] = old_site if previous and old_site in previous: data[old_site] = previous[old_site] logger.warning(f"站点 {old_site} 暂时沿用上一次成功加载的配置") continue site = payload["site"] if site in files_map: broken[path] = f"与 {files_map[site].name} 重复声明站点 {site}" logger.error(f"站点配置文件重复声明 {site},已忽略: {path.name}") continue for error in site_file_errors(payload)[:5]: logger.warning(f"站点配置 Schema 告警({path.name}): {error}") data[site] = payload["config"] files_map[site] = path meta[site] = {key: value for key, value in payload.items() if key != "config"} on_disk[site] = canonical_config_text(payload["config"]) self._files, self._meta, self._on_disk = files_map, meta, on_disk self._broken, self._broken_sites = broken, broken_sites return data # ------------------------------------------------------------------ 保存 def save(self, full_config: Dict[str, Any]) -> List[str]: """保存旧 sites.json 结构的字典,返回实际重写的站点列表。""" self.sites_dir.mkdir(parents=True, exist_ok=True) pending: List[Tuple[str, Path, Path, Dict[str, Any], str]] = [] taken = {path for path in self._files.values()} | set(self._broken) try: for site, config in full_config.items(): if not isinstance(config, dict): continue text = canonical_config_text(config) path = self._files.get(site) if path is not None and self._on_disk.get(site) == text and path.is_file(): continue if path is None: path = self._unique_path(site, taken) taken.add(path) if path in self._broken: logger.error(f"站点 {site} 对应的文件 {path.name} 无法解析,为避免覆盖用户手改内容,本次跳过保存") continue payload = build_envelope(site, config, self._meta.get(site) or self._default_meta()) tmp = path.with_name(path.name + ".tmp") with open(tmp, "w", encoding="utf-8", newline="\n") as handle: handle.write(_dump(payload)) handle.flush() os.fsync(handle.fileno()) pending.append((site, path, tmp, payload, text)) except Exception: # broad-except: 清理已写出的临时文件后原样重新抛出 for _, _, tmp, _, _ in pending: try: tmp.unlink() except OSError: pass raise written = [] for site, path, tmp, payload, text in pending: os.replace(tmp, path) self._files[site] = path self._on_disk[site] = text self._meta[site] = {key: value for key, value in payload.items() if key != "config"} written.append(site) for site in [s for s in self._files if s not in full_config]: path = self._files.pop(site) self._on_disk.pop(site, None) self._meta.pop(site, None) if path in self._broken or not path.is_file(): continue trash = self.sites_dir / TRASH_DIR_NAME trash.mkdir(exist_ok=True) target = trash / f"{path.stem}.{_timestamp()}.json" os.replace(path, target) logger.info(f"站点 {site} 已删除,配置文件移入回收站: {target}") return written def _default_meta(self) -> Dict[str, Any]: return {"min_app_version": DEFAULT_MIN_APP_VERSION} def _unique_path(self, site: str, taken: set) -> Path: base = self.sites_dir / site_filename(site) path, counter = base, 2 while path in taken or (path.exists() and path not in self._files.values()): path = base.with_name(f"{base.stem}~{counter}.json") counter += 1 return path # ------------------------------------------------------------------ 迁移 def migrate_legacy(self) -> Optional[Path]: """把旧的 config/sites.json 迁移进目录,返回备份文件路径;旧文件无法解析时保持原样并返回 None。""" try: raw = self.legacy_file.read_text(encoding="utf-8-sig").strip() legacy = json.loads(raw) if raw else {} if not isinstance(legacy, dict): raise ValueError("顶层不是对象") except (OSError, ValueError) as exc: logger.error(f"旧版站点配置 {self.legacy_file} 无法解析,暂不迁移(原文件保持不变): {exc}") return None from updater import merge_site_records # 与更新器合并 sites.json 的语义保持一致 self.sites_dir.mkdir(parents=True, exist_ok=True) existing: Dict[str, Tuple[Path, Dict[str, Any]]] = {} for path in self.site_files(): try: payload = self.read_envelope(path) except (OSError, ValueError): continue existing[payload["site"]] = (path, payload) taken = {path for path, _ in existing.values()} merged_count = created_count = 0 for site, config in legacy.items(): if not isinstance(config, dict) or (site.startswith("_") and site != "_global"): continue if site in existing: path, payload = existing[site] payload = dict(payload) merged_config = merge_site_records(config, payload["config"]) if ( site == "_global" and config.get("selector_definitions") == [] and payload["config"].get("selector_definitions") ): merged_config["selector_definitions"] = payload["config"]["selector_definitions"] payload["config"] = merged_config merged_count += 1 else: path = self._unique_path(site, taken) taken.add(path) payload = build_envelope(site, config, self._default_meta()) created_count += 1 _write_text_atomic(path, _dump(payload)) backup = self.legacy_file.with_name(f"{self.legacy_file.name}.migrated-{_timestamp()}.bak") os.replace(self.legacy_file, backup) logger.warning( f"已把旧版站点配置迁移到 {self.sites_dir}:新建 {created_count} 个、与发布版合并 {merged_count} 个;" f"原文件已备份为 {backup.name}" ) return backup def load_sites_dict(sites_dir: os.PathLike | str) -> Dict[str, Any]: """只读地拼出目录里的全部站点(不迁移、不告警),供测试、脚本与导出使用。""" store = SiteStore(sites_dir, Path(sites_dir) / "__no_legacy__.json") return store.load() or {}