Initial commit

This commit is contained in:
Alex committed 2026-11-02 22:54:14 +08:00
commit 57fc3525eb
7311 files changed
+1291398

No files matched your search

+1
View File
@@ -0,0 +1 @@
"""R2-5:运行时数据存储。"""
+206
View File
@@ -0,0 +1,206 @@
"""R2-5:运行时数据的 SQLite 存储(请求历史、累计统计)。
以前请求历史每次保存都把整份 JSON(最多 2000 条、每条可能带几十 KB 的捕获内容)重写一遍;
现在改为 SQLite(WAL 模式):
- ``sync_history`` 按每条记录的内容摘要只写入新增或变化的记录,删除已从内存中淘汰的记录;
- ``synchronous=NORMAL``:WAL 模式下提交不会每次都落盘同步(保持去掉 fsync 的性能意图),
进程崩溃也不会损坏数据库,最多丢失最后一次提交;
- 旧的 ``config/request_history.json`` / ``config/app_stats.json`` 由 RequestManager 在首次加载时自动导入,
原文件改名为 ``*.migrated-<时间>.bak``。
线程安全:单连接加可重入锁(写入频率很低,每次都是一个短事务)。
"""
from __future__ import annotations
import hashlib
import json
import os
import sqlite3
import threading
import time
from typing import Any, Dict, Iterable, List, Optional, Tuple
SCHEMA_VERSION = 1
_SCHEMA = (
"CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT NOT NULL)",
"CREATE TABLE IF NOT EXISTS stats (key TEXT PRIMARY KEY, value INTEGER NOT NULL)",
"""CREATE TABLE IF NOT EXISTS request_history (
request_id TEXT PRIMARY KEY,
created_at REAL NOT NULL,
status TEXT,
updated_at REAL NOT NULL,
record TEXT NOT NULL
)""",
"CREATE INDEX IF NOT EXISTS idx_request_history_created ON request_history(created_at)",
"CREATE INDEX IF NOT EXISTS idx_request_history_status ON request_history(status)",
)
def _record_key(record: Dict[str, Any], index: int) -> str:
key = str(record.get("request_id") or "").strip()
if key:
return key
digest = hashlib.sha1(json.dumps(record, sort_keys=True, ensure_ascii=False, default=str).encode("utf-8")).hexdigest()
return f"anon-{record.get('created_at', 0)}-{index}-{digest[:12]}"
def _created_at(record: Dict[str, Any]) -> float:
try:
return float(record.get("created_at") or 0.0)
except (TypeError, ValueError):
return 0.0
class RuntimeStore:
def __init__(self, path: str):
self.path = str(path)
self._lock = threading.RLock()
self._conn: Optional[sqlite3.Connection] = None
self._history_digests: Dict[str, str] = {}
# ------------------------------------------------------------------ 连接与迁移
def _connection(self) -> sqlite3.Connection:
if self._conn is None:
directory = os.path.dirname(os.path.abspath(self.path))
os.makedirs(directory, exist_ok=True)
conn = sqlite3.connect(self.path, timeout=10, check_same_thread=False, isolation_level=None)
conn.execute("PRAGMA journal_mode=WAL")
conn.execute("PRAGMA synchronous=NORMAL")
conn.execute("PRAGMA busy_timeout=5000")
with conn:
for statement in _SCHEMA:
conn.execute(statement)
conn.execute(
"INSERT INTO meta(key, value) VALUES('schema_version', ?) "
"ON CONFLICT(key) DO UPDATE SET value=excluded.value",
(str(SCHEMA_VERSION),),
)
self._conn = conn
return self._conn
def close(self) -> None:
with self._lock:
if self._conn is not None:
try:
self._conn.close()
finally:
self._conn = None
# ------------------------------------------------------------------ 统计
def load_stats(self) -> Dict[str, int]:
with self._lock:
rows = self._connection().execute("SELECT key, value FROM stats").fetchall()
return {key: int(value) for key, value in rows}
def save_stats(self, values: Dict[str, int]) -> None:
with self._lock:
conn = self._connection()
conn.execute("BEGIN")
try:
conn.executemany(
"INSERT INTO stats(key, value) VALUES(?, ?) ON CONFLICT(key) DO UPDATE SET value=excluded.value",
[(str(k), int(v)) for k, v in values.items()],
)
conn.execute("COMMIT")
except Exception: # broad-except: 回滚事务后原样重新抛出
conn.execute("ROLLBACK")
raise
# ------------------------------------------------------------------ 请求历史
def history_count(self) -> int:
with self._lock:
return int(self._connection().execute("SELECT COUNT(*) FROM request_history").fetchone()[0])
def load_history(self, limit: int) -> List[Dict[str, Any]]:
"""按创建时间升序返回最近 ``limit`` 条;同时记下各条摘要,供后续增量同步。"""
if limit <= 0:
return []
with self._lock:
rows = self._connection().execute(
"SELECT request_id, record FROM request_history ORDER BY created_at DESC, rowid DESC LIMIT ?",
(int(limit),),
).fetchall()
records = []
for request_id, text in reversed(rows):
try:
record = json.loads(text)
except json.JSONDecodeError:
continue
if isinstance(record, dict):
records.append(record)
self._history_digests[request_id] = hashlib.sha1(text.encode("utf-8")).hexdigest()
return records
def sync_history(self, records: Iterable[Dict[str, Any]]) -> Tuple[int, int]:
"""让数据库与给定的内存记录一致:只写入新增/变化的记录,删除不在其中的记录。返回 (写入数, 删除数)。"""
now = time.time()
with self._lock:
conn = self._connection()
current: Dict[str, Tuple[str, Dict[str, Any], str]] = {}
for index, record in enumerate(records):
if not isinstance(record, dict):
continue
text = json.dumps(record, ensure_ascii=False, separators=(",", ":"), default=str)
current[_record_key(record, index)] = (text, record, hashlib.sha1(text.encode("utf-8")).hexdigest())
if not self._history_digests: # 首次同步:以数据库现有内容为基准
for request_id, text in conn.execute("SELECT request_id, record FROM request_history"):
self._history_digests[request_id] = hashlib.sha1(text.encode("utf-8")).hexdigest()
changed = [
(key, _created_at(record), str(record.get("status") or ""), now, text)
for key, (text, record, digest) in current.items()
if self._history_digests.get(key) != digest
]
removed = [key for key in self._history_digests if key not in current]
if not changed and not removed:
return 0, 0
conn.execute("BEGIN")
try:
conn.executemany(
"INSERT INTO request_history(request_id, created_at, status, updated_at, record) VALUES(?, ?, ?, ?, ?) "
"ON CONFLICT(request_id) DO UPDATE SET created_at=excluded.created_at, status=excluded.status, "
"updated_at=excluded.updated_at, record=excluded.record",
changed,
)
for start in range(0, len(removed), 500):
chunk = removed[start:start + 500]
conn.execute(
f"DELETE FROM request_history WHERE request_id IN ({','.join('?' * len(chunk))})", chunk
)
conn.execute("COMMIT")
except Exception: # broad-except: 回滚事务后原样重新抛出
conn.execute("ROLLBACK")
raise
for key, _, _, _, text in changed:
self._history_digests[key] = hashlib.sha1(text.encode("utf-8")).hexdigest()
for key in removed:
self._history_digests.pop(key, None)
return len(changed), len(removed)
def query_history(self, *, status: Optional[str] = None, since: Optional[float] = None,
limit: int = 100, offset: int = 0) -> List[Dict[str, Any]]:
"""按状态/时间过滤查询(最新在前),供面板或导出使用。"""
clauses, params = [], []
if status:
clauses.append("status = ?")
params.append(status)
if since is not None:
clauses.append("created_at >= ?")
params.append(float(since))
where = f"WHERE {' AND '.join(clauses)}" if clauses else ""
params += [max(0, int(limit)), max(0, int(offset))]
with self._lock:
rows = self._connection().execute(
f"SELECT record FROM request_history {where} ORDER BY created_at DESC LIMIT ? OFFSET ?", params
).fetchall()
return [json.loads(text) for (text,) in rows]
def migrate_legacy_file(path: str) -> Optional[str]:
"""把已导入的旧 JSON 文件改名为 .migrated-<时间>.bak,返回新文件名。"""
if not os.path.exists(path):
return None
backup = f"{path}.migrated-{time.strftime('%Y%m%d-%H%M%S')}.bak"
os.replace(path, backup)
return backup