Files
universal-web-api/app/services/request_manager.py
T
2026-11-02 22:54:14 +08:00

2626 lines
109 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""
request_manager.py - 请求生命周期管理器(v2.0)
v2.0 改动:
- 移除全局执行锁(锁转移到 TabPoolManager)
- 保留请求追踪、状态管理、取消信号功能
- acquire/release 改为标记状态,不再阻塞
"""
import asyncio
import atexit
import threading
import json
import os
import time
import uuid
import re
import math
import hashlib
import copy
import random
from enum import Enum
from dataclasses import dataclass, field
from typing import Optional, Dict, Any, List
from collections import OrderedDict
from app.core.config import BrowserConstants, get_logger, _request_context
from app.services.sse_utils import iter_sse_payloads
from app.utils.site_url import get_canonical_route_domain
logger = get_logger("REQUEST")
MAX_CAPTURED_RESPONSE_CHARS = 30000
MAX_SSE_CHUNK_BUFFER_CHARS = 262144
# 修复#3a:data:URI 与超长 base64 正则改为模块级预编译,供文本清洗与 token 估算共用
_DATA_URI_PATTERN = re.compile(
r"data:(?:image|audio|video)/[a-zA-Z0-9.+-]+;base64,[A-Za-z0-9+/=\r\n]{1024,}",
re.IGNORECASE,
)
_LONG_BASE64_PATTERN = re.compile(
r"(?<![A-Za-z0-9+/])[A-Za-z0-9+/]{8192,}={0,2}(?![A-Za-z0-9+/])"
)
# 两个模式合并为一次扫描(先匹配 data URI,再匹配裸 base64),供惰性截断使用
_MEDIA_BLOB_PATTERN = re.compile(
"(?:" + _DATA_URI_PATTERN.pattern + ")|(?:" + _LONG_BASE64_PATTERN.pattern + ")",
re.IGNORECASE,
)
# 最短匹配长度的上界(data URI 前缀 + 1024 / 裸 base64 8192),用于限定搜索窗口
_MEDIA_BLOB_MIN_MATCH_SLACK = 8192 + 256
_MEDIA_PLACEHOLDER = "[图片占位符]"
def _history_save_debounce_sec() -> float:
try:
return max(0.0, min(60.0, float(os.getenv("REQUEST_HISTORY_SAVE_DEBOUNCE_SEC", "5") or 5)))
except (TypeError, ValueError):
return 5.0
def _get_positive_int_env(name: str, default: int) -> int:
try:
value = int(os.getenv(name, str(default)))
except Exception:
return default
return value if value > 0 else default
def _browser_bool(key: str, default: bool = True) -> bool:
value = BrowserConstants.get(key)
if value is None:
return bool(default)
if isinstance(value, bool):
return value
if isinstance(value, (int, float)):
return bool(value)
text = str(value).strip().lower()
if text in {"1", "true", "yes", "y", "on"}:
return True
if text in {"0", "false", "no", "n", "off"}:
return False
return bool(default)
def _browser_int(
key: str,
default: int,
*,
min_value: int = 0,
max_value: Optional[int] = None,
) -> int:
try:
value = int(BrowserConstants.get(key))
except Exception:
value = int(default)
value = max(int(min_value), value)
if max_value is not None:
value = min(int(max_value), value)
return value
# ================= 客户端高频取消防抖熔断器 =================
class CancelStormGuard:
"""Bound rapid zero-output abort loops without delaying normal requests.
Cleanup correctness is the primary protection. As a second line of defense,
four transport cancellations in five seconds open a five-second circuit for
that client only. Rejected retries create no workflow/request record. This is
not an authorization boundary or a global/DDoS rate limiter.
"""
def __init__(self, window_sec: float = 5.0, max_backoff_sec: float = 0.5,
threshold: int = 4, cooldown_sec: float = 5.0, max_clients: int = 2048):
self.window_sec = max(.1, float(window_sec))
self.max_backoff_sec = max_backoff_sec # Legacy constructor compatibility.
self.threshold = max(2, int(threshold))
self.cooldown_sec = max(.1, float(cooldown_sec))
self.max_clients = max(1, int(max_clients))
self._lock = threading.Lock()
self._cancel_records = OrderedDict()
self._blocked_until = {}
def get_client_fingerprint(self, request: Any = None, auth_header: Optional[str] = None) -> str:
client = getattr(request, 'client', None)
headers = getattr(request, 'headers', {}) or {}
parts = [str(getattr(client, 'host', '') or ''),
str(headers.get('authorization') or headers.get('x-api-key') or auth_header or ''),
str(headers.get('user-agent') or '')]
# Never log token fragments, raw credentials, or request content. Do not
# trust arbitrary X-Forwarded-For values as a client-controlled bypass.
return hashlib.sha256(json.dumps(parts, ensure_ascii=True).encode()).hexdigest()[:24]
def _prune_expired_unlocked(self, now: float):
for key, records in list(self._cancel_records.items()):
if self._blocked_until.get(key, 0) > now:
continue
if not records or now - records[-1] > self.window_sec:
self._cancel_records.pop(key, None)
self._blocked_until.pop(key, None)
def record_cancel(self, fingerprint: str, has_output: bool = False, reason: str = 'unknown'):
if not fingerprint:
return
if has_output:
self.record_success(fingerprint)
return
if reason not in {'coroutine_cancelled', 'client_disconnected', 'stream_closed', 'unknown', 'cleanup'}:
return
now = time.monotonic()
opened = False
with self._lock:
if self._blocked_until.get(fingerprint, 0) > now:
return # Already in-flight failures must not extend the cooldown indefinitely.
self._prune_expired_unlocked(now)
records = self._cancel_records.pop(fingerprint, [])
records = [t for t in records if now - t <= self.window_sec]
records.append(now)
self._cancel_records[fingerprint] = records[-self.threshold:]
if len(records) >= self.threshold:
self._blocked_until[fingerprint] = now + self.cooldown_sec
opened = True
while len(self._cancel_records) > self.max_clients:
key, _ = self._cancel_records.popitem(last=False)
self._blocked_until.pop(key, None)
if opened:
logger.warning(
f'[CANCEL_STORM_GUARD] client={fingerprint[:12]} '
f'{self.window_sec:g}s 内 {self.threshold} 次零输出异常取消;'
f'冷却 {self.cooldown_sec:g}s,后续请求返回 429 / Retry-After,不启动网页工作流'
)
def record_success(self, fingerprint: str):
with self._lock:
self._cancel_records.pop(fingerprint, None)
self._blocked_until.pop(fingerprint, None)
async def maybe_backoff(self, fingerprint: str) -> float:
"""Compatibility entry point: reject an open circuit instead of queuing sleeps."""
from fastapi import HTTPException
now = time.monotonic()
with self._lock:
until = self._blocked_until.get(fingerprint, 0)
remaining = until - now
if until and remaining <= 0:
self._blocked_until.pop(fingerprint, None)
self._cancel_records.pop(fingerprint, None)
if remaining > 0:
retry_after = max(1, math.ceil(remaining))
raise HTTPException(
status_code=429,
detail={'error': {'code': 'client_cancel_storm', 'type': 'rate_limit_error',
'message': '检测到连续零输出取消,请停止自动重试并在冷却后重试。',
'retry_after_seconds': retry_after}},
headers={'Retry-After': str(retry_after)},
)
return 0.0
cancel_storm_guard = CancelStormGuard()
class RequestStatus(Enum):
"""请求状态枚举"""
QUEUED = "queued"
RUNNING = "running"
COMPLETED = "completed"
CANCELLED = "cancelled"
FAILED = "failed"
@dataclass
class RequestContext:
"""请求上下文"""
request_id: str
status: RequestStatus = RequestStatus.QUEUED
created_at: float = field(default_factory=time.time)
started_at: Optional[float] = None
finished_at: Optional[float] = None
_cancel_flag: bool = field(default=False, repr=False)
cancel_reason: Optional[str] = None
_lock: threading.Lock = field(default_factory=threading.Lock, repr=False)
# 修复#1:历史写入专用锁,串行化 _append_monitor_history 的"判重→插入"全过程;
# 必须最外层获取,持有期间允许再拿 ctx._lock / manager._history_lock(内层顺序不变)
_history_append_lock: threading.Lock = field(default_factory=threading.Lock, repr=False)
_sse_chunk_buffer: str = field(default="", repr=False)
# v2.0 新增:关联的标签页 ID
tab_id: Optional[str] = None
client_fp: str = ""
protocol: str = "openai.chat" # R2-2:openai.chat / anthropic.messages / openai.responses
http_request_id: str = "" # R2-7:客户端可见的 X-Request-ID,把 HTTP 请求与 req-xxx 日志关联起来
monitor: Dict[str, Any] = field(default_factory=dict)
started_at_monotonic: Optional[float] = field(default=None, repr=False)
last_activity_at: Optional[float] = field(default=None, repr=False)
def should_stop(self) -> bool:
with self._lock:
return self._cancel_flag
def request_cancel(self, reason: str = "unknown"):
with self._lock:
if self.status in (
RequestStatus.COMPLETED,
RequestStatus.CANCELLED,
RequestStatus.FAILED,
):
return
if self._cancel_flag:
return
self._cancel_flag = True
self.cancel_reason = reason
if self.finished_at is None:
self.finished_at = time.time()
self.status = RequestStatus.CANCELLED
logger.info(f"[{self.request_id}] 取消 ({reason})")
def mark_worker_stop_requested(self, reason: str = "worker_stop_requested"):
with self._lock:
self._cancel_flag = True
self.cancel_reason = reason
if self.finished_at is None:
self.finished_at = time.time()
def mark_running(self, tab_id: str = None):
with self._lock:
if self.status in (RequestStatus.COMPLETED, RequestStatus.CANCELLED, RequestStatus.FAILED):
return
self.started_at = time.time()
self.started_at_monotonic = time.monotonic()
self.last_activity_at = self.started_at
self.finished_at = None
self.tab_id = tab_id
if self._cancel_flag:
self.status = RequestStatus.CANCELLED
if self.finished_at is None or self.finished_at < self.started_at:
self.finished_at = self.started_at
else:
self.status = RequestStatus.RUNNING
def reset_timeout(self):
with self._lock:
self.started_at_monotonic = time.monotonic()
self.last_activity_at = time.time()
logger.debug(f"[{self.request_id}] 重置请求绝对超时计时")
def mark_activity(self) -> None:
"""Refresh idle activity without extending the request's hard timeout."""
with self._lock:
if self.status == RequestStatus.RUNNING:
self.last_activity_at = time.time()
def mark_completed(self):
with self._lock:
if self.status == RequestStatus.RUNNING or (
self.status == RequestStatus.CANCELLED
and self.cancel_reason in {
"audio_media_fast_return",
"stop_sequence",
"stream_done",
}
):
self.status = RequestStatus.COMPLETED
self.finished_at = time.time()
def mark_failed(self, reason: str = None):
with self._lock:
# 修复#2:终态保护——已 COMPLETED/FAILED/CANCELLED 的请求不再改写为 FAILED
if self.status in (
RequestStatus.COMPLETED,
RequestStatus.FAILED,
RequestStatus.CANCELLED,
):
logger.debug(
f"[{self.request_id}] mark_failed 忽略:已处于终态 "
f"{self.status.value} (reason={reason or '-'})"
)
return
self._cancel_flag = True
self.status = RequestStatus.FAILED
self.finished_at = time.time()
if reason:
self.cancel_reason = reason
def get_duration(self) -> float:
with self._lock:
end = self.finished_at or time.time()
start = self.started_at or self.created_at
return max(0.0, end - start)
def is_terminal(self) -> bool:
with self._lock:
return self.status in (
RequestStatus.COMPLETED,
RequestStatus.CANCELLED,
RequestStatus.FAILED,
)
def is_running_for_tab(self, tab_id: str = None) -> bool:
target_tab_id = str(tab_id or "").strip()
with self._lock:
if self.status != RequestStatus.RUNNING:
return False
if target_tab_id and str(self.tab_id or "").strip() != target_tab_id:
return False
return True
def status_summary(self, now: Optional[float] = None) -> Dict[str, Any]:
current_time = time.time() if now is None else float(now)
with self._lock:
status = self.status
tab_id = self.tab_id
if status == RequestStatus.RUNNING:
duration_end = self.finished_at or current_time
duration_start = self.started_at or self.created_at
duration = max(0.0, duration_end - duration_start)
else:
duration = 0.0
return {
"request_id": self.request_id,
"status": status,
"tab_id": tab_id,
"duration": duration,
}
def snapshot(self, now: Optional[float] = None) -> Dict[str, Any]:
current_time = time.time() if now is None else float(now)
with self._lock:
status = self.status
created_at = self.created_at
started_at = self.started_at
finished_at = self.finished_at
tab_id = self.tab_id
cancel_reason = self.cancel_reason
last_activity_at = self.last_activity_at
duration_end = finished_at or current_time
duration_start = started_at or created_at
return {
"request_id": self.request_id,
"status": status,
"created_at": created_at,
"started_at": started_at,
"finished_at": finished_at,
"tab_id": tab_id,
"cancel_reason": cancel_reason,
"last_activity_at": last_activity_at,
"duration": max(0.0, duration_end - duration_start),
"is_terminal": status in (
RequestStatus.COMPLETED,
RequestStatus.CANCELLED,
RequestStatus.FAILED,
),
}
class RequestManager:
"""
请求管理器(v2.0 - 纯追踪模式)
v2.0 改动:
- 不再持有执行锁
- 只负责请求追踪和取消信号
"""
_instance: Optional['RequestManager'] = None
_instance_lock = threading.Lock()
# 僵尸请求超时时间(秒)- 超过此时间的 RUNNING 请求将被强制清理
# 注意:该类属性在 import 时求值,仅作兼容兜底;运行时请使用 _zombie_ttl() 以支持热配置
ZOMBIE_TTL = _get_positive_int_env("REQUEST_ZOMBIE_TTL", 600)
def __new__(cls):
if cls._instance is None:
with cls._instance_lock:
if cls._instance is None:
instance = super().__new__(cls)
instance._initialized = False
cls._instance = instance
return cls._instance
def __init__(self):
if self._initialized:
return
self._requests: OrderedDict[str, RequestContext] = OrderedDict()
self._requests_lock = threading.Lock()
self._request_counter = 0 # 请求计数器
self._max_history = 100
self._stats_file = "config/app_stats.json"
self._history_file = "config/request_history.json"
# R2-5:单例被重新初始化(测试会这样做)时,丢弃上一次缓存的存储连接,按当前路径/环境重新打开
previous_store = getattr(self, "_store", None)
if previous_store is not None:
previous_store.close()
self._store = None
self._monitor_history: List[Dict[str, Any]] = []
self._history_lock = threading.Lock()
self._history_save_lock = threading.Lock()
self._stats_save_lock = threading.Lock()
self._save_schedule_lock = threading.Lock()
# 修复#5:缓存三元组 (cache_key, revision, 记录对象引用列表),
# 持有引用保证 id() 在缓存有效期内不被 CPython 地址复用
self._history_revision_cache: Optional[tuple[tuple[Any, ...], str, List[Dict[str, Any]]]] = None
self._stats_save_requested = False
self._history_save_requested = False
self._stats_save_worker: Optional[threading.Thread] = None
self._history_save_worker: Optional[threading.Thread] = None
# P0-7:历史落盘去抖(REQUEST_HISTORY_SAVE_DEBOUNCE_SEC,默认 5s),进程退出时强制刷盘
self._history_flush_event = threading.Event()
atexit.register(self.flush_pending_saves)
self.total_requests = 0
self.total_input_tokens = 0
self.total_output_tokens = 0
self._cleanup_stale_temp_files()
self._load_stats()
self._load_history()
# 修复#4:启动定时僵尸清扫 daemon 线程,低流量时段也能清理超时请求
# (保留 create_request 里的现有触发;daemon 线程随进程退出,无需 shutdown 处理)
self._zombie_sweep_thread = threading.Thread(
target=self._run_zombie_sweep_loop,
daemon=True,
name="request-zombie-sweep",
)
self._zombie_sweep_thread.start()
self._initialized = True
logger.debug("RequestManager 初始化完成")
def _runtime_store(self):
"""R2-5:请求历史与统计的 SQLite 存储;默认与旧 JSON 文件同目录(config/runtime.sqlite3)。"""
store = getattr(self, "_store", None)
if store is None:
from app.services.storage.runtime_store import RuntimeStore
history_file = getattr(self, "_history_file", "config/request_history.json")
path = os.getenv("RUNTIME_DB_PATH") or os.path.join(os.path.dirname(history_file) or ".", "runtime.sqlite3")
store = RuntimeStore(path)
self._store = store
return store
def _load_stats(self):
try:
from app.services.storage.runtime_store import migrate_legacy_file
store = self._runtime_store()
data: Dict[str, Any] = store.load_stats()
stats_file = getattr(self, "_stats_file", "")
if not data and stats_file and os.path.exists(stats_file):
# 首次启用 SQLite:导入旧的 config/app_stats.json,原文件改名备份
with open(stats_file, "r", encoding="utf-8") as f:
legacy = json.load(f)
data = {
key: self._coerce_token_count(legacy.get(key, 0))
for key in ("total_requests", "total_input_tokens", "total_output_tokens")
} if isinstance(legacy, dict) else {}
if data:
store.save_stats(data)
migrate_legacy_file(stats_file)
self.total_requests = self._coerce_token_count(data.get("total_requests", 0))
self.total_input_tokens = self._coerce_token_count(data.get("total_input_tokens", 0))
self.total_output_tokens = self._coerce_token_count(data.get("total_output_tokens", 0))
except Exception as e:
logger.debug(f"加载状态失败: {e}")
def _save_stats(self):
try:
with self._stats_save_lock:
self._runtime_store().save_stats({
"total_requests": self.total_requests,
"total_input_tokens": self.total_input_tokens,
"total_output_tokens": self.total_output_tokens,
})
except Exception as e:
logger.debug(f"保存状态失败: {e}")
def _cleanup_stale_temp_files(self):
for path in (self._stats_file + ".tmp", self._history_file + ".tmp"):
try:
if os.path.exists(path):
os.remove(path)
logger.debug(f"清理遗留临时文件: {path}")
except Exception as e:
logger.debug(f"清理遗留临时文件失败: {path}, {e}")
def _request_monitor_max_records(self) -> int:
return _browser_int("REQUEST_MONITOR_MAX_RECORDS", 200, min_value=0, max_value=2000)
def _zombie_ttl(self) -> int:
"""僵尸请求判定阈值(秒)。优先读 BrowserConstants,便于前端热配置。"""
return _browser_int("REQUEST_ZOMBIE_TTL", int(self.ZOMBIE_TTL or 600), min_value=30, max_value=3600)
def _request_monitor_enabled(self) -> bool:
return _browser_bool("REQUEST_MONITOR_ENABLED", True) and self._request_monitor_max_records() > 0
def _request_monitor_detail_enabled(self) -> bool:
return _browser_bool("REQUEST_MONITOR_DETAIL_ENABLED", True)
def _request_monitor_save_to_file(self) -> bool:
return _browser_bool("REQUEST_MONITOR_SAVE_TO_FILE", True)
def _request_monitor_capture_chars(self) -> int:
return _browser_int(
"REQUEST_MONITOR_MAX_CAPTURED_RESPONSE_CHARS",
MAX_CAPTURED_RESPONSE_CHARS,
min_value=0,
max_value=200000,
)
def _trim_monitor_history_unlocked(self) -> None:
max_records = self._request_monitor_max_records()
if max_records <= 0:
if self._monitor_history:
self._monitor_history = []
self._history_revision_cache = None
return
if len(self._monitor_history) > max_records:
self._monitor_history = self._sort_history_records(self._monitor_history)[-max_records:]
self._history_revision_cache = None
def _load_history(self):
try:
if not self._request_monitor_enabled():
self._monitor_history = []
return
from app.services.storage.runtime_store import migrate_legacy_file
store = self._runtime_store()
max_to_load = self._request_monitor_max_records()
if store.history_count() > 0:
records = store.load_history(max_to_load)
elif os.path.exists(self._history_file):
# R2-5 首次启用 SQLite:导入旧的 JSON 历史(兼容两种旧格式),原文件改名备份
with open(self._history_file, "r", encoding="utf-8") as f:
data = json.load(f)
# B2:旧版历史文件是顶层数组,必须先按类型分支再取值——
# 旧实现先调用 data.get(...),list 没有 .get 直接抛错,兼容分支永远走不到,恢复 0 条。
if isinstance(data, list):
records = data
elif isinstance(data, dict):
records = data.get("records", [])
else:
records = []
if isinstance(records, list):
store.sync_history([item for item in records if isinstance(item, dict)])
migrate_legacy_file(self._history_file)
else:
return
if isinstance(records, list):
raw_records = [item for item in records if isinstance(item, dict)]
max_records = self._request_monitor_max_records()
if max_records <= 0:
# 注意 lst[-0:] 等于整个列表,max_records=0 时必须显式清空
raw_records = []
elif self._history_records_are_ordered(raw_records):
raw_records = raw_records[-max_records:]
normalized_records = [
self._normalize_history_record(item) for item in raw_records
]
self._monitor_history = (
self._sort_history_records(normalized_records)[-max_records:]
if max_records > 0 else []
)
# 如果没有持久化的 token 统计,从已有的 200 条历史请求中求和做初次填充
if self.total_input_tokens == 0 and self.total_output_tokens == 0:
for record in self._monitor_history:
estimate = record.get("token_estimate") or {}
self.total_input_tokens += estimate.get("prompt", 0)
self.total_output_tokens += estimate.get("response", 0)
except Exception as e:
logger.debug(f"加载请求历史失败: {e}")
self._monitor_history = []
def _save_history(self):
if not self._request_monitor_enabled() or not self._request_monitor_save_to_file():
return
try:
with self._history_save_lock:
with self._history_lock:
max_records = self._request_monitor_max_records()
self._trim_monitor_history_unlocked()
records = list(self._monitor_history[-max_records:]) if max_records > 0 else []
# R2-5:增量同步到 SQLite——只写新增/变化的记录,删除已淘汰的记录(不再整文件重写)
self._runtime_store().sync_history(records)
except Exception as e:
logger.debug(f"保存请求历史失败: {e}")
def _schedule_stats_save(self) -> None:
worker_to_start = None
with self._save_schedule_lock:
self._stats_save_requested = True
if self._stats_save_worker and self._stats_save_worker.is_alive():
return
worker_to_start = threading.Thread(
target=self._run_stats_save_worker,
daemon=True,
name="request-stats-save",
)
self._stats_save_worker = worker_to_start
worker_to_start.start()
def _run_stats_save_worker(self) -> None:
while True:
with self._save_schedule_lock:
if not self._stats_save_requested:
self._stats_save_worker = None
return
self._stats_save_requested = False
self._save_stats()
def _schedule_history_save(self) -> None:
worker_to_start = None
with self._save_schedule_lock:
self._history_save_requested = True
if self._history_save_worker and self._history_save_worker.is_alive():
return
worker_to_start = threading.Thread(
target=self._run_history_save_worker,
daemon=True,
name="request-history-save",
)
self._history_save_worker = worker_to_start
worker_to_start.start()
def _run_history_save_worker(self) -> None:
while True:
# 去抖:窗口内的多次更新合并为一次整文件重写;flush_pending_saves 可提前唤醒
delay = _history_save_debounce_sec()
if delay > 0 and not self._history_flush_event.is_set():
self._history_flush_event.wait(delay)
with self._save_schedule_lock:
if not self._history_save_requested:
self._history_save_worker = None
return
self._history_save_requested = False
self._save_history()
def flush_pending_saves(self, timeout: float = 5.0) -> None:
"""立即写出尚在去抖窗口内的历史 / 统计(atexit 与关闭流程调用)。"""
self._history_flush_event.set()
try:
for worker in (self._history_save_worker, self._stats_save_worker):
if worker is not None and worker.is_alive() and worker is not threading.current_thread():
worker.join(timeout)
with self._save_schedule_lock:
pending_history = self._history_save_requested and not (
self._history_save_worker and self._history_save_worker.is_alive()
)
if pending_history:
self._history_save_requested = False
if pending_history:
self._save_history()
finally:
self._history_flush_event.clear()
@staticmethod
def _sanitize_text_for_storage(value: Any, max_chars: int = 80000) -> str:
text = "" if value is None else str(value)
if not text:
return ""
if "[内容已截断,原始长度" in text:
return text
max_chars = max(0, int(max_chars))
# 短文本:一次替换即可
if len(text) <= max_chars + _MEDIA_BLOB_MIN_MATCH_SLACK:
text = _MEDIA_BLOB_PATTERN.sub(_MEDIA_PLACEHOLDER, text)
if len(text) > max_chars:
return text[:max_chars] + f"\n\n[内容已截断,原始长度 {len(text)} 字符]"
return text
# P0-7:长文本(prompt 可达 1M 字)不再“先整段跑正则再截断”:
# 边替换边输出,凑够 max_chars 即停止扫描。图片后面的文字仍会保留
# (与整段替换的结果一致,只是不再处理截断点之后的内容)。
total_len = len(text)
parts: List[str] = []
out_len = 0
pos = 0
while pos < total_len and out_len <= max_chars:
budget = max_chars - out_len + 1
window_end = min(total_len, pos + budget + _MEDIA_BLOB_MIN_MATCH_SLACK)
match = _MEDIA_BLOB_PATTERN.search(text, pos, window_end)
if match is None or match.start() >= pos + budget:
chunk = text[pos:min(total_len, pos + budget)]
parts.append(chunk)
out_len += len(chunk)
pos += len(chunk)
continue
# 窗口可能截断了匹配:从匹配起点重新做一次不设上界的 match,跳过完整的媒体块
full = _MEDIA_BLOB_PATTERN.match(text, match.start()) or match
prefix = text[pos:match.start()]
parts.append(prefix)
parts.append(_MEDIA_PLACEHOLDER)
out_len += len(prefix) + len(_MEDIA_PLACEHOLDER)
pos = max(full.end(), match.start() + 1)
result = "".join(parts)
if pos < total_len or len(result) > max_chars:
return result[:max_chars] + f"\n\n[内容已截断,原始长度 {total_len} 字符]"
return result
@classmethod
def _sanitize_for_storage(cls, value: Any, max_depth: int = 6) -> Any:
if max_depth < 0:
return "[内容已折叠]"
if isinstance(value, str):
return cls._sanitize_text_for_storage(value)
if isinstance(value, list):
return [cls._sanitize_for_storage(item, max_depth - 1) for item in value[:100]]
if isinstance(value, dict):
result: Dict[str, Any] = {}
for key, item in list(value.items())[:100]:
result[str(key)] = cls._sanitize_for_storage(item, max_depth - 1)
return result
return value
@classmethod
def _normalize_history_record(cls, record: Dict[str, Any]) -> Dict[str, Any]:
normalized = dict(record)
prompt_text = cls._sanitize_text_for_storage(normalized.get("prompt", ""), max_chars=20000)
response_text = cls._sanitize_text_for_storage(normalized.get("response", ""), max_chars=30000)
error_message = cls._sanitize_text_for_storage(normalized.get("error_message", ""), max_chars=10000)
error_stack = cls._sanitize_text_for_storage(normalized.get("error_stack", ""), max_chars=20000)
try:
media_count = int(normalized.get("media_count") or 0)
except Exception:
media_count = 0
normalized["prompt"] = prompt_text
normalized["response"] = response_text
normalized["summary"] = cls._sanitize_text_for_storage(
normalized.get("summary") or response_text[:180],
max_chars=500,
)
normalized["error_message"] = error_message
normalized["error_stack"] = error_stack or error_message
normalized["media_count"] = media_count
canonical_route_domain = cls._canonical_monitor_domain(normalized.get("route_domain"))
canonical_target_domain = cls._canonical_monitor_domain(normalized.get("target_domain"))
if canonical_route_domain:
normalized["route_domain"] = canonical_route_domain
if canonical_target_domain:
normalized["target_domain"] = canonical_target_domain
elif canonical_route_domain:
normalized["target_domain"] = canonical_route_domain
status_value = str(normalized.get("status") or "").strip()
token_estimate_for_success = normalized.get("token_estimate")
if not isinstance(token_estimate_for_success, dict):
token_estimate_for_success = {}
has_meaningful_response = (
bool(response_text.strip())
or media_count > 0
or bool(normalized.get("has_response_text"))
or cls._coerce_token_count(token_estimate_for_success.get("response")) > 0
)
stop_sequence_completed = (
status_value == RequestStatus.COMPLETED.value
and (
str(normalized.get("error_code") or "").strip() == "stop_sequence"
or str(normalized.get("error_message") or "").strip() == "stop_sequence"
or str(normalized.get("cancel_reason") or "").strip()
in {"audio_media_fast_return", "stop_sequence", "stream_done"}
)
)
if (
status_value == RequestStatus.COMPLETED.value
and not has_meaningful_response
and not stop_sequence_completed
):
normalized["status"] = RequestStatus.FAILED.value
normalized["success"] = False
normalized["error_code"] = "empty_response"
normalized["error_message"] = error_message or "请求流程已结束,但没有捕获到 AI 响应文本或媒体结果。"
normalized["error_stack"] = normalized["error_stack"] or normalized["error_message"]
elif status_value == RequestStatus.COMPLETED.value:
normalized["success"] = True
if stop_sequence_completed and str(normalized.get("error_code") or "").strip() in {"", "stop_sequence"}:
normalized["error_code"] = ""
elif status_value:
normalized["success"] = False
token_estimate = normalized.get("token_estimate")
if not isinstance(token_estimate, dict):
prompt_tokens = cls._estimate_tokens(prompt_text)
response_tokens = cls._estimate_tokens(response_text)
normalized["token_estimate"] = {
"prompt": prompt_tokens,
"response": response_tokens,
"total": prompt_tokens + response_tokens,
"chars": len(prompt_text) + len(response_text),
}
else:
prompt_tokens = cls._coerce_token_count(token_estimate.get("prompt"))
response_tokens = cls._coerce_token_count(token_estimate.get("response"))
chars = cls._coerce_token_count(token_estimate.get("chars"))
if not chars:
chars = len(prompt_text) + len(response_text)
normalized["token_estimate"] = {
"prompt": prompt_tokens,
"response": response_tokens,
"total": prompt_tokens + response_tokens,
"chars": chars,
}
normalized["history_key"] = cls._make_history_key(normalized)
return normalized
@staticmethod
def _canonical_monitor_domain(value: Any) -> str:
text = str(value or "").strip()
if not text or text == "未知域名":
return ""
try:
return get_canonical_route_domain(text) or text
except Exception:
return text
@staticmethod
def _content_to_text(content: Any) -> str:
if content is None:
return ""
if isinstance(content, str):
return content
if isinstance(content, list):
parts: List[str] = []
for item in content:
if isinstance(item, dict):
item_type = str(item.get("type") or "").strip()
if item_type in {"text", "input_text", "output_text"}:
parts.append(str(item.get("text") or ""))
elif item_type in {"image_url", "input_image", "output_image"}:
parts.append("[图片占位符]")
elif item_type in {"input_audio", "audio_url", "output_audio"}:
parts.append("[音频占位符]")
elif item_type in {"input_video", "video_url", "output_video"}:
parts.append("[视频占位符]")
else:
parts.append(json.dumps(item, ensure_ascii=False))
else:
parts.append(str(item))
return "\n".join(part for part in parts if part)
return json.dumps(content, ensure_ascii=False)
@classmethod
def _messages_to_prompt_text(cls, messages: Any) -> str:
if not isinstance(messages, list):
return cls._sanitize_text_for_storage(messages)
lines: List[str] = []
for message in messages:
if not isinstance(message, dict):
continue
role = str(message.get("role") or "message")
text = cls._content_to_text(message.get("content"))
if not text:
continue
lines.append(f"{role}: {text}")
return cls._sanitize_text_for_storage("\n\n".join(lines))
@classmethod
def _has_multimodal_payload(cls, value: Any, max_depth: int = 6) -> bool:
if max_depth < 0:
return False
if isinstance(value, str):
lowered = value.lower()
return "image_url" in lowered or "data:image" in lowered or "data:video" in lowered or "data:audio" in lowered
if isinstance(value, list):
return any(cls._has_multimodal_payload(item, max_depth - 1) for item in value)
if isinstance(value, dict):
if str(value.get("type") or "").strip() in {
"image_url",
"input_image",
"output_image",
"input_audio",
"audio_url",
"output_audio",
"input_video",
"video_url",
"output_video",
}:
return True
if any(
key in value
for key in (
"image_url",
"images",
"media",
"input_audio",
"audio_url",
"input_video",
"video_url",
)
):
return True
return any(cls._has_multimodal_payload(item, max_depth - 1) for item in value.values())
return False
@staticmethod
def _estimate_tokens(text: str) -> int:
if not text:
return 0
# 修复#3a:先把 data:URI 与超长 base64 替换成短占位符(空格),
# 避免三个 findall 在大 base64 上全量执行;被剥离的字节按每 4 字符≈1 token 折算补偿
stripped_tokens = 0
if len(text) >= 1024:
original_len = len(text)
text = _DATA_URI_PATTERN.sub(" ", text)
text = _LONG_BASE64_PATTERN.sub(" ", text)
stripped_chars = original_len - len(text)
if stripped_chars > 0:
stripped_tokens = stripped_chars // 4
ascii_words = re.findall(r"[A-Za-z0-9_]+", text)
non_ascii_chars = re.findall(r"[^\x00-\x7F\s]", text)
punctuation = re.findall(r"[^\w\s]", text, flags=re.UNICODE)
return max(
1,
int(len(ascii_words) * 1.25 + len(non_ascii_chars) * 0.8 + len(punctuation) * 0.25)
+ stripped_tokens,
)
@staticmethod
def _coerce_token_count(value: Any) -> int:
try:
count = int(value)
except (TypeError, ValueError, OverflowError):
return 0
return max(0, count)
@classmethod
def _history_record_token_counts(cls, record: Dict[str, Any]) -> tuple[int, int]:
if not isinstance(record, dict):
return (0, 0)
token_estimate = record.get("token_estimate")
if not isinstance(token_estimate, dict):
return (0, 0)
return (
cls._coerce_token_count(token_estimate.get("prompt")),
cls._coerce_token_count(token_estimate.get("response")),
)
@staticmethod
def _coerce_timestamp(value: Any) -> float:
try:
timestamp = float(value)
except (TypeError, ValueError, OverflowError):
return 0.0
if not math.isfinite(timestamp) or timestamp <= 0:
return 0.0
if timestamp > 1_000_000_000_000:
return timestamp / 1000.0
return timestamp
@classmethod
def _history_sort_value(cls, record: Dict[str, Any]) -> float:
if not isinstance(record, dict):
return 0.0
return (
cls._coerce_timestamp(record.get("finished_at"))
or cls._coerce_timestamp(record.get("started_at"))
or cls._coerce_timestamp(record.get("created_at"))
)
@classmethod
def _make_history_key(cls, record: Dict[str, Any]) -> str:
request_id = str(record.get("id") or "").strip() or "request"
created_at = cls._coerce_timestamp(record.get("created_at"))
finished_at = cls._coerce_timestamp(record.get("finished_at"))
return f"{request_id}:{created_at:.6f}:{finished_at:.6f}"
@classmethod
def _sort_history_records(cls, records: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
return sorted(records, key=cls._history_order_key)
@classmethod
def _history_records_are_ordered(cls, records: List[Dict[str, Any]]) -> bool:
previous_key: Optional[tuple[float, float, str]] = None
for record in records:
current_key = cls._history_order_key(record)
if previous_key is not None and current_key < previous_key:
return False
previous_key = current_key
return True
@classmethod
def _history_order_key(cls, record: Dict[str, Any]) -> tuple[float, float, str]:
if not isinstance(record, dict):
return (0.0, 0.0, "")
return (
cls._history_sort_value(record),
cls._coerce_timestamp(record.get("created_at")),
str(record.get("history_key") or record.get("id") or ""),
)
@classmethod
def _append_sorted_history_record_unlocked(
cls,
records: List[Dict[str, Any]],
record: Dict[str, Any],
*,
max_records: int = 200, # 兜底值;调用方应显式传入 _request_monitor_max_records()
) -> None:
key = cls._history_order_key(record)
left = 0
right = len(records)
while left < right:
mid = (left + right) // 2
if cls._history_order_key(records[mid]) <= key:
left = mid + 1
else:
right = mid
records.insert(left, record)
overflow = len(records) - max(1, int(max_records or 200))
if overflow > 0:
del records[:overflow]
def _replace_monitor_history_record_unlocked(
self,
record: Dict[str, Any],
*,
previous_history_key: str = "",
) -> Optional[Dict[str, Any]]:
"""Replace a previously recorded monitor item, preserving sorted order."""
target_key = str(previous_history_key or "").strip()
target_id = str(record.get("id") or "").strip()
target_created_at = self._coerce_timestamp(record.get("created_at"))
replace_index: Optional[int] = None
if target_key:
for index, item in enumerate(self._monitor_history):
if str(item.get("history_key") or "").strip() == target_key:
replace_index = index
break
if replace_index is None and target_id:
for index in range(len(self._monitor_history) - 1, -1, -1):
item = self._monitor_history[index]
if str(item.get("id") or "").strip() != target_id:
continue
item_created_at = self._coerce_timestamp(item.get("created_at"))
if (
not target_created_at
or not item_created_at
or abs(item_created_at - target_created_at) < 0.001
):
replace_index = index
break
previous_record = None
if replace_index is not None:
previous_record = self._monitor_history.pop(replace_index)
# 修复:显式传入配置上限,否则回落到默认 200 条,与 REQUEST_MONITOR_MAX_RECORDS 冲突并批量截断历史
self._append_sorted_history_record_unlocked(
self._monitor_history, record, max_records=self._request_monitor_max_records()
)
self._history_revision_cache = None
return previous_record
@staticmethod
def _extract_response_text(payload: Any) -> str:
if isinstance(payload, str):
return payload
if not isinstance(payload, dict):
return "" if payload is None else str(payload)
if isinstance(payload.get("response"), dict):
nested_text = RequestManager._extract_response_text(payload["response"])
if nested_text:
return nested_text
responses_event_text = RequestManager._extract_responses_event_text(payload)
if responses_event_text:
return responses_event_text
try:
choices = payload.get("choices") or []
if choices:
first = choices[0] or {}
message = first.get("message") or {}
delta = first.get("delta") or {}
content = message.get("content")
if content is None:
content = delta.get("content")
text = RequestManager._content_to_text(content)
if text:
return text
tool_text = RequestManager._extract_tool_calls_text(
message.get("tool_calls") or delta.get("tool_calls")
)
if tool_text:
return tool_text
except Exception:
pass
responses_text = RequestManager._extract_responses_output_text(payload)
if responses_text:
return responses_text
anthropic_text = RequestManager._extract_anthropic_content_text(payload)
if anthropic_text:
return anthropic_text
if "error" in payload:
error = payload.get("error") or {}
if isinstance(error, dict):
return str(error.get("message") or "")
return ""
@staticmethod
def _extract_tool_calls_text(tool_calls: Any) -> str:
if not isinstance(tool_calls, list):
return ""
parts: List[str] = []
for index, item in enumerate(tool_calls):
if not isinstance(item, dict):
continue
function_data = item.get("function") if isinstance(item.get("function"), dict) else {}
name = str(
function_data.get("name")
or item.get("name")
or item.get("tool_name")
or ""
).strip()
call_id = str(item.get("id") or "").strip()
label = name or call_id or f"tool_call_{index}"
parts.append(f"[tool_call] {label}")
return "\n".join(parts)
@staticmethod
def _extract_error_code(payload: Any) -> str:
if not isinstance(payload, dict):
return ""
error = payload.get("error")
if not isinstance(error, dict):
return ""
return str(error.get("code") or error.get("type") or "").strip()
@staticmethod
def _extract_error_payload(payload: Any) -> Dict[str, str]:
if not isinstance(payload, dict):
return {}
if isinstance(payload.get("response"), dict):
nested_error = RequestManager._extract_error_payload(payload["response"])
if nested_error:
return nested_error
error = payload.get("error")
if isinstance(error, str) and error.strip():
message = error.strip()
return {
"message": message,
"code": "execution_error",
"stack": message,
}
if not isinstance(error, dict):
return {}
message = str(error.get("message") or "").strip()
code = str(error.get("code") or error.get("type") or "execution_error").strip()
stack = json.dumps(error, ensure_ascii=False)
return {
"message": message,
"code": code or "execution_error",
"stack": stack,
}
@staticmethod
def _extract_media_items(payload: Any) -> List[Dict[str, Any]]:
if not isinstance(payload, dict):
return []
if isinstance(payload.get("response"), dict):
return RequestManager._extract_media_items(payload["response"])
media_items: List[Dict[str, Any]] = []
top_media = payload.get("media")
if isinstance(top_media, list):
media_items.extend(item for item in top_media if isinstance(item, dict))
choices = payload.get("choices")
if isinstance(choices, list) and choices:
first = choices[0] or {}
delta = first.get("delta") or {}
message = first.get("message") or {}
for source in (delta, message):
if not isinstance(source, dict):
continue
nested_media = source.get("media")
if isinstance(nested_media, list):
media_items.extend(item for item in nested_media if isinstance(item, dict))
content_media = RequestManager._extract_chat_content_media(source.get("content"))
if content_media:
media_items.extend(content_media)
media_items.extend(RequestManager._extract_responses_event_media(payload))
media_items.extend(RequestManager._extract_responses_output_media(payload))
return media_items
@staticmethod
def _extract_chat_content_media(content: Any) -> List[Dict[str, Any]]:
if isinstance(content, dict):
content = [content]
if not isinstance(content, list):
return []
media_items: List[Dict[str, Any]] = []
for part in content:
if not isinstance(part, dict):
continue
part_type = str(part.get("type") or "").strip()
ref = ""
media_type = ""
if part_type in {"image_url", "input_image", "output_image"}:
image_value = part.get("image_url") or part.get("url") or {}
ref = str(
image_value.get("url") if isinstance(image_value, dict) else image_value
).strip()
media_type = "image"
elif part_type in {"input_audio", "audio_url", "output_audio"}:
audio_value = part.get("audio_url") or part.get("input_audio") or part.get("url") or {}
ref = str(
audio_value.get("url") if isinstance(audio_value, dict) else audio_value
).strip()
media_type = "audio"
elif part_type in {"input_video", "video_url", "output_video"}:
video_value = part.get("video_url") or part.get("url") or {}
ref = str(
video_value.get("url") if isinstance(video_value, dict) else video_value
).strip()
media_type = "video"
if not ref:
continue
media_item: Dict[str, Any] = {"media_type": media_type}
if ref.startswith("data:"):
media_item["data_uri"] = ref
else:
media_item["url"] = ref
detail = str(part.get("detail") or "").strip()
if detail:
media_item["detail"] = detail
mime = str(part.get("mime_type") or part.get("mime") or "").strip()
if mime:
media_item["mime"] = mime
label = str(part.get("label") or "").strip()
if label:
media_item["label"] = label
media_items.append(media_item)
return media_items
@staticmethod
def _extract_responses_output_text(payload: Dict[str, Any]) -> str:
output = payload.get("output")
if not isinstance(output, list):
return ""
parts: List[str] = []
for item in output:
if not isinstance(item, dict):
continue
content = item.get("content")
if isinstance(content, list):
for part in content:
if not isinstance(part, dict):
continue
part_type = str(part.get("type") or "").strip()
if part_type == "output_text":
text = str(part.get("text") or "")
if text:
parts.append(text)
elif part_type in {"output_image", "output_audio", "output_video"}:
ref = RequestManager._extract_media_ref(
part.get("image_url") or part.get("audio_url") or part.get("video_url")
)
if ref:
parts.append(ref)
elif item.get("type") == "function_call":
name = str(item.get("name") or "").strip()
if name:
parts.append(f"[function_call] {name}")
return "\n".join(parts)
@staticmethod
def _extract_responses_event_text(payload: Dict[str, Any]) -> str:
event_type = str(payload.get("type") or "").strip()
if event_type == "response.output_text.delta":
return str(payload.get("delta") or "")
if event_type == "response.content_part.added":
part = payload.get("part")
if not isinstance(part, dict):
return ""
part_type = str(part.get("type") or "").strip()
if part_type == "output_text":
return str(part.get("text") or "")
if event_type == "response.function_call_arguments.done":
name = str(payload.get("name") or "").strip()
if name:
return f"[function_call] {name}"
return ""
@staticmethod
def _extract_responses_event_media(payload: Dict[str, Any]) -> List[Dict[str, Any]]:
event_type = str(payload.get("type") or "").strip()
if event_type != "response.content_part.added":
return []
part = payload.get("part")
if not isinstance(part, dict):
return []
return RequestManager._extract_responses_output_media({
"output": [{
"content": [part],
}],
})
@staticmethod
def _extract_anthropic_content_text(payload: Dict[str, Any]) -> str:
content = payload.get("content")
if not isinstance(content, list):
return ""
parts: List[str] = []
for item in content:
if not isinstance(item, dict):
continue
item_type = str(item.get("type") or "").strip()
if item_type == "text":
text = str(item.get("text") or "")
if text:
parts.append(text)
elif item_type == "tool_use":
name = str(item.get("name") or "").strip()
if name:
parts.append(f"[tool_use] {name}")
return "\n".join(parts)
@staticmethod
def _extract_media_ref(value: Any) -> str:
if isinstance(value, dict):
return str(value.get("url") or value.get("data_uri") or "").strip()
return str(value or "").strip()
@staticmethod
def _extract_responses_output_media(payload: Dict[str, Any]) -> List[Dict[str, Any]]:
output = payload.get("output")
if not isinstance(output, list):
return []
media_items: List[Dict[str, Any]] = []
for item in output:
if not isinstance(item, dict):
continue
content = item.get("content")
if not isinstance(content, list):
continue
for part in content:
if not isinstance(part, dict):
continue
part_type = str(part.get("type") or "").strip()
if part_type == "output_image":
ref = RequestManager._extract_media_ref(part.get("image_url"))
media_type = "image"
elif part_type == "output_audio":
ref = RequestManager._extract_media_ref(part.get("audio_url"))
media_type = "audio"
elif part_type == "output_video":
ref = RequestManager._extract_media_ref(part.get("video_url"))
media_type = "video"
else:
continue
if not ref:
continue
media_item: Dict[str, Any] = {"media_type": media_type}
if ref.startswith("data:"):
media_item["data_uri"] = ref
else:
media_item["url"] = ref
mime = str(part.get("mime_type") or "").strip()
if mime:
media_item["mime"] = mime
label = str(part.get("label") or "").strip()
if label:
media_item["label"] = label
media_items.append(media_item)
return media_items
@staticmethod
def _iter_sse_payloads(chunk: str) -> List[Dict[str, Any]]:
return iter_sse_payloads(chunk)
def _iter_sse_payloads_for_context(
self,
ctx: RequestContext,
chunk: str,
) -> List[Dict[str, Any]]:
"""Parse complete SSE frames while preserving split-frame tails per request."""
if not isinstance(chunk, str):
return []
text = chunk.replace("\r\n", "\n").replace("\r", "\n")
dropped_chars = 0
complete_segments: List[str] = []
with ctx._lock:
combined = f"{ctx._sse_chunk_buffer}{text}"
if "\n\n" not in combined:
# 修复#6:超限时清空缓冲而非截尾——截断片段可能与后续数据拼出坏帧
if len(combined) > MAX_SSE_CHUNK_BUFFER_CHARS:
dropped_chars = len(combined)
ctx._sse_chunk_buffer = ""
else:
ctx._sse_chunk_buffer = combined
else:
segments = combined.split("\n\n")
complete_segments = segments[:-1]
tail = segments[-1]
# 修复#6:尾部超限同样清空丢弃
if tail and len(tail) > MAX_SSE_CHUNK_BUFFER_CHARS:
dropped_chars = len(tail)
ctx._sse_chunk_buffer = ""
else:
ctx._sse_chunk_buffer = tail or ""
if dropped_chars:
logger.debug(
f"[{ctx.request_id}] SSE 监控缓冲超过 {MAX_SSE_CHUNK_BUFFER_CHARS} 字符"
f"(实际 {dropped_chars}),已整体清空丢弃(仅影响监控采集,不影响转发)"
)
if not complete_segments:
return []
return self._iter_sse_payloads("\n\n".join(complete_segments) + "\n\n")
def _flush_sse_payloads_for_context(self, ctx: RequestContext) -> List[Dict[str, Any]]:
"""Parse a final unterminated SSE frame before request history is written."""
if ctx is None or not hasattr(ctx, "_sse_chunk_buffer"):
return []
with ctx._lock:
tail = ctx._sse_chunk_buffer
ctx._sse_chunk_buffer = ""
if not tail:
return []
return self._iter_sse_payloads(f"{tail}\n\n")
def record_request_input(
self,
ctx: RequestContext,
payload: Any,
*,
endpoint: str = "",
route_domain: str = "",
route_group: str = "",
tab_index: Optional[int] = None,
preset_name: Optional[str] = None,
) -> None:
if ctx is None:
return
payload_dict = payload if isinstance(payload, dict) else {}
messages = payload_dict.get("messages") if payload_dict else None
prompt_text = self._messages_to_prompt_text(messages)
is_stream = bool(payload_dict.get("stream")) if payload_dict else False
has_tools = bool(payload_dict.get("tools") or payload_dict.get("functions")) if payload_dict else False
request_type = "工具调用" if has_tools else ("流式" if is_stream else "非流式")
model = str(payload_dict.get("model") or "").strip()
resolved_preset = str(preset_name or payload_dict.get("preset_name") or "").strip()
# 计算完整、未截断的 Prompt 估算 Token
raw_prompt_for_tokens = ""
if isinstance(messages, list):
raw_prompt_for_tokens = "\n\n".join(
f"{msg.get('role', 'message')}: {self._content_to_text(msg.get('content'))}"
for msg in messages if isinstance(msg, dict)
)
else:
raw_prompt_for_tokens = str(messages or "")
prompt_tokens = self._estimate_tokens(raw_prompt_for_tokens)
detail_enabled = self._request_monitor_detail_enabled()
prompt_capture_limit = min(20000, self._request_monitor_capture_chars())
# 修复#3b:重清洗(正则替换/递归拷贝)全部移到 ctx._lock 之外执行,锁内只做赋值。
# payload_dict / prompt_text / messages 均为本地变量,锁外清洗无竞态。
monitor_route_domain = self._canonical_monitor_domain(route_domain)
monitor_update = {
"endpoint": endpoint,
"route_domain": monitor_route_domain,
"target_domain": monitor_route_domain,
"route_group": str(route_group or "").strip(),
"tab_index": tab_index,
"preset_name": resolved_preset,
"model": model,
"request_type": request_type,
"is_stream": is_stream,
"is_multimodal": self._has_multimodal_payload(messages),
"prompt_tokens": prompt_tokens,
}
if detail_enabled and prompt_capture_limit > 0:
monitor_update["prompt"] = self._sanitize_text_for_storage(
prompt_text,
max_chars=prompt_capture_limit,
)
monitor_update["payload"] = self._sanitize_for_storage(payload_dict)
with ctx._lock:
ctx.monitor.update(monitor_update)
def update_request_metadata(self, request_id: str, **metadata: Any) -> bool:
request_key = str(request_id or "").strip()
if not request_key:
return False
with self._requests_lock:
ctx = self._requests.get(request_key)
if not ctx:
return False
with ctx._lock:
for key, value in metadata.items():
if value is None:
continue
if key == "target_domain" and not str(value or "").strip():
continue
if key in {"target_domain", "route_domain"}:
value = self._canonical_monitor_domain(value)
if not value and key == "route_domain":
continue
ctx.monitor[key] = self._sanitize_for_storage(value)
return True
def append_workflow_trace(self, request_id: str, trace: list) -> bool:
"""Append bounded structural traces across chunks/tool rounds; no values."""
ctx = self.get_request(str(request_id or ""))
if ctx is None:
return False
allowed = {"path", "action", "status", "branch", "attempt", "variable", "error_type", "reason"}
with ctx._lock:
run = int(ctx.monitor.get("_workflow_trace_run", 0)) + 1
ctx.monitor["_workflow_trace_run"] = run
previous = list(ctx.monitor.get("workflow_trace") or [])[:500]
for entry in list(trace or [])[:500 - len(previous)]:
if not isinstance(entry, dict):
continue
safe = {k: self._sanitize_for_storage(v) for k, v in entry.items()
if k in allowed and isinstance(v, (str, int, bool))}
previous.append({**safe, "run": run})
ctx.monitor["workflow_trace"] = previous
return True
def capture_external_response(
self,
request_id: str,
response_text: Any,
*,
metadata: Optional[Dict[str, Any]] = None,
append_if_matched: Optional[Any] = None,
strategy: str = "replace",
) -> bool:
"""Attach externally observed response text to a tracked request.
Command-loop helpers can finish independently from page/network listeners.
This lets a late network bridge update the same monitor row and refresh the
history record when it has already been written.
"""
request_key = str(request_id or "").strip()
if not request_key:
return False
with self._requests_lock:
ctx = self._requests.get(request_key)
if not ctx:
return False
text = str(response_text or "")
if not text.strip() and not metadata:
return False
capture_limit = self._request_monitor_capture_chars()
terminal = False
with ctx._lock:
if ctx.status == RequestStatus.RUNNING:
ctx.last_activity_at = time.time()
if text:
stored_text = self._sanitize_text_for_storage(
text,
max_chars=capture_limit if capture_limit > 0 else MAX_CAPTURED_RESPONSE_CHARS,
)
existing_text = str(ctx.monitor.get("response_text") or "").strip()
should_append = False
if strategy == "append":
should_append = True
elif callable(append_if_matched):
try:
should_append = bool(append_if_matched(existing_text))
except Exception:
should_append = False
elif isinstance(append_if_matched, str) and append_if_matched:
should_append = append_if_matched in existing_text
elif bool(append_if_matched):
should_append = True
if existing_text and should_append and stored_text:
stored_text = self._sanitize_text_for_storage(
f"{existing_text}\n\n{stored_text}",
max_chars=capture_limit if capture_limit > 0 else MAX_CAPTURED_RESPONSE_CHARS,
)
ctx.monitor["response_text"] = stored_text
ctx.monitor["has_response_text"] = True
ctx.monitor["response_tokens"] = self._estimate_tokens(stored_text)
if metadata:
for key, value in metadata.items():
if value is None:
continue
ctx.monitor[str(key)] = self._sanitize_for_storage(value)
terminal = ctx.status in (
RequestStatus.COMPLETED,
RequestStatus.CANCELLED,
RequestStatus.FAILED,
)
if terminal:
try:
self._append_monitor_history(ctx)
except Exception as e:
logger.debug(f"刷新外部响应请求监控历史失败: {e}")
return True
def capture_response_chunk(self, ctx: RequestContext, chunk: Any) -> None:
if ctx is None or not isinstance(chunk, str):
return
if chunk:
ctx.mark_activity()
payloads = self._iter_sse_payloads_for_context(ctx, chunk)
if not payloads:
return
self._capture_response_payloads(ctx, payloads)
def _capture_response_payloads(
self,
ctx: RequestContext,
payloads: List[Dict[str, Any]],
) -> None:
if ctx is None or not payloads:
return
capture_limit = self._request_monitor_capture_chars()
detail_enabled = self._request_monitor_detail_enabled()
with ctx._lock:
for payload in payloads:
error = payload.get("error") if isinstance(payload, dict) else None
payload_error = self._extract_error_payload(payload)
text = "" if payload_error else self._extract_response_text(payload)
media_items = self._extract_media_items(payload)
if text:
ctx.monitor["has_response_text"] = True
ctx.monitor["response_tokens"] = self._coerce_token_count(
ctx.monitor.get("response_tokens")
) + self._estimate_tokens(text)
if detail_enabled and capture_limit > 0:
parts = ctx.monitor.setdefault("response_parts", [])
if isinstance(parts, list):
current_len = ctx.monitor.get("response_parts_chars")
if not isinstance(current_len, int):
current_len = sum(len(str(item)) for item in parts)
current_len = max(0, current_len)
remaining = max(0, capture_limit - current_len)
if remaining > 0:
captured = text[:remaining]
parts.append(captured)
ctx.monitor["response_parts_chars"] = current_len + len(captured)
else:
ctx.monitor["response_parts_chars"] = current_len
if media_items:
existing = ctx.monitor.setdefault("media_items", [])
if isinstance(existing, list):
existing.extend(self._sanitize_for_storage(media_items))
ctx.monitor["has_response_media"] = True
ctx.monitor["is_multimodal"] = True
if payload_error:
error_message = str(payload_error.get("message") or "")
if error_message:
ctx.monitor["error_message"] = self._sanitize_text_for_storage(error_message, max_chars=20000)
ctx.monitor["error_code"] = str(payload_error.get("code") or "execution_error")
ctx.monitor["error_stack"] = self._sanitize_text_for_storage(
payload_error.get("stack") or error_message,
max_chars=40000,
)
def capture_response_payload(
self,
ctx: RequestContext,
payload: Any,
*,
error_code: str = "",
) -> None:
if ctx is None:
return
ctx.mark_activity()
text = self._extract_response_text(payload)
media_items = self._extract_media_items(payload)
detail_enabled = self._request_monitor_detail_enabled()
capture_limit = self._request_monitor_capture_chars()
sanitized_payload = self._sanitize_for_storage(payload) if detail_enabled else {}
payload_error = self._extract_error_payload(payload)
payload_error_code = str(payload_error.get("code") or self._extract_error_code(payload)).strip()
# 计算完整、未截断的 Completion 估算 Token
response_tokens = self._estimate_tokens(text)
with ctx._lock:
if text:
ctx.monitor["has_response_text"] = True
if text and detail_enabled and capture_limit > 0:
ctx.monitor["response_text"] = self._sanitize_text_for_storage(text, max_chars=capture_limit)
if media_items:
ctx.monitor["media_items"] = self._sanitize_for_storage(media_items)
ctx.monitor["has_response_media"] = True
ctx.monitor["is_multimodal"] = True
if detail_enabled:
ctx.monitor["response_payload"] = sanitized_payload
ctx.monitor["response_tokens"] = response_tokens
if payload_error:
error_message = str(payload_error.get("message") or "")
if error_message:
ctx.monitor["error_message"] = self._sanitize_text_for_storage(error_message, max_chars=20000)
ctx.monitor["error_code"] = payload_error_code or str(error_code or "execution_error")
ctx.monitor["error_stack"] = self._sanitize_text_for_storage(
payload_error.get("stack") or error_message,
max_chars=40000,
)
if payload_error_code or error_code:
ctx.monitor["error_code"] = payload_error_code or error_code
def capture_error(
self,
ctx: RequestContext,
error: Any,
*,
code: str = "execution_error",
stack: str = "",
) -> None:
if ctx is None:
return
message = str(error or "")
with ctx._lock:
ctx.monitor["error_message"] = self._sanitize_text_for_storage(message, max_chars=20000)
ctx.monitor["error_code"] = str(code or "execution_error")
ctx.monitor["error_stack"] = self._sanitize_text_for_storage(stack or message, max_chars=40000)
def _compact_context_monitor_after_history_append(
self,
monitor: Dict[str, Any],
*,
prompt_text: str,
response_text: str,
prompt_tokens: int,
response_tokens: int,
) -> None:
"""Release heavy request payload references after history has a durable summary."""
monitor["prompt"] = prompt_text
monitor["prompt_tokens"] = prompt_tokens
if response_text:
monitor["response_text"] = response_text
monitor["response_tokens"] = response_tokens
for key in (
"payload",
"response_payload",
"response_parts",
"response_parts_chars",
"media_items",
):
monitor.pop(key, None)
if not response_text:
monitor.pop("response_text", None)
def _append_monitor_history(self, ctx: RequestContext) -> None:
# 修复#1:整个"判重→插入完成"流程持 ctx 专用历史锁执行,消除
# finish_request / cancel_request / capture_external_response 并发时
# 判重(ctx._lock 内)与插入(_history_lock 内)之间的跨锁竞态窗口。
# 该锁为最外层锁;锁内可再获取 ctx._lock / _history_lock,内层顺序保持不变。
with ctx._history_append_lock:
self._append_monitor_history_inner(ctx)
def _append_monitor_history_inner(self, ctx: RequestContext) -> None:
tail_payloads = self._flush_sse_payloads_for_context(ctx)
if tail_payloads:
self._capture_response_payloads(ctx, tail_payloads)
if not self._request_monitor_enabled():
with self._history_lock:
if self._monitor_history:
self._monitor_history = []
self._history_revision_cache = None
with ctx._lock:
ctx.monitor["_history_recorded"] = True
for key in (
"payload",
"response_payload",
"response_parts",
"response_parts_chars",
"media_items",
):
ctx.monitor.pop(key, None)
return
with ctx._lock:
already_recorded = bool(ctx.monitor.get("_history_recorded"))
previous_history_key = str(ctx.monitor.get("_history_key") or "")
if not already_recorded:
ctx.monitor["_history_recorded"] = True
monitor = dict(ctx.monitor)
snapshot = ctx.snapshot()
created_at = float(snapshot["created_at"] or time.time())
started_at = float(snapshot["started_at"] or created_at)
finished_at = float(snapshot["finished_at"] or time.time())
queue_ms = max(0, int((started_at - created_at) * 1000))
generation_ms = max(0, int((finished_at - started_at) * 1000))
duration_ms = max(0, int((finished_at - created_at) * 1000))
response_text = monitor.get("response_text")
if not response_text:
response_parts = monitor.get("response_parts")
if isinstance(response_parts, list):
response_text = "".join(str(item) for item in response_parts)
detail_enabled = self._request_monitor_detail_enabled()
capture_limit = self._request_monitor_capture_chars()
response_capture_limit = capture_limit if capture_limit > 0 else 0
prompt_capture_limit = min(20000, response_capture_limit) if response_capture_limit > 0 else 0
response_text = self._sanitize_text_for_storage(
response_text,
max_chars=response_capture_limit or 1,
) if detail_enabled and response_capture_limit > 0 else ""
prompt_text = self._sanitize_text_for_storage(
monitor.get("prompt", ""),
max_chars=prompt_capture_limit or 1,
) if detail_enabled and prompt_capture_limit > 0 else ""
prompt_tokens = monitor.get("prompt_tokens")
if prompt_tokens is None:
prompt_tokens = self._estimate_tokens(prompt_text)
else:
prompt_tokens = self._coerce_token_count(prompt_tokens)
response_tokens = monitor.get("response_tokens")
if response_tokens is None:
response_tokens = self._estimate_tokens(response_text)
else:
response_tokens = self._coerce_token_count(response_tokens)
media_items = monitor.get("media_items") if isinstance(monitor.get("media_items"), list) else []
has_meaningful_response = (
bool(response_text.strip())
or len(media_items) > 0
or bool(monitor.get("has_response_text"))
or response_tokens > 0
)
status = snapshot["status"]
status_value = status.value if isinstance(status, RequestStatus) else str(status)
stop_sequence_completed = (
status_value == RequestStatus.COMPLETED.value
and str(snapshot["cancel_reason"] or "").strip()
in {"audio_media_fast_return", "stop_sequence", "stream_done"}
)
success = (
status_value == RequestStatus.COMPLETED.value
and (has_meaningful_response or stop_sequence_completed)
)
error_message = self._sanitize_text_for_storage(
monitor.get("error_message") or snapshot["cancel_reason"] or "",
max_chars=10000,
)
error_code = str(monitor.get("error_code") or (status_value if not success else "") or "").strip()
if success and stop_sequence_completed:
error_message = ""
error_code = ""
if (
status_value == RequestStatus.COMPLETED.value
and not has_meaningful_response
and not stop_sequence_completed
):
status_value = RequestStatus.FAILED.value
success = False
error_code = "empty_response"
error_message = "请求流程已结束,但没有捕获到 AI 响应文本或媒体结果。"
error_stack = self._sanitize_text_for_storage(
monitor.get("error_stack") or error_message,
max_chars=20000,
)
record = {
"id": ctx.request_id,
"status": status_value,
"success": success,
"created_at": created_at,
"started_at": started_at,
"finished_at": finished_at,
"duration_ms": duration_ms,
"queue_ms": queue_ms,
"generation_ms": generation_ms,
"target_domain": self._canonical_monitor_domain(monitor.get("target_domain") or monitor.get("route_domain")) or "未知域名",
"route_domain": self._canonical_monitor_domain(monitor.get("route_domain")),
"route_group": str(monitor.get("route_group") or ""),
"preset_name": str(monitor.get("preset_name") or "默认预设"),
"tab_index": monitor.get("tab_index"),
"tab_id": snapshot["tab_id"] or monitor.get("tab_id") or "",
"model": str(monitor.get("model") or ""),
"endpoint": str(monitor.get("endpoint") or ""),
"protocol": str(getattr(ctx, "protocol", "") or "openai.chat"),
"http_request_id": str(getattr(ctx, "http_request_id", "") or ""),
"request_type": str(monitor.get("request_type") or ""),
"is_stream": bool(monitor.get("is_stream")),
"is_multimodal": bool(monitor.get("is_multimodal") or monitor.get("has_response_media")),
"has_response_text": bool(monitor.get("has_response_text")) or response_tokens > 0,
"prompt": prompt_text,
"response": response_text,
"summary": response_text[:180] or ("已完成(响应详情未保存)" if has_meaningful_response else ""),
"error_code": error_code,
"error_message": error_message,
"error_stack": error_stack,
"workflow_trace": list(monitor.get("workflow_trace") or [])[:500],
"cancel_reason": str(snapshot["cancel_reason"] or ""),
"media_count": len(media_items),
"token_estimate": {
"prompt": prompt_tokens,
"response": response_tokens,
"total": prompt_tokens + response_tokens,
"chars": len(prompt_text) + len(response_text),
},
}
record = self._normalize_history_record(record)
with ctx._lock:
ctx.monitor["_history_key"] = str(record.get("history_key") or "")
self._compact_context_monitor_after_history_append(
ctx.monitor,
prompt_text=prompt_text,
response_text=response_text,
prompt_tokens=prompt_tokens,
response_tokens=response_tokens,
)
previous_record = None
with self._history_lock:
if already_recorded:
previous_record = self._replace_monitor_history_record_unlocked(
record,
previous_history_key=previous_history_key,
)
else:
# 修复:显式传入配置上限,否则回落到默认 200 条,与 REQUEST_MONITOR_MAX_RECORDS 冲突并批量截断历史
self._append_sorted_history_record_unlocked(
self._monitor_history, record, max_records=self._request_monitor_max_records()
)
self._history_revision_cache = None
self._trim_monitor_history_unlocked()
if not already_recorded:
prompt_delta = prompt_tokens
response_delta = response_tokens
else:
previous_prompt_tokens, previous_response_tokens = self._history_record_token_counts(
previous_record or {}
)
prompt_delta = prompt_tokens - previous_prompt_tokens
response_delta = response_tokens - previous_response_tokens
if prompt_delta or response_delta:
with self._requests_lock:
self.total_input_tokens = max(0, self.total_input_tokens + prompt_delta)
self.total_output_tokens = max(0, self.total_output_tokens + response_delta)
self._schedule_stats_save()
self._schedule_history_save()
@staticmethod
def _history_revision_unlocked(records: List[Dict[str, Any]]) -> str:
if not records:
return "0::0"
latest = max(
(item for item in records if isinstance(item, dict)),
key=RequestManager._history_sort_value,
default={},
)
latest_time = RequestManager._history_sort_value(latest)
latest_key = latest.get("history_key") or RequestManager._make_history_key(latest)
digest = hashlib.blake2s(digest_size=8)
for item in records:
if not isinstance(item, dict):
continue
for part in RequestManager._history_revision_parts(item):
digest.update(str(part).encode("utf-8", errors="replace"))
digest.update(b"\x1f")
digest.update(b"\x1e")
return f"{len(records)}:{latest_key}:{latest_time}:{digest.hexdigest()}"
@staticmethod
def _history_revision_cache_key(records: List[Dict[str, Any]]) -> tuple[Any, ...]:
if not records:
return (0,)
return tuple(
(
id(item),
RequestManager._history_revision_cache_parts(item),
)
for item in records
if isinstance(item, dict)
)
@staticmethod
def _history_revision_text_cache_marker(value: Any) -> tuple[int, int]:
if value is None or value == "":
return (0, 0)
text = str(value)
return (id(value), len(text))
@staticmethod
def _history_revision_cache_parts(record: Dict[str, Any]) -> tuple[Any, ...]:
token_estimate = record.get("token_estimate")
if not isinstance(token_estimate, dict):
token_estimate = {}
return (
record.get("history_key") or RequestManager._make_history_key(record),
RequestManager._history_sort_value(record),
record.get("status"),
record.get("success"),
record.get("target_domain"),
record.get("route_domain"),
record.get("route_group"),
record.get("preset_name"),
record.get("tab_index"),
record.get("tab_id"),
record.get("model"),
record.get("endpoint"),
record.get("request_type"),
record.get("is_stream"),
record.get("is_multimodal"),
record.get("duration_ms"),
record.get("queue_ms"),
record.get("generation_ms"),
RequestManager._history_revision_text_cache_marker(record.get("summary")),
record.get("error_code"),
RequestManager._history_revision_text_cache_marker(record.get("error_message")),
RequestManager._history_revision_text_cache_marker(record.get("prompt")),
RequestManager._history_revision_text_cache_marker(record.get("response")),
RequestManager._history_revision_text_cache_marker(record.get("error_stack")),
record.get("media_count"),
token_estimate.get("prompt"),
token_estimate.get("response"),
token_estimate.get("total"),
token_estimate.get("chars"),
)
def _get_history_revision_cached(self, records: List[Dict[str, Any]]) -> str:
cache_key = self._history_revision_cache_key(records)
cache = getattr(self, "_history_revision_cache", None)
if cache and cache[0] == cache_key:
return cache[1]
revision = self._history_revision_unlocked(records)
# 修复#5:缓存 value 同时持有记录对象引用(侵入最小方案)。
# 由于旧记录对象被缓存持有而保持存活,其 id() 不可能被新对象复用;
# 因此 cache_key 中 id 相等即等价于 is 同一对象,消除 not_modified 误报。
# (记录 dict 插入历史后不做原地修改,替换总是生成新 dict,故内部字符串 id 同样稳定)
self._history_revision_cache = (cache_key, revision, list(records))
return revision
@staticmethod
def _history_revision_text_part(value: Any) -> str:
text = str(value or "")
if not text:
return ""
digest = hashlib.blake2s(text.encode("utf-8", errors="replace"), digest_size=8).hexdigest()
return f"{len(text)}:{digest}"
@staticmethod
def _history_revision_parts(record: Dict[str, Any]) -> tuple[Any, ...]:
token_estimate = record.get("token_estimate")
if not isinstance(token_estimate, dict):
token_estimate = {}
return (
record.get("history_key") or RequestManager._make_history_key(record),
RequestManager._history_sort_value(record),
record.get("status"),
record.get("success"),
record.get("target_domain"),
record.get("route_domain"),
record.get("route_group"),
record.get("preset_name"),
record.get("tab_index"),
record.get("tab_id"),
record.get("model"),
record.get("endpoint"),
record.get("request_type"),
record.get("is_stream"),
record.get("is_multimodal"),
record.get("duration_ms"),
record.get("queue_ms"),
record.get("generation_ms"),
record.get("summary"),
record.get("error_code"),
RequestManager._history_revision_text_part(record.get("error_message")),
RequestManager._history_revision_text_part(record.get("prompt")),
RequestManager._history_revision_text_part(record.get("response")),
RequestManager._history_revision_text_part(record.get("error_stack")),
record.get("media_count"),
token_estimate.get("prompt"),
token_estimate.get("response"),
token_estimate.get("total"),
token_estimate.get("chars"),
)
@staticmethod
def _history_preview_text(value: Any, max_chars: int = 220) -> str:
text = str(value or "").replace("\r\n", "\n").replace("\r", "\n")
text = re.sub(r"\s+", " ", text).strip()
if len(text) > max_chars:
return text[:max_chars] + "..."
return text
@classmethod
def _to_history_list_record(cls, record: Dict[str, Any]) -> Dict[str, Any]:
item = dict(record)
token_estimate = item.get("token_estimate")
if isinstance(token_estimate, dict):
item["token_estimate"] = dict(token_estimate)
prompt_text = str(item.pop("prompt", "") or "")
response_text = str(item.pop("response", "") or "")
error_stack = str(item.pop("error_stack", "") or "")
item.pop("payload", None)
item.pop("workflow_trace", None)
item.pop("response_payload", None)
item.pop("response_parts", None)
item.pop("response_parts_chars", None)
summary = str(item.get("summary") or "").strip()
if not summary:
summary = response_text or str(item.get("error_message") or "")
item["summary"] = cls._history_preview_text(summary, max_chars=240)
item["prompt_preview"] = cls._history_preview_text(prompt_text, max_chars=240)
item["response_preview"] = cls._history_preview_text(response_text, max_chars=300)
item["has_detail"] = bool(prompt_text or response_text or error_stack)
item["detail_loaded"] = False
item["detail_text_lengths"] = {
"prompt": len(prompt_text),
"response": len(response_text),
"error_stack": len(error_stack),
}
error_message = str(item.get("error_message") or "")
if len(error_message) > 800:
item["error_message"] = error_message[:800] + "..."
return item
def get_request_history_payload(
self,
limit: int = 200,
include_detail: bool = False,
if_revision: Optional[str] = None,
) -> Dict[str, Any]:
max_records = self._request_monitor_max_records()
if not self._request_monitor_enabled():
return {
"records": [],
"count": 0,
"max_records": max_records,
"revision": "0::0",
"not_modified": False,
"enabled": False,
}
try:
count = max(1, min(max_records, int(limit or max_records or 1)))
except Exception:
count = max(1, max_records)
requested_revision = str(if_revision or "").strip()
with self._history_lock:
self._trim_monitor_history_unlocked()
history_refs_all = [
item
for item in self._monitor_history[-count:]
if isinstance(item, dict)
]
if not self._history_records_are_ordered(self._monitor_history):
history_refs_all = [
item
for item in self._monitor_history
if isinstance(item, dict)
]
history_refs_all = self._sort_history_records(history_refs_all)
history_refs = history_refs_all[-count:]
revision = self._get_history_revision_cached(history_refs)
if not include_detail and requested_revision and requested_revision == revision:
return {
"records": [],
"count": 0,
"max_records": max_records,
"revision": revision,
"not_modified": True,
"enabled": True,
}
history = (
[copy.deepcopy(item) for item in history_refs]
if include_detail
else list(history_refs)
)
if include_detail:
records = list(reversed(history))
else:
records = [self._to_history_list_record(item) for item in reversed(history)]
return {
"records": records,
"count": len(records),
"max_records": max_records,
"revision": revision,
"not_modified": False,
"enabled": True,
}
def get_request_history(self, limit: int = 200) -> List[Dict[str, Any]]:
return self.get_request_history_payload(limit, include_detail=True)["records"]
def get_request_history_record(self, request_id: str) -> Optional[Dict[str, Any]]:
request_key = str(request_id or "").strip()
if not request_key or not self._request_monitor_enabled():
return None
with self._history_lock:
if self._history_records_are_ordered(self._monitor_history):
history_iterable = reversed(self._monitor_history)
else:
history = [
item
for item in self._monitor_history
if isinstance(item, dict)
]
history_iterable = reversed(self._sort_history_records(history))
id_match = None
for item in history_iterable:
if not isinstance(item, dict):
continue
if str(item.get("history_key") or "").strip() == request_key:
return copy.deepcopy(item)
if id_match is None and str(item.get("id") or "").strip() == request_key:
id_match = item
if id_match is not None:
return copy.deepcopy(id_match)
return None
def create_request(self, client_fp: str = "") -> RequestContext:
"""创建新请求"""
request_id = self._generate_id()
ctx = RequestContext(request_id=request_id, client_fp=str(client_fp or ""))
try: # R2-2:按协议入口设置的 ChatJob 标注协议(未设置时为 openai.chat)
from app.core.chat_job import current_protocol
ctx.protocol = current_protocol()
except Exception:
pass
try: # R2-7:记录本次 HTTP 请求的 X-Request-ID(由 RequestMetricsMiddleware 设置)
from app.services.metrics import REQUEST_ID
ctx.http_request_id = REQUEST_ID.get()
except Exception:
pass
cleanup_history_contexts: List[RequestContext] = []
with self._requests_lock:
self._requests[request_id] = ctx
cleanup_history_contexts = self._cleanup_old_requests()
for stale_ctx in cleanup_history_contexts:
try:
self._append_monitor_history(stale_ctx)
except Exception as e:
logger.debug(f"写入清理请求监控历史失败: {e}")
# 设置上下文后记录日志
token = _request_context.set(request_id)
try:
logger.info(f"创建 (X-Request-ID={ctx.http_request_id})" if ctx.http_request_id else "创建")
finally:
_request_context.reset(token)
return ctx
def _generate_id(self) -> str:
"""生成简短的请求 ID"""
with self._requests_lock:
self._request_counter += 1
return f"req-{self._request_counter:03d}"
def _cleanup_old_requests(self) -> List[RequestContext]:
"""清理旧请求(修复版:不因单个未完成请求阻塞所有清理)"""
now = time.time()
# 修复:改为运行时读取,支持前端热配置(原类属性在 import 时求值,改配置需重启)
zombie_ttl = self._zombie_ttl()
over_capacity = len(self._requests) > self._max_history
to_delete = []
history_contexts: List[RequestContext] = []
for req_id, ctx in list(self._requests.items()):
snapshot = ctx.snapshot(now)
status = snapshot["status"]
# Keep a live worker discoverable by ownership/recovery even when
# its HTTP request has already reached a terminal state.
worker = getattr(ctx, "worker_thread", None)
worker_alive = isinstance(worker, threading.Thread) and worker.is_alive()
# 已终态的可以删除
if snapshot["is_terminal"]:
if worker_alive:
continue
if over_capacity and len(self._requests) - len(to_delete) > self._max_history:
to_delete.append(req_id)
history_contexts.append(ctx)
# 超时的 RUNNING 请求视为僵尸,强制标记失败
elif status == RequestStatus.RUNNING:
started = snapshot["started_at"] or snapshot["created_at"]
active_at = snapshot.get("last_activity_at") or started
if now - active_at > zombie_ttl:
logger.warning(
f"[{req_id}] 僵尸请求 (无活动 {now - active_at:.0f}s, 运行 {now - started:.0f}s),强制清理"
)
ctx.mark_failed("zombie_timeout")
if not worker_alive:
to_delete.append(req_id)
history_contexts.append(ctx)
elif status == RequestStatus.QUEUED:
queued_at = snapshot["created_at"] or now
if now - queued_at > zombie_ttl:
logger.warning(
f"[{req_id}] 排队请求超时 (等待 {now - queued_at:.0f}s),强制清理"
)
ctx.mark_failed("queued_timeout")
if not worker_alive:
to_delete.append(req_id)
history_contexts.append(ctx)
# 批量删除
for req_id in to_delete:
del self._requests[req_id]
if to_delete:
logger.debug(f"清理了 {len(to_delete)} 个旧请求")
return history_contexts
def _run_zombie_sweep_loop(self) -> None:
"""修复#4:每 60 秒清扫一次僵尸/超时请求(daemon 线程,捕获所有异常防止线程死亡)"""
while True:
time.sleep(60)
try:
with self._requests_lock:
cleanup_history_contexts = self._cleanup_old_requests()
for stale_ctx in cleanup_history_contexts:
try:
self._append_monitor_history(stale_ctx)
except Exception as e:
logger.debug(f"写入清理请求监控历史失败: {e}")
except Exception as e:
logger.debug(f"僵尸请求定时清扫异常: {e}")
def start_request(self, ctx: RequestContext, tab_id: str = None):
"""标记请求开始执行"""
ctx.mark_running(tab_id)
with self._requests_lock:
self.total_requests += 1
self._schedule_stats_save()
# 日志由调用方在上下文中记录,这里不再重复
def bind_tab(self, request_id: str, tab_id: str) -> bool:
"""为已创建/运行中的请求补绑真实标签页 ID。"""
if not request_id or not tab_id:
return False
with self._requests_lock:
ctx = self._requests.get(request_id)
if not ctx or ctx.is_terminal():
return False
with ctx._lock:
previous_tab_id = str(ctx.tab_id or "").strip()
ctx.tab_id = tab_id
logger.debug(
f"[{request_id}] 绑定标签页: {previous_tab_id or '-'} -> {tab_id} "
f"(status={ctx.status.value}, cancel_reason={ctx.cancel_reason or '-'})"
)
return True
def finish_request(self, ctx: RequestContext, success: bool = True):
"""标记请求结束"""
with ctx._lock:
if ctx.status in (
RequestStatus.COMPLETED,
RequestStatus.FAILED,
):
pass
elif ctx._cancel_flag or ctx.status == RequestStatus.CANCELLED:
ctx.status = RequestStatus.CANCELLED
elif ctx.status == RequestStatus.RUNNING:
ctx.status = RequestStatus.COMPLETED if success else RequestStatus.FAILED
elif ctx.status not in (
RequestStatus.COMPLETED,
RequestStatus.FAILED,
):
ctx.status = RequestStatus.COMPLETED if success else RequestStatus.FAILED
if ctx.finished_at is None:
ctx.finished_at = time.time()
snapshot = ctx.snapshot()
# 设置上下文后记录日志
token = _request_context.set(ctx.request_id)
try:
logger.info(
f"完成 ({snapshot['duration']:.1f}s, status={snapshot['status'].value}, "
f"tab={snapshot['tab_id'] or '-'}, reason={snapshot['cancel_reason'] or '-'})"
)
finally:
_request_context.reset(token)
try:
self._append_monitor_history(ctx)
except Exception as e:
logger.debug(f"写入请求监控历史失败: {e}")
try: # R2-2 / R2-7:按协议与最终状态计数(每个请求只记一次)
with ctx._lock:
already_reported = getattr(ctx, "_metrics_reported", False)
ctx._metrics_reported = True
if not already_reported:
from app.services.metrics import CHAT_REQUEST_DURATION, CHAT_REQUESTS
protocol = str(getattr(ctx, "protocol", "") or "openai.chat")
CHAT_REQUESTS.inc(protocol=protocol, status=snapshot["status"].value)
CHAT_REQUEST_DURATION.observe(float(snapshot.get("duration") or 0.0), protocol=protocol)
except Exception as e:
logger.debug(f"记录请求指标失败: {e}")
# Count a terminal request once, not every cleanup/wrapper invocation.
# Partial text, tool text and media are real output; empty SSE keepalives are not.
with ctx._lock:
reported = getattr(ctx, '_cancel_storm_reported', False)
ctx._cancel_storm_reported = True
client_fp = ctx.client_fp
has_output = bool(ctx.monitor.get('has_response_text') or
ctx.monitor.get('has_response_media') or
int(ctx.monitor.get('response_tokens') or 0) > 0)
status, reason = ctx.status, str(ctx.cancel_reason or '')
if client_fp and not reported:
if status == RequestStatus.COMPLETED or has_output:
cancel_storm_guard.record_success(client_fp)
elif status == RequestStatus.CANCELLED:
cancel_storm_guard.record_cancel(client_fp, has_output=False, reason=reason)
def cancel_request(self, request_id: str, reason: str = "manual") -> bool:
"""取消指定请求"""
with self._requests_lock:
ctx = self._requests.get(request_id)
if not ctx:
return False
if ctx.is_terminal():
return False
ctx.request_cancel(reason)
try:
self._append_monitor_history(ctx)
except Exception as e:
logger.debug(f"写入取消请求监控历史失败: {e}")
return True
def get_request(self, request_id: str) -> Optional[RequestContext]:
with self._requests_lock:
return self._requests.get(request_id)
def get_running_requests(self, tab_id: str = None) -> list:
"""获取所有正在执行的请求"""
with self._requests_lock:
contexts = list(self._requests.values())
return [ctx for ctx in contexts if ctx.is_running_for_tab(tab_id)]
def get_status(self) -> Dict[str, Any]:
"""获取管理器状态"""
with self._requests_lock:
contexts = list(self._requests.values())
status_counts = {}
running = []
now = time.time()
for ctx in contexts:
summary = ctx.status_summary(now)
status = summary["status"]
status_value = status.value if isinstance(status, RequestStatus) else str(status)
status_counts[status_value] = status_counts.get(status_value, 0) + 1
if status == RequestStatus.RUNNING:
running.append({
"request_id": summary["request_id"],
"tab_id": summary["tab_id"],
"duration": round(summary["duration"], 1),
})
return {
"running_count": len(running),
"running_requests": running,
"total_tracked": len(contexts),
"status_counts": status_counts,
}
# ================= 兼容旧接口 =================
def is_locked(self) -> bool:
"""兼容旧接口 - 检查是否有正在执行的请求"""
with self._requests_lock:
contexts = list(self._requests.values())
return any(ctx.is_running_for_tab() for ctx in contexts)
def get_current_request_id(self, tab_id: str = None) -> Optional[str]:
"""兼容旧接口 - 获取当前执行的请求ID(返回第一个)"""
with self._requests_lock:
contexts = list(self._requests.values())
for ctx in contexts:
if ctx.is_running_for_tab(tab_id):
return ctx.request_id
return None
def cancel_current(self, reason: str = "manual", tab_id: str = None) -> bool:
"""取消当前正在执行的请求。传入 tab_id 时仅取消该标签页关联请求。"""
cancelled = False
for ctx in self.get_running_requests(tab_id=tab_id):
if self.cancel_request(ctx.request_id, reason):
cancelled = True
return cancelled
def force_release(self, tab_id: str = None) -> bool:
"""兼容旧接口 - 强制取消运行中的请求。传入 tab_id 时仅取消该标签页请求。"""
return self.cancel_current("force_release", tab_id=tab_id)
# ================= 全局单例 =================
request_manager = RequestManager()
# ================= 辅助函数 =================
async def watch_client_disconnect(request, ctx: RequestContext,
check_interval: float = 0.5):
"""监控客户端连接状态"""
try:
while not ctx.is_terminal():
if await request.is_disconnected():
ctx.request_cancel("client_disconnected")
break
await asyncio.sleep(check_interval)
except asyncio.CancelledError:
pass
except Exception as e:
logger.debug(f"断开检测异常: {e}")