""" request_manager.py - 请求生命周期管理器(v2.0) v2.0 改动: - 移除全局执行锁(锁转移到 TabPoolManager) - 保留请求追踪、状态管理、取消信号功能 - acquire/release 改为标记状态,不再阻塞 """ import asyncio import atexit import threading import json import os import time import uuid import re import math import hashlib import copy import random from enum import Enum from dataclasses import dataclass, field from typing import Optional, Dict, Any, List from collections import OrderedDict from app.core.config import BrowserConstants, get_logger, _request_context from app.services.sse_utils import iter_sse_payloads from app.utils.site_url import get_canonical_route_domain logger = get_logger("REQUEST") MAX_CAPTURED_RESPONSE_CHARS = 30000 MAX_SSE_CHUNK_BUFFER_CHARS = 262144 # 修复#3a:data:URI 与超长 base64 正则改为模块级预编译,供文本清洗与 token 估算共用 _DATA_URI_PATTERN = re.compile( r"data:(?:image|audio|video)/[a-zA-Z0-9.+-]+;base64,[A-Za-z0-9+/=\r\n]{1024,}", re.IGNORECASE, ) _LONG_BASE64_PATTERN = re.compile( r"(? float: try: return max(0.0, min(60.0, float(os.getenv("REQUEST_HISTORY_SAVE_DEBOUNCE_SEC", "5") or 5))) except (TypeError, ValueError): return 5.0 def _get_positive_int_env(name: str, default: int) -> int: try: value = int(os.getenv(name, str(default))) except Exception: return default return value if value > 0 else default def _browser_bool(key: str, default: bool = True) -> bool: value = BrowserConstants.get(key) if value is None: return bool(default) if isinstance(value, bool): return value if isinstance(value, (int, float)): return bool(value) text = str(value).strip().lower() if text in {"1", "true", "yes", "y", "on"}: return True if text in {"0", "false", "no", "n", "off"}: return False return bool(default) def _browser_int( key: str, default: int, *, min_value: int = 0, max_value: Optional[int] = None, ) -> int: try: value = int(BrowserConstants.get(key)) except Exception: value = int(default) value = max(int(min_value), value) if max_value is not None: value = min(int(max_value), value) return value # ================= 客户端高频取消防抖熔断器 ================= class CancelStormGuard: """Bound rapid zero-output abort loops without delaying normal requests. Cleanup correctness is the primary protection. As a second line of defense, four transport cancellations in five seconds open a five-second circuit for that client only. Rejected retries create no workflow/request record. This is not an authorization boundary or a global/DDoS rate limiter. """ def __init__(self, window_sec: float = 5.0, max_backoff_sec: float = 0.5, threshold: int = 4, cooldown_sec: float = 5.0, max_clients: int = 2048): self.window_sec = max(.1, float(window_sec)) self.max_backoff_sec = max_backoff_sec # Legacy constructor compatibility. self.threshold = max(2, int(threshold)) self.cooldown_sec = max(.1, float(cooldown_sec)) self.max_clients = max(1, int(max_clients)) self._lock = threading.Lock() self._cancel_records = OrderedDict() self._blocked_until = {} def get_client_fingerprint(self, request: Any = None, auth_header: Optional[str] = None) -> str: client = getattr(request, 'client', None) headers = getattr(request, 'headers', {}) or {} parts = [str(getattr(client, 'host', '') or ''), str(headers.get('authorization') or headers.get('x-api-key') or auth_header or ''), str(headers.get('user-agent') or '')] # Never log token fragments, raw credentials, or request content. Do not # trust arbitrary X-Forwarded-For values as a client-controlled bypass. return hashlib.sha256(json.dumps(parts, ensure_ascii=True).encode()).hexdigest()[:24] def _prune_expired_unlocked(self, now: float): for key, records in list(self._cancel_records.items()): if self._blocked_until.get(key, 0) > now: continue if not records or now - records[-1] > self.window_sec: self._cancel_records.pop(key, None) self._blocked_until.pop(key, None) def record_cancel(self, fingerprint: str, has_output: bool = False, reason: str = 'unknown'): if not fingerprint: return if has_output: self.record_success(fingerprint) return if reason not in {'coroutine_cancelled', 'client_disconnected', 'stream_closed', 'unknown', 'cleanup'}: return now = time.monotonic() opened = False with self._lock: if self._blocked_until.get(fingerprint, 0) > now: return # Already in-flight failures must not extend the cooldown indefinitely. self._prune_expired_unlocked(now) records = self._cancel_records.pop(fingerprint, []) records = [t for t in records if now - t <= self.window_sec] records.append(now) self._cancel_records[fingerprint] = records[-self.threshold:] if len(records) >= self.threshold: self._blocked_until[fingerprint] = now + self.cooldown_sec opened = True while len(self._cancel_records) > self.max_clients: key, _ = self._cancel_records.popitem(last=False) self._blocked_until.pop(key, None) if opened: logger.warning( f'[CANCEL_STORM_GUARD] client={fingerprint[:12]} ' f'{self.window_sec:g}s 内 {self.threshold} 次零输出异常取消;' f'冷却 {self.cooldown_sec:g}s,后续请求返回 429 / Retry-After,不启动网页工作流' ) def record_success(self, fingerprint: str): with self._lock: self._cancel_records.pop(fingerprint, None) self._blocked_until.pop(fingerprint, None) async def maybe_backoff(self, fingerprint: str) -> float: """Compatibility entry point: reject an open circuit instead of queuing sleeps.""" from fastapi import HTTPException now = time.monotonic() with self._lock: until = self._blocked_until.get(fingerprint, 0) remaining = until - now if until and remaining <= 0: self._blocked_until.pop(fingerprint, None) self._cancel_records.pop(fingerprint, None) if remaining > 0: retry_after = max(1, math.ceil(remaining)) raise HTTPException( status_code=429, detail={'error': {'code': 'client_cancel_storm', 'type': 'rate_limit_error', 'message': '检测到连续零输出取消,请停止自动重试并在冷却后重试。', 'retry_after_seconds': retry_after}}, headers={'Retry-After': str(retry_after)}, ) return 0.0 cancel_storm_guard = CancelStormGuard() class RequestStatus(Enum): """请求状态枚举""" QUEUED = "queued" RUNNING = "running" COMPLETED = "completed" CANCELLED = "cancelled" FAILED = "failed" @dataclass class RequestContext: """请求上下文""" request_id: str status: RequestStatus = RequestStatus.QUEUED created_at: float = field(default_factory=time.time) started_at: Optional[float] = None finished_at: Optional[float] = None _cancel_flag: bool = field(default=False, repr=False) cancel_reason: Optional[str] = None _lock: threading.Lock = field(default_factory=threading.Lock, repr=False) # 修复#1:历史写入专用锁,串行化 _append_monitor_history 的"判重→插入"全过程; # 必须最外层获取,持有期间允许再拿 ctx._lock / manager._history_lock(内层顺序不变) _history_append_lock: threading.Lock = field(default_factory=threading.Lock, repr=False) _sse_chunk_buffer: str = field(default="", repr=False) # v2.0 新增:关联的标签页 ID tab_id: Optional[str] = None client_fp: str = "" protocol: str = "openai.chat" # R2-2:openai.chat / anthropic.messages / openai.responses http_request_id: str = "" # R2-7:客户端可见的 X-Request-ID,把 HTTP 请求与 req-xxx 日志关联起来 monitor: Dict[str, Any] = field(default_factory=dict) started_at_monotonic: Optional[float] = field(default=None, repr=False) last_activity_at: Optional[float] = field(default=None, repr=False) def should_stop(self) -> bool: with self._lock: return self._cancel_flag def request_cancel(self, reason: str = "unknown"): with self._lock: if self.status in ( RequestStatus.COMPLETED, RequestStatus.CANCELLED, RequestStatus.FAILED, ): return if self._cancel_flag: return self._cancel_flag = True self.cancel_reason = reason if self.finished_at is None: self.finished_at = time.time() self.status = RequestStatus.CANCELLED logger.info(f"[{self.request_id}] 取消 ({reason})") def mark_worker_stop_requested(self, reason: str = "worker_stop_requested"): with self._lock: self._cancel_flag = True self.cancel_reason = reason if self.finished_at is None: self.finished_at = time.time() def mark_running(self, tab_id: str = None): with self._lock: if self.status in (RequestStatus.COMPLETED, RequestStatus.CANCELLED, RequestStatus.FAILED): return self.started_at = time.time() self.started_at_monotonic = time.monotonic() self.last_activity_at = self.started_at self.finished_at = None self.tab_id = tab_id if self._cancel_flag: self.status = RequestStatus.CANCELLED if self.finished_at is None or self.finished_at < self.started_at: self.finished_at = self.started_at else: self.status = RequestStatus.RUNNING def reset_timeout(self): with self._lock: self.started_at_monotonic = time.monotonic() self.last_activity_at = time.time() logger.debug(f"[{self.request_id}] 重置请求绝对超时计时") def mark_activity(self) -> None: """Refresh idle activity without extending the request's hard timeout.""" with self._lock: if self.status == RequestStatus.RUNNING: self.last_activity_at = time.time() def mark_completed(self): with self._lock: if self.status == RequestStatus.RUNNING or ( self.status == RequestStatus.CANCELLED and self.cancel_reason in { "audio_media_fast_return", "stop_sequence", "stream_done", } ): self.status = RequestStatus.COMPLETED self.finished_at = time.time() def mark_failed(self, reason: str = None): with self._lock: # 修复#2:终态保护——已 COMPLETED/FAILED/CANCELLED 的请求不再改写为 FAILED if self.status in ( RequestStatus.COMPLETED, RequestStatus.FAILED, RequestStatus.CANCELLED, ): logger.debug( f"[{self.request_id}] mark_failed 忽略:已处于终态 " f"{self.status.value} (reason={reason or '-'})" ) return self._cancel_flag = True self.status = RequestStatus.FAILED self.finished_at = time.time() if reason: self.cancel_reason = reason def get_duration(self) -> float: with self._lock: end = self.finished_at or time.time() start = self.started_at or self.created_at return max(0.0, end - start) def is_terminal(self) -> bool: with self._lock: return self.status in ( RequestStatus.COMPLETED, RequestStatus.CANCELLED, RequestStatus.FAILED, ) def is_running_for_tab(self, tab_id: str = None) -> bool: target_tab_id = str(tab_id or "").strip() with self._lock: if self.status != RequestStatus.RUNNING: return False if target_tab_id and str(self.tab_id or "").strip() != target_tab_id: return False return True def status_summary(self, now: Optional[float] = None) -> Dict[str, Any]: current_time = time.time() if now is None else float(now) with self._lock: status = self.status tab_id = self.tab_id if status == RequestStatus.RUNNING: duration_end = self.finished_at or current_time duration_start = self.started_at or self.created_at duration = max(0.0, duration_end - duration_start) else: duration = 0.0 return { "request_id": self.request_id, "status": status, "tab_id": tab_id, "duration": duration, } def snapshot(self, now: Optional[float] = None) -> Dict[str, Any]: current_time = time.time() if now is None else float(now) with self._lock: status = self.status created_at = self.created_at started_at = self.started_at finished_at = self.finished_at tab_id = self.tab_id cancel_reason = self.cancel_reason last_activity_at = self.last_activity_at duration_end = finished_at or current_time duration_start = started_at or created_at return { "request_id": self.request_id, "status": status, "created_at": created_at, "started_at": started_at, "finished_at": finished_at, "tab_id": tab_id, "cancel_reason": cancel_reason, "last_activity_at": last_activity_at, "duration": max(0.0, duration_end - duration_start), "is_terminal": status in ( RequestStatus.COMPLETED, RequestStatus.CANCELLED, RequestStatus.FAILED, ), } class RequestManager: """ 请求管理器(v2.0 - 纯追踪模式) v2.0 改动: - 不再持有执行锁 - 只负责请求追踪和取消信号 """ _instance: Optional['RequestManager'] = None _instance_lock = threading.Lock() # 僵尸请求超时时间(秒)- 超过此时间的 RUNNING 请求将被强制清理 # 注意:该类属性在 import 时求值,仅作兼容兜底;运行时请使用 _zombie_ttl() 以支持热配置 ZOMBIE_TTL = _get_positive_int_env("REQUEST_ZOMBIE_TTL", 600) def __new__(cls): if cls._instance is None: with cls._instance_lock: if cls._instance is None: instance = super().__new__(cls) instance._initialized = False cls._instance = instance return cls._instance def __init__(self): if self._initialized: return self._requests: OrderedDict[str, RequestContext] = OrderedDict() self._requests_lock = threading.Lock() self._request_counter = 0 # 请求计数器 self._max_history = 100 self._stats_file = "config/app_stats.json" self._history_file = "config/request_history.json" # R2-5:单例被重新初始化(测试会这样做)时,丢弃上一次缓存的存储连接,按当前路径/环境重新打开 previous_store = getattr(self, "_store", None) if previous_store is not None: previous_store.close() self._store = None self._monitor_history: List[Dict[str, Any]] = [] self._history_lock = threading.Lock() self._history_save_lock = threading.Lock() self._stats_save_lock = threading.Lock() self._save_schedule_lock = threading.Lock() # 修复#5:缓存三元组 (cache_key, revision, 记录对象引用列表), # 持有引用保证 id() 在缓存有效期内不被 CPython 地址复用 self._history_revision_cache: Optional[tuple[tuple[Any, ...], str, List[Dict[str, Any]]]] = None self._stats_save_requested = False self._history_save_requested = False self._stats_save_worker: Optional[threading.Thread] = None self._history_save_worker: Optional[threading.Thread] = None # P0-7:历史落盘去抖(REQUEST_HISTORY_SAVE_DEBOUNCE_SEC,默认 5s),进程退出时强制刷盘 self._history_flush_event = threading.Event() atexit.register(self.flush_pending_saves) self.total_requests = 0 self.total_input_tokens = 0 self.total_output_tokens = 0 self._cleanup_stale_temp_files() self._load_stats() self._load_history() # 修复#4:启动定时僵尸清扫 daemon 线程,低流量时段也能清理超时请求 # (保留 create_request 里的现有触发;daemon 线程随进程退出,无需 shutdown 处理) self._zombie_sweep_thread = threading.Thread( target=self._run_zombie_sweep_loop, daemon=True, name="request-zombie-sweep", ) self._zombie_sweep_thread.start() self._initialized = True logger.debug("RequestManager 初始化完成") def _runtime_store(self): """R2-5:请求历史与统计的 SQLite 存储;默认与旧 JSON 文件同目录(config/runtime.sqlite3)。""" store = getattr(self, "_store", None) if store is None: from app.services.storage.runtime_store import RuntimeStore history_file = getattr(self, "_history_file", "config/request_history.json") path = os.getenv("RUNTIME_DB_PATH") or os.path.join(os.path.dirname(history_file) or ".", "runtime.sqlite3") store = RuntimeStore(path) self._store = store return store def _load_stats(self): try: from app.services.storage.runtime_store import migrate_legacy_file store = self._runtime_store() data: Dict[str, Any] = store.load_stats() stats_file = getattr(self, "_stats_file", "") if not data and stats_file and os.path.exists(stats_file): # 首次启用 SQLite:导入旧的 config/app_stats.json,原文件改名备份 with open(stats_file, "r", encoding="utf-8") as f: legacy = json.load(f) data = { key: self._coerce_token_count(legacy.get(key, 0)) for key in ("total_requests", "total_input_tokens", "total_output_tokens") } if isinstance(legacy, dict) else {} if data: store.save_stats(data) migrate_legacy_file(stats_file) self.total_requests = self._coerce_token_count(data.get("total_requests", 0)) self.total_input_tokens = self._coerce_token_count(data.get("total_input_tokens", 0)) self.total_output_tokens = self._coerce_token_count(data.get("total_output_tokens", 0)) except Exception as e: logger.debug(f"加载状态失败: {e}") def _save_stats(self): try: with self._stats_save_lock: self._runtime_store().save_stats({ "total_requests": self.total_requests, "total_input_tokens": self.total_input_tokens, "total_output_tokens": self.total_output_tokens, }) except Exception as e: logger.debug(f"保存状态失败: {e}") def _cleanup_stale_temp_files(self): for path in (self._stats_file + ".tmp", self._history_file + ".tmp"): try: if os.path.exists(path): os.remove(path) logger.debug(f"清理遗留临时文件: {path}") except Exception as e: logger.debug(f"清理遗留临时文件失败: {path}, {e}") def _request_monitor_max_records(self) -> int: return _browser_int("REQUEST_MONITOR_MAX_RECORDS", 200, min_value=0, max_value=2000) def _zombie_ttl(self) -> int: """僵尸请求判定阈值(秒)。优先读 BrowserConstants,便于前端热配置。""" return _browser_int("REQUEST_ZOMBIE_TTL", int(self.ZOMBIE_TTL or 600), min_value=30, max_value=3600) def _request_monitor_enabled(self) -> bool: return _browser_bool("REQUEST_MONITOR_ENABLED", True) and self._request_monitor_max_records() > 0 def _request_monitor_detail_enabled(self) -> bool: return _browser_bool("REQUEST_MONITOR_DETAIL_ENABLED", True) def _request_monitor_save_to_file(self) -> bool: return _browser_bool("REQUEST_MONITOR_SAVE_TO_FILE", True) def _request_monitor_capture_chars(self) -> int: return _browser_int( "REQUEST_MONITOR_MAX_CAPTURED_RESPONSE_CHARS", MAX_CAPTURED_RESPONSE_CHARS, min_value=0, max_value=200000, ) def _trim_monitor_history_unlocked(self) -> None: max_records = self._request_monitor_max_records() if max_records <= 0: if self._monitor_history: self._monitor_history = [] self._history_revision_cache = None return if len(self._monitor_history) > max_records: self._monitor_history = self._sort_history_records(self._monitor_history)[-max_records:] self._history_revision_cache = None def _load_history(self): try: if not self._request_monitor_enabled(): self._monitor_history = [] return from app.services.storage.runtime_store import migrate_legacy_file store = self._runtime_store() max_to_load = self._request_monitor_max_records() if store.history_count() > 0: records = store.load_history(max_to_load) elif os.path.exists(self._history_file): # R2-5 首次启用 SQLite:导入旧的 JSON 历史(兼容两种旧格式),原文件改名备份 with open(self._history_file, "r", encoding="utf-8") as f: data = json.load(f) # B2:旧版历史文件是顶层数组,必须先按类型分支再取值—— # 旧实现先调用 data.get(...),list 没有 .get 直接抛错,兼容分支永远走不到,恢复 0 条。 if isinstance(data, list): records = data elif isinstance(data, dict): records = data.get("records", []) else: records = [] if isinstance(records, list): store.sync_history([item for item in records if isinstance(item, dict)]) migrate_legacy_file(self._history_file) else: return if isinstance(records, list): raw_records = [item for item in records if isinstance(item, dict)] max_records = self._request_monitor_max_records() if max_records <= 0: # 注意 lst[-0:] 等于整个列表,max_records=0 时必须显式清空 raw_records = [] elif self._history_records_are_ordered(raw_records): raw_records = raw_records[-max_records:] normalized_records = [ self._normalize_history_record(item) for item in raw_records ] self._monitor_history = ( self._sort_history_records(normalized_records)[-max_records:] if max_records > 0 else [] ) # 如果没有持久化的 token 统计,从已有的 200 条历史请求中求和做初次填充 if self.total_input_tokens == 0 and self.total_output_tokens == 0: for record in self._monitor_history: estimate = record.get("token_estimate") or {} self.total_input_tokens += estimate.get("prompt", 0) self.total_output_tokens += estimate.get("response", 0) except Exception as e: logger.debug(f"加载请求历史失败: {e}") self._monitor_history = [] def _save_history(self): if not self._request_monitor_enabled() or not self._request_monitor_save_to_file(): return try: with self._history_save_lock: with self._history_lock: max_records = self._request_monitor_max_records() self._trim_monitor_history_unlocked() records = list(self._monitor_history[-max_records:]) if max_records > 0 else [] # R2-5:增量同步到 SQLite——只写新增/变化的记录,删除已淘汰的记录(不再整文件重写) self._runtime_store().sync_history(records) except Exception as e: logger.debug(f"保存请求历史失败: {e}") def _schedule_stats_save(self) -> None: worker_to_start = None with self._save_schedule_lock: self._stats_save_requested = True if self._stats_save_worker and self._stats_save_worker.is_alive(): return worker_to_start = threading.Thread( target=self._run_stats_save_worker, daemon=True, name="request-stats-save", ) self._stats_save_worker = worker_to_start worker_to_start.start() def _run_stats_save_worker(self) -> None: while True: with self._save_schedule_lock: if not self._stats_save_requested: self._stats_save_worker = None return self._stats_save_requested = False self._save_stats() def _schedule_history_save(self) -> None: worker_to_start = None with self._save_schedule_lock: self._history_save_requested = True if self._history_save_worker and self._history_save_worker.is_alive(): return worker_to_start = threading.Thread( target=self._run_history_save_worker, daemon=True, name="request-history-save", ) self._history_save_worker = worker_to_start worker_to_start.start() def _run_history_save_worker(self) -> None: while True: # 去抖:窗口内的多次更新合并为一次整文件重写;flush_pending_saves 可提前唤醒 delay = _history_save_debounce_sec() if delay > 0 and not self._history_flush_event.is_set(): self._history_flush_event.wait(delay) with self._save_schedule_lock: if not self._history_save_requested: self._history_save_worker = None return self._history_save_requested = False self._save_history() def flush_pending_saves(self, timeout: float = 5.0) -> None: """立即写出尚在去抖窗口内的历史 / 统计(atexit 与关闭流程调用)。""" self._history_flush_event.set() try: for worker in (self._history_save_worker, self._stats_save_worker): if worker is not None and worker.is_alive() and worker is not threading.current_thread(): worker.join(timeout) with self._save_schedule_lock: pending_history = self._history_save_requested and not ( self._history_save_worker and self._history_save_worker.is_alive() ) if pending_history: self._history_save_requested = False if pending_history: self._save_history() finally: self._history_flush_event.clear() @staticmethod def _sanitize_text_for_storage(value: Any, max_chars: int = 80000) -> str: text = "" if value is None else str(value) if not text: return "" if "[内容已截断,原始长度" in text: return text max_chars = max(0, int(max_chars)) # 短文本:一次替换即可 if len(text) <= max_chars + _MEDIA_BLOB_MIN_MATCH_SLACK: text = _MEDIA_BLOB_PATTERN.sub(_MEDIA_PLACEHOLDER, text) if len(text) > max_chars: return text[:max_chars] + f"\n\n[内容已截断,原始长度 {len(text)} 字符]" return text # P0-7:长文本(prompt 可达 1M 字)不再“先整段跑正则再截断”: # 边替换边输出,凑够 max_chars 即停止扫描。图片后面的文字仍会保留 # (与整段替换的结果一致,只是不再处理截断点之后的内容)。 total_len = len(text) parts: List[str] = [] out_len = 0 pos = 0 while pos < total_len and out_len <= max_chars: budget = max_chars - out_len + 1 window_end = min(total_len, pos + budget + _MEDIA_BLOB_MIN_MATCH_SLACK) match = _MEDIA_BLOB_PATTERN.search(text, pos, window_end) if match is None or match.start() >= pos + budget: chunk = text[pos:min(total_len, pos + budget)] parts.append(chunk) out_len += len(chunk) pos += len(chunk) continue # 窗口可能截断了匹配:从匹配起点重新做一次不设上界的 match,跳过完整的媒体块 full = _MEDIA_BLOB_PATTERN.match(text, match.start()) or match prefix = text[pos:match.start()] parts.append(prefix) parts.append(_MEDIA_PLACEHOLDER) out_len += len(prefix) + len(_MEDIA_PLACEHOLDER) pos = max(full.end(), match.start() + 1) result = "".join(parts) if pos < total_len or len(result) > max_chars: return result[:max_chars] + f"\n\n[内容已截断,原始长度 {total_len} 字符]" return result @classmethod def _sanitize_for_storage(cls, value: Any, max_depth: int = 6) -> Any: if max_depth < 0: return "[内容已折叠]" if isinstance(value, str): return cls._sanitize_text_for_storage(value) if isinstance(value, list): return [cls._sanitize_for_storage(item, max_depth - 1) for item in value[:100]] if isinstance(value, dict): result: Dict[str, Any] = {} for key, item in list(value.items())[:100]: result[str(key)] = cls._sanitize_for_storage(item, max_depth - 1) return result return value @classmethod def _normalize_history_record(cls, record: Dict[str, Any]) -> Dict[str, Any]: normalized = dict(record) prompt_text = cls._sanitize_text_for_storage(normalized.get("prompt", ""), max_chars=20000) response_text = cls._sanitize_text_for_storage(normalized.get("response", ""), max_chars=30000) error_message = cls._sanitize_text_for_storage(normalized.get("error_message", ""), max_chars=10000) error_stack = cls._sanitize_text_for_storage(normalized.get("error_stack", ""), max_chars=20000) try: media_count = int(normalized.get("media_count") or 0) except Exception: media_count = 0 normalized["prompt"] = prompt_text normalized["response"] = response_text normalized["summary"] = cls._sanitize_text_for_storage( normalized.get("summary") or response_text[:180], max_chars=500, ) normalized["error_message"] = error_message normalized["error_stack"] = error_stack or error_message normalized["media_count"] = media_count canonical_route_domain = cls._canonical_monitor_domain(normalized.get("route_domain")) canonical_target_domain = cls._canonical_monitor_domain(normalized.get("target_domain")) if canonical_route_domain: normalized["route_domain"] = canonical_route_domain if canonical_target_domain: normalized["target_domain"] = canonical_target_domain elif canonical_route_domain: normalized["target_domain"] = canonical_route_domain status_value = str(normalized.get("status") or "").strip() token_estimate_for_success = normalized.get("token_estimate") if not isinstance(token_estimate_for_success, dict): token_estimate_for_success = {} has_meaningful_response = ( bool(response_text.strip()) or media_count > 0 or bool(normalized.get("has_response_text")) or cls._coerce_token_count(token_estimate_for_success.get("response")) > 0 ) stop_sequence_completed = ( status_value == RequestStatus.COMPLETED.value and ( str(normalized.get("error_code") or "").strip() == "stop_sequence" or str(normalized.get("error_message") or "").strip() == "stop_sequence" or str(normalized.get("cancel_reason") or "").strip() in {"audio_media_fast_return", "stop_sequence", "stream_done"} ) ) if ( status_value == RequestStatus.COMPLETED.value and not has_meaningful_response and not stop_sequence_completed ): normalized["status"] = RequestStatus.FAILED.value normalized["success"] = False normalized["error_code"] = "empty_response" normalized["error_message"] = error_message or "请求流程已结束,但没有捕获到 AI 响应文本或媒体结果。" normalized["error_stack"] = normalized["error_stack"] or normalized["error_message"] elif status_value == RequestStatus.COMPLETED.value: normalized["success"] = True if stop_sequence_completed and str(normalized.get("error_code") or "").strip() in {"", "stop_sequence"}: normalized["error_code"] = "" elif status_value: normalized["success"] = False token_estimate = normalized.get("token_estimate") if not isinstance(token_estimate, dict): prompt_tokens = cls._estimate_tokens(prompt_text) response_tokens = cls._estimate_tokens(response_text) normalized["token_estimate"] = { "prompt": prompt_tokens, "response": response_tokens, "total": prompt_tokens + response_tokens, "chars": len(prompt_text) + len(response_text), } else: prompt_tokens = cls._coerce_token_count(token_estimate.get("prompt")) response_tokens = cls._coerce_token_count(token_estimate.get("response")) chars = cls._coerce_token_count(token_estimate.get("chars")) if not chars: chars = len(prompt_text) + len(response_text) normalized["token_estimate"] = { "prompt": prompt_tokens, "response": response_tokens, "total": prompt_tokens + response_tokens, "chars": chars, } normalized["history_key"] = cls._make_history_key(normalized) return normalized @staticmethod def _canonical_monitor_domain(value: Any) -> str: text = str(value or "").strip() if not text or text == "未知域名": return "" try: return get_canonical_route_domain(text) or text except Exception: return text @staticmethod def _content_to_text(content: Any) -> str: if content is None: return "" if isinstance(content, str): return content if isinstance(content, list): parts: List[str] = [] for item in content: if isinstance(item, dict): item_type = str(item.get("type") or "").strip() if item_type in {"text", "input_text", "output_text"}: parts.append(str(item.get("text") or "")) elif item_type in {"image_url", "input_image", "output_image"}: parts.append("[图片占位符]") elif item_type in {"input_audio", "audio_url", "output_audio"}: parts.append("[音频占位符]") elif item_type in {"input_video", "video_url", "output_video"}: parts.append("[视频占位符]") else: parts.append(json.dumps(item, ensure_ascii=False)) else: parts.append(str(item)) return "\n".join(part for part in parts if part) return json.dumps(content, ensure_ascii=False) @classmethod def _messages_to_prompt_text(cls, messages: Any) -> str: if not isinstance(messages, list): return cls._sanitize_text_for_storage(messages) lines: List[str] = [] for message in messages: if not isinstance(message, dict): continue role = str(message.get("role") or "message") text = cls._content_to_text(message.get("content")) if not text: continue lines.append(f"{role}: {text}") return cls._sanitize_text_for_storage("\n\n".join(lines)) @classmethod def _has_multimodal_payload(cls, value: Any, max_depth: int = 6) -> bool: if max_depth < 0: return False if isinstance(value, str): lowered = value.lower() return "image_url" in lowered or "data:image" in lowered or "data:video" in lowered or "data:audio" in lowered if isinstance(value, list): return any(cls._has_multimodal_payload(item, max_depth - 1) for item in value) if isinstance(value, dict): if str(value.get("type") or "").strip() in { "image_url", "input_image", "output_image", "input_audio", "audio_url", "output_audio", "input_video", "video_url", "output_video", }: return True if any( key in value for key in ( "image_url", "images", "media", "input_audio", "audio_url", "input_video", "video_url", ) ): return True return any(cls._has_multimodal_payload(item, max_depth - 1) for item in value.values()) return False @staticmethod def _estimate_tokens(text: str) -> int: if not text: return 0 # 修复#3a:先把 data:URI 与超长 base64 替换成短占位符(空格), # 避免三个 findall 在大 base64 上全量执行;被剥离的字节按每 4 字符≈1 token 折算补偿 stripped_tokens = 0 if len(text) >= 1024: original_len = len(text) text = _DATA_URI_PATTERN.sub(" ", text) text = _LONG_BASE64_PATTERN.sub(" ", text) stripped_chars = original_len - len(text) if stripped_chars > 0: stripped_tokens = stripped_chars // 4 ascii_words = re.findall(r"[A-Za-z0-9_]+", text) non_ascii_chars = re.findall(r"[^\x00-\x7F\s]", text) punctuation = re.findall(r"[^\w\s]", text, flags=re.UNICODE) return max( 1, int(len(ascii_words) * 1.25 + len(non_ascii_chars) * 0.8 + len(punctuation) * 0.25) + stripped_tokens, ) @staticmethod def _coerce_token_count(value: Any) -> int: try: count = int(value) except (TypeError, ValueError, OverflowError): return 0 return max(0, count) @classmethod def _history_record_token_counts(cls, record: Dict[str, Any]) -> tuple[int, int]: if not isinstance(record, dict): return (0, 0) token_estimate = record.get("token_estimate") if not isinstance(token_estimate, dict): return (0, 0) return ( cls._coerce_token_count(token_estimate.get("prompt")), cls._coerce_token_count(token_estimate.get("response")), ) @staticmethod def _coerce_timestamp(value: Any) -> float: try: timestamp = float(value) except (TypeError, ValueError, OverflowError): return 0.0 if not math.isfinite(timestamp) or timestamp <= 0: return 0.0 if timestamp > 1_000_000_000_000: return timestamp / 1000.0 return timestamp @classmethod def _history_sort_value(cls, record: Dict[str, Any]) -> float: if not isinstance(record, dict): return 0.0 return ( cls._coerce_timestamp(record.get("finished_at")) or cls._coerce_timestamp(record.get("started_at")) or cls._coerce_timestamp(record.get("created_at")) ) @classmethod def _make_history_key(cls, record: Dict[str, Any]) -> str: request_id = str(record.get("id") or "").strip() or "request" created_at = cls._coerce_timestamp(record.get("created_at")) finished_at = cls._coerce_timestamp(record.get("finished_at")) return f"{request_id}:{created_at:.6f}:{finished_at:.6f}" @classmethod def _sort_history_records(cls, records: List[Dict[str, Any]]) -> List[Dict[str, Any]]: return sorted(records, key=cls._history_order_key) @classmethod def _history_records_are_ordered(cls, records: List[Dict[str, Any]]) -> bool: previous_key: Optional[tuple[float, float, str]] = None for record in records: current_key = cls._history_order_key(record) if previous_key is not None and current_key < previous_key: return False previous_key = current_key return True @classmethod def _history_order_key(cls, record: Dict[str, Any]) -> tuple[float, float, str]: if not isinstance(record, dict): return (0.0, 0.0, "") return ( cls._history_sort_value(record), cls._coerce_timestamp(record.get("created_at")), str(record.get("history_key") or record.get("id") or ""), ) @classmethod def _append_sorted_history_record_unlocked( cls, records: List[Dict[str, Any]], record: Dict[str, Any], *, max_records: int = 200, # 兜底值;调用方应显式传入 _request_monitor_max_records() ) -> None: key = cls._history_order_key(record) left = 0 right = len(records) while left < right: mid = (left + right) // 2 if cls._history_order_key(records[mid]) <= key: left = mid + 1 else: right = mid records.insert(left, record) overflow = len(records) - max(1, int(max_records or 200)) if overflow > 0: del records[:overflow] def _replace_monitor_history_record_unlocked( self, record: Dict[str, Any], *, previous_history_key: str = "", ) -> Optional[Dict[str, Any]]: """Replace a previously recorded monitor item, preserving sorted order.""" target_key = str(previous_history_key or "").strip() target_id = str(record.get("id") or "").strip() target_created_at = self._coerce_timestamp(record.get("created_at")) replace_index: Optional[int] = None if target_key: for index, item in enumerate(self._monitor_history): if str(item.get("history_key") or "").strip() == target_key: replace_index = index break if replace_index is None and target_id: for index in range(len(self._monitor_history) - 1, -1, -1): item = self._monitor_history[index] if str(item.get("id") or "").strip() != target_id: continue item_created_at = self._coerce_timestamp(item.get("created_at")) if ( not target_created_at or not item_created_at or abs(item_created_at - target_created_at) < 0.001 ): replace_index = index break previous_record = None if replace_index is not None: previous_record = self._monitor_history.pop(replace_index) # 修复:显式传入配置上限,否则回落到默认 200 条,与 REQUEST_MONITOR_MAX_RECORDS 冲突并批量截断历史 self._append_sorted_history_record_unlocked( self._monitor_history, record, max_records=self._request_monitor_max_records() ) self._history_revision_cache = None return previous_record @staticmethod def _extract_response_text(payload: Any) -> str: if isinstance(payload, str): return payload if not isinstance(payload, dict): return "" if payload is None else str(payload) if isinstance(payload.get("response"), dict): nested_text = RequestManager._extract_response_text(payload["response"]) if nested_text: return nested_text responses_event_text = RequestManager._extract_responses_event_text(payload) if responses_event_text: return responses_event_text try: choices = payload.get("choices") or [] if choices: first = choices[0] or {} message = first.get("message") or {} delta = first.get("delta") or {} content = message.get("content") if content is None: content = delta.get("content") text = RequestManager._content_to_text(content) if text: return text tool_text = RequestManager._extract_tool_calls_text( message.get("tool_calls") or delta.get("tool_calls") ) if tool_text: return tool_text except Exception: pass responses_text = RequestManager._extract_responses_output_text(payload) if responses_text: return responses_text anthropic_text = RequestManager._extract_anthropic_content_text(payload) if anthropic_text: return anthropic_text if "error" in payload: error = payload.get("error") or {} if isinstance(error, dict): return str(error.get("message") or "") return "" @staticmethod def _extract_tool_calls_text(tool_calls: Any) -> str: if not isinstance(tool_calls, list): return "" parts: List[str] = [] for index, item in enumerate(tool_calls): if not isinstance(item, dict): continue function_data = item.get("function") if isinstance(item.get("function"), dict) else {} name = str( function_data.get("name") or item.get("name") or item.get("tool_name") or "" ).strip() call_id = str(item.get("id") or "").strip() label = name or call_id or f"tool_call_{index}" parts.append(f"[tool_call] {label}") return "\n".join(parts) @staticmethod def _extract_error_code(payload: Any) -> str: if not isinstance(payload, dict): return "" error = payload.get("error") if not isinstance(error, dict): return "" return str(error.get("code") or error.get("type") or "").strip() @staticmethod def _extract_error_payload(payload: Any) -> Dict[str, str]: if not isinstance(payload, dict): return {} if isinstance(payload.get("response"), dict): nested_error = RequestManager._extract_error_payload(payload["response"]) if nested_error: return nested_error error = payload.get("error") if isinstance(error, str) and error.strip(): message = error.strip() return { "message": message, "code": "execution_error", "stack": message, } if not isinstance(error, dict): return {} message = str(error.get("message") or "").strip() code = str(error.get("code") or error.get("type") or "execution_error").strip() stack = json.dumps(error, ensure_ascii=False) return { "message": message, "code": code or "execution_error", "stack": stack, } @staticmethod def _extract_media_items(payload: Any) -> List[Dict[str, Any]]: if not isinstance(payload, dict): return [] if isinstance(payload.get("response"), dict): return RequestManager._extract_media_items(payload["response"]) media_items: List[Dict[str, Any]] = [] top_media = payload.get("media") if isinstance(top_media, list): media_items.extend(item for item in top_media if isinstance(item, dict)) choices = payload.get("choices") if isinstance(choices, list) and choices: first = choices[0] or {} delta = first.get("delta") or {} message = first.get("message") or {} for source in (delta, message): if not isinstance(source, dict): continue nested_media = source.get("media") if isinstance(nested_media, list): media_items.extend(item for item in nested_media if isinstance(item, dict)) content_media = RequestManager._extract_chat_content_media(source.get("content")) if content_media: media_items.extend(content_media) media_items.extend(RequestManager._extract_responses_event_media(payload)) media_items.extend(RequestManager._extract_responses_output_media(payload)) return media_items @staticmethod def _extract_chat_content_media(content: Any) -> List[Dict[str, Any]]: if isinstance(content, dict): content = [content] if not isinstance(content, list): return [] media_items: List[Dict[str, Any]] = [] for part in content: if not isinstance(part, dict): continue part_type = str(part.get("type") or "").strip() ref = "" media_type = "" if part_type in {"image_url", "input_image", "output_image"}: image_value = part.get("image_url") or part.get("url") or {} ref = str( image_value.get("url") if isinstance(image_value, dict) else image_value ).strip() media_type = "image" elif part_type in {"input_audio", "audio_url", "output_audio"}: audio_value = part.get("audio_url") or part.get("input_audio") or part.get("url") or {} ref = str( audio_value.get("url") if isinstance(audio_value, dict) else audio_value ).strip() media_type = "audio" elif part_type in {"input_video", "video_url", "output_video"}: video_value = part.get("video_url") or part.get("url") or {} ref = str( video_value.get("url") if isinstance(video_value, dict) else video_value ).strip() media_type = "video" if not ref: continue media_item: Dict[str, Any] = {"media_type": media_type} if ref.startswith("data:"): media_item["data_uri"] = ref else: media_item["url"] = ref detail = str(part.get("detail") or "").strip() if detail: media_item["detail"] = detail mime = str(part.get("mime_type") or part.get("mime") or "").strip() if mime: media_item["mime"] = mime label = str(part.get("label") or "").strip() if label: media_item["label"] = label media_items.append(media_item) return media_items @staticmethod def _extract_responses_output_text(payload: Dict[str, Any]) -> str: output = payload.get("output") if not isinstance(output, list): return "" parts: List[str] = [] for item in output: if not isinstance(item, dict): continue content = item.get("content") if isinstance(content, list): for part in content: if not isinstance(part, dict): continue part_type = str(part.get("type") or "").strip() if part_type == "output_text": text = str(part.get("text") or "") if text: parts.append(text) elif part_type in {"output_image", "output_audio", "output_video"}: ref = RequestManager._extract_media_ref( part.get("image_url") or part.get("audio_url") or part.get("video_url") ) if ref: parts.append(ref) elif item.get("type") == "function_call": name = str(item.get("name") or "").strip() if name: parts.append(f"[function_call] {name}") return "\n".join(parts) @staticmethod def _extract_responses_event_text(payload: Dict[str, Any]) -> str: event_type = str(payload.get("type") or "").strip() if event_type == "response.output_text.delta": return str(payload.get("delta") or "") if event_type == "response.content_part.added": part = payload.get("part") if not isinstance(part, dict): return "" part_type = str(part.get("type") or "").strip() if part_type == "output_text": return str(part.get("text") or "") if event_type == "response.function_call_arguments.done": name = str(payload.get("name") or "").strip() if name: return f"[function_call] {name}" return "" @staticmethod def _extract_responses_event_media(payload: Dict[str, Any]) -> List[Dict[str, Any]]: event_type = str(payload.get("type") or "").strip() if event_type != "response.content_part.added": return [] part = payload.get("part") if not isinstance(part, dict): return [] return RequestManager._extract_responses_output_media({ "output": [{ "content": [part], }], }) @staticmethod def _extract_anthropic_content_text(payload: Dict[str, Any]) -> str: content = payload.get("content") if not isinstance(content, list): return "" parts: List[str] = [] for item in content: if not isinstance(item, dict): continue item_type = str(item.get("type") or "").strip() if item_type == "text": text = str(item.get("text") or "") if text: parts.append(text) elif item_type == "tool_use": name = str(item.get("name") or "").strip() if name: parts.append(f"[tool_use] {name}") return "\n".join(parts) @staticmethod def _extract_media_ref(value: Any) -> str: if isinstance(value, dict): return str(value.get("url") or value.get("data_uri") or "").strip() return str(value or "").strip() @staticmethod def _extract_responses_output_media(payload: Dict[str, Any]) -> List[Dict[str, Any]]: output = payload.get("output") if not isinstance(output, list): return [] media_items: List[Dict[str, Any]] = [] for item in output: if not isinstance(item, dict): continue content = item.get("content") if not isinstance(content, list): continue for part in content: if not isinstance(part, dict): continue part_type = str(part.get("type") or "").strip() if part_type == "output_image": ref = RequestManager._extract_media_ref(part.get("image_url")) media_type = "image" elif part_type == "output_audio": ref = RequestManager._extract_media_ref(part.get("audio_url")) media_type = "audio" elif part_type == "output_video": ref = RequestManager._extract_media_ref(part.get("video_url")) media_type = "video" else: continue if not ref: continue media_item: Dict[str, Any] = {"media_type": media_type} if ref.startswith("data:"): media_item["data_uri"] = ref else: media_item["url"] = ref mime = str(part.get("mime_type") or "").strip() if mime: media_item["mime"] = mime label = str(part.get("label") or "").strip() if label: media_item["label"] = label media_items.append(media_item) return media_items @staticmethod def _iter_sse_payloads(chunk: str) -> List[Dict[str, Any]]: return iter_sse_payloads(chunk) def _iter_sse_payloads_for_context( self, ctx: RequestContext, chunk: str, ) -> List[Dict[str, Any]]: """Parse complete SSE frames while preserving split-frame tails per request.""" if not isinstance(chunk, str): return [] text = chunk.replace("\r\n", "\n").replace("\r", "\n") dropped_chars = 0 complete_segments: List[str] = [] with ctx._lock: combined = f"{ctx._sse_chunk_buffer}{text}" if "\n\n" not in combined: # 修复#6:超限时清空缓冲而非截尾——截断片段可能与后续数据拼出坏帧 if len(combined) > MAX_SSE_CHUNK_BUFFER_CHARS: dropped_chars = len(combined) ctx._sse_chunk_buffer = "" else: ctx._sse_chunk_buffer = combined else: segments = combined.split("\n\n") complete_segments = segments[:-1] tail = segments[-1] # 修复#6:尾部超限同样清空丢弃 if tail and len(tail) > MAX_SSE_CHUNK_BUFFER_CHARS: dropped_chars = len(tail) ctx._sse_chunk_buffer = "" else: ctx._sse_chunk_buffer = tail or "" if dropped_chars: logger.debug( f"[{ctx.request_id}] SSE 监控缓冲超过 {MAX_SSE_CHUNK_BUFFER_CHARS} 字符" f"(实际 {dropped_chars}),已整体清空丢弃(仅影响监控采集,不影响转发)" ) if not complete_segments: return [] return self._iter_sse_payloads("\n\n".join(complete_segments) + "\n\n") def _flush_sse_payloads_for_context(self, ctx: RequestContext) -> List[Dict[str, Any]]: """Parse a final unterminated SSE frame before request history is written.""" if ctx is None or not hasattr(ctx, "_sse_chunk_buffer"): return [] with ctx._lock: tail = ctx._sse_chunk_buffer ctx._sse_chunk_buffer = "" if not tail: return [] return self._iter_sse_payloads(f"{tail}\n\n") def record_request_input( self, ctx: RequestContext, payload: Any, *, endpoint: str = "", route_domain: str = "", route_group: str = "", tab_index: Optional[int] = None, preset_name: Optional[str] = None, ) -> None: if ctx is None: return payload_dict = payload if isinstance(payload, dict) else {} messages = payload_dict.get("messages") if payload_dict else None prompt_text = self._messages_to_prompt_text(messages) is_stream = bool(payload_dict.get("stream")) if payload_dict else False has_tools = bool(payload_dict.get("tools") or payload_dict.get("functions")) if payload_dict else False request_type = "工具调用" if has_tools else ("流式" if is_stream else "非流式") model = str(payload_dict.get("model") or "").strip() resolved_preset = str(preset_name or payload_dict.get("preset_name") or "").strip() # 计算完整、未截断的 Prompt 估算 Token raw_prompt_for_tokens = "" if isinstance(messages, list): raw_prompt_for_tokens = "\n\n".join( f"{msg.get('role', 'message')}: {self._content_to_text(msg.get('content'))}" for msg in messages if isinstance(msg, dict) ) else: raw_prompt_for_tokens = str(messages or "") prompt_tokens = self._estimate_tokens(raw_prompt_for_tokens) detail_enabled = self._request_monitor_detail_enabled() prompt_capture_limit = min(20000, self._request_monitor_capture_chars()) # 修复#3b:重清洗(正则替换/递归拷贝)全部移到 ctx._lock 之外执行,锁内只做赋值。 # payload_dict / prompt_text / messages 均为本地变量,锁外清洗无竞态。 monitor_route_domain = self._canonical_monitor_domain(route_domain) monitor_update = { "endpoint": endpoint, "route_domain": monitor_route_domain, "target_domain": monitor_route_domain, "route_group": str(route_group or "").strip(), "tab_index": tab_index, "preset_name": resolved_preset, "model": model, "request_type": request_type, "is_stream": is_stream, "is_multimodal": self._has_multimodal_payload(messages), "prompt_tokens": prompt_tokens, } if detail_enabled and prompt_capture_limit > 0: monitor_update["prompt"] = self._sanitize_text_for_storage( prompt_text, max_chars=prompt_capture_limit, ) monitor_update["payload"] = self._sanitize_for_storage(payload_dict) with ctx._lock: ctx.monitor.update(monitor_update) def update_request_metadata(self, request_id: str, **metadata: Any) -> bool: request_key = str(request_id or "").strip() if not request_key: return False with self._requests_lock: ctx = self._requests.get(request_key) if not ctx: return False with ctx._lock: for key, value in metadata.items(): if value is None: continue if key == "target_domain" and not str(value or "").strip(): continue if key in {"target_domain", "route_domain"}: value = self._canonical_monitor_domain(value) if not value and key == "route_domain": continue ctx.monitor[key] = self._sanitize_for_storage(value) return True def append_workflow_trace(self, request_id: str, trace: list) -> bool: """Append bounded structural traces across chunks/tool rounds; no values.""" ctx = self.get_request(str(request_id or "")) if ctx is None: return False allowed = {"path", "action", "status", "branch", "attempt", "variable", "error_type", "reason"} with ctx._lock: run = int(ctx.monitor.get("_workflow_trace_run", 0)) + 1 ctx.monitor["_workflow_trace_run"] = run previous = list(ctx.monitor.get("workflow_trace") or [])[:500] for entry in list(trace or [])[:500 - len(previous)]: if not isinstance(entry, dict): continue safe = {k: self._sanitize_for_storage(v) for k, v in entry.items() if k in allowed and isinstance(v, (str, int, bool))} previous.append({**safe, "run": run}) ctx.monitor["workflow_trace"] = previous return True def capture_external_response( self, request_id: str, response_text: Any, *, metadata: Optional[Dict[str, Any]] = None, append_if_matched: Optional[Any] = None, strategy: str = "replace", ) -> bool: """Attach externally observed response text to a tracked request. Command-loop helpers can finish independently from page/network listeners. This lets a late network bridge update the same monitor row and refresh the history record when it has already been written. """ request_key = str(request_id or "").strip() if not request_key: return False with self._requests_lock: ctx = self._requests.get(request_key) if not ctx: return False text = str(response_text or "") if not text.strip() and not metadata: return False capture_limit = self._request_monitor_capture_chars() terminal = False with ctx._lock: if ctx.status == RequestStatus.RUNNING: ctx.last_activity_at = time.time() if text: stored_text = self._sanitize_text_for_storage( text, max_chars=capture_limit if capture_limit > 0 else MAX_CAPTURED_RESPONSE_CHARS, ) existing_text = str(ctx.monitor.get("response_text") or "").strip() should_append = False if strategy == "append": should_append = True elif callable(append_if_matched): try: should_append = bool(append_if_matched(existing_text)) except Exception: should_append = False elif isinstance(append_if_matched, str) and append_if_matched: should_append = append_if_matched in existing_text elif bool(append_if_matched): should_append = True if existing_text and should_append and stored_text: stored_text = self._sanitize_text_for_storage( f"{existing_text}\n\n{stored_text}", max_chars=capture_limit if capture_limit > 0 else MAX_CAPTURED_RESPONSE_CHARS, ) ctx.monitor["response_text"] = stored_text ctx.monitor["has_response_text"] = True ctx.monitor["response_tokens"] = self._estimate_tokens(stored_text) if metadata: for key, value in metadata.items(): if value is None: continue ctx.monitor[str(key)] = self._sanitize_for_storage(value) terminal = ctx.status in ( RequestStatus.COMPLETED, RequestStatus.CANCELLED, RequestStatus.FAILED, ) if terminal: try: self._append_monitor_history(ctx) except Exception as e: logger.debug(f"刷新外部响应请求监控历史失败: {e}") return True def capture_response_chunk(self, ctx: RequestContext, chunk: Any) -> None: if ctx is None or not isinstance(chunk, str): return if chunk: ctx.mark_activity() payloads = self._iter_sse_payloads_for_context(ctx, chunk) if not payloads: return self._capture_response_payloads(ctx, payloads) def _capture_response_payloads( self, ctx: RequestContext, payloads: List[Dict[str, Any]], ) -> None: if ctx is None or not payloads: return capture_limit = self._request_monitor_capture_chars() detail_enabled = self._request_monitor_detail_enabled() with ctx._lock: for payload in payloads: error = payload.get("error") if isinstance(payload, dict) else None payload_error = self._extract_error_payload(payload) text = "" if payload_error else self._extract_response_text(payload) media_items = self._extract_media_items(payload) if text: ctx.monitor["has_response_text"] = True ctx.monitor["response_tokens"] = self._coerce_token_count( ctx.monitor.get("response_tokens") ) + self._estimate_tokens(text) if detail_enabled and capture_limit > 0: parts = ctx.monitor.setdefault("response_parts", []) if isinstance(parts, list): current_len = ctx.monitor.get("response_parts_chars") if not isinstance(current_len, int): current_len = sum(len(str(item)) for item in parts) current_len = max(0, current_len) remaining = max(0, capture_limit - current_len) if remaining > 0: captured = text[:remaining] parts.append(captured) ctx.monitor["response_parts_chars"] = current_len + len(captured) else: ctx.monitor["response_parts_chars"] = current_len if media_items: existing = ctx.monitor.setdefault("media_items", []) if isinstance(existing, list): existing.extend(self._sanitize_for_storage(media_items)) ctx.monitor["has_response_media"] = True ctx.monitor["is_multimodal"] = True if payload_error: error_message = str(payload_error.get("message") or "") if error_message: ctx.monitor["error_message"] = self._sanitize_text_for_storage(error_message, max_chars=20000) ctx.monitor["error_code"] = str(payload_error.get("code") or "execution_error") ctx.monitor["error_stack"] = self._sanitize_text_for_storage( payload_error.get("stack") or error_message, max_chars=40000, ) def capture_response_payload( self, ctx: RequestContext, payload: Any, *, error_code: str = "", ) -> None: if ctx is None: return ctx.mark_activity() text = self._extract_response_text(payload) media_items = self._extract_media_items(payload) detail_enabled = self._request_monitor_detail_enabled() capture_limit = self._request_monitor_capture_chars() sanitized_payload = self._sanitize_for_storage(payload) if detail_enabled else {} payload_error = self._extract_error_payload(payload) payload_error_code = str(payload_error.get("code") or self._extract_error_code(payload)).strip() # 计算完整、未截断的 Completion 估算 Token response_tokens = self._estimate_tokens(text) with ctx._lock: if text: ctx.monitor["has_response_text"] = True if text and detail_enabled and capture_limit > 0: ctx.monitor["response_text"] = self._sanitize_text_for_storage(text, max_chars=capture_limit) if media_items: ctx.monitor["media_items"] = self._sanitize_for_storage(media_items) ctx.monitor["has_response_media"] = True ctx.monitor["is_multimodal"] = True if detail_enabled: ctx.monitor["response_payload"] = sanitized_payload ctx.monitor["response_tokens"] = response_tokens if payload_error: error_message = str(payload_error.get("message") or "") if error_message: ctx.monitor["error_message"] = self._sanitize_text_for_storage(error_message, max_chars=20000) ctx.monitor["error_code"] = payload_error_code or str(error_code or "execution_error") ctx.monitor["error_stack"] = self._sanitize_text_for_storage( payload_error.get("stack") or error_message, max_chars=40000, ) if payload_error_code or error_code: ctx.monitor["error_code"] = payload_error_code or error_code def capture_error( self, ctx: RequestContext, error: Any, *, code: str = "execution_error", stack: str = "", ) -> None: if ctx is None: return message = str(error or "") with ctx._lock: ctx.monitor["error_message"] = self._sanitize_text_for_storage(message, max_chars=20000) ctx.monitor["error_code"] = str(code or "execution_error") ctx.monitor["error_stack"] = self._sanitize_text_for_storage(stack or message, max_chars=40000) def _compact_context_monitor_after_history_append( self, monitor: Dict[str, Any], *, prompt_text: str, response_text: str, prompt_tokens: int, response_tokens: int, ) -> None: """Release heavy request payload references after history has a durable summary.""" monitor["prompt"] = prompt_text monitor["prompt_tokens"] = prompt_tokens if response_text: monitor["response_text"] = response_text monitor["response_tokens"] = response_tokens for key in ( "payload", "response_payload", "response_parts", "response_parts_chars", "media_items", ): monitor.pop(key, None) if not response_text: monitor.pop("response_text", None) def _append_monitor_history(self, ctx: RequestContext) -> None: # 修复#1:整个"判重→插入完成"流程持 ctx 专用历史锁执行,消除 # finish_request / cancel_request / capture_external_response 并发时 # 判重(ctx._lock 内)与插入(_history_lock 内)之间的跨锁竞态窗口。 # 该锁为最外层锁;锁内可再获取 ctx._lock / _history_lock,内层顺序保持不变。 with ctx._history_append_lock: self._append_monitor_history_inner(ctx) def _append_monitor_history_inner(self, ctx: RequestContext) -> None: tail_payloads = self._flush_sse_payloads_for_context(ctx) if tail_payloads: self._capture_response_payloads(ctx, tail_payloads) if not self._request_monitor_enabled(): with self._history_lock: if self._monitor_history: self._monitor_history = [] self._history_revision_cache = None with ctx._lock: ctx.monitor["_history_recorded"] = True for key in ( "payload", "response_payload", "response_parts", "response_parts_chars", "media_items", ): ctx.monitor.pop(key, None) return with ctx._lock: already_recorded = bool(ctx.monitor.get("_history_recorded")) previous_history_key = str(ctx.monitor.get("_history_key") or "") if not already_recorded: ctx.monitor["_history_recorded"] = True monitor = dict(ctx.monitor) snapshot = ctx.snapshot() created_at = float(snapshot["created_at"] or time.time()) started_at = float(snapshot["started_at"] or created_at) finished_at = float(snapshot["finished_at"] or time.time()) queue_ms = max(0, int((started_at - created_at) * 1000)) generation_ms = max(0, int((finished_at - started_at) * 1000)) duration_ms = max(0, int((finished_at - created_at) * 1000)) response_text = monitor.get("response_text") if not response_text: response_parts = monitor.get("response_parts") if isinstance(response_parts, list): response_text = "".join(str(item) for item in response_parts) detail_enabled = self._request_monitor_detail_enabled() capture_limit = self._request_monitor_capture_chars() response_capture_limit = capture_limit if capture_limit > 0 else 0 prompt_capture_limit = min(20000, response_capture_limit) if response_capture_limit > 0 else 0 response_text = self._sanitize_text_for_storage( response_text, max_chars=response_capture_limit or 1, ) if detail_enabled and response_capture_limit > 0 else "" prompt_text = self._sanitize_text_for_storage( monitor.get("prompt", ""), max_chars=prompt_capture_limit or 1, ) if detail_enabled and prompt_capture_limit > 0 else "" prompt_tokens = monitor.get("prompt_tokens") if prompt_tokens is None: prompt_tokens = self._estimate_tokens(prompt_text) else: prompt_tokens = self._coerce_token_count(prompt_tokens) response_tokens = monitor.get("response_tokens") if response_tokens is None: response_tokens = self._estimate_tokens(response_text) else: response_tokens = self._coerce_token_count(response_tokens) media_items = monitor.get("media_items") if isinstance(monitor.get("media_items"), list) else [] has_meaningful_response = ( bool(response_text.strip()) or len(media_items) > 0 or bool(monitor.get("has_response_text")) or response_tokens > 0 ) status = snapshot["status"] status_value = status.value if isinstance(status, RequestStatus) else str(status) stop_sequence_completed = ( status_value == RequestStatus.COMPLETED.value and str(snapshot["cancel_reason"] or "").strip() in {"audio_media_fast_return", "stop_sequence", "stream_done"} ) success = ( status_value == RequestStatus.COMPLETED.value and (has_meaningful_response or stop_sequence_completed) ) error_message = self._sanitize_text_for_storage( monitor.get("error_message") or snapshot["cancel_reason"] or "", max_chars=10000, ) error_code = str(monitor.get("error_code") or (status_value if not success else "") or "").strip() if success and stop_sequence_completed: error_message = "" error_code = "" if ( status_value == RequestStatus.COMPLETED.value and not has_meaningful_response and not stop_sequence_completed ): status_value = RequestStatus.FAILED.value success = False error_code = "empty_response" error_message = "请求流程已结束,但没有捕获到 AI 响应文本或媒体结果。" error_stack = self._sanitize_text_for_storage( monitor.get("error_stack") or error_message, max_chars=20000, ) record = { "id": ctx.request_id, "status": status_value, "success": success, "created_at": created_at, "started_at": started_at, "finished_at": finished_at, "duration_ms": duration_ms, "queue_ms": queue_ms, "generation_ms": generation_ms, "target_domain": self._canonical_monitor_domain(monitor.get("target_domain") or monitor.get("route_domain")) or "未知域名", "route_domain": self._canonical_monitor_domain(monitor.get("route_domain")), "route_group": str(monitor.get("route_group") or ""), "preset_name": str(monitor.get("preset_name") or "默认预设"), "tab_index": monitor.get("tab_index"), "tab_id": snapshot["tab_id"] or monitor.get("tab_id") or "", "model": str(monitor.get("model") or ""), "endpoint": str(monitor.get("endpoint") or ""), "protocol": str(getattr(ctx, "protocol", "") or "openai.chat"), "http_request_id": str(getattr(ctx, "http_request_id", "") or ""), "request_type": str(monitor.get("request_type") or ""), "is_stream": bool(monitor.get("is_stream")), "is_multimodal": bool(monitor.get("is_multimodal") or monitor.get("has_response_media")), "has_response_text": bool(monitor.get("has_response_text")) or response_tokens > 0, "prompt": prompt_text, "response": response_text, "summary": response_text[:180] or ("已完成(响应详情未保存)" if has_meaningful_response else ""), "error_code": error_code, "error_message": error_message, "error_stack": error_stack, "workflow_trace": list(monitor.get("workflow_trace") or [])[:500], "cancel_reason": str(snapshot["cancel_reason"] or ""), "media_count": len(media_items), "token_estimate": { "prompt": prompt_tokens, "response": response_tokens, "total": prompt_tokens + response_tokens, "chars": len(prompt_text) + len(response_text), }, } record = self._normalize_history_record(record) with ctx._lock: ctx.monitor["_history_key"] = str(record.get("history_key") or "") self._compact_context_monitor_after_history_append( ctx.monitor, prompt_text=prompt_text, response_text=response_text, prompt_tokens=prompt_tokens, response_tokens=response_tokens, ) previous_record = None with self._history_lock: if already_recorded: previous_record = self._replace_monitor_history_record_unlocked( record, previous_history_key=previous_history_key, ) else: # 修复:显式传入配置上限,否则回落到默认 200 条,与 REQUEST_MONITOR_MAX_RECORDS 冲突并批量截断历史 self._append_sorted_history_record_unlocked( self._monitor_history, record, max_records=self._request_monitor_max_records() ) self._history_revision_cache = None self._trim_monitor_history_unlocked() if not already_recorded: prompt_delta = prompt_tokens response_delta = response_tokens else: previous_prompt_tokens, previous_response_tokens = self._history_record_token_counts( previous_record or {} ) prompt_delta = prompt_tokens - previous_prompt_tokens response_delta = response_tokens - previous_response_tokens if prompt_delta or response_delta: with self._requests_lock: self.total_input_tokens = max(0, self.total_input_tokens + prompt_delta) self.total_output_tokens = max(0, self.total_output_tokens + response_delta) self._schedule_stats_save() self._schedule_history_save() @staticmethod def _history_revision_unlocked(records: List[Dict[str, Any]]) -> str: if not records: return "0::0" latest = max( (item for item in records if isinstance(item, dict)), key=RequestManager._history_sort_value, default={}, ) latest_time = RequestManager._history_sort_value(latest) latest_key = latest.get("history_key") or RequestManager._make_history_key(latest) digest = hashlib.blake2s(digest_size=8) for item in records: if not isinstance(item, dict): continue for part in RequestManager._history_revision_parts(item): digest.update(str(part).encode("utf-8", errors="replace")) digest.update(b"\x1f") digest.update(b"\x1e") return f"{len(records)}:{latest_key}:{latest_time}:{digest.hexdigest()}" @staticmethod def _history_revision_cache_key(records: List[Dict[str, Any]]) -> tuple[Any, ...]: if not records: return (0,) return tuple( ( id(item), RequestManager._history_revision_cache_parts(item), ) for item in records if isinstance(item, dict) ) @staticmethod def _history_revision_text_cache_marker(value: Any) -> tuple[int, int]: if value is None or value == "": return (0, 0) text = str(value) return (id(value), len(text)) @staticmethod def _history_revision_cache_parts(record: Dict[str, Any]) -> tuple[Any, ...]: token_estimate = record.get("token_estimate") if not isinstance(token_estimate, dict): token_estimate = {} return ( record.get("history_key") or RequestManager._make_history_key(record), RequestManager._history_sort_value(record), record.get("status"), record.get("success"), record.get("target_domain"), record.get("route_domain"), record.get("route_group"), record.get("preset_name"), record.get("tab_index"), record.get("tab_id"), record.get("model"), record.get("endpoint"), record.get("request_type"), record.get("is_stream"), record.get("is_multimodal"), record.get("duration_ms"), record.get("queue_ms"), record.get("generation_ms"), RequestManager._history_revision_text_cache_marker(record.get("summary")), record.get("error_code"), RequestManager._history_revision_text_cache_marker(record.get("error_message")), RequestManager._history_revision_text_cache_marker(record.get("prompt")), RequestManager._history_revision_text_cache_marker(record.get("response")), RequestManager._history_revision_text_cache_marker(record.get("error_stack")), record.get("media_count"), token_estimate.get("prompt"), token_estimate.get("response"), token_estimate.get("total"), token_estimate.get("chars"), ) def _get_history_revision_cached(self, records: List[Dict[str, Any]]) -> str: cache_key = self._history_revision_cache_key(records) cache = getattr(self, "_history_revision_cache", None) if cache and cache[0] == cache_key: return cache[1] revision = self._history_revision_unlocked(records) # 修复#5:缓存 value 同时持有记录对象引用(侵入最小方案)。 # 由于旧记录对象被缓存持有而保持存活,其 id() 不可能被新对象复用; # 因此 cache_key 中 id 相等即等价于 is 同一对象,消除 not_modified 误报。 # (记录 dict 插入历史后不做原地修改,替换总是生成新 dict,故内部字符串 id 同样稳定) self._history_revision_cache = (cache_key, revision, list(records)) return revision @staticmethod def _history_revision_text_part(value: Any) -> str: text = str(value or "") if not text: return "" digest = hashlib.blake2s(text.encode("utf-8", errors="replace"), digest_size=8).hexdigest() return f"{len(text)}:{digest}" @staticmethod def _history_revision_parts(record: Dict[str, Any]) -> tuple[Any, ...]: token_estimate = record.get("token_estimate") if not isinstance(token_estimate, dict): token_estimate = {} return ( record.get("history_key") or RequestManager._make_history_key(record), RequestManager._history_sort_value(record), record.get("status"), record.get("success"), record.get("target_domain"), record.get("route_domain"), record.get("route_group"), record.get("preset_name"), record.get("tab_index"), record.get("tab_id"), record.get("model"), record.get("endpoint"), record.get("request_type"), record.get("is_stream"), record.get("is_multimodal"), record.get("duration_ms"), record.get("queue_ms"), record.get("generation_ms"), record.get("summary"), record.get("error_code"), RequestManager._history_revision_text_part(record.get("error_message")), RequestManager._history_revision_text_part(record.get("prompt")), RequestManager._history_revision_text_part(record.get("response")), RequestManager._history_revision_text_part(record.get("error_stack")), record.get("media_count"), token_estimate.get("prompt"), token_estimate.get("response"), token_estimate.get("total"), token_estimate.get("chars"), ) @staticmethod def _history_preview_text(value: Any, max_chars: int = 220) -> str: text = str(value or "").replace("\r\n", "\n").replace("\r", "\n") text = re.sub(r"\s+", " ", text).strip() if len(text) > max_chars: return text[:max_chars] + "..." return text @classmethod def _to_history_list_record(cls, record: Dict[str, Any]) -> Dict[str, Any]: item = dict(record) token_estimate = item.get("token_estimate") if isinstance(token_estimate, dict): item["token_estimate"] = dict(token_estimate) prompt_text = str(item.pop("prompt", "") or "") response_text = str(item.pop("response", "") or "") error_stack = str(item.pop("error_stack", "") or "") item.pop("payload", None) item.pop("workflow_trace", None) item.pop("response_payload", None) item.pop("response_parts", None) item.pop("response_parts_chars", None) summary = str(item.get("summary") or "").strip() if not summary: summary = response_text or str(item.get("error_message") or "") item["summary"] = cls._history_preview_text(summary, max_chars=240) item["prompt_preview"] = cls._history_preview_text(prompt_text, max_chars=240) item["response_preview"] = cls._history_preview_text(response_text, max_chars=300) item["has_detail"] = bool(prompt_text or response_text or error_stack) item["detail_loaded"] = False item["detail_text_lengths"] = { "prompt": len(prompt_text), "response": len(response_text), "error_stack": len(error_stack), } error_message = str(item.get("error_message") or "") if len(error_message) > 800: item["error_message"] = error_message[:800] + "..." return item def get_request_history_payload( self, limit: int = 200, include_detail: bool = False, if_revision: Optional[str] = None, ) -> Dict[str, Any]: max_records = self._request_monitor_max_records() if not self._request_monitor_enabled(): return { "records": [], "count": 0, "max_records": max_records, "revision": "0::0", "not_modified": False, "enabled": False, } try: count = max(1, min(max_records, int(limit or max_records or 1))) except Exception: count = max(1, max_records) requested_revision = str(if_revision or "").strip() with self._history_lock: self._trim_monitor_history_unlocked() history_refs_all = [ item for item in self._monitor_history[-count:] if isinstance(item, dict) ] if not self._history_records_are_ordered(self._monitor_history): history_refs_all = [ item for item in self._monitor_history if isinstance(item, dict) ] history_refs_all = self._sort_history_records(history_refs_all) history_refs = history_refs_all[-count:] revision = self._get_history_revision_cached(history_refs) if not include_detail and requested_revision and requested_revision == revision: return { "records": [], "count": 0, "max_records": max_records, "revision": revision, "not_modified": True, "enabled": True, } history = ( [copy.deepcopy(item) for item in history_refs] if include_detail else list(history_refs) ) if include_detail: records = list(reversed(history)) else: records = [self._to_history_list_record(item) for item in reversed(history)] return { "records": records, "count": len(records), "max_records": max_records, "revision": revision, "not_modified": False, "enabled": True, } def get_request_history(self, limit: int = 200) -> List[Dict[str, Any]]: return self.get_request_history_payload(limit, include_detail=True)["records"] def get_request_history_record(self, request_id: str) -> Optional[Dict[str, Any]]: request_key = str(request_id or "").strip() if not request_key or not self._request_monitor_enabled(): return None with self._history_lock: if self._history_records_are_ordered(self._monitor_history): history_iterable = reversed(self._monitor_history) else: history = [ item for item in self._monitor_history if isinstance(item, dict) ] history_iterable = reversed(self._sort_history_records(history)) id_match = None for item in history_iterable: if not isinstance(item, dict): continue if str(item.get("history_key") or "").strip() == request_key: return copy.deepcopy(item) if id_match is None and str(item.get("id") or "").strip() == request_key: id_match = item if id_match is not None: return copy.deepcopy(id_match) return None def create_request(self, client_fp: str = "") -> RequestContext: """创建新请求""" request_id = self._generate_id() ctx = RequestContext(request_id=request_id, client_fp=str(client_fp or "")) try: # R2-2:按协议入口设置的 ChatJob 标注协议(未设置时为 openai.chat) from app.core.chat_job import current_protocol ctx.protocol = current_protocol() except Exception: pass try: # R2-7:记录本次 HTTP 请求的 X-Request-ID(由 RequestMetricsMiddleware 设置) from app.services.metrics import REQUEST_ID ctx.http_request_id = REQUEST_ID.get() except Exception: pass cleanup_history_contexts: List[RequestContext] = [] with self._requests_lock: self._requests[request_id] = ctx cleanup_history_contexts = self._cleanup_old_requests() for stale_ctx in cleanup_history_contexts: try: self._append_monitor_history(stale_ctx) except Exception as e: logger.debug(f"写入清理请求监控历史失败: {e}") # 设置上下文后记录日志 token = _request_context.set(request_id) try: logger.info(f"创建 (X-Request-ID={ctx.http_request_id})" if ctx.http_request_id else "创建") finally: _request_context.reset(token) return ctx def _generate_id(self) -> str: """生成简短的请求 ID""" with self._requests_lock: self._request_counter += 1 return f"req-{self._request_counter:03d}" def _cleanup_old_requests(self) -> List[RequestContext]: """清理旧请求(修复版:不因单个未完成请求阻塞所有清理)""" now = time.time() # 修复:改为运行时读取,支持前端热配置(原类属性在 import 时求值,改配置需重启) zombie_ttl = self._zombie_ttl() over_capacity = len(self._requests) > self._max_history to_delete = [] history_contexts: List[RequestContext] = [] for req_id, ctx in list(self._requests.items()): snapshot = ctx.snapshot(now) status = snapshot["status"] # Keep a live worker discoverable by ownership/recovery even when # its HTTP request has already reached a terminal state. worker = getattr(ctx, "worker_thread", None) worker_alive = isinstance(worker, threading.Thread) and worker.is_alive() # 已终态的可以删除 if snapshot["is_terminal"]: if worker_alive: continue if over_capacity and len(self._requests) - len(to_delete) > self._max_history: to_delete.append(req_id) history_contexts.append(ctx) # 超时的 RUNNING 请求视为僵尸,强制标记失败 elif status == RequestStatus.RUNNING: started = snapshot["started_at"] or snapshot["created_at"] active_at = snapshot.get("last_activity_at") or started if now - active_at > zombie_ttl: logger.warning( f"[{req_id}] 僵尸请求 (无活动 {now - active_at:.0f}s, 运行 {now - started:.0f}s),强制清理" ) ctx.mark_failed("zombie_timeout") if not worker_alive: to_delete.append(req_id) history_contexts.append(ctx) elif status == RequestStatus.QUEUED: queued_at = snapshot["created_at"] or now if now - queued_at > zombie_ttl: logger.warning( f"[{req_id}] 排队请求超时 (等待 {now - queued_at:.0f}s),强制清理" ) ctx.mark_failed("queued_timeout") if not worker_alive: to_delete.append(req_id) history_contexts.append(ctx) # 批量删除 for req_id in to_delete: del self._requests[req_id] if to_delete: logger.debug(f"清理了 {len(to_delete)} 个旧请求") return history_contexts def _run_zombie_sweep_loop(self) -> None: """修复#4:每 60 秒清扫一次僵尸/超时请求(daemon 线程,捕获所有异常防止线程死亡)""" while True: time.sleep(60) try: with self._requests_lock: cleanup_history_contexts = self._cleanup_old_requests() for stale_ctx in cleanup_history_contexts: try: self._append_monitor_history(stale_ctx) except Exception as e: logger.debug(f"写入清理请求监控历史失败: {e}") except Exception as e: logger.debug(f"僵尸请求定时清扫异常: {e}") def start_request(self, ctx: RequestContext, tab_id: str = None): """标记请求开始执行""" ctx.mark_running(tab_id) with self._requests_lock: self.total_requests += 1 self._schedule_stats_save() # 日志由调用方在上下文中记录,这里不再重复 def bind_tab(self, request_id: str, tab_id: str) -> bool: """为已创建/运行中的请求补绑真实标签页 ID。""" if not request_id or not tab_id: return False with self._requests_lock: ctx = self._requests.get(request_id) if not ctx or ctx.is_terminal(): return False with ctx._lock: previous_tab_id = str(ctx.tab_id or "").strip() ctx.tab_id = tab_id logger.debug( f"[{request_id}] 绑定标签页: {previous_tab_id or '-'} -> {tab_id} " f"(status={ctx.status.value}, cancel_reason={ctx.cancel_reason or '-'})" ) return True def finish_request(self, ctx: RequestContext, success: bool = True): """标记请求结束""" with ctx._lock: if ctx.status in ( RequestStatus.COMPLETED, RequestStatus.FAILED, ): pass elif ctx._cancel_flag or ctx.status == RequestStatus.CANCELLED: ctx.status = RequestStatus.CANCELLED elif ctx.status == RequestStatus.RUNNING: ctx.status = RequestStatus.COMPLETED if success else RequestStatus.FAILED elif ctx.status not in ( RequestStatus.COMPLETED, RequestStatus.FAILED, ): ctx.status = RequestStatus.COMPLETED if success else RequestStatus.FAILED if ctx.finished_at is None: ctx.finished_at = time.time() snapshot = ctx.snapshot() # 设置上下文后记录日志 token = _request_context.set(ctx.request_id) try: logger.info( f"完成 ({snapshot['duration']:.1f}s, status={snapshot['status'].value}, " f"tab={snapshot['tab_id'] or '-'}, reason={snapshot['cancel_reason'] or '-'})" ) finally: _request_context.reset(token) try: self._append_monitor_history(ctx) except Exception as e: logger.debug(f"写入请求监控历史失败: {e}") try: # R2-2 / R2-7:按协议与最终状态计数(每个请求只记一次) with ctx._lock: already_reported = getattr(ctx, "_metrics_reported", False) ctx._metrics_reported = True if not already_reported: from app.services.metrics import CHAT_REQUEST_DURATION, CHAT_REQUESTS protocol = str(getattr(ctx, "protocol", "") or "openai.chat") CHAT_REQUESTS.inc(protocol=protocol, status=snapshot["status"].value) CHAT_REQUEST_DURATION.observe(float(snapshot.get("duration") or 0.0), protocol=protocol) except Exception as e: logger.debug(f"记录请求指标失败: {e}") # Count a terminal request once, not every cleanup/wrapper invocation. # Partial text, tool text and media are real output; empty SSE keepalives are not. with ctx._lock: reported = getattr(ctx, '_cancel_storm_reported', False) ctx._cancel_storm_reported = True client_fp = ctx.client_fp has_output = bool(ctx.monitor.get('has_response_text') or ctx.monitor.get('has_response_media') or int(ctx.monitor.get('response_tokens') or 0) > 0) status, reason = ctx.status, str(ctx.cancel_reason or '') if client_fp and not reported: if status == RequestStatus.COMPLETED or has_output: cancel_storm_guard.record_success(client_fp) elif status == RequestStatus.CANCELLED: cancel_storm_guard.record_cancel(client_fp, has_output=False, reason=reason) def cancel_request(self, request_id: str, reason: str = "manual") -> bool: """取消指定请求""" with self._requests_lock: ctx = self._requests.get(request_id) if not ctx: return False if ctx.is_terminal(): return False ctx.request_cancel(reason) try: self._append_monitor_history(ctx) except Exception as e: logger.debug(f"写入取消请求监控历史失败: {e}") return True def get_request(self, request_id: str) -> Optional[RequestContext]: with self._requests_lock: return self._requests.get(request_id) def get_running_requests(self, tab_id: str = None) -> list: """获取所有正在执行的请求""" with self._requests_lock: contexts = list(self._requests.values()) return [ctx for ctx in contexts if ctx.is_running_for_tab(tab_id)] def get_status(self) -> Dict[str, Any]: """获取管理器状态""" with self._requests_lock: contexts = list(self._requests.values()) status_counts = {} running = [] now = time.time() for ctx in contexts: summary = ctx.status_summary(now) status = summary["status"] status_value = status.value if isinstance(status, RequestStatus) else str(status) status_counts[status_value] = status_counts.get(status_value, 0) + 1 if status == RequestStatus.RUNNING: running.append({ "request_id": summary["request_id"], "tab_id": summary["tab_id"], "duration": round(summary["duration"], 1), }) return { "running_count": len(running), "running_requests": running, "total_tracked": len(contexts), "status_counts": status_counts, } # ================= 兼容旧接口 ================= def is_locked(self) -> bool: """兼容旧接口 - 检查是否有正在执行的请求""" with self._requests_lock: contexts = list(self._requests.values()) return any(ctx.is_running_for_tab() for ctx in contexts) def get_current_request_id(self, tab_id: str = None) -> Optional[str]: """兼容旧接口 - 获取当前执行的请求ID(返回第一个)""" with self._requests_lock: contexts = list(self._requests.values()) for ctx in contexts: if ctx.is_running_for_tab(tab_id): return ctx.request_id return None def cancel_current(self, reason: str = "manual", tab_id: str = None) -> bool: """取消当前正在执行的请求。传入 tab_id 时仅取消该标签页关联请求。""" cancelled = False for ctx in self.get_running_requests(tab_id=tab_id): if self.cancel_request(ctx.request_id, reason): cancelled = True return cancelled def force_release(self, tab_id: str = None) -> bool: """兼容旧接口 - 强制取消运行中的请求。传入 tab_id 时仅取消该标签页请求。""" return self.cancel_current("force_release", tab_id=tab_id) # ================= 全局单例 ================= request_manager = RequestManager() # ================= 辅助函数 ================= async def watch_client_disconnect(request, ctx: RequestContext, check_interval: float = 0.5): """监控客户端连接状态""" try: while not ctx.is_terminal(): if await request.is_disconnected(): ctx.request_cancel("client_disconnected") break await asyncio.sleep(check_interval) except asyncio.CancelledError: pass except Exception as e: logger.debug(f"断开检测异常: {e}")