""" app/api/chat.py - 核心聊天 API 职责: - OpenAI 兼容的 /v1/chat/completions 接口 - 流式/非流式响应处理 - 模型列表 """ import json import codecs import contextlib # 修复(7):用于 aclosing 确保内部异步生成器及时清理 import copy import os import random import re import time import asyncio import queue import threading import uuid from collections import OrderedDict from typing import Optional, Any, Dict, List, Callable, AsyncIterator from fastapi import APIRouter, Request, HTTPException, Header, Depends from fastapi.responses import JSONResponse, StreamingResponse as StarletteStreamingResponse from app.api.streaming_response import RequestStreamingResponse as StreamingResponse from app.core.chat_job import PROTOCOL_OPENAI_RESPONSES, job_for, use_chat_job, with_chat_job from pydantic import BaseModel, Field, field_validator from app.core.config import _request_context, get_logger, SSEFormatter from app.core import get_browser from app.services.request_manager import ( request_manager, RequestContext, RequestStatus, watch_client_disconnect, cancel_storm_guard, ) from app.services.request_lifecycle import ( TrackedWorkerExecutionCancelled, request_finalization, get_max_request_execute_time_sec, cleanup_worker_thread_after_request, mark_request_hard_timeout, put_worker_queue_item, run_in_request_context, run_tracked_blocking_call, wait_worker_queue_item, ) from app.services.tool_calling import ( build_tool_completion_response, complete_tool_calling_roundtrip_async, decode_browser_non_stream_payload, extract_tool_calling_assistant_content, get_tool_calling_allow_media_postprocess, has_tool_calling_request, iter_tool_stream_chunks, normalize_tool_request, summarize_messages_for_debug, ) from app.api.openai_stop import ( apply_stop_sequences_to_text, build_stop_sequence_stream_state, extract_openai_sse_error_message, filter_openai_stop_sse_chunk, flush_openai_stop_state, iter_openai_sse_payloads, sse_chunk_has_done, sse_frame_data_text, ) from app.api.deps import ( verify_dashboard_auth, verify_service_auth, verify_service_token, ) from app.services.catalog_routing import catalog_router, select_catalog_tab from app.services.error_metadata import ( resolve_error_metadata, build_error_response, is_manual_terminate, is_prompt_rejection_code, ) from app.services.text_filter import sanitize_response_text from app.utils.model_routing import collect_route_domain_models, inspect_model_route from app.utils.site_url import route_domain_matches logger = get_logger("API.CHAT") router = APIRouter() MODEL_LIST_CREATED = int(time.time()) STREAM_QUEUE_POLL_TIMEOUT = 0.5 SSE_HEARTBEAT_INTERVAL = 5.0 RESPONSES_STATE_MAX_ENTRIES = 1024 RESPONSES_STATE_TTL_SEC = 3600.0 def _responses_state_env_bytes(name: str, default_mib: float) -> int: raw = os.getenv(name) try: value = float(str(raw).strip()) if raw not in (None, "") else float(default_mib) except (TypeError, ValueError): value = float(default_mib) if value != value or value <= 0: # NaN / 非正数回落默认 value = float(default_mib) return int(value * 1024 * 1024) # B3:Responses 续接历史的字节预算(单位 MiB,可用环境变量覆盖)。 # 单条超限 → 不存储并留下「过大」墓碑,续接时返回 413 明确提示;总量超限 → 按 LRU 淘汰。 RESPONSES_STATE_MAX_ENTRY_BYTES = _responses_state_env_bytes("RESPONSES_STATE_MAX_ENTRY_MB", 8) RESPONSES_STATE_MAX_TOTAL_BYTES = _responses_state_env_bytes("RESPONSES_STATE_MAX_TOTAL_MB", 64) _responses_state_lock = threading.RLock() # 修复(4):会话历史改存 json.dumps 序列化字符串(不可变), # 读写两侧不再对含 base64 图片的完整消息历史做 deepcopy,序列化/反序列化均在锁外执行 # B3:条目 = (stored_at, serialized 或 None(过大墓碑), owner 主体指纹, 字节数) _responses_state_by_id: "OrderedDict[str, tuple[float, Optional[str], str, int]]" = OrderedDict() _responses_state_total_bytes = 0 def _responses_principal_from_request(request: Any) -> str: """B3:续接状态按调用主体隔离——取请求携带的 API 令牌(Bearer / X-API-Key)的 SHA-256 指纹。 未携带令牌(认证关闭的本机场景)时为空串,所有匿名调用共享一个命名空间,与旧行为一致。 """ import hashlib headers = getattr(request, "headers", None) if headers is None: return "" raw = str(headers.get("authorization") or "").strip() if raw.lower().startswith("bearer "): raw = raw[7:].strip() if not raw: raw = str(headers.get("x-api-key") or "").strip() if not raw: return "" return hashlib.sha256(raw.encode("utf-8")).hexdigest() class _ToolCallingExecutionCancelled(Exception): """Raised when a non-stream tool-calling worker is still running after cancellation.""" def _get_tool_calling_cancel_reason(ctx: RequestContext) -> str: reason = str(ctx.cancel_reason or "").strip() return reason or "tool_calling_cancelled" def _is_manual_terminate(ctx: RequestContext) -> bool: return is_manual_terminate(ctx) def _manual_terminate_response() -> JSONResponse: return JSONResponse( content={ "error": { "message": "请求已被手动中断", "type": "request_cancelled", "code": "manual_terminate", } }, status_code=499, headers={"x-should-retry": "false"}, ) def _request_cancelled_response() -> JSONResponse: # 修复(2):非手动终止的内部取消也返回与手动终止一致的结构化 499,而非无结构 500 return JSONResponse( content={ "error": { "message": "请求已被取消", "type": "request_cancelled", "code": "request_cancelled", } }, status_code=499, headers={"x-should-retry": "false"}, ) # ========================================================================= # 兼容性别名与适配函数(维持既有测试用例与下游依赖契约) # ========================================================================= from app.services.error_metadata import ( _PROMPT_REJECTION_CODES as _ARENA_PROMPT_REJECTION_REASONS, _KNOWN_NON_RETRYABLE_CODES as _ARENA_NON_RETRYABLE_REASONS, ARENA_PROMPT_REJECTED_CODE, ARENA_NON_RETRYABLE_CODES, ARENA_PAGE_ERROR, ARENA_SEND_NO_TARGET_AFTER_RETRY, ) # ================= 向前兼容适配层(供外部测试与兼容调用) ================= _select_arena_catalog_tab = select_catalog_tab def _match_arena_catalog_tab( browser: Any, config_engine: Any, tabs: List[Dict[str, Any]], requested_model: Any, ) -> Optional[Dict[str, Any]]: res = catalog_router.match_route(browser, config_engine, tabs, requested_model) if not res: return None return { "tab": res.get("catalog_tab"), "model": res.get("matched_model"), "preset_name": res.get("preset_name"), } _match_arena_catalog_tab._is_default_wrapper = True def _is_arena_prompt_rejection(ctx: RequestContext) -> bool: meta = resolve_error_metadata(ctx) return meta is not None and is_prompt_rejection_code(meta.code) def _is_arena_non_retryable(ctx: RequestContext) -> bool: meta = resolve_error_metadata(ctx) return meta is not None and not meta.retryable and meta.status_code == 422 def _is_arena_prompt_rejection_payload(payload: Any) -> bool: meta = resolve_error_metadata(payload) return meta is not None and is_prompt_rejection_code(meta.code) def _is_arena_non_retryable_payload(payload: Any) -> bool: meta = resolve_error_metadata(payload) return meta is not None and not meta.retryable and meta.status_code == 422 def _arena_prompt_rejection_response( message: str = "Arena 拒绝了该提示词:内容违反 Terms of Use", ) -> JSONResponse: from app.services.error_metadata import ErrorMetadata meta = resolve_error_metadata(ARENA_PROMPT_REJECTED_CODE, default_message=message) return build_error_response(meta or ErrorMetadata(code=ARENA_PROMPT_REJECTED_CODE, message=message, status_code=422, retryable=False)) def _arena_non_retryable_response(message: str, code: str) -> JSONResponse: from app.services.error_metadata import ErrorMetadata meta = ErrorMetadata(code=code, message=message, status_code=422, retryable=False) return build_error_response(meta) # 动态目录函数符号导出(供 monkeypatch / 兼容测试使用) from app.services.arena_direct_models import ( # noqa: E402 build_openai_model_entries, get_arena_direct_catalog_for_tab, get_arena_direct_model_public_id, list_arena_direct_models, match_arena_direct_model, normalize_model_catalog_config, ) def _is_genuine_task_cancellation() -> bool: """修复(2):区分「当前协程真的被外部 cancel」与「内部路径把取消转换成的 CancelledError」。 cancelling() > 0 说明本任务确实收到了外部取消(如服务关闭、连接层取消), 此时必须继续向上抛;否则视为内部转换信号,可安全地做结构化收尾。 """ task = asyncio.current_task() if task is None: return True # 保守:无法判断时按真实取消处理 try: return task.cancelling() > 0 except AttributeError: return True def _is_absolute_request_timeout_error(error: Any) -> bool: return str(error or "").strip() == "absolute_request_timeout" def _format_tool_calling_error(error: Any) -> tuple[str, str]: if _is_absolute_request_timeout_error(error): return "请求执行超过最大绝对超时,已强制中断", "absolute_request_timeout" return f"执行错误: {error}", str(getattr(error, "code", None) or "tool_calling_failed") async def _run_tracked_tool_calling_worker( worker_fn: Callable[[], Any], *, ctx: RequestContext, worker_state: Dict[str, Any], label: str, ) -> Any: try: return await run_tracked_blocking_call( worker_fn, ctx=ctx, worker_state=worker_state, label=label, poll_timeout=STREAM_QUEUE_POLL_TIMEOUT, ) except TrackedWorkerExecutionCancelled as e: raise _ToolCallingExecutionCancelled(str(e) or _get_tool_calling_cancel_reason(ctx)) def _put_worker_queue_item( chunk_queue: queue.Queue, ctx: RequestContext, item: Any, *, final: bool = False, ) -> bool: return put_worker_queue_item( chunk_queue, ctx, item, final=final, poll_timeout=STREAM_QUEUE_POLL_TIMEOUT, ) def _extract_stream_error_message(chunk: Any) -> str: return extract_openai_sse_error_message(chunk) def _extract_chunk_media_items(data: Dict[str, Any]) -> List[Dict[str, Any]]: media_items: List[Dict[str, Any]] = [] top_level_media = data.get("media") if isinstance(top_level_media, list): media_items.extend(item for item in top_level_media if isinstance(item, dict)) choices = data.get("choices") if isinstance(choices, list) and choices: delta = choices[0].get("delta", {}) if isinstance(delta, dict): delta_media = delta.get("media") if isinstance(delta_media, list): media_items.extend(item for item in delta_media if isinstance(item, dict)) media_items.extend(_extract_content_part_media_items(delta.get("content"))) return media_items def _extract_media_part_ref(value: Any) -> str: if isinstance(value, dict): return str(value.get("url") or value.get("data_uri") or "").strip() return str(value or "").strip() def _extract_content_part_media_items(content: Any) -> List[Dict[str, Any]]: if isinstance(content, dict): content = [content] if not isinstance(content, list): return [] media_items: List[Dict[str, Any]] = [] for part in content: if not isinstance(part, dict): continue part_type = str(part.get("type") or "").strip().lower() ref = "" media_type = "" if part_type in {"image_url", "input_image", "output_image"}: ref = _extract_media_part_ref( part.get("image_url") or part.get("url") or part.get("data_uri") ) media_type = "image" elif part_type in {"audio_url", "input_audio", "output_audio"}: ref = _extract_media_part_ref( part.get("audio_url") or part.get("input_audio") or part.get("url") ) media_type = "audio" elif part_type in {"video_url", "input_video", "output_video"}: ref = _extract_media_part_ref( part.get("video_url") or part.get("input_video") or part.get("url") ) media_type = "video" if not ref: continue media_item: Dict[str, Any] = {"media_type": media_type} if ref.startswith("data:"): media_item["data_uri"] = ref else: media_item["url"] = ref detail = str(part.get("detail") or "").strip() if detail: media_item["detail"] = detail mime = str(part.get("mime_type") or part.get("mime") or "").strip() if mime: media_item["mime"] = mime label = str(part.get("label") or "").strip() if label: media_item["label"] = label media_items.append(media_item) return media_items def _extract_delta_content_text(content: Any) -> str: if content is None: return "" if isinstance(content, str): return content if isinstance(content, list): parts: List[str] = [] for item in content: if isinstance(item, str): parts.append(item) continue if not isinstance(item, dict): continue item_type = str(item.get("type") or "").strip().lower() if item_type in {"text", "input_text", "output_text"}: parts.append(str(item.get("text") or "")) return "".join(parts) if isinstance(content, dict): item_type = str(content.get("type") or "").strip().lower() if item_type in {"text", "input_text", "output_text"}: return str(content.get("text") or "") return "" def _dedupe_media_items(media_items: List[Dict[str, Any]]) -> List[Dict[str, Any]]: deduped: List[Dict[str, Any]] = [] seen = set() for item in media_items or []: media_type = str(item.get("media_type") or "").strip().lower() ref = str(item.get("url") or item.get("data_uri") or "").strip() key = (media_type, ref) if key in seen: continue seen.add(key) deduped.append(item) return deduped def _validate_image_inputs(messages: list) -> None: """Legacy name: validate all recognized attachment sources, never partial success.""" from app.utils.attachments import normalize_messages, AttachmentError try: normalize_messages(messages) except AttachmentError as exc: raise HTTPException(status_code=400, detail=exc.detail()) from exc # ================= 请求模型 ================= class ChatRequest(BaseModel): """聊天请求模型""" workflow_variables: Dict[str, Any] = Field(default_factory=dict) @field_validator('workflow_variables') @classmethod def validate_workflow_variables(cls, value): from app.core.workflow.flow_runtime import validate_inputs, FlowValidationError try: return validate_inputs(value) except FlowValidationError as exc: raise ValueError(str(exc)) from exc model: str = Field(default="未知") messages: list = Field(...) stream: Optional[bool] = Field(default=False) temperature: Optional[float] = Field(default=0.7, ge=0, le=2) max_tokens: Optional[int] = Field(default=None, ge=1) n: Optional[int] = Field(default=1, ge=1) response_format: Optional[dict] = Field(default=None) @field_validator('n') @classmethod def validate_n(cls, value): # B4:网页端一次只产出一个回复,原先接受 n>1 却只返回 1 个 choice(静默降级)。 # 现在明确拒绝,客户端可自行并发多次请求。 if value is not None and int(value) > 1: raise ValueError("仅支持 n=1(网页端每次只生成一个回复;需要多个结果请并发发起多次请求)") return value stop: Optional[Any] = Field(default=None) tools: Optional[list] = Field(default=None) tool_choice: Optional[Any] = Field(default=None) parallel_tool_calls: Optional[bool] = Field(default=None) functions: Optional[list] = Field(default=None) function_call: Optional[Any] = Field(default=None) preset_name: Optional[str] = Field(default=None) stream_options: Optional[dict] = Field(default=None) class ResponsesRequest(BaseModel): """Responses API 请求模型(兼容 Codex / OpenAI Responses wire format)""" workflow_variables: Dict[str, Any] = Field(default_factory=dict) @field_validator('workflow_variables') @classmethod def validate_workflow_variables(cls, value): from app.core.workflow.flow_runtime import validate_inputs, FlowValidationError try: return validate_inputs(value) except FlowValidationError as exc: raise ValueError(str(exc)) from exc model: str = Field(default="未知") input: Optional[Any] = Field(default="") instructions: Optional[str] = Field(default=None) stream: Optional[bool] = Field(default=False) temperature: Optional[float] = Field(default=1.0, ge=0, le=2) max_output_tokens: Optional[int] = Field(default=None, ge=1) tools: Optional[list] = Field(default=None) tool_choice: Optional[Any] = Field(default=None) parallel_tool_calls: Optional[bool] = Field(default=None) text: Optional[dict] = Field(default=None) metadata: Optional[dict] = Field(default=None) prompt: Optional[Any] = Field(default=None) previous_response_id: Optional[str] = Field(default=None) reasoning: Optional[dict] = Field(default=None) store: Optional[bool] = Field(default=None) top_p: Optional[float] = Field(default=None, ge=0, le=1) truncation: Optional[str] = Field(default=None) user: Optional[str] = Field(default=None) stop: Optional[Any] = Field(default=None) model_config = { "extra": "allow", } def _is_single_choice_request(body: ChatRequest) -> bool: try: return int(getattr(body, "n", 1) or 1) <= 1 except (TypeError, ValueError): return True def _new_response_id() -> str: return f"resp_{int(time.time() * 1000)}_{uuid.uuid4().hex[:10]}" def _new_response_item_id(prefix: str) -> str: return f"{prefix}_{uuid.uuid4().hex[:12]}" def _pop_responses_state_locked(response_id: str) -> None: global _responses_state_total_bytes entry = _responses_state_by_id.pop(response_id, None) if entry is not None: _responses_state_total_bytes = max(0, _responses_state_total_bytes - int(entry[3] or 0)) def _prune_responses_state_locked(now: Optional[float] = None) -> None: cutoff = float(now if now is not None else time.time()) - RESPONSES_STATE_TTL_SEC expired_ids = [ response_id for response_id, entry in _responses_state_by_id.items() if entry[0] < cutoff ] for response_id in expired_ids: _pop_responses_state_locked(response_id) while len(_responses_state_by_id) > RESPONSES_STATE_MAX_ENTRIES: _pop_responses_state_locked(next(iter(_responses_state_by_id))) # B3:总字节预算,按 LRU(OrderedDict 头部最久未用)淘汰 while _responses_state_by_id and _responses_state_total_bytes > RESPONSES_STATE_MAX_TOTAL_BYTES: _pop_responses_state_locked(next(iter(_responses_state_by_id))) def _load_responses_state( previous_response_id: Optional[str], principal: str = "", ) -> List[Dict[str, Any]]: response_id = str(previous_response_id or "").strip() if not response_id: return [] with _responses_state_lock: _prune_responses_state_locked() entry = _responses_state_by_id.get(response_id) # B3:不同主体一律当作不存在(不泄露 ID 是否有效) if entry is None or str(entry[2] or "") != str(principal or ""): raise HTTPException( status_code=404, detail=f"previous_response_id not found or expired: {response_id}", ) if entry[1] is None: raise HTTPException( status_code=413, detail=( f"previous_response_id 的会话历史超过存储上限 " f"({RESPONSES_STATE_MAX_ENTRY_BYTES // (1024 * 1024)} MiB),未被保存;" "请在 input 中携带完整上下文重新发送" ), ) _responses_state_by_id.move_to_end(response_id) serialized = entry[1] # 修复(4):锁内只做 dict 读写与 LRU 维护;大 payload 的反序列化放到锁外, # 避免含 base64 图片的历史在全局锁持有期间做重拷贝阻塞其他请求。 # json round-trip 对该数据(本就来自 JSON 请求体)与 deepcopy 语义等价 return json.loads(serialized) def _store_responses_state( response_id: str, request_messages: List[Dict[str, Any]], chat_payload: Dict[str, Any], *, enabled: bool, principal: str = "", ) -> None: global _responses_state_total_bytes if not enabled: return choices = chat_payload.get("choices") if isinstance(chat_payload, dict) else None choice = choices[0] if isinstance(choices, list) and choices and isinstance(choices[0], dict) else {} assistant = choice.get("message") if isinstance(choice.get("message"), dict) else None if assistant is None: return # 修复(4):不再 deepcopy 完整历史;仅浅拷贝 assistant 消息以安全改写 role, # 随后整体 json.dumps 成不可变字符串存储(序列化在锁外执行,dumps 本身不改动源数据) history = list(request_messages or []) assistant_message = dict(assistant) assistant_message["role"] = "assistant" history.append(assistant_message) key = str(response_id or "").strip() if not key: return try: serialized = json.dumps(history, ensure_ascii=False) except (TypeError, ValueError) as e: # 数据理论上均来自 JSON 请求体/响应体,不可序列化属异常场景:跳过存储而非让请求 500 logger.warning(f"responses 会话历史序列化失败(跳过存储): {e}") return nbytes = len(serialized.encode("utf-8")) stored: Optional[str] = serialized if nbytes > RESPONSES_STATE_MAX_ENTRY_BYTES: # B3:单条超限不入内存,只留极小的墓碑,续接时给出可预测的 413 logger.warning( f"responses 会话历史 {nbytes / (1024 * 1024):.1f} MiB 超过单条上限 " f"{RESPONSES_STATE_MAX_ENTRY_BYTES / (1024 * 1024):.0f} MiB,未存储: {key}" ) stored = None nbytes = 0 with _responses_state_lock: _prune_responses_state_locked() _pop_responses_state_locked(key) _responses_state_by_id[key] = (time.time(), stored, str(principal or ""), nbytes) _responses_state_total_bytes += nbytes _responses_state_by_id.move_to_end(key) _prune_responses_state_locked() def _pack_responses_sse(event: str, data: Dict[str, Any]) -> str: return ( f"event: {event}\n" f"data: {json.dumps(data, ensure_ascii=False)}\n\n" ) def _format_rfc3339_timestamp(timestamp: Optional[float] = None) -> str: return time.strftime( "%Y-%m-%dT%H:%M:%SZ", time.gmtime(float(timestamp if timestamp is not None else time.time())), ) def _normalize_response_tool_choice(value: Any) -> Any: if isinstance(value, dict): value_type = str(value.get("type") or "").strip().lower() if value_type == "function": function_block = value.get("function") if isinstance(value.get("function"), dict) else {} name = str(function_block.get("name") or value.get("name") or "").strip() if name: return { "type": "function", "function": { "name": name, }, } return value def _normalize_responses_tools(tools: Any) -> Optional[List[Dict[str, Any]]]: if not isinstance(tools, list): return None normalized: List[Dict[str, Any]] = [] for item in tools: if not isinstance(item, dict): continue item_type = str(item.get("type") or "").strip().lower() if item_type != "function": continue if isinstance(item.get("function"), dict): fn = item["function"] name = str(fn.get("name") or "").strip() if not name: continue normalized.append( { "type": "function", "function": { "name": name, "description": str(fn.get("description") or "").strip(), "parameters": fn.get("parameters", {"type": "object", "properties": {}}), }, } ) continue name = str(item.get("name") or "").strip() if not name: continue normalized.append( { "type": "function", "function": { "name": name, "description": str(item.get("description") or "").strip(), "parameters": item.get("parameters", {"type": "object", "properties": {}}), }, } ) return normalized or None def _normalize_responses_text_format(text_config: Any) -> Optional[Dict[str, Any]]: if not isinstance(text_config, dict): return None format_config = text_config.get("format") if not isinstance(format_config, dict): return None format_type = str(format_config.get("type") or "text").strip().lower() if format_type == "json_schema": if isinstance(format_config.get("json_schema"), dict): payload = format_config["json_schema"] else: payload = {} if "name" in format_config: payload["name"] = format_config.get("name") if "schema" in format_config: payload["schema"] = format_config.get("schema") if "strict" in format_config: payload["strict"] = format_config.get("strict") return { "type": "json_schema", "json_schema": payload, } if format_type == "json_object": return {"type": "json_object"} return {"type": format_type} def _normalize_response_message_content(content: Any) -> Any: if content is None: return "" if isinstance(content, str): return content if isinstance(content, dict): return _normalize_response_message_content([content]) if not isinstance(content, list): return str(content) normalized_parts: List[Dict[str, Any]] = [] leading_text_parts: List[str] = [] for part in content: if isinstance(part, str): if part: leading_text_parts.append(part) continue if not isinstance(part, dict): text = str(part or "") if text: leading_text_parts.append(text) continue from app.utils.attachments import normalize_attachment_part, AttachmentError try: attachment = normalize_attachment_part(part) except AttachmentError as exc: raise HTTPException(status_code=400, detail=exc.detail()) from exc if attachment is not None: normalized_parts.append(attachment) continue part_type = str(part.get("type") or "").strip().lower() if part_type in {"input_text", "output_text", "text"}: text = str(part.get("text") or "") if not text: continue if normalized_parts: normalized_parts.append({"type": "text", "text": text}) else: leading_text_parts.append(text) continue if part_type in {"input_image", "image_url", "output_image"}: image_value = part.get("image_url") if isinstance(image_value, dict): image_url = str(image_value.get("url") or "").strip() detail = str(image_value.get("detail") or "").strip() else: image_url = str(image_value or part.get("url") or "").strip() detail = str(part.get("detail") or "").strip() if not image_url: continue image_payload: Dict[str, Any] = {"url": image_url} if detail: image_payload["detail"] = detail normalized_parts.append( { "type": "image_url", "image_url": image_payload, } ) continue if part_type in {"output_audio", "output_video"}: if part_type == "output_audio": media_value = part.get("audio_url") or part.get("input_audio") or part.get("url") or "" media_label = "audio" else: media_value = part.get("video_url") or part.get("url") or "" media_label = "video" media_url = str( media_value.get("url") if isinstance(media_value, dict) else media_value ).strip() if media_url: media_text = f"[{media_label}]({media_url})" if normalized_parts: normalized_parts.append({"type": "text", "text": media_text}) else: leading_text_parts.append(media_text) continue text_fallback_value = part.get("text") if text_fallback_value is None: text_fallback_value = part.get("output") if text_fallback_value is None: text_fallback_value = part.get("content") if text_fallback_value is None: text_fallback = json.dumps(part, ensure_ascii=False) elif isinstance(text_fallback_value, (dict, list)): text_fallback = json.dumps(text_fallback_value, ensure_ascii=False) else: text_fallback = str(text_fallback_value).strip() if text_fallback: if normalized_parts: normalized_parts.append({"type": "text", "text": text_fallback}) else: leading_text_parts.append(text_fallback) joined_text = "\n".join(part for part in leading_text_parts if part) if not normalized_parts: return joined_text if joined_text: normalized_parts.insert( 0, { "type": "text", "text": joined_text, }, ) return normalized_parts def _normalize_response_input_role(role: Any) -> str: normalized_role = str(role or "user").strip().lower() if normalized_role == "developer": return "system" if normalized_role in {"system", "user", "assistant", "tool"}: return normalized_role return "user" def _normalize_response_tool_output_content(content: Any) -> Any: normalized = _normalize_response_message_content(content) if normalized in ("", None): return "" return normalized def _responses_tool_output_can_follow_openai( messages: List[Dict[str, Any]], tool_call_id: str, ) -> bool: if not tool_call_id: return False for message in reversed(messages): if not isinstance(message, dict): continue role = str(message.get("role") or "").strip().lower() if role == "tool": continue if role != "assistant": return False tool_calls = message.get("tool_calls") if not isinstance(tool_calls, list) or not tool_calls: return False return any( isinstance(item, dict) and str(item.get("id") or "").strip() == tool_call_id for item in tool_calls ) return False def _responses_tool_output_fallback_content( item: Dict[str, Any], output: Any, ) -> Any: call_id = str(item.get("call_id") or item.get("tool_call_id") or item.get("id") or "").strip() name = str(item.get("name") or "").strip() header = "[Function Call Output" if name: header += f": {name}" if call_id: header += f" ({call_id})" header += "]" if isinstance(output, list): return [{"type": "text", "text": f"{header}\n"}] + output if isinstance(output, dict): output = json.dumps(output, ensure_ascii=False) return f"{header}\n{str(output or '')}".strip() def _response_function_call_to_tool_call(item: Dict[str, Any]) -> Dict[str, Any]: arguments = item.get("arguments") if not isinstance(arguments, str): arguments = json.dumps(arguments if arguments is not None else {}, ensure_ascii=False) return { "id": str(item.get("call_id") or item.get("id") or _new_response_item_id("call")).strip(), "type": "function", "function": { "name": str(item.get("name") or "").strip(), "arguments": arguments, }, } def _normalize_chat_style_tool_calls(tool_calls: Any) -> Optional[List[Any]]: if not isinstance(tool_calls, list): return None normalized: List[Any] = [] for item in tool_calls: if not isinstance(item, dict): normalized.append(item) continue next_item = dict(item) function_data = next_item.get("function") if isinstance(function_data, dict): next_function = dict(function_data) arguments = next_function.get("arguments") if not isinstance(arguments, str): next_function["arguments"] = json.dumps( arguments if arguments is not None else {}, ensure_ascii=False, ) next_item["function"] = next_function normalized.append(next_item) return normalized def _append_response_input_item(messages: List[Dict[str, Any]], item: Any) -> None: if item is None: return if isinstance(item, str): if item: messages.append({"role": "user", "content": item}) return if not isinstance(item, dict): text = str(item) if text: messages.append({"role": "user", "content": text}) return role = str(item.get("role") or "").strip().lower() item_type = str(item.get("type") or "").strip().lower() if role or item_type == "message": normalized_role = _normalize_response_input_role(role) content = item.get("content") if content is None and "text" in item: content = item.get("text") message_payload: Dict[str, Any] = { "role": normalized_role, "content": _normalize_response_message_content(content), } if normalized_role == "assistant": tool_calls = _normalize_chat_style_tool_calls(item.get("tool_calls")) if tool_calls is not None: message_payload["tool_calls"] = tool_calls if normalized_role == "tool": tool_call_id = str( item.get("tool_call_id") or item.get("call_id") or item.get("id") or "" ).strip() if tool_call_id: message_payload["tool_call_id"] = tool_call_id name = str(item.get("name") or "").strip() if name: message_payload["name"] = name messages.append(message_payload) return if item_type == "function_call": messages.append( { "role": "assistant", "content": "", "tool_calls": [_response_function_call_to_tool_call(item)], } ) return if item_type in {"function_call_output", "tool_result"}: output_source = item.get("output") if output_source is None and "content" in item: output_source = item.get("content") output = _normalize_response_tool_output_content(output_source) tool_call_id = str( item.get("call_id") or item.get("tool_call_id") or item.get("id") or "" ).strip() if _responses_tool_output_can_follow_openai(messages, tool_call_id): tool_message: Dict[str, Any] = { "role": "tool", "tool_call_id": tool_call_id, "content": output, } name = str(item.get("name") or "").strip() if name: tool_message["name"] = name messages.append(tool_message) else: messages.append( { "role": "user", "content": _responses_tool_output_fallback_content(item, output), } ) return content = item.get("content") if content is None and "text" in item: content = item.get("text") normalized_content = _normalize_response_message_content(content) if normalized_content not in ("", [], None): messages.append({"role": "user", "content": normalized_content}) def _responses_input_to_messages(body: ResponsesRequest, principal: str = "") -> List[Dict[str, Any]]: messages: List[Dict[str, Any]] = [] instructions = str(body.instructions or "").strip() if instructions: messages.append({"role": "system", "content": instructions}) messages.extend(_load_responses_state(body.previous_response_id, principal)) source = body.input if source in (None, "") and body.prompt not in (None, ""): source = body.prompt if isinstance(source, list): pending_tool_calls: List[Dict[str, Any]] = [] def _flush_pending_tool_calls() -> None: nonlocal pending_tool_calls if not pending_tool_calls: return messages.append( { "role": "assistant", "content": "", "tool_calls": pending_tool_calls, } ) pending_tool_calls = [] for item in source: if isinstance(item, dict) and str(item.get("type") or "").strip().lower() == "function_call": pending_tool_calls.append(_response_function_call_to_tool_call(item)) continue _flush_pending_tool_calls() _append_response_input_item(messages, item) _flush_pending_tool_calls() else: _append_response_input_item(messages, source) if not messages: messages.append({"role": "user", "content": ""}) return messages def _responses_request_to_chat_request( body: ResponsesRequest, *, stream: bool, principal: str = "", ) -> ChatRequest: return ChatRequest( workflow_variables=body.workflow_variables, model=body.model, messages=_responses_input_to_messages(body, principal), stream=stream, temperature=body.temperature, max_tokens=body.max_output_tokens, response_format=_normalize_responses_text_format(body.text), tools=_normalize_responses_tools(body.tools), tool_choice=_normalize_response_tool_choice(body.tool_choice), parallel_tool_calls=body.parallel_tool_calls, stop=body.stop, ) def _response_message_item_from_text(text: str) -> Dict[str, Any]: return { "id": _new_response_item_id("msg"), "type": "message", "status": "completed", "role": "assistant", "content": [ { "type": "output_text", "text": str(text or ""), "annotations": [], } ], } def _response_content_part_from_media_item(item: Any, index: int) -> Optional[Dict[str, Any]]: if not isinstance(item, dict): return None media_type = str(item.get("media_type") or item.get("type") or "image").strip().lower() ref = str(item.get("url") or item.get("data_uri") or "").strip() if not ref: return None if media_type == "audio": payload: Dict[str, Any] = {"type": "output_audio", "audio_url": ref} elif media_type == "video": payload = {"type": "output_video", "video_url": ref} else: payload = {"type": "output_image", "image_url": ref} payload["annotations"] = [] payload["index"] = index mime = str(item.get("mime") or "").strip() if mime: payload["mime_type"] = mime label = str(item.get("label") or "").strip() if label: payload["label"] = label return payload def _response_message_item_from_content( text: str, media_items: Optional[List[Dict[str, Any]]] = None, ) -> Dict[str, Any]: content: List[Dict[str, Any]] = [] if text: content.append( { "type": "output_text", "text": str(text or ""), "annotations": [], } ) for item in media_items or []: media_part = _response_content_part_from_media_item(item, len(content)) if media_part is not None: content.append(media_part) if not content: content.append( { "type": "output_text", "text": "", "annotations": [], } ) return { "id": _new_response_item_id("msg"), "type": "message", "status": "completed", "role": "assistant", "content": content, } def _response_function_call_item_from_tool_call(tool_call: Dict[str, Any]) -> Dict[str, Any]: function_data = tool_call.get("function") if isinstance(tool_call.get("function"), dict) else {} call_id = str(tool_call.get("id") or _new_response_item_id("call")).strip() item_id = ( f"fc_{call_id[5:]}" if call_id.startswith("call_") else _new_response_item_id("fc") ) arguments = function_data.get("arguments") if not isinstance(arguments, str): arguments = json.dumps(arguments if arguments is not None else {}, ensure_ascii=False) return { "id": item_id, "type": "function_call", "status": "completed", "call_id": call_id, "name": str(function_data.get("name") or "").strip(), "arguments": arguments, } def _chat_payload_to_responses_output(chat_payload: Dict[str, Any]) -> List[Dict[str, Any]]: choices = chat_payload.get("choices") if isinstance(chat_payload.get("choices"), list) else [] if not choices: return [] message = choices[0].get("message") if isinstance(choices[0], dict) else {} if not isinstance(message, dict): return [] output: List[Dict[str, Any]] = [] tool_calls = message.get("tool_calls") if isinstance(message.get("tool_calls"), list) else [] for tool_call in tool_calls: if not isinstance(tool_call, dict): continue output.append(_response_function_call_item_from_tool_call(tool_call)) content = message.get("content") media_items = message.get("media") if not isinstance(media_items, list): media_items = chat_payload.get("media") if isinstance(chat_payload.get("media"), list) else [] if content not in ("", None) or media_items: output.append(_response_message_item_from_content(str(content or ""), media_items)) return output def _build_in_progress_response_item(item: Dict[str, Any]) -> Dict[str, Any]: item_type = str(item.get("type") or "").strip().lower() if item_type == "function_call": return { "id": item.get("id"), "type": "function_call", "status": "in_progress", "call_id": item.get("call_id"), "name": item.get("name"), "arguments": "", } if item_type == "message": return { "id": item.get("id"), "type": "message", "status": "in_progress", "role": item.get("role") or "assistant", "content": [], } return dict(item) def _build_responses_usage(chat_payload: Dict[str, Any]) -> Dict[str, int]: usage = chat_payload.get("usage") if isinstance(chat_payload.get("usage"), dict) else {} return { "input_tokens": int(usage.get("prompt_tokens") or 0), "output_tokens": int(usage.get("completion_tokens") or 0), "total_tokens": int(usage.get("total_tokens") or 0), } def _build_responses_text_payload(body: ResponsesRequest) -> Dict[str, Any]: format_payload = _normalize_responses_text_format(body.text) text_payload = { "format": format_payload if isinstance(format_payload, dict) else {"type": "text"}, "verbosity": "medium", } if isinstance(body.text, dict) and body.text.get("verbosity") is not None: text_payload["verbosity"] = body.text.get("verbosity") return text_payload def _responses_parallel_tool_calls(body: ResponsesRequest) -> bool: if body.parallel_tool_calls is None: return True return bool(body.parallel_tool_calls) def _responses_tool_choice(body: ResponsesRequest) -> Any: if body.tool_choice is None: return "auto" return body.tool_choice def _responses_metadata(body: ResponsesRequest) -> Dict[str, Any]: if isinstance(body.metadata, dict): return body.metadata return {} def _responses_reasoning_payload(body: ResponsesRequest) -> Dict[str, Any]: if isinstance(body.reasoning, dict): return body.reasoning return {"effort": None, "summary": None} def _responses_request_settings(body: ResponsesRequest) -> Dict[str, Any]: return { "max_output_tokens": body.max_output_tokens, "parallel_tool_calls": _responses_parallel_tool_calls(body), "previous_response_id": body.previous_response_id, "reasoning": _responses_reasoning_payload(body), "store": True if body.store is None else bool(body.store), "temperature": body.temperature if body.temperature is not None else 1.0, "tool_choice": _responses_tool_choice(body), "top_p": body.top_p if body.top_p is not None else 1.0, "truncation": body.truncation if body.truncation is not None else "disabled", "user": body.user, "metadata": _responses_metadata(body), } def _build_responses_object( body: ResponsesRequest, chat_payload: Dict[str, Any], *, response_id: Optional[str] = None, created_at: Optional[int] = None, status: str = "completed", error: Optional[Dict[str, Any]] = None, incomplete_details: Optional[Dict[str, Any]] = None, ) -> Dict[str, Any]: response_obj: Dict[str, Any] = { "id": response_id or _new_response_id(), "object": "response", "created_at": int(created_at if created_at is not None else time.time()), "status": status, "error": error, "incomplete_details": incomplete_details, "instructions": body.instructions, "model": body.model, "output": _chat_payload_to_responses_output(chat_payload), "tools": body.tools or [], "text": _build_responses_text_payload(body), "usage": _build_responses_usage(chat_payload), } response_obj.update(_responses_request_settings(body)) if status == "completed": response_obj["completed_at"] = int(time.time()) return response_obj def _responses_completion_status_from_chat_payload(chat_payload: Dict[str, Any]) -> tuple[str, Optional[Dict[str, Any]], str]: choices = chat_payload.get("choices") if isinstance(chat_payload.get("choices"), list) else [] for choice in choices: if not isinstance(choice, dict): continue finish_reason = str(choice.get("finish_reason") or "").strip().lower() if finish_reason == "length": return "incomplete", {"reason": "max_output_tokens"}, "response.incomplete" if finish_reason == "content_filter": return "incomplete", {"reason": "content_filter"}, "response.incomplete" return "completed", None, "response.completed" def _responses_error_payload(payload: Dict[str, Any]) -> Dict[str, Any]: error = payload.get("error") if isinstance(payload, dict) else None if isinstance(error, dict): error_payload = dict(error) error_payload["message"] = str(error_payload.get("message") or "responses_backing_request_failed") error_payload["type"] = str(error_payload.get("type") or "execution_error") if error_payload.get("code") is None: error_payload["code"] = "responses_backing_request_failed" return error_payload if isinstance(error, str) and error.strip(): return { "message": error.strip(), "type": "execution_error", "code": "responses_backing_request_failed", } return { "message": "responses_backing_request_failed", "type": "execution_error", "code": "responses_backing_request_failed", } def _decode_json_response(response: JSONResponse) -> Dict[str, Any]: raw = response.body if isinstance(raw, bytes): text = raw.decode("utf-8") else: text = str(raw or "") data = json.loads(text or "{}") return data if isinstance(data, dict) else {} async def _run_chat_completion_final( request: Request, body: ChatRequest, authenticated: bool, ) -> tuple[int, Dict[str, Any]]: effective_body = body if body.stream is False else body.model_copy(update={"stream": False}) response = await chat_completions( request=request, body=effective_body, authenticated=authenticated, ) if not isinstance(response, JSONResponse): raise RuntimeError("responses_backing_request_unexpected_response_type") return int(response.status_code), _decode_json_response(response) async def _stream_responses_compat( request: Request, body: ResponsesRequest, chat_body: ChatRequest, authenticated: bool, ): response_id = _new_response_id() created_at = int(time.time()) sequence_number = 0 next_output_index = 0 message_item_id = "" message_output_index: Optional[int] = None text_part_started = False message_item_done = False collected_text_parts: List[str] = [] collected_media: List[Dict[str, Any]] = [] tool_call_states: Dict[int, Dict[str, Any]] = {} finish_reason: Optional[str] = None usage_payload: Dict[str, Any] = {} sse_buffer = "" utf8_decoder = codecs.getincrementaldecoder("utf-8")("ignore") def _pack_response_event(event: str, data: Dict[str, Any]) -> str: nonlocal sequence_number sequence_number += 1 payload = dict(data) if event in {"response.created", "response.completed", "response.failed", "response.incomplete"}: payload = {"response": payload} payload.setdefault("type", event) payload["sequence_number"] = sequence_number return _pack_responses_sse(event, payload) def _next_output_index() -> int: nonlocal next_output_index value = next_output_index next_output_index += 1 return value def _decode_stream_chunk(chunk: Any, *, final: bool = False) -> str: if isinstance(chunk, bytes): return utf8_decoder.decode(chunk, final=final) if final: return utf8_decoder.decode(b"", final=True) return str(chunk or "") def _iter_buffered_openai_sse_payloads(chunk: str) -> List[Dict[str, Any]]: nonlocal sse_buffer if not chunk: return [] sse_buffer += chunk.replace("\r\n", "\n").replace("\r", "\n") payloads: List[Dict[str, Any]] = [] while "\n\n" in sse_buffer: frame, sse_buffer = sse_buffer.split("\n\n", 1) if not frame.strip(): continue payloads.extend(iter_openai_sse_payloads(frame + "\n\n")) return payloads def _flush_buffered_openai_sse_payloads() -> List[Dict[str, Any]]: nonlocal sse_buffer tail = sse_buffer sse_buffer = "" if not tail.strip(): return [] return iter_openai_sse_payloads(tail + "\n\n") def _first_choice(payload: Dict[str, Any]) -> Dict[str, Any]: choices = payload.get("choices") if isinstance(payload.get("choices"), list) else [] if not choices or not isinstance(choices[0], dict): return {} return choices[0] def _ensure_message_item() -> List[str]: nonlocal message_item_id, message_output_index if message_output_index is not None: return [] message_item_id = _new_response_item_id("msg") message_output_index = _next_output_index() return [ _pack_response_event( "response.output_item.added", { "type": "response.output_item.added", "response_id": response_id, "output_index": message_output_index, "item": { "id": message_item_id, "type": "message", "status": "in_progress", "role": "assistant", "content": [], }, }, ) ] def _ensure_text_part() -> List[str]: nonlocal text_part_started events = _ensure_message_item() if text_part_started: return events text_part_started = True events.append( _pack_response_event( "response.content_part.added", { "type": "response.content_part.added", "response_id": response_id, "item_id": message_item_id, "output_index": message_output_index, "content_index": 0, "part": { "type": "output_text", "text": "", "annotations": [], }, }, ) ) return events def _final_message_item() -> Dict[str, Any]: content: List[Dict[str, Any]] = [] text = "".join(collected_text_parts) if text_part_started or text: content.append( { "type": "output_text", "text": text, "annotations": [], } ) for media_item in _dedupe_media_items(collected_media): media_part = _response_content_part_from_media_item(media_item, len(content)) if media_part is not None: content.append(media_part) if not content: content.append( { "type": "output_text", "text": "", "annotations": [], } ) return { "id": message_item_id, "type": "message", "status": "completed", "role": "assistant", "content": content, } def _final_tool_item(state: Dict[str, Any]) -> Dict[str, Any]: return { "id": state["item_id"], "type": "function_call", "status": "completed", "call_id": state["call_id"], "name": state["name"], "arguments": state["arguments"], } def _ensure_tool_state(index: int, tool_delta: Dict[str, Any]) -> tuple[Dict[str, Any], List[str]]: function_delta = tool_delta.get("function") if isinstance(tool_delta.get("function"), dict) else {} state = tool_call_states.get(index) if state is not None: if tool_delta.get("id"): state["call_id"] = str(tool_delta.get("id") or state["call_id"]) if function_delta.get("name"): state["name"] = str(function_delta.get("name") or state["name"]) return state, [] call_id = str(tool_delta.get("id") or _new_response_item_id("call")).strip() name = str(function_delta.get("name") or "").strip() item_id = _response_function_call_item_from_tool_call( {"id": call_id, "function": {"name": name, "arguments": ""}} )["id"] state = { "output_index": _next_output_index(), "item_id": item_id, "call_id": call_id, "name": name, "arguments": "", "done": False, } tool_call_states[index] = state return state, [ _pack_response_event( "response.output_item.added", { "type": "response.output_item.added", "response_id": response_id, "output_index": state["output_index"], "item": { "id": state["item_id"], "type": "function_call", "status": "in_progress", "call_id": state["call_id"], "name": state["name"], "arguments": "", }, }, ) ] def _completed_output() -> List[Dict[str, Any]]: output: List[Optional[Dict[str, Any]]] = [None] * max(next_output_index, 0) for state in tool_call_states.values(): output[int(state["output_index"])] = _final_tool_item(state) if message_output_index is not None: output[int(message_output_index)] = _final_message_item() return [item for item in output if isinstance(item, dict)] def _completed_chat_payload() -> Dict[str, Any]: message: Dict[str, Any] = { "role": "assistant", "content": "".join(collected_text_parts), } if collected_media: message["media"] = _dedupe_media_items(collected_media) if tool_call_states: message["tool_calls"] = [ { "id": state["call_id"], "type": "function", "function": { "name": state["name"], "arguments": state["arguments"], }, } for _index, state in sorted(tool_call_states.items()) ] return { "model": body.model, "choices": [ { "message": message, "finish_reason": finish_reason or "stop", } ], "usage": usage_payload, "media": _dedupe_media_items(collected_media), } def _failed_response(payload: Dict[str, Any], error: Optional[Dict[str, Any]] = None) -> Dict[str, Any]: return _build_responses_object( body, payload if isinstance(payload, dict) else {"choices": [], "usage": {}}, response_id=response_id, created_at=created_at, status="failed", error=error or _responses_error_payload(payload), ) def _complete_tool_events(state: Dict[str, Any]) -> List[str]: if state.get("done"): return [] state["done"] = True return [ _pack_response_event( "response.function_call_arguments.done", { "type": "response.function_call_arguments.done", "response_id": response_id, "item_id": state["item_id"], "output_index": state["output_index"], "arguments": state["arguments"], "name": state["name"], "call_id": state["call_id"], }, ), _pack_response_event( "response.output_item.done", { "type": "response.output_item.done", "response_id": response_id, "output_index": state["output_index"], "item": _final_tool_item(state), }, ), ] def _complete_message_events() -> List[str]: nonlocal message_item_done if message_output_index is None or message_item_done: return [] message_item_done = True text = "".join(collected_text_parts) events: List[str] = [] if text_part_started: events.extend( [ _pack_response_event( "response.output_text.done", { "type": "response.output_text.done", "response_id": response_id, "item_id": message_item_id, "output_index": message_output_index, "content_index": 0, "text": text, }, ), _pack_response_event( "response.content_part.done", { "type": "response.content_part.done", "response_id": response_id, "item_id": message_item_id, "output_index": message_output_index, "content_index": 0, "part": { "type": "output_text", "text": text, "annotations": [], }, }, ), ] ) content_index = 1 if text_part_started else 0 for media_item in _dedupe_media_items(collected_media): media_part = _response_content_part_from_media_item(media_item, content_index) if media_part is None: continue events.append( _pack_response_event( "response.content_part.added", { "type": "response.content_part.added", "response_id": response_id, "item_id": message_item_id, "output_index": message_output_index, "content_index": content_index, "part": media_part, }, ) ) events.append( _pack_response_event( "response.content_part.done", { "type": "response.content_part.done", "response_id": response_id, "item_id": message_item_id, "output_index": message_output_index, "content_index": content_index, "part": media_part, }, ) ) content_index += 1 events.append( _pack_response_event( "response.output_item.done", { "type": "response.output_item.done", "response_id": response_id, "output_index": message_output_index, "item": _final_message_item(), }, ) ) return events async def _iter_backing_chunks(streaming_response: StreamingResponse): iterator = streaming_response.body_iterator.__aiter__() next_task = asyncio.create_task(iterator.__anext__()) try: while True: done, _pending = await asyncio.wait( {next_task}, timeout=SSE_HEARTBEAT_INTERVAL, return_when=asyncio.FIRST_COMPLETED, ) if not done: if await request.is_disconnected(): next_task.cancel() try: await next_task except asyncio.CancelledError: pass return yield ": keepalive\n\n" continue try: chunk = next_task.result() except StopAsyncIteration: break yield chunk next_task = asyncio.create_task(iterator.__anext__()) finally: if not next_task.done(): next_task.cancel() try: await next_task except asyncio.CancelledError: pass close = getattr(iterator, "aclose", None) if callable(close): try: await close() except Exception: pass async def _emit_openai_payload_events(payloads: List[Dict[str, Any]]) -> AsyncIterator[str]: nonlocal finish_reason, usage_payload for payload in payloads: if "error" in payload: yield _pack_response_event("response.failed", _failed_response(payload)) return if isinstance(payload.get("usage"), dict): usage_payload = dict(payload.get("usage") or {}) choice = _first_choice(payload) if not choice: continue if choice.get("finish_reason") is not None: finish_reason = str(choice.get("finish_reason") or "") delta = choice.get("delta") if isinstance(choice.get("delta"), dict) else {} collected_media.extend(_extract_chunk_media_items(payload)) content_text = _extract_delta_content_text(delta.get("content")) if content_text: collected_text_parts.append(content_text) for event_chunk in _ensure_text_part(): yield event_chunk yield _pack_response_event( "response.output_text.delta", { "type": "response.output_text.delta", "response_id": response_id, "item_id": message_item_id, "output_index": message_output_index, "content_index": 0, "delta": content_text, }, ) tool_call_deltas = delta.get("tool_calls") if not isinstance(tool_call_deltas, list): continue for fallback_index, tool_delta in enumerate(tool_call_deltas): if not isinstance(tool_delta, dict): continue try: tool_index = int(tool_delta.get("index", fallback_index) or fallback_index) except Exception: tool_index = fallback_index state, added_events = _ensure_tool_state(tool_index, tool_delta) for event_chunk in added_events: yield event_chunk function_delta = tool_delta.get("function") if isinstance(tool_delta.get("function"), dict) else {} if function_delta.get("name"): state["name"] = str(function_delta.get("name") or state["name"]) arguments_delta = function_delta.get("arguments") if arguments_delta: text_delta = str(arguments_delta) state["arguments"] = str(state.get("arguments") or "") + text_delta yield _pack_response_event( "response.function_call_arguments.delta", { "type": "response.function_call_arguments.delta", "response_id": response_id, "item_id": state["item_id"], "output_index": state["output_index"], "delta": text_delta, }, ) yield _pack_response_event( "response.created", { "id": response_id, "object": "response", "created_at": created_at, "status": "in_progress", "error": None, "incomplete_details": None, "instructions": body.instructions, "model": body.model, "output": [], "tools": body.tools or [], "text": _build_responses_text_payload(body), **_responses_request_settings(body), }, ) try: chat_response = await chat_completions( request=request, body=chat_body, authenticated=authenticated, ) if isinstance(chat_response, JSONResponse): yield _pack_response_event( "response.failed", _failed_response(_decode_json_response(chat_response)), ) return if not isinstance(chat_response, StarletteStreamingResponse): raise RuntimeError("responses_backing_request_unexpected_response_type") async for raw_chunk in _iter_backing_chunks(chat_response): if await request.is_disconnected(): return chunk = _decode_stream_chunk(raw_chunk) if not chunk: continue if chunk.lstrip().startswith(":") and "data:" not in chunk: yield chunk continue async for event_chunk in _emit_openai_payload_events(_iter_buffered_openai_sse_payloads(chunk)): yield event_chunk if event_chunk.startswith("event: response.failed"): return decoder_tail = _decode_stream_chunk(b"", final=True) if decoder_tail: async for event_chunk in _emit_openai_payload_events(_iter_buffered_openai_sse_payloads(decoder_tail)): yield event_chunk if event_chunk.startswith("event: response.failed"): return async for event_chunk in _emit_openai_payload_events(_flush_buffered_openai_sse_payloads()): yield event_chunk if event_chunk.startswith("event: response.failed"): return if collected_media and message_output_index is None: for event_chunk in _ensure_message_item(): yield event_chunk for _index, state in sorted(tool_call_states.items()): for event_chunk in _complete_tool_events(state): yield event_chunk for event_chunk in _complete_message_events(): yield event_chunk chat_payload = _completed_chat_payload() response_status, incomplete_details, terminal_event = _responses_completion_status_from_chat_payload(chat_payload) completed = _build_responses_object( body, chat_payload, response_id=response_id, created_at=created_at, status=response_status, error=None, incomplete_details=incomplete_details, ) completed["output"] = _completed_output() _store_responses_state( response_id, chat_body.messages, chat_payload, enabled=body.store is not False, principal=_responses_principal_from_request(request), ) yield _pack_response_event(terminal_event, completed) except Exception as e: yield _pack_response_event( "response.failed", _failed_response( {"choices": [], "usage": {}}, error={ "message": str(e), "type": "execution_error", "code": "responses_backing_request_failed", }, ), ) # ================= response_format 转化 ================= DEFAULT_RESPONSE_FORMAT_HINTS = { "json_object": "\n\n[系统指令:请以 JSON 格式输出你的回复。确保输出是有效的 JSON 对象,不要包含 ```json 代码块标记或任何其他非 JSON 文字。]", "json_schema": "\n\n[系统指令:请严格按照以下 JSON Schema 格式输出你的回复,确保输出是有效的 JSON,不要包含代码块标记:\n{schema}]", "text": "" } def _get_response_format_hint(format_type: str) -> str: """获取指定格式类型的提示词模板""" format_type = str(format_type or "text").strip().lower() or "text" try: from app.services.config_engine import config_engine hints = config_engine.global_config.get("response_format_hints") if hints and isinstance(hints, dict) and format_type in hints: return hints[format_type] except Exception: pass return DEFAULT_RESPONSE_FORMAT_HINTS.get(format_type, "") def _apply_response_format(messages: list, response_format: dict) -> list: """将 response_format 转化为提示词并追加到最后一条用户消息""" if not isinstance(response_format, dict) or not response_format: return messages format_type = str(response_format.get("type") or "text").strip().lower() or "text" hint_template = _get_response_format_hint(format_type) if not hint_template: return messages hint = hint_template if format_type == "json_schema": json_schema = response_format.get("json_schema", {}) schema_content = ( json_schema.get("schema", json_schema) if isinstance(json_schema, dict) else json_schema ) try: schema_str = json.dumps(schema_content, ensure_ascii=False, indent=2) hint = hint_template.replace("{schema}", schema_str) except Exception: hint = hint_template.replace("{schema}", str(schema_content)) import copy new_messages = copy.deepcopy(messages) for i in range(len(new_messages) - 1, -1, -1): msg = new_messages[i] if msg.get("role") == "user": content = msg.get("content", "") if isinstance(content, str): msg["content"] = content + hint break elif isinstance(content, list): for j in range(len(content) - 1, -1, -1): item = content[j] if isinstance(item, dict) and item.get("type") == "text": item["text"] = item.get("text", "") + hint break else: content.append({"type": "text", "text": hint}) break return new_messages def _should_include_stream_usage(body: ChatRequest) -> bool: stream_options = body.stream_options return isinstance(stream_options, dict) and bool(stream_options.get("include_usage")) def _pack_stream_usage_chunk(model: str, ctx: Optional[RequestContext] = None) -> str: # 修复(8):usage 不再硬编码 0。request_manager 已在 ctx.monitor 上维护估算值: # prompt_tokens 由 record_request_input 写入,response_tokens 由 # capture_response_chunk/capture_response_payload 累计,此处低成本读取填入 prompt_tokens = 0 completion_tokens = 0 if ctx is not None: try: prompt_tokens = max(0, int(ctx.monitor.get("prompt_tokens") or 0)) completion_tokens = max(0, int(ctx.monitor.get("response_tokens") or 0)) except (TypeError, ValueError, AttributeError): prompt_tokens = 0 completion_tokens = 0 data = { "id": f"chatcmpl-usage-{int(time.time() * 1000)}", "object": "chat.completion.chunk", "created": int(time.time()), "model": model, "choices": [], "usage": { "prompt_tokens": prompt_tokens, "completion_tokens": completion_tokens, "total_tokens": prompt_tokens + completion_tokens, }, } return f"data: {json.dumps(data, ensure_ascii=False)}\n\n" def _maybe_pack_stream_usage_chunk( body: ChatRequest, ctx: Optional[RequestContext] = None, ) -> Optional[str]: if not _should_include_stream_usage(body): return None return _pack_stream_usage_chunk(body.model, ctx) def _split_sse_done_frame(chunk: Any) -> tuple[str, bool]: if not isinstance(chunk, str) or not sse_chunk_has_done(chunk): return str(chunk or ""), False frames = chunk.replace("\r\n", "\n").replace("\r", "\n").split("\n\n") kept_frames = [] had_done = False for frame in frames: if not frame: continue if sse_frame_data_text(frame).strip() == "[DONE]": had_done = True continue kept_frames.append(f"{frame}\n\n") return "".join(kept_frames), had_done def _iter_stream_chunks_with_optional_usage( body: ChatRequest, chunks, ctx: Optional[RequestContext] = None, ): usage_emitted = False for chunk in chunks: emit_chunk, chunk_had_done = _split_sse_done_frame(chunk) if emit_chunk: yield emit_chunk if chunk_had_done: if not usage_emitted: # 修复(8):透传 ctx 以便 usage 携带已有的 token 估算 usage_chunk = _maybe_pack_stream_usage_chunk(body, ctx) if usage_chunk: usage_emitted = True yield usage_chunk yield _pack_done() # ================= 认证依赖 ================= async def verify_auth( authorization: Optional[str] = Header(None), x_api_key: Optional[str] = Header(None, alias="X-API-Key"), ) -> bool: """验证对外服务 API 的 Bearer Token 或 X-API-Key。""" return await verify_service_auth( authorization=authorization, x_api_key=x_api_key, ) # ================= 核心聊天 API ================= @router.post("/v1/chat/completions") async def chat_completions( request: Request, body: ChatRequest, authenticated: bool = Depends(verify_auth) ): """ OpenAI 兼容的聊天补全接口 """ client_fp = cancel_storm_guard.get_client_fingerprint(request) await cancel_storm_guard.maybe_backoff(client_fp) logger.info( f"[CHAT:ENTRY] 收到聊天补全请求: model={body.model!r}, preset={body.preset_name!r}, " f"stream={body.stream}, messages_count={len(body.messages)}" ) from app.utils.attachments import normalize_messages, AttachmentError try: body.messages = normalize_messages(body.messages) except AttachmentError as exc: raise HTTPException(status_code=400, detail=exc.detail()) from exc if isinstance(body.response_format, dict) and body.response_format: format_type = str(body.response_format.get("type") or "text").strip().lower() or "text" if format_type != "text": logger.debug(f"检测到 response_format.type={format_type},转化为提示词") body.messages = _apply_response_format(body.messages, body.response_format) setattr(body, "_response_format_applied", True) catalog_tab_index: Optional[int] = None try: browser = get_browser(auto_connect=False) # get_tabs_with_index 可能触发同步 CDP 扫描(阻塞网络 I/O), # 不能在事件循环线程直接调用,否则浏览器卡顿时全服务假死。 tabs = await asyncio.to_thread(browser.tab_pool.get_tabs_with_index) tab_models = collect_route_domain_models(tabs) # 阶段 1:先尝试标签页完全精确匹配(暂不检查通用 ID 和前缀模糊,以便 Arena Catalog 优先于通用 ID 和前缀) route_info = inspect_model_route( body.model, tabs, allow_prefix=False, allow_generic=False, models=tab_models, ) route_domain = str(route_info.get("route_domain") or "") if route_info.get("match_type") == "none": from app.services.config_engine import config_engine # 阶段 2:通用动态模型目录匹配(只在标签页当前绑定的预设内匹配,不跨预设自动切换) catalog_match_result = await asyncio.to_thread( catalog_router.match_route, browser, config_engine, tabs, body.model, body.preset_name, ) if browser else None if catalog_match_result: route_domain = str(catalog_match_result.get("route_domain") or "") catalog_tab_index = catalog_match_result.get("catalog_tab_index") route_info.update({ "route_domain": route_domain, "route_type": catalog_match_result.get("route_type", "model_catalog"), "model_name": catalog_match_result.get("model_name") or str(body.model or ""), "matched_id": catalog_match_result.get("matched_id") or str(body.model or ""), "match_type": catalog_match_result.get("match_type", "catalog"), "preset_name": catalog_match_result.get("preset_name"), "available_model_ids": catalog_match_result.get("available_model_ids") or [], }) else: logger.debug( f"[CATALOG_ROUTER:NOMATCH] 当前标签页预设目录不包含请求模型: requested_model={body.model!r}" ) # 阶段 3:若动态目录仍未命中,检查通用模型 ID(Generic)与标签页前缀模糊匹配(Prefix)兜底 if route_info.get("match_type") == "none": fallback_info = inspect_model_route( body.model, tabs, allow_prefix=True, allow_generic=True, models=tab_models, ) if fallback_info.get("match_type") in {"generic", "prefix"}: route_info = fallback_info route_domain = str(route_info.get("route_domain") or "") except Exception as e: logger.debug(f"模型路由解析失败(已忽略): {e}") route_info = { "normalized_model": str(body.model or "").strip().lower(), "route_domain": "", "matched_id": "", "match_type": "error", "available_model_ids": [], } route_domain = "" if route_info.get("route_type") == "model_name" and route_info.get("model_name"): model_name = str(route_info.get("model_name") or "") logger.info( "模型显示名称路由命中: " f"model={body.model!r}, normalized={route_info.get('normalized_model')!r}, " f"matched_id={route_info.get('matched_id')!r}, match_type={route_info.get('match_type')}, " f"model_name={model_name}, available={route_info.get('available_model_ids')}" ) from app.api import tab_routes as tab_routes_api route_body = tab_routes_api.ChatRequest(**body.model_dump()) if bool(getattr(body, "_response_format_applied", False)): setattr(route_body, "_response_format_applied", True) return await tab_routes_api.chat_with_exposed_model_name( model_name=model_name, request=request, body=route_body, authenticated=authenticated, ) if route_domain: logger.info( "模型路由命中: " f"model={body.model!r}, normalized={route_info.get('normalized_model')!r}, " f"matched_id={route_info.get('matched_id')!r}, match_type={route_info.get('match_type')}, " f"route_domain={route_domain}, available={route_info.get('available_model_ids')}" ) from app.api import tab_routes as tab_routes_api route_body = tab_routes_api.ChatRequest(**body.model_dump()) if bool(getattr(body, "_response_format_applied", False)): setattr(route_body, "_response_format_applied", True) return await tab_routes_api.chat_with_route_domain( route_domain=route_domain, request=request, body=route_body, tab_index=(catalog_tab_index if route_info.get("match_type") == "catalog" else None), selector=None, preset_name=(route_info.get("preset_name") or body.preset_name), authenticated=authenticated, ) elif route_info.get("match_type") == "none" and route_info.get("normalized_model"): logger.debug( "模型路由未命中: " f"model={body.model!r}, normalized={route_info.get('normalized_model')!r}, " f"available={route_info.get('available_model_ids')}" ) ctx = request_manager.create_request(client_fp=client_fp) try: raw_input_len = sum(len(str(msg.get("content") or "")) for msg in body.messages if isinstance(msg, dict)) logger.info(f"[DIAG] 接收到的原始请求 messages 总字符长度: {raw_input_len} 字符, 消息数: {len(body.messages)}") except Exception as e: logger.debug(f"[DIAG] 估算原始请求长度失败: {e}") # 修复(5):create_request 之后、生命周期函数(其内部 finally 才会 finish_request) # 接管之前若抛异常,主动 finish_request 清理,避免 ctx 永久滞留 QUEUED 状态 try: # 修复(6):record_request_input 内部对完整消息执行重正则/大文本处理, # 用 to_thread 移到线程池执行,避免阻塞事件循环;调用顺序保持不变 await asyncio.to_thread( request_manager.record_request_input, ctx, body.model_dump(), endpoint="/v1/chat/completions", route_domain=route_domain, preset_name=body.preset_name, ) with logger.context(ctx.request_id): logger.info("开始") try: logger.debug( "[chat] 请求消息摘要: " f"{summarize_messages_for_debug(body.messages)}" ) except Exception as e: logger.debug(f"[chat] 请求消息摘要生成失败: {e}") is_tool_calling = has_tool_calling_request( messages=body.messages, tools=body.tools, functions=body.functions, ) except asyncio.CancelledError: ctx.request_cancel("coroutine_cancelled") with request_finalization(ctx): pass raise except Exception: request_manager.finish_request(ctx, success=False) raise with logger.context(ctx.request_id): if is_tool_calling: if body.stream: return StreamingResponse( _stream_tool_calling_with_lifecycle(request, body, ctx), media_type="text/event-stream", headers={ "Cache-Control": "no-cache, no-transform", "Connection": "keep-alive", "X-Accel-Buffering": "no" }, request_context=ctx, ) return await _non_stream_tool_calling_with_lifecycle(request, body, ctx) if body.stream: return StreamingResponse( _stream_with_lifecycle(request, body, ctx), media_type="text/event-stream", headers={ "Cache-Control": "no-cache, no-transform", "Connection": "keep-alive", "X-Accel-Buffering": "no" }, request_context=ctx, ) else: return await _non_stream_with_lifecycle(request, body, ctx) @router.post("/v1/responses") async def create_response( request: Request, body: ResponsesRequest, authenticated: bool = Depends(verify_auth) ): """OpenAI Responses API 兼容入口。""" principal = _responses_principal_from_request(request) chat_body = _responses_request_to_chat_request(body, stream=bool(body.stream), principal=principal) # R2-2:标注协议,执行入口据此给 RequestContext 打标签并按协议记录指标 job = job_for(PROTOCOL_OPENAI_RESPONSES, model=body.model, stream=bool(body.stream)) if body.stream: return StreamingResponse( with_chat_job(job, _stream_responses_compat( request=request, body=body, chat_body=chat_body, authenticated=authenticated, )), media_type="text/event-stream", headers={ "Cache-Control": "no-cache, no-transform", "Connection": "keep-alive", "X-Accel-Buffering": "no", }, ) try: with use_chat_job(job): status_code, payload = await _run_chat_completion_final( request=request, body=chat_body, authenticated=authenticated, ) except Exception as e: failed = _build_responses_object( body, {"choices": [], "usage": {}}, status="failed", error={ "message": str(e), "type": "execution_error", "code": "responses_backing_request_failed", }, ) return JSONResponse(content=failed, status_code=500) if status_code >= 400 or "error" in payload: failed = _build_responses_object( body, payload, status="failed", error=_responses_error_payload(payload), ) return JSONResponse(content=failed, status_code=status_code) response_status, incomplete_details, _terminal_event = _responses_completion_status_from_chat_payload(payload) response_obj = _build_responses_object( body, payload, status=response_status, error=None, incomplete_details=incomplete_details, ) _store_responses_state( str(response_obj.get("id") or ""), chat_body.messages, payload, enabled=body.store is not False, principal=principal, ) return JSONResponse(content=response_obj) async def _stream_with_lifecycle( request: Request, body: ChatRequest, ctx: RequestContext ): """流式响应 + 完整生命周期管理""" request_context_token = _request_context.set(ctx.request_id) disconnect_task = None worker_thread = None chunk_queue: Optional[queue.Queue] = None try: request_manager.start_request(ctx) disconnect_task = asyncio.create_task( watch_client_disconnect(request, ctx, check_interval=0.3) ) browser = get_browser(auto_connect=False) chunk_queue = queue.Queue(maxsize=100) def worker(): gen = None chunk_counter = 0 try: gen = browser.execute_workflow( body.messages, stream=True, task_id=ctx.request_id, stop_checker=ctx.should_stop, requested_model=body.model, **({"workflow_variables": body.workflow_variables} if body.workflow_variables else {}), ) for chunk in gen: chunk_counter += 1 if ctx.should_stop(): cancel_reason = str(ctx.cancel_reason or "unknown") if cancel_reason in {"cleanup", "client_disconnected", "coroutine_cancelled"}: logger.debug(f"工作线程检测到停止: {cancel_reason}") else: logger.info(f"工作线程检测到取消: {cancel_reason}") break if not _put_worker_queue_item(chunk_queue, ctx, chunk): logger.debug("工作线程停止入队,结束流式生产") break except Exception as e: logger.error(f"工作线程异常: {e}") _put_worker_queue_item(chunk_queue, ctx, ("ERROR", str(e)), final=True) finally: if gen is not None: try: gen.close() except Exception as e: logger.debug(f"关闭工作流生成器失败(忽略): {e}") _put_worker_queue_item(chunk_queue, ctx, None, final=True) logger.debug("工作线程结束") worker_thread = threading.Thread( target=run_in_request_context, args=(ctx, worker), daemon=True, ) # 供 TabRecovery 判定旧 worker 是否退出(setattr 规避 dataclass 字段限制) setattr(ctx, "worker_thread", worker_thread) worker_thread.start() # Keep the connection alive while the browser workflow is acquiring a # tab or waiting for its first upstream chunk. keepalive_id = f"chatcmpl-{ctx.request_id}" initial_keepalive = SSEFormatter.pack_keepalive( model=body.model, completion_id=keepalive_id, ) request_manager.capture_response_chunk(ctx, initial_keepalive) yield initial_keepalive last_sse_emit_at = time.monotonic() request_started_at = time.monotonic() max_execute_time_sec = get_max_request_execute_time_sec() done_emitted = False client_disconnected = False stop_state = build_stop_sequence_stream_state( body.stop, single_choice=_is_single_choice_request(body), upstream_single_choice=True, ) while True: if mark_request_hard_timeout( ctx, request_started_at, max_execute_time_sec, label="chat_stream", ): request_manager.capture_error(ctx, "请求执行超过最大绝对超时", code="absolute_request_timeout") ctx.mark_failed("absolute_request_timeout") yield _pack_error_done("请求执行超过最大绝对超时,已强制中断", "absolute_request_timeout") done_emitted = True break if await request.is_disconnected(): logger.debug("客户端断开") ctx.request_cancel("client_disconnected") client_disconnected = True break try: chunk = await wait_worker_queue_item( chunk_queue, timeout=STREAM_QUEUE_POLL_TIMEOUT, ) except queue.Empty: # 兜底:worker 的结束哨兵(None)在队列持续打满时可能被 # put_worker_queue_item 丢弃(final 宽限超时/取消时立即丢弃)。 # 若 worker 线程已退出且队列已排空,说明流已结束,直接收尾, # 避免空转到 300 秒绝对超时并把已完成的响应误报为错误。 if not worker_thread.is_alive() and chunk_queue.empty(): logger.warning("工作线程已退出且队列为空(结束标记疑似丢失),提前收尾流式消费") break if time.monotonic() - last_sse_emit_at >= SSE_HEARTBEAT_INTERVAL: yield SSEFormatter.pack_keepalive( model=body.model, completion_id=keepalive_id, ) last_sse_emit_at = time.monotonic() continue if chunk is None: logger.debug("收到结束标记") break if isinstance(chunk, tuple) and chunk[0] == "ERROR": logger.error(f"错误: {chunk[1]}") request_manager.capture_error(ctx, chunk[1], code="worker_error") ctx.mark_failed(chunk[1]) yield _pack_error_done(f"执行错误: {chunk[1]}", "internal_error") break # 🔍 探针:记录发送给客户端的 chunk has_images = '"images"' in chunk if isinstance(chunk, str) else False if has_images: logger.info(f"[SEND] 发送包含图片的 chunk 给客户端") outgoing_chunks = filter_openai_stop_sse_chunk(chunk, stop_state, body.model) for outgoing_chunk in outgoing_chunks: emit_chunk, chunk_had_done = _split_sse_done_frame(outgoing_chunk) if emit_chunk: request_manager.capture_response_chunk(ctx, emit_chunk) yield emit_chunk last_sse_emit_at = time.monotonic() error_message = _extract_stream_error_message(emit_chunk or outgoing_chunk) if error_message: logger.error(f"流式响应返回错误事件: {error_message}") error_code = next((str(payload["error"].get("code") or "stream_error") for payload in iter_openai_sse_payloads(emit_chunk or outgoing_chunk) if isinstance(payload.get("error"), dict)), "stream_error") request_manager.capture_error(ctx, error_message, code=error_code) ctx.mark_failed(error_code) done_chunk = _pack_done() request_manager.capture_response_chunk(ctx, done_chunk) yield done_chunk done_emitted = True break if chunk_had_done: # 修复(8):透传 ctx 以便 usage 携带已有的 token 估算 usage_chunk = _maybe_pack_stream_usage_chunk(body, ctx) if usage_chunk: request_manager.capture_response_chunk(ctx, usage_chunk) yield usage_chunk done_chunk = _pack_done() request_manager.capture_response_chunk(ctx, done_chunk) yield done_chunk done_emitted = True ctx.request_cancel("stream_done") ctx.mark_completed() if done_emitted: break if stop_state.stopped: done_emitted = True ctx.request_cancel("stop_sequence") ctx.mark_completed() break if ctx.status == RequestStatus.FAILED: break await asyncio.sleep(0) # 修复(3):外部取消(如手动终止)后不再补发 usage+[DONE] 伪装成正常完成; # stream_done/stop_sequence/audio_media_fast_return 是正常完成路径设置的取消原因, # 仍走原有 flush 收尾 cancelled_midway = ( ctx.should_stop() and str(ctx.cancel_reason or "") not in { "stream_done", "stop_sequence", "audio_media_fast_return", } ) if ( not client_disconnected and not done_emitted and ctx.status != RequestStatus.FAILED ): if cancelled_midway: # 明确告知客户端响应因取消而不完整,避免半截回复被当成完整响应 error_meta = resolve_error_metadata(ctx) is_manual = _is_manual_terminate(ctx) if error_meta and not error_meta.retryable: cancel_code = error_meta.code cancel_message = error_meta.message request_manager.capture_error(ctx, cancel_message, code=cancel_code) cancel_chunk = _pack_error( cancel_message, cancel_code, error_type=error_meta.error_type, status_code=error_meta.status_code, retryable=error_meta.retryable, ) else: cancel_code = "manual_terminate" if is_manual else "request_cancelled" cancel_message = "请求已被取消,响应不完整" cancel_chunk = _pack_error( cancel_message, cancel_code, error_type="execution_error", status_code=None, retryable=None, ) request_manager.capture_response_chunk(ctx, cancel_chunk) yield cancel_chunk done_chunk = _pack_done() request_manager.capture_response_chunk(ctx, done_chunk) yield done_chunk else: for tail_chunk in flush_openai_stop_state(stop_state, body.model): request_manager.capture_response_chunk(ctx, tail_chunk) yield tail_chunk usage_chunk = _maybe_pack_stream_usage_chunk(body, ctx) if usage_chunk: request_manager.capture_response_chunk(ctx, usage_chunk) yield usage_chunk done_chunk = _pack_done() request_manager.capture_response_chunk(ctx, done_chunk) yield done_chunk if not ctx.should_stop() and ctx.status == RequestStatus.RUNNING: ctx.mark_completed() except GeneratorExit: if not ctx.is_terminal(): ctx.request_cancel("stream_closed") raise except asyncio.CancelledError: logger.debug("协程取消") ctx.request_cancel("coroutine_cancelled") raise except Exception as e: logger.error(f"异常: {e}") request_manager.capture_error(ctx, e, code="internal_error") ctx.mark_failed(str(getattr(e, "code", None) or e)) yield _pack_error(f"执行错误: {str(e)}", "internal_error") yield _pack_done() finally: with request_finalization(ctx, request_context_token): if worker_thread and worker_thread.is_alive(): await cleanup_worker_thread_after_request( worker_thread, ctx, completed=ctx.status == RequestStatus.COMPLETED, cancel_reason="cleanup", join_timeout=5.0, retire_reason="worker_cleanup_timeout", completed_join_timeout=0.5, ) if chunk_queue is not None: try: while not chunk_queue.empty(): chunk_queue.get_nowait() except Exception: pass if disconnect_task: disconnect_task.cancel() try: await disconnect_task except asyncio.CancelledError: pass async def _non_stream_with_lifecycle( request: Request, body: ChatRequest, ctx: RequestContext ) -> JSONResponse: """非流式响应 + 生命周期管理""" collected_content = [] collected_media = [] error_data = None sse_buffer = "" def _iter_buffered_stream_payloads(chunk: str) -> List[Dict[str, Any]]: nonlocal sse_buffer if not chunk: return [] sse_buffer += chunk.replace("\r\n", "\n").replace("\r", "\n") payloads: List[Dict[str, Any]] = [] while "\n\n" in sse_buffer: frame, sse_buffer = sse_buffer.split("\n\n", 1) if not frame.strip(): continue payloads.extend(iter_openai_sse_payloads(frame + "\n\n")) return payloads def _flush_buffered_stream_payloads() -> List[Dict[str, Any]]: nonlocal sse_buffer tail = sse_buffer sse_buffer = "" if not tail.strip(): return [] return iter_openai_sse_payloads(tail + "\n\n") def _consume_stream_payload(data: Dict[str, Any]) -> bool: nonlocal error_data if "error" in data: error_data = data return False collected_media.extend(_extract_chunk_media_items(data)) if "choices" in data and data["choices"]: delta = data["choices"][0].get("delta", {}) content = delta.get("content", "") content_text = _extract_delta_content_text(content) if content_text: collected_content.append(content_text) return True # 修复(7):error 时 break 提前退出 async for,内部生成器的 finally(含 finish_request、 # worker 清理)会被推迟到 GC 才执行;用 aclosing 保证退出本块时同步 aclose async with contextlib.aclosing(_stream_with_lifecycle(request, body, ctx)) as stream_gen: async for chunk in stream_gen: if isinstance(chunk, str): for data in _iter_buffered_stream_payloads(chunk): if not _consume_stream_payload(data): break if error_data: break if not error_data: for data in _flush_buffered_stream_payloads(): if not _consume_stream_payload(data): break # 优先解析错误响应(如 400 安全违规、422 提示词拒绝、429 限流等) error_meta = resolve_error_metadata(error_data) or resolve_error_metadata(ctx) if error_meta: return build_error_response(error_meta) if _is_manual_terminate(ctx): return _manual_terminate_response() if error_data: fallback_meta = resolve_error_metadata(error_data) if fallback_meta: return build_error_response(fallback_meta) return JSONResponse(content=error_data, status_code=500) full_content = sanitize_response_text("".join(collected_content), stop_sequences=body.stop) response = SSEFormatter.pack_non_stream( full_content, model=body.model, media=_dedupe_media_items(collected_media), ) request_manager.capture_response_payload(ctx, response) return JSONResponse(content=response) def _execute_browser_non_stream_messages( browser, messages: List[Dict[str, Any]], request_id: str, stop_checker=None, requested_model: Optional[str] = None, workflow_variables: Optional[Dict[str, Any]] = None, ) -> Dict[str, Any]: payload = None for chunk in browser.execute_workflow( messages, stream=False, task_id=request_id, stop_checker=stop_checker, allow_media_postprocess=get_tool_calling_allow_media_postprocess(), requested_model=requested_model, **({"workflow_variables": workflow_variables} if workflow_variables else {}), ): payload = chunk if not payload: raise RuntimeError("empty_browser_response") data = decode_browser_non_stream_payload(payload) if "error" in data: error = data.get("error") or {} from app.services.error_metadata import BrowserExecutionError raise BrowserExecutionError(error) return data def _extract_assistant_content(response: Dict[str, Any]) -> str: try: return extract_tool_calling_assistant_content(response) except Exception: return "" async def _run_tool_calling_async( browser, body: ChatRequest, request_id: str, stop_checker=None, worker_state: Optional[Dict[str, Any]] = None, ) -> Dict[str, Any]: legacy_function_call = bool(body.functions) and not bool(body.tools) tools, tool_choice = normalize_tool_request( tools=body.tools, tool_choice=body.tool_choice, functions=body.functions, function_call=body.function_call, ) tracked_worker_state = worker_state if isinstance(worker_state, dict) else {} async def _round_executor(browser_messages: List[Dict[str, str]]) -> str: worker_fn = lambda: _extract_assistant_content( _execute_browser_non_stream_messages( browser=browser, messages=browser_messages, request_id=request_id, stop_checker=stop_checker, requested_model=body.model, **({"workflow_variables": body.workflow_variables} if body.workflow_variables else {}), ) ) if isinstance(tracked_worker_state.get("ctx"), RequestContext): return await _run_tracked_tool_calling_worker( worker_fn, ctx=tracked_worker_state["ctx"], worker_state=tracked_worker_state, label=f"{request_id[:8]}-round", ) return await asyncio.to_thread(worker_fn) parsed = await complete_tool_calling_roundtrip_async( messages=body.messages, tools=tools, tool_choice=tool_choice, parallel_tool_calls=( False if legacy_function_call else body.parallel_tool_calls ), round_executor=_round_executor, stop_checker=stop_checker, ) if not parsed.get("tool_calls"): parsed = dict(parsed) parsed["content"] = apply_stop_sequences_to_text( str(parsed.get("content") or ""), body.stop, ) return build_tool_completion_response( body.model, parsed, legacy_function_call=legacy_function_call, ) async def _complete_tool_calling_with_lifecycle( request: Request, body: ChatRequest, ctx: RequestContext, ) -> Dict[str, Any]: disconnect_task = None worker_state: Dict[str, Any] = {"thread": None, "label": None, "ctx": ctx} try: disconnect_task = asyncio.create_task( watch_client_disconnect(request, ctx, check_interval=0.3) ) browser = get_browser(auto_connect=False) request_manager.start_request(ctx) response = await _run_tool_calling_async( browser, body, ctx.request_id, ctx.should_stop, worker_state=worker_state, ) if _is_manual_terminate(ctx): raise _ToolCallingExecutionCancelled(_get_tool_calling_cancel_reason(ctx)) request_manager.capture_response_payload(ctx, response) if not ctx.should_stop() and ctx.status == RequestStatus.RUNNING: ctx.mark_completed() return response except _ToolCallingExecutionCancelled: cancel_reason = _get_tool_calling_cancel_reason(ctx) if cancel_reason == "absolute_request_timeout": request_manager.capture_error(ctx, "请求执行超过最大绝对超时", code="absolute_request_timeout") ctx.mark_failed("absolute_request_timeout") raise RuntimeError("absolute_request_timeout") if not ctx.should_stop(): ctx.request_cancel(cancel_reason or "tool_calling_cancelled") if _is_manual_terminate(ctx): raise raise asyncio.CancelledError() except RuntimeError as e: if _is_manual_terminate(ctx): raise _ToolCallingExecutionCancelled( _get_tool_calling_cancel_reason(ctx) ) from e if str(e) == "tool_calling_cancelled" and ctx.should_stop(): raise asyncio.CancelledError() logger.error(f"tool_calling_failed: {e}") request_manager.capture_error(ctx, e, code=getattr(e, "code", "tool_calling_failed")) ctx.mark_failed(str(getattr(e, "code", None) or e)) raise except asyncio.CancelledError: ctx.request_cancel("coroutine_cancelled") raise asyncio.CancelledError() except Exception as e: logger.error(f"tool_calling_failed: {e}") request_manager.capture_error(ctx, e, code=getattr(e, "code", "tool_calling_failed")) ctx.mark_failed(str(getattr(e, "code", None) or e)) raise finally: with request_finalization(ctx): if disconnect_task: disconnect_task.cancel() try: await disconnect_task except asyncio.CancelledError: pass worker_thread = worker_state.get("thread") if isinstance(worker_thread, threading.Thread) and worker_thread.is_alive(): await cleanup_worker_thread_after_request( worker_thread, ctx, completed=ctx.status == RequestStatus.COMPLETED, cancel_reason="cleanup", join_timeout=5.0, retire_reason="worker_cleanup_timeout", ) worker_state["thread"] = None worker_state["label"] = None async def _non_stream_tool_calling_with_lifecycle( request: Request, body: ChatRequest, ctx: RequestContext, ) -> JSONResponse: try: response = await _complete_tool_calling_with_lifecycle(request, body, ctx) return JSONResponse(content=response) except _ToolCallingExecutionCancelled: return _manual_terminate_response() except asyncio.CancelledError: if _is_manual_terminate(ctx): return _manual_terminate_response() if _is_genuine_task_cancellation(): raise return _request_cancelled_response() except Exception as e: if _is_manual_terminate(ctx): return _manual_terminate_response() message, code = _format_tool_calling_error(e) ctx.mark_failed(message) request_manager.capture_error(ctx, message, code=code) return JSONResponse( content={ "error": { "message": message, "type": "execution_error", "code": code, } }, status_code=500, ) async def _stream_tool_calling_with_lifecycle( request: Request, body: ChatRequest, ctx: RequestContext ): response_task = None keepalive_id = f"chatcmpl-{ctx.request_id}" try: yield SSEFormatter.pack_keepalive( model=body.model, completion_id=keepalive_id, ) response_task = asyncio.create_task( _complete_tool_calling_with_lifecycle(request, body, ctx) ) while True: try: response = await asyncio.wait_for( asyncio.shield(response_task), timeout=SSE_HEARTBEAT_INTERVAL, ) break except asyncio.TimeoutError: yield SSEFormatter.pack_keepalive( model=body.model, completion_id=keepalive_id, ) message = response.get("choices", [{}])[0].get("message", {}) or {} legacy_function_call = bool(body.functions) and not bool(body.tools) response_tool_calls = message.get("tool_calls") or [] if legacy_function_call and not response_tool_calls and isinstance(message.get("function_call"), dict): response_tool_calls = [ {"type": "function", "function": message["function_call"]} ] parsed = { "content": message.get("content"), "tool_calls": response_tool_calls, } for chunk in _iter_stream_chunks_with_optional_usage( body, iter_tool_stream_chunks( body.model, parsed, legacy_function_call=legacy_function_call, ), ctx=ctx, ): if await request.is_disconnected(): ctx.request_cancel("client_disconnected") break yield chunk await asyncio.sleep(0) except GeneratorExit: if not ctx.is_terminal(): ctx.request_cancel("stream_closed") raise except asyncio.CancelledError: if _is_genuine_task_cancellation(): raise cancel_code = "manual_terminate" if _is_manual_terminate(ctx) else "request_cancelled" yield _pack_error("请求已被取消", cancel_code) yield _pack_done() except Exception as e: message, code = _format_tool_calling_error(e) ctx.mark_failed(message) request_manager.capture_error(ctx, message, code=code) yield _pack_error(message, code) yield _pack_done() finally: with request_finalization(ctx): if response_task is not None and not response_task.done(): response_task.cancel() try: await response_task except asyncio.CancelledError: pass def _pack_error( message: str, code: str = "error", *, error_type: str = "execution_error", status_code: Optional[int] = None, retryable: Optional[bool] = None, param: Optional[str] = None, extra: Optional[Dict[str, Any]] = None, ) -> str: return SSEFormatter.pack_error( message=message, error_type=error_type, code=code, status_code=status_code, retryable=retryable, param=param, extra=extra, ) def _pack_done() -> str: return "data: [DONE]\n\n" def _pack_error_done(message: str, code: str = "error") -> str: return f"{_pack_error(message, code)}{_pack_done()}" def _collect_model_entries() -> List[Dict[str, Any]]: entries: List[Dict[str, Any]] = [] seen_model_ids = set() def _append_entry(model_id: Any, owned_by: Any = "universal-web-api", display_name: Any = "") -> None: clean_id = str(model_id or "").strip() normalized_id = clean_id.lower() if not clean_id or normalized_id in seen_model_ids: return seen_model_ids.add(normalized_id) entries.append( { "id": clean_id, "object": "model", "type": "model", "created": MODEL_LIST_CREATED, "owned_by": str(owned_by or "universal-web-api"), "display_name": str(display_name or clean_id), } ) try: browser = get_browser(auto_connect=False) tabs = browser.tab_pool.get_tabs_with_index() from app.services.config_engine import config_engine catalog_router.collect_models_for_catalog( browser=browser, config_engine=config_engine, tabs=tabs, append_entry=_append_entry, created=MODEL_LIST_CREATED, ) except Exception as e: logger.debug(f"构建模型列表失败(已忽略): {e}") if not entries: _append_entry("web-browser") return entries def _build_anthropic_models_response(entries: List[Dict[str, Any]]) -> Dict[str, Any]: data = [] for item in entries: model_id = str(item.get("id") or "").strip() if not model_id: continue data.append( { "type": "model", "id": model_id, "display_name": str(item.get("display_name") or model_id), "created_at": _format_rfc3339_timestamp(item.get("created")), } ) first_id = data[0]["id"] if data else None last_id = data[-1]["id"] if data else None return { "data": data, "has_more": False, "first_id": first_id, "last_id": last_id, } def _verify_models_auth( authorization: Optional[str], x_api_key: Optional[str], ) -> None: verify_service_token(authorization=authorization, x_api_key=x_api_key) # ================= 模型列表 ================= @router.get("/models") @router.get("/model") @router.get("/v1/v1/models") @router.get("/v1/v1/model") @router.get("/v1/models") @router.get("/v1/model") async def list_models( authorization: Optional[str] = Header(None), x_api_key: Optional[str] = Header(None, alias="x-api-key"), anthropic_version: Optional[str] = Header(None, alias="anthropic-version"), ): _verify_models_auth(authorization, x_api_key) data = await asyncio.to_thread(_collect_model_entries) if anthropic_version: return _build_anthropic_models_response(data) return { "object": "list", "data": data, } @router.get("/api/pool/status") async def get_pool_status(authenticated: bool = Depends(verify_dashboard_auth)): """获取标签页池状态""" try: browser = get_browser(auto_connect=False) return browser.get_pool_status() except Exception as e: return {"error": str(e), "initialized": False}