Files
2026-11-02 22:54:14 +08:00

3300 lines
120 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""
app/api/chat.py - 核心聊天 API
职责:
- OpenAI 兼容的 /v1/chat/completions 接口
- 流式/非流式响应处理
- 模型列表
"""
import json
import codecs
import contextlib # 修复(7):用于 aclosing 确保内部异步生成器及时清理
import copy
import os
import random
import re
import time
import asyncio
import queue
import threading
import uuid
from collections import OrderedDict
from typing import Optional, Any, Dict, List, Callable, AsyncIterator
from fastapi import APIRouter, Request, HTTPException, Header, Depends
from fastapi.responses import JSONResponse, StreamingResponse as StarletteStreamingResponse
from app.api.streaming_response import RequestStreamingResponse as StreamingResponse
from app.core.chat_job import PROTOCOL_OPENAI_RESPONSES, job_for, use_chat_job, with_chat_job
from pydantic import BaseModel, Field, field_validator
from app.core.config import _request_context, get_logger, SSEFormatter
from app.core import get_browser
from app.services.request_manager import (
request_manager,
RequestContext,
RequestStatus,
watch_client_disconnect,
cancel_storm_guard,
)
from app.services.request_lifecycle import (
TrackedWorkerExecutionCancelled,
request_finalization,
get_max_request_execute_time_sec,
cleanup_worker_thread_after_request,
mark_request_hard_timeout,
put_worker_queue_item,
run_in_request_context,
run_tracked_blocking_call,
wait_worker_queue_item,
)
from app.services.tool_calling import (
build_tool_completion_response,
complete_tool_calling_roundtrip_async,
decode_browser_non_stream_payload,
extract_tool_calling_assistant_content,
get_tool_calling_allow_media_postprocess,
has_tool_calling_request,
iter_tool_stream_chunks,
normalize_tool_request,
summarize_messages_for_debug,
)
from app.api.openai_stop import (
apply_stop_sequences_to_text,
build_stop_sequence_stream_state,
extract_openai_sse_error_message,
filter_openai_stop_sse_chunk,
flush_openai_stop_state,
iter_openai_sse_payloads,
sse_chunk_has_done,
sse_frame_data_text,
)
from app.api.deps import (
verify_dashboard_auth,
verify_service_auth,
verify_service_token,
)
from app.services.catalog_routing import catalog_router, select_catalog_tab
from app.services.error_metadata import (
resolve_error_metadata,
build_error_response,
is_manual_terminate,
is_prompt_rejection_code,
)
from app.services.text_filter import sanitize_response_text
from app.utils.model_routing import collect_route_domain_models, inspect_model_route
from app.utils.site_url import route_domain_matches
logger = get_logger("API.CHAT")
router = APIRouter()
MODEL_LIST_CREATED = int(time.time())
STREAM_QUEUE_POLL_TIMEOUT = 0.5
SSE_HEARTBEAT_INTERVAL = 5.0
RESPONSES_STATE_MAX_ENTRIES = 1024
RESPONSES_STATE_TTL_SEC = 3600.0
def _responses_state_env_bytes(name: str, default_mib: float) -> int:
raw = os.getenv(name)
try:
value = float(str(raw).strip()) if raw not in (None, "") else float(default_mib)
except (TypeError, ValueError):
value = float(default_mib)
if value != value or value <= 0: # NaN / 非正数回落默认
value = float(default_mib)
return int(value * 1024 * 1024)
# B3:Responses 续接历史的字节预算(单位 MiB,可用环境变量覆盖)。
# 单条超限 → 不存储并留下「过大」墓碑,续接时返回 413 明确提示;总量超限 → 按 LRU 淘汰。
RESPONSES_STATE_MAX_ENTRY_BYTES = _responses_state_env_bytes("RESPONSES_STATE_MAX_ENTRY_MB", 8)
RESPONSES_STATE_MAX_TOTAL_BYTES = _responses_state_env_bytes("RESPONSES_STATE_MAX_TOTAL_MB", 64)
_responses_state_lock = threading.RLock()
# 修复(4):会话历史改存 json.dumps 序列化字符串(不可变),
# 读写两侧不再对含 base64 图片的完整消息历史做 deepcopy,序列化/反序列化均在锁外执行
# B3:条目 = (stored_at, serialized 或 None(过大墓碑), owner 主体指纹, 字节数)
_responses_state_by_id: "OrderedDict[str, tuple[float, Optional[str], str, int]]" = OrderedDict()
_responses_state_total_bytes = 0
def _responses_principal_from_request(request: Any) -> str:
"""B3:续接状态按调用主体隔离——取请求携带的 API 令牌(Bearer / X-API-Key)的 SHA-256 指纹。
未携带令牌(认证关闭的本机场景)时为空串,所有匿名调用共享一个命名空间,与旧行为一致。
"""
import hashlib
headers = getattr(request, "headers", None)
if headers is None:
return ""
raw = str(headers.get("authorization") or "").strip()
if raw.lower().startswith("bearer "):
raw = raw[7:].strip()
if not raw:
raw = str(headers.get("x-api-key") or "").strip()
if not raw:
return ""
return hashlib.sha256(raw.encode("utf-8")).hexdigest()
class _ToolCallingExecutionCancelled(Exception):
"""Raised when a non-stream tool-calling worker is still running after cancellation."""
def _get_tool_calling_cancel_reason(ctx: RequestContext) -> str:
reason = str(ctx.cancel_reason or "").strip()
return reason or "tool_calling_cancelled"
def _is_manual_terminate(ctx: RequestContext) -> bool:
return is_manual_terminate(ctx)
def _manual_terminate_response() -> JSONResponse:
return JSONResponse(
content={
"error": {
"message": "请求已被手动中断",
"type": "request_cancelled",
"code": "manual_terminate",
}
},
status_code=499,
headers={"x-should-retry": "false"},
)
def _request_cancelled_response() -> JSONResponse:
# 修复(2):非手动终止的内部取消也返回与手动终止一致的结构化 499,而非无结构 500
return JSONResponse(
content={
"error": {
"message": "请求已被取消",
"type": "request_cancelled",
"code": "request_cancelled",
}
},
status_code=499,
headers={"x-should-retry": "false"},
)
# =========================================================================
# 兼容性别名与适配函数(维持既有测试用例与下游依赖契约)
# =========================================================================
from app.services.error_metadata import (
_PROMPT_REJECTION_CODES as _ARENA_PROMPT_REJECTION_REASONS,
_KNOWN_NON_RETRYABLE_CODES as _ARENA_NON_RETRYABLE_REASONS,
ARENA_PROMPT_REJECTED_CODE,
ARENA_NON_RETRYABLE_CODES,
ARENA_PAGE_ERROR,
ARENA_SEND_NO_TARGET_AFTER_RETRY,
)
# ================= 向前兼容适配层(供外部测试与兼容调用) =================
_select_arena_catalog_tab = select_catalog_tab
def _match_arena_catalog_tab(
browser: Any,
config_engine: Any,
tabs: List[Dict[str, Any]],
requested_model: Any,
) -> Optional[Dict[str, Any]]:
res = catalog_router.match_route(browser, config_engine, tabs, requested_model)
if not res:
return None
return {
"tab": res.get("catalog_tab"),
"model": res.get("matched_model"),
"preset_name": res.get("preset_name"),
}
_match_arena_catalog_tab._is_default_wrapper = True
def _is_arena_prompt_rejection(ctx: RequestContext) -> bool:
meta = resolve_error_metadata(ctx)
return meta is not None and is_prompt_rejection_code(meta.code)
def _is_arena_non_retryable(ctx: RequestContext) -> bool:
meta = resolve_error_metadata(ctx)
return meta is not None and not meta.retryable and meta.status_code == 422
def _is_arena_prompt_rejection_payload(payload: Any) -> bool:
meta = resolve_error_metadata(payload)
return meta is not None and is_prompt_rejection_code(meta.code)
def _is_arena_non_retryable_payload(payload: Any) -> bool:
meta = resolve_error_metadata(payload)
return meta is not None and not meta.retryable and meta.status_code == 422
def _arena_prompt_rejection_response(
message: str = "Arena 拒绝了该提示词:内容违反 Terms of Use",
) -> JSONResponse:
from app.services.error_metadata import ErrorMetadata
meta = resolve_error_metadata(ARENA_PROMPT_REJECTED_CODE, default_message=message)
return build_error_response(meta or ErrorMetadata(code=ARENA_PROMPT_REJECTED_CODE, message=message, status_code=422, retryable=False))
def _arena_non_retryable_response(message: str, code: str) -> JSONResponse:
from app.services.error_metadata import ErrorMetadata
meta = ErrorMetadata(code=code, message=message, status_code=422, retryable=False)
return build_error_response(meta)
# 动态目录函数符号导出(供 monkeypatch / 兼容测试使用)
from app.services.arena_direct_models import ( # noqa: E402
build_openai_model_entries,
get_arena_direct_catalog_for_tab,
get_arena_direct_model_public_id,
list_arena_direct_models,
match_arena_direct_model,
normalize_model_catalog_config,
)
def _is_genuine_task_cancellation() -> bool:
"""修复(2):区分「当前协程真的被外部 cancel」与「内部路径把取消转换成的 CancelledError」。
cancelling() > 0 说明本任务确实收到了外部取消(如服务关闭、连接层取消),
此时必须继续向上抛;否则视为内部转换信号,可安全地做结构化收尾。
"""
task = asyncio.current_task()
if task is None:
return True # 保守:无法判断时按真实取消处理
try:
return task.cancelling() > 0
except AttributeError:
return True
def _is_absolute_request_timeout_error(error: Any) -> bool:
return str(error or "").strip() == "absolute_request_timeout"
def _format_tool_calling_error(error: Any) -> tuple[str, str]:
if _is_absolute_request_timeout_error(error):
return "请求执行超过最大绝对超时,已强制中断", "absolute_request_timeout"
return f"执行错误: {error}", str(getattr(error, "code", None) or "tool_calling_failed")
async def _run_tracked_tool_calling_worker(
worker_fn: Callable[[], Any],
*,
ctx: RequestContext,
worker_state: Dict[str, Any],
label: str,
) -> Any:
try:
return await run_tracked_blocking_call(
worker_fn,
ctx=ctx,
worker_state=worker_state,
label=label,
poll_timeout=STREAM_QUEUE_POLL_TIMEOUT,
)
except TrackedWorkerExecutionCancelled as e:
raise _ToolCallingExecutionCancelled(str(e) or _get_tool_calling_cancel_reason(ctx))
def _put_worker_queue_item(
chunk_queue: queue.Queue,
ctx: RequestContext,
item: Any,
*,
final: bool = False,
) -> bool:
return put_worker_queue_item(
chunk_queue,
ctx,
item,
final=final,
poll_timeout=STREAM_QUEUE_POLL_TIMEOUT,
)
def _extract_stream_error_message(chunk: Any) -> str:
return extract_openai_sse_error_message(chunk)
def _extract_chunk_media_items(data: Dict[str, Any]) -> List[Dict[str, Any]]:
media_items: List[Dict[str, Any]] = []
top_level_media = data.get("media")
if isinstance(top_level_media, list):
media_items.extend(item for item in top_level_media if isinstance(item, dict))
choices = data.get("choices")
if isinstance(choices, list) and choices:
delta = choices[0].get("delta", {})
if isinstance(delta, dict):
delta_media = delta.get("media")
if isinstance(delta_media, list):
media_items.extend(item for item in delta_media if isinstance(item, dict))
media_items.extend(_extract_content_part_media_items(delta.get("content")))
return media_items
def _extract_media_part_ref(value: Any) -> str:
if isinstance(value, dict):
return str(value.get("url") or value.get("data_uri") or "").strip()
return str(value or "").strip()
def _extract_content_part_media_items(content: Any) -> List[Dict[str, Any]]:
if isinstance(content, dict):
content = [content]
if not isinstance(content, list):
return []
media_items: List[Dict[str, Any]] = []
for part in content:
if not isinstance(part, dict):
continue
part_type = str(part.get("type") or "").strip().lower()
ref = ""
media_type = ""
if part_type in {"image_url", "input_image", "output_image"}:
ref = _extract_media_part_ref(
part.get("image_url") or part.get("url") or part.get("data_uri")
)
media_type = "image"
elif part_type in {"audio_url", "input_audio", "output_audio"}:
ref = _extract_media_part_ref(
part.get("audio_url") or part.get("input_audio") or part.get("url")
)
media_type = "audio"
elif part_type in {"video_url", "input_video", "output_video"}:
ref = _extract_media_part_ref(
part.get("video_url") or part.get("input_video") or part.get("url")
)
media_type = "video"
if not ref:
continue
media_item: Dict[str, Any] = {"media_type": media_type}
if ref.startswith("data:"):
media_item["data_uri"] = ref
else:
media_item["url"] = ref
detail = str(part.get("detail") or "").strip()
if detail:
media_item["detail"] = detail
mime = str(part.get("mime_type") or part.get("mime") or "").strip()
if mime:
media_item["mime"] = mime
label = str(part.get("label") or "").strip()
if label:
media_item["label"] = label
media_items.append(media_item)
return media_items
def _extract_delta_content_text(content: Any) -> str:
if content is None:
return ""
if isinstance(content, str):
return content
if isinstance(content, list):
parts: List[str] = []
for item in content:
if isinstance(item, str):
parts.append(item)
continue
if not isinstance(item, dict):
continue
item_type = str(item.get("type") or "").strip().lower()
if item_type in {"text", "input_text", "output_text"}:
parts.append(str(item.get("text") or ""))
return "".join(parts)
if isinstance(content, dict):
item_type = str(content.get("type") or "").strip().lower()
if item_type in {"text", "input_text", "output_text"}:
return str(content.get("text") or "")
return ""
def _dedupe_media_items(media_items: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
deduped: List[Dict[str, Any]] = []
seen = set()
for item in media_items or []:
media_type = str(item.get("media_type") or "").strip().lower()
ref = str(item.get("url") or item.get("data_uri") or "").strip()
key = (media_type, ref)
if key in seen:
continue
seen.add(key)
deduped.append(item)
return deduped
def _validate_image_inputs(messages: list) -> None:
"""Legacy name: validate all recognized attachment sources, never partial success."""
from app.utils.attachments import normalize_messages, AttachmentError
try:
normalize_messages(messages)
except AttachmentError as exc:
raise HTTPException(status_code=400, detail=exc.detail()) from exc
# ================= 请求模型 =================
class ChatRequest(BaseModel):
"""聊天请求模型"""
workflow_variables: Dict[str, Any] = Field(default_factory=dict)
@field_validator('workflow_variables')
@classmethod
def validate_workflow_variables(cls, value):
from app.core.workflow.flow_runtime import validate_inputs, FlowValidationError
try:
return validate_inputs(value)
except FlowValidationError as exc:
raise ValueError(str(exc)) from exc
model: str = Field(default="未知")
messages: list = Field(...)
stream: Optional[bool] = Field(default=False)
temperature: Optional[float] = Field(default=0.7, ge=0, le=2)
max_tokens: Optional[int] = Field(default=None, ge=1)
n: Optional[int] = Field(default=1, ge=1)
response_format: Optional[dict] = Field(default=None)
@field_validator('n')
@classmethod
def validate_n(cls, value):
# B4:网页端一次只产出一个回复,原先接受 n>1 却只返回 1 个 choice(静默降级)。
# 现在明确拒绝,客户端可自行并发多次请求。
if value is not None and int(value) > 1:
raise ValueError("仅支持 n=1(网页端每次只生成一个回复;需要多个结果请并发发起多次请求)")
return value
stop: Optional[Any] = Field(default=None)
tools: Optional[list] = Field(default=None)
tool_choice: Optional[Any] = Field(default=None)
parallel_tool_calls: Optional[bool] = Field(default=None)
functions: Optional[list] = Field(default=None)
function_call: Optional[Any] = Field(default=None)
preset_name: Optional[str] = Field(default=None)
stream_options: Optional[dict] = Field(default=None)
class ResponsesRequest(BaseModel):
"""Responses API 请求模型(兼容 Codex / OpenAI Responses wire format)"""
workflow_variables: Dict[str, Any] = Field(default_factory=dict)
@field_validator('workflow_variables')
@classmethod
def validate_workflow_variables(cls, value):
from app.core.workflow.flow_runtime import validate_inputs, FlowValidationError
try:
return validate_inputs(value)
except FlowValidationError as exc:
raise ValueError(str(exc)) from exc
model: str = Field(default="未知")
input: Optional[Any] = Field(default="")
instructions: Optional[str] = Field(default=None)
stream: Optional[bool] = Field(default=False)
temperature: Optional[float] = Field(default=1.0, ge=0, le=2)
max_output_tokens: Optional[int] = Field(default=None, ge=1)
tools: Optional[list] = Field(default=None)
tool_choice: Optional[Any] = Field(default=None)
parallel_tool_calls: Optional[bool] = Field(default=None)
text: Optional[dict] = Field(default=None)
metadata: Optional[dict] = Field(default=None)
prompt: Optional[Any] = Field(default=None)
previous_response_id: Optional[str] = Field(default=None)
reasoning: Optional[dict] = Field(default=None)
store: Optional[bool] = Field(default=None)
top_p: Optional[float] = Field(default=None, ge=0, le=1)
truncation: Optional[str] = Field(default=None)
user: Optional[str] = Field(default=None)
stop: Optional[Any] = Field(default=None)
model_config = {
"extra": "allow",
}
def _is_single_choice_request(body: ChatRequest) -> bool:
try:
return int(getattr(body, "n", 1) or 1) <= 1
except (TypeError, ValueError):
return True
def _new_response_id() -> str:
return f"resp_{int(time.time() * 1000)}_{uuid.uuid4().hex[:10]}"
def _new_response_item_id(prefix: str) -> str:
return f"{prefix}_{uuid.uuid4().hex[:12]}"
def _pop_responses_state_locked(response_id: str) -> None:
global _responses_state_total_bytes
entry = _responses_state_by_id.pop(response_id, None)
if entry is not None:
_responses_state_total_bytes = max(0, _responses_state_total_bytes - int(entry[3] or 0))
def _prune_responses_state_locked(now: Optional[float] = None) -> None:
cutoff = float(now if now is not None else time.time()) - RESPONSES_STATE_TTL_SEC
expired_ids = [
response_id
for response_id, entry in _responses_state_by_id.items()
if entry[0] < cutoff
]
for response_id in expired_ids:
_pop_responses_state_locked(response_id)
while len(_responses_state_by_id) > RESPONSES_STATE_MAX_ENTRIES:
_pop_responses_state_locked(next(iter(_responses_state_by_id)))
# B3:总字节预算,按 LRU(OrderedDict 头部最久未用)淘汰
while _responses_state_by_id and _responses_state_total_bytes > RESPONSES_STATE_MAX_TOTAL_BYTES:
_pop_responses_state_locked(next(iter(_responses_state_by_id)))
def _load_responses_state(
previous_response_id: Optional[str],
principal: str = "",
) -> List[Dict[str, Any]]:
response_id = str(previous_response_id or "").strip()
if not response_id:
return []
with _responses_state_lock:
_prune_responses_state_locked()
entry = _responses_state_by_id.get(response_id)
# B3:不同主体一律当作不存在(不泄露 ID 是否有效)
if entry is None or str(entry[2] or "") != str(principal or ""):
raise HTTPException(
status_code=404,
detail=f"previous_response_id not found or expired: {response_id}",
)
if entry[1] is None:
raise HTTPException(
status_code=413,
detail=(
f"previous_response_id 的会话历史超过存储上限 "
f"({RESPONSES_STATE_MAX_ENTRY_BYTES // (1024 * 1024)} MiB),未被保存;"
"请在 input 中携带完整上下文重新发送"
),
)
_responses_state_by_id.move_to_end(response_id)
serialized = entry[1]
# 修复(4):锁内只做 dict 读写与 LRU 维护;大 payload 的反序列化放到锁外,
# 避免含 base64 图片的历史在全局锁持有期间做重拷贝阻塞其他请求。
# json round-trip 对该数据(本就来自 JSON 请求体)与 deepcopy 语义等价
return json.loads(serialized)
def _store_responses_state(
response_id: str,
request_messages: List[Dict[str, Any]],
chat_payload: Dict[str, Any],
*,
enabled: bool,
principal: str = "",
) -> None:
global _responses_state_total_bytes
if not enabled:
return
choices = chat_payload.get("choices") if isinstance(chat_payload, dict) else None
choice = choices[0] if isinstance(choices, list) and choices and isinstance(choices[0], dict) else {}
assistant = choice.get("message") if isinstance(choice.get("message"), dict) else None
if assistant is None:
return
# 修复(4):不再 deepcopy 完整历史;仅浅拷贝 assistant 消息以安全改写 role,
# 随后整体 json.dumps 成不可变字符串存储(序列化在锁外执行,dumps 本身不改动源数据)
history = list(request_messages or [])
assistant_message = dict(assistant)
assistant_message["role"] = "assistant"
history.append(assistant_message)
key = str(response_id or "").strip()
if not key:
return
try:
serialized = json.dumps(history, ensure_ascii=False)
except (TypeError, ValueError) as e:
# 数据理论上均来自 JSON 请求体/响应体,不可序列化属异常场景:跳过存储而非让请求 500
logger.warning(f"responses 会话历史序列化失败(跳过存储): {e}")
return
nbytes = len(serialized.encode("utf-8"))
stored: Optional[str] = serialized
if nbytes > RESPONSES_STATE_MAX_ENTRY_BYTES:
# B3:单条超限不入内存,只留极小的墓碑,续接时给出可预测的 413
logger.warning(
f"responses 会话历史 {nbytes / (1024 * 1024):.1f} MiB 超过单条上限 "
f"{RESPONSES_STATE_MAX_ENTRY_BYTES / (1024 * 1024):.0f} MiB,未存储: {key}"
)
stored = None
nbytes = 0
with _responses_state_lock:
_prune_responses_state_locked()
_pop_responses_state_locked(key)
_responses_state_by_id[key] = (time.time(), stored, str(principal or ""), nbytes)
_responses_state_total_bytes += nbytes
_responses_state_by_id.move_to_end(key)
_prune_responses_state_locked()
def _pack_responses_sse(event: str, data: Dict[str, Any]) -> str:
return (
f"event: {event}\n"
f"data: {json.dumps(data, ensure_ascii=False)}\n\n"
)
def _format_rfc3339_timestamp(timestamp: Optional[float] = None) -> str:
return time.strftime(
"%Y-%m-%dT%H:%M:%SZ",
time.gmtime(float(timestamp if timestamp is not None else time.time())),
)
def _normalize_response_tool_choice(value: Any) -> Any:
if isinstance(value, dict):
value_type = str(value.get("type") or "").strip().lower()
if value_type == "function":
function_block = value.get("function") if isinstance(value.get("function"), dict) else {}
name = str(function_block.get("name") or value.get("name") or "").strip()
if name:
return {
"type": "function",
"function": {
"name": name,
},
}
return value
def _normalize_responses_tools(tools: Any) -> Optional[List[Dict[str, Any]]]:
if not isinstance(tools, list):
return None
normalized: List[Dict[str, Any]] = []
for item in tools:
if not isinstance(item, dict):
continue
item_type = str(item.get("type") or "").strip().lower()
if item_type != "function":
continue
if isinstance(item.get("function"), dict):
fn = item["function"]
name = str(fn.get("name") or "").strip()
if not name:
continue
normalized.append(
{
"type": "function",
"function": {
"name": name,
"description": str(fn.get("description") or "").strip(),
"parameters": fn.get("parameters", {"type": "object", "properties": {}}),
},
}
)
continue
name = str(item.get("name") or "").strip()
if not name:
continue
normalized.append(
{
"type": "function",
"function": {
"name": name,
"description": str(item.get("description") or "").strip(),
"parameters": item.get("parameters", {"type": "object", "properties": {}}),
},
}
)
return normalized or None
def _normalize_responses_text_format(text_config: Any) -> Optional[Dict[str, Any]]:
if not isinstance(text_config, dict):
return None
format_config = text_config.get("format")
if not isinstance(format_config, dict):
return None
format_type = str(format_config.get("type") or "text").strip().lower()
if format_type == "json_schema":
if isinstance(format_config.get("json_schema"), dict):
payload = format_config["json_schema"]
else:
payload = {}
if "name" in format_config:
payload["name"] = format_config.get("name")
if "schema" in format_config:
payload["schema"] = format_config.get("schema")
if "strict" in format_config:
payload["strict"] = format_config.get("strict")
return {
"type": "json_schema",
"json_schema": payload,
}
if format_type == "json_object":
return {"type": "json_object"}
return {"type": format_type}
def _normalize_response_message_content(content: Any) -> Any:
if content is None:
return ""
if isinstance(content, str):
return content
if isinstance(content, dict):
return _normalize_response_message_content([content])
if not isinstance(content, list):
return str(content)
normalized_parts: List[Dict[str, Any]] = []
leading_text_parts: List[str] = []
for part in content:
if isinstance(part, str):
if part:
leading_text_parts.append(part)
continue
if not isinstance(part, dict):
text = str(part or "")
if text:
leading_text_parts.append(text)
continue
from app.utils.attachments import normalize_attachment_part, AttachmentError
try:
attachment = normalize_attachment_part(part)
except AttachmentError as exc:
raise HTTPException(status_code=400, detail=exc.detail()) from exc
if attachment is not None:
normalized_parts.append(attachment)
continue
part_type = str(part.get("type") or "").strip().lower()
if part_type in {"input_text", "output_text", "text"}:
text = str(part.get("text") or "")
if not text:
continue
if normalized_parts:
normalized_parts.append({"type": "text", "text": text})
else:
leading_text_parts.append(text)
continue
if part_type in {"input_image", "image_url", "output_image"}:
image_value = part.get("image_url")
if isinstance(image_value, dict):
image_url = str(image_value.get("url") or "").strip()
detail = str(image_value.get("detail") or "").strip()
else:
image_url = str(image_value or part.get("url") or "").strip()
detail = str(part.get("detail") or "").strip()
if not image_url:
continue
image_payload: Dict[str, Any] = {"url": image_url}
if detail:
image_payload["detail"] = detail
normalized_parts.append(
{
"type": "image_url",
"image_url": image_payload,
}
)
continue
if part_type in {"output_audio", "output_video"}:
if part_type == "output_audio":
media_value = part.get("audio_url") or part.get("input_audio") or part.get("url") or ""
media_label = "audio"
else:
media_value = part.get("video_url") or part.get("url") or ""
media_label = "video"
media_url = str(
media_value.get("url") if isinstance(media_value, dict) else media_value
).strip()
if media_url:
media_text = f"[{media_label}]({media_url})"
if normalized_parts:
normalized_parts.append({"type": "text", "text": media_text})
else:
leading_text_parts.append(media_text)
continue
text_fallback_value = part.get("text")
if text_fallback_value is None:
text_fallback_value = part.get("output")
if text_fallback_value is None:
text_fallback_value = part.get("content")
if text_fallback_value is None:
text_fallback = json.dumps(part, ensure_ascii=False)
elif isinstance(text_fallback_value, (dict, list)):
text_fallback = json.dumps(text_fallback_value, ensure_ascii=False)
else:
text_fallback = str(text_fallback_value).strip()
if text_fallback:
if normalized_parts:
normalized_parts.append({"type": "text", "text": text_fallback})
else:
leading_text_parts.append(text_fallback)
joined_text = "\n".join(part for part in leading_text_parts if part)
if not normalized_parts:
return joined_text
if joined_text:
normalized_parts.insert(
0,
{
"type": "text",
"text": joined_text,
},
)
return normalized_parts
def _normalize_response_input_role(role: Any) -> str:
normalized_role = str(role or "user").strip().lower()
if normalized_role == "developer":
return "system"
if normalized_role in {"system", "user", "assistant", "tool"}:
return normalized_role
return "user"
def _normalize_response_tool_output_content(content: Any) -> Any:
normalized = _normalize_response_message_content(content)
if normalized in ("", None):
return ""
return normalized
def _responses_tool_output_can_follow_openai(
messages: List[Dict[str, Any]],
tool_call_id: str,
) -> bool:
if not tool_call_id:
return False
for message in reversed(messages):
if not isinstance(message, dict):
continue
role = str(message.get("role") or "").strip().lower()
if role == "tool":
continue
if role != "assistant":
return False
tool_calls = message.get("tool_calls")
if not isinstance(tool_calls, list) or not tool_calls:
return False
return any(
isinstance(item, dict)
and str(item.get("id") or "").strip() == tool_call_id
for item in tool_calls
)
return False
def _responses_tool_output_fallback_content(
item: Dict[str, Any],
output: Any,
) -> Any:
call_id = str(item.get("call_id") or item.get("tool_call_id") or item.get("id") or "").strip()
name = str(item.get("name") or "").strip()
header = "[Function Call Output"
if name:
header += f": {name}"
if call_id:
header += f" ({call_id})"
header += "]"
if isinstance(output, list):
return [{"type": "text", "text": f"{header}\n"}] + output
if isinstance(output, dict):
output = json.dumps(output, ensure_ascii=False)
return f"{header}\n{str(output or '')}".strip()
def _response_function_call_to_tool_call(item: Dict[str, Any]) -> Dict[str, Any]:
arguments = item.get("arguments")
if not isinstance(arguments, str):
arguments = json.dumps(arguments if arguments is not None else {}, ensure_ascii=False)
return {
"id": str(item.get("call_id") or item.get("id") or _new_response_item_id("call")).strip(),
"type": "function",
"function": {
"name": str(item.get("name") or "").strip(),
"arguments": arguments,
},
}
def _normalize_chat_style_tool_calls(tool_calls: Any) -> Optional[List[Any]]:
if not isinstance(tool_calls, list):
return None
normalized: List[Any] = []
for item in tool_calls:
if not isinstance(item, dict):
normalized.append(item)
continue
next_item = dict(item)
function_data = next_item.get("function")
if isinstance(function_data, dict):
next_function = dict(function_data)
arguments = next_function.get("arguments")
if not isinstance(arguments, str):
next_function["arguments"] = json.dumps(
arguments if arguments is not None else {},
ensure_ascii=False,
)
next_item["function"] = next_function
normalized.append(next_item)
return normalized
def _append_response_input_item(messages: List[Dict[str, Any]], item: Any) -> None:
if item is None:
return
if isinstance(item, str):
if item:
messages.append({"role": "user", "content": item})
return
if not isinstance(item, dict):
text = str(item)
if text:
messages.append({"role": "user", "content": text})
return
role = str(item.get("role") or "").strip().lower()
item_type = str(item.get("type") or "").strip().lower()
if role or item_type == "message":
normalized_role = _normalize_response_input_role(role)
content = item.get("content")
if content is None and "text" in item:
content = item.get("text")
message_payload: Dict[str, Any] = {
"role": normalized_role,
"content": _normalize_response_message_content(content),
}
if normalized_role == "assistant":
tool_calls = _normalize_chat_style_tool_calls(item.get("tool_calls"))
if tool_calls is not None:
message_payload["tool_calls"] = tool_calls
if normalized_role == "tool":
tool_call_id = str(
item.get("tool_call_id") or item.get("call_id") or item.get("id") or ""
).strip()
if tool_call_id:
message_payload["tool_call_id"] = tool_call_id
name = str(item.get("name") or "").strip()
if name:
message_payload["name"] = name
messages.append(message_payload)
return
if item_type == "function_call":
messages.append(
{
"role": "assistant",
"content": "",
"tool_calls": [_response_function_call_to_tool_call(item)],
}
)
return
if item_type in {"function_call_output", "tool_result"}:
output_source = item.get("output")
if output_source is None and "content" in item:
output_source = item.get("content")
output = _normalize_response_tool_output_content(output_source)
tool_call_id = str(
item.get("call_id") or item.get("tool_call_id") or item.get("id") or ""
).strip()
if _responses_tool_output_can_follow_openai(messages, tool_call_id):
tool_message: Dict[str, Any] = {
"role": "tool",
"tool_call_id": tool_call_id,
"content": output,
}
name = str(item.get("name") or "").strip()
if name:
tool_message["name"] = name
messages.append(tool_message)
else:
messages.append(
{
"role": "user",
"content": _responses_tool_output_fallback_content(item, output),
}
)
return
content = item.get("content")
if content is None and "text" in item:
content = item.get("text")
normalized_content = _normalize_response_message_content(content)
if normalized_content not in ("", [], None):
messages.append({"role": "user", "content": normalized_content})
def _responses_input_to_messages(body: ResponsesRequest, principal: str = "") -> List[Dict[str, Any]]:
messages: List[Dict[str, Any]] = []
instructions = str(body.instructions or "").strip()
if instructions:
messages.append({"role": "system", "content": instructions})
messages.extend(_load_responses_state(body.previous_response_id, principal))
source = body.input
if source in (None, "") and body.prompt not in (None, ""):
source = body.prompt
if isinstance(source, list):
pending_tool_calls: List[Dict[str, Any]] = []
def _flush_pending_tool_calls() -> None:
nonlocal pending_tool_calls
if not pending_tool_calls:
return
messages.append(
{
"role": "assistant",
"content": "",
"tool_calls": pending_tool_calls,
}
)
pending_tool_calls = []
for item in source:
if isinstance(item, dict) and str(item.get("type") or "").strip().lower() == "function_call":
pending_tool_calls.append(_response_function_call_to_tool_call(item))
continue
_flush_pending_tool_calls()
_append_response_input_item(messages, item)
_flush_pending_tool_calls()
else:
_append_response_input_item(messages, source)
if not messages:
messages.append({"role": "user", "content": ""})
return messages
def _responses_request_to_chat_request(
body: ResponsesRequest,
*,
stream: bool,
principal: str = "",
) -> ChatRequest:
return ChatRequest(
workflow_variables=body.workflow_variables,
model=body.model,
messages=_responses_input_to_messages(body, principal),
stream=stream,
temperature=body.temperature,
max_tokens=body.max_output_tokens,
response_format=_normalize_responses_text_format(body.text),
tools=_normalize_responses_tools(body.tools),
tool_choice=_normalize_response_tool_choice(body.tool_choice),
parallel_tool_calls=body.parallel_tool_calls,
stop=body.stop,
)
def _response_message_item_from_text(text: str) -> Dict[str, Any]:
return {
"id": _new_response_item_id("msg"),
"type": "message",
"status": "completed",
"role": "assistant",
"content": [
{
"type": "output_text",
"text": str(text or ""),
"annotations": [],
}
],
}
def _response_content_part_from_media_item(item: Any, index: int) -> Optional[Dict[str, Any]]:
if not isinstance(item, dict):
return None
media_type = str(item.get("media_type") or item.get("type") or "image").strip().lower()
ref = str(item.get("url") or item.get("data_uri") or "").strip()
if not ref:
return None
if media_type == "audio":
payload: Dict[str, Any] = {"type": "output_audio", "audio_url": ref}
elif media_type == "video":
payload = {"type": "output_video", "video_url": ref}
else:
payload = {"type": "output_image", "image_url": ref}
payload["annotations"] = []
payload["index"] = index
mime = str(item.get("mime") or "").strip()
if mime:
payload["mime_type"] = mime
label = str(item.get("label") or "").strip()
if label:
payload["label"] = label
return payload
def _response_message_item_from_content(
text: str,
media_items: Optional[List[Dict[str, Any]]] = None,
) -> Dict[str, Any]:
content: List[Dict[str, Any]] = []
if text:
content.append(
{
"type": "output_text",
"text": str(text or ""),
"annotations": [],
}
)
for item in media_items or []:
media_part = _response_content_part_from_media_item(item, len(content))
if media_part is not None:
content.append(media_part)
if not content:
content.append(
{
"type": "output_text",
"text": "",
"annotations": [],
}
)
return {
"id": _new_response_item_id("msg"),
"type": "message",
"status": "completed",
"role": "assistant",
"content": content,
}
def _response_function_call_item_from_tool_call(tool_call: Dict[str, Any]) -> Dict[str, Any]:
function_data = tool_call.get("function") if isinstance(tool_call.get("function"), dict) else {}
call_id = str(tool_call.get("id") or _new_response_item_id("call")).strip()
item_id = (
f"fc_{call_id[5:]}" if call_id.startswith("call_") else _new_response_item_id("fc")
)
arguments = function_data.get("arguments")
if not isinstance(arguments, str):
arguments = json.dumps(arguments if arguments is not None else {}, ensure_ascii=False)
return {
"id": item_id,
"type": "function_call",
"status": "completed",
"call_id": call_id,
"name": str(function_data.get("name") or "").strip(),
"arguments": arguments,
}
def _chat_payload_to_responses_output(chat_payload: Dict[str, Any]) -> List[Dict[str, Any]]:
choices = chat_payload.get("choices") if isinstance(chat_payload.get("choices"), list) else []
if not choices:
return []
message = choices[0].get("message") if isinstance(choices[0], dict) else {}
if not isinstance(message, dict):
return []
output: List[Dict[str, Any]] = []
tool_calls = message.get("tool_calls") if isinstance(message.get("tool_calls"), list) else []
for tool_call in tool_calls:
if not isinstance(tool_call, dict):
continue
output.append(_response_function_call_item_from_tool_call(tool_call))
content = message.get("content")
media_items = message.get("media")
if not isinstance(media_items, list):
media_items = chat_payload.get("media") if isinstance(chat_payload.get("media"), list) else []
if content not in ("", None) or media_items:
output.append(_response_message_item_from_content(str(content or ""), media_items))
return output
def _build_in_progress_response_item(item: Dict[str, Any]) -> Dict[str, Any]:
item_type = str(item.get("type") or "").strip().lower()
if item_type == "function_call":
return {
"id": item.get("id"),
"type": "function_call",
"status": "in_progress",
"call_id": item.get("call_id"),
"name": item.get("name"),
"arguments": "",
}
if item_type == "message":
return {
"id": item.get("id"),
"type": "message",
"status": "in_progress",
"role": item.get("role") or "assistant",
"content": [],
}
return dict(item)
def _build_responses_usage(chat_payload: Dict[str, Any]) -> Dict[str, int]:
usage = chat_payload.get("usage") if isinstance(chat_payload.get("usage"), dict) else {}
return {
"input_tokens": int(usage.get("prompt_tokens") or 0),
"output_tokens": int(usage.get("completion_tokens") or 0),
"total_tokens": int(usage.get("total_tokens") or 0),
}
def _build_responses_text_payload(body: ResponsesRequest) -> Dict[str, Any]:
format_payload = _normalize_responses_text_format(body.text)
text_payload = {
"format": format_payload if isinstance(format_payload, dict) else {"type": "text"},
"verbosity": "medium",
}
if isinstance(body.text, dict) and body.text.get("verbosity") is not None:
text_payload["verbosity"] = body.text.get("verbosity")
return text_payload
def _responses_parallel_tool_calls(body: ResponsesRequest) -> bool:
if body.parallel_tool_calls is None:
return True
return bool(body.parallel_tool_calls)
def _responses_tool_choice(body: ResponsesRequest) -> Any:
if body.tool_choice is None:
return "auto"
return body.tool_choice
def _responses_metadata(body: ResponsesRequest) -> Dict[str, Any]:
if isinstance(body.metadata, dict):
return body.metadata
return {}
def _responses_reasoning_payload(body: ResponsesRequest) -> Dict[str, Any]:
if isinstance(body.reasoning, dict):
return body.reasoning
return {"effort": None, "summary": None}
def _responses_request_settings(body: ResponsesRequest) -> Dict[str, Any]:
return {
"max_output_tokens": body.max_output_tokens,
"parallel_tool_calls": _responses_parallel_tool_calls(body),
"previous_response_id": body.previous_response_id,
"reasoning": _responses_reasoning_payload(body),
"store": True if body.store is None else bool(body.store),
"temperature": body.temperature if body.temperature is not None else 1.0,
"tool_choice": _responses_tool_choice(body),
"top_p": body.top_p if body.top_p is not None else 1.0,
"truncation": body.truncation if body.truncation is not None else "disabled",
"user": body.user,
"metadata": _responses_metadata(body),
}
def _build_responses_object(
body: ResponsesRequest,
chat_payload: Dict[str, Any],
*,
response_id: Optional[str] = None,
created_at: Optional[int] = None,
status: str = "completed",
error: Optional[Dict[str, Any]] = None,
incomplete_details: Optional[Dict[str, Any]] = None,
) -> Dict[str, Any]:
response_obj: Dict[str, Any] = {
"id": response_id or _new_response_id(),
"object": "response",
"created_at": int(created_at if created_at is not None else time.time()),
"status": status,
"error": error,
"incomplete_details": incomplete_details,
"instructions": body.instructions,
"model": body.model,
"output": _chat_payload_to_responses_output(chat_payload),
"tools": body.tools or [],
"text": _build_responses_text_payload(body),
"usage": _build_responses_usage(chat_payload),
}
response_obj.update(_responses_request_settings(body))
if status == "completed":
response_obj["completed_at"] = int(time.time())
return response_obj
def _responses_completion_status_from_chat_payload(chat_payload: Dict[str, Any]) -> tuple[str, Optional[Dict[str, Any]], str]:
choices = chat_payload.get("choices") if isinstance(chat_payload.get("choices"), list) else []
for choice in choices:
if not isinstance(choice, dict):
continue
finish_reason = str(choice.get("finish_reason") or "").strip().lower()
if finish_reason == "length":
return "incomplete", {"reason": "max_output_tokens"}, "response.incomplete"
if finish_reason == "content_filter":
return "incomplete", {"reason": "content_filter"}, "response.incomplete"
return "completed", None, "response.completed"
def _responses_error_payload(payload: Dict[str, Any]) -> Dict[str, Any]:
error = payload.get("error") if isinstance(payload, dict) else None
if isinstance(error, dict):
error_payload = dict(error)
error_payload["message"] = str(error_payload.get("message") or "responses_backing_request_failed")
error_payload["type"] = str(error_payload.get("type") or "execution_error")
if error_payload.get("code") is None:
error_payload["code"] = "responses_backing_request_failed"
return error_payload
if isinstance(error, str) and error.strip():
return {
"message": error.strip(),
"type": "execution_error",
"code": "responses_backing_request_failed",
}
return {
"message": "responses_backing_request_failed",
"type": "execution_error",
"code": "responses_backing_request_failed",
}
def _decode_json_response(response: JSONResponse) -> Dict[str, Any]:
raw = response.body
if isinstance(raw, bytes):
text = raw.decode("utf-8")
else:
text = str(raw or "")
data = json.loads(text or "{}")
return data if isinstance(data, dict) else {}
async def _run_chat_completion_final(
request: Request,
body: ChatRequest,
authenticated: bool,
) -> tuple[int, Dict[str, Any]]:
effective_body = body if body.stream is False else body.model_copy(update={"stream": False})
response = await chat_completions(
request=request,
body=effective_body,
authenticated=authenticated,
)
if not isinstance(response, JSONResponse):
raise RuntimeError("responses_backing_request_unexpected_response_type")
return int(response.status_code), _decode_json_response(response)
async def _stream_responses_compat(
request: Request,
body: ResponsesRequest,
chat_body: ChatRequest,
authenticated: bool,
):
response_id = _new_response_id()
created_at = int(time.time())
sequence_number = 0
next_output_index = 0
message_item_id = ""
message_output_index: Optional[int] = None
text_part_started = False
message_item_done = False
collected_text_parts: List[str] = []
collected_media: List[Dict[str, Any]] = []
tool_call_states: Dict[int, Dict[str, Any]] = {}
finish_reason: Optional[str] = None
usage_payload: Dict[str, Any] = {}
sse_buffer = ""
utf8_decoder = codecs.getincrementaldecoder("utf-8")("ignore")
def _pack_response_event(event: str, data: Dict[str, Any]) -> str:
nonlocal sequence_number
sequence_number += 1
payload = dict(data)
if event in {"response.created", "response.completed", "response.failed", "response.incomplete"}:
payload = {"response": payload}
payload.setdefault("type", event)
payload["sequence_number"] = sequence_number
return _pack_responses_sse(event, payload)
def _next_output_index() -> int:
nonlocal next_output_index
value = next_output_index
next_output_index += 1
return value
def _decode_stream_chunk(chunk: Any, *, final: bool = False) -> str:
if isinstance(chunk, bytes):
return utf8_decoder.decode(chunk, final=final)
if final:
return utf8_decoder.decode(b"", final=True)
return str(chunk or "")
def _iter_buffered_openai_sse_payloads(chunk: str) -> List[Dict[str, Any]]:
nonlocal sse_buffer
if not chunk:
return []
sse_buffer += chunk.replace("\r\n", "\n").replace("\r", "\n")
payloads: List[Dict[str, Any]] = []
while "\n\n" in sse_buffer:
frame, sse_buffer = sse_buffer.split("\n\n", 1)
if not frame.strip():
continue
payloads.extend(iter_openai_sse_payloads(frame + "\n\n"))
return payloads
def _flush_buffered_openai_sse_payloads() -> List[Dict[str, Any]]:
nonlocal sse_buffer
tail = sse_buffer
sse_buffer = ""
if not tail.strip():
return []
return iter_openai_sse_payloads(tail + "\n\n")
def _first_choice(payload: Dict[str, Any]) -> Dict[str, Any]:
choices = payload.get("choices") if isinstance(payload.get("choices"), list) else []
if not choices or not isinstance(choices[0], dict):
return {}
return choices[0]
def _ensure_message_item() -> List[str]:
nonlocal message_item_id, message_output_index
if message_output_index is not None:
return []
message_item_id = _new_response_item_id("msg")
message_output_index = _next_output_index()
return [
_pack_response_event(
"response.output_item.added",
{
"type": "response.output_item.added",
"response_id": response_id,
"output_index": message_output_index,
"item": {
"id": message_item_id,
"type": "message",
"status": "in_progress",
"role": "assistant",
"content": [],
},
},
)
]
def _ensure_text_part() -> List[str]:
nonlocal text_part_started
events = _ensure_message_item()
if text_part_started:
return events
text_part_started = True
events.append(
_pack_response_event(
"response.content_part.added",
{
"type": "response.content_part.added",
"response_id": response_id,
"item_id": message_item_id,
"output_index": message_output_index,
"content_index": 0,
"part": {
"type": "output_text",
"text": "",
"annotations": [],
},
},
)
)
return events
def _final_message_item() -> Dict[str, Any]:
content: List[Dict[str, Any]] = []
text = "".join(collected_text_parts)
if text_part_started or text:
content.append(
{
"type": "output_text",
"text": text,
"annotations": [],
}
)
for media_item in _dedupe_media_items(collected_media):
media_part = _response_content_part_from_media_item(media_item, len(content))
if media_part is not None:
content.append(media_part)
if not content:
content.append(
{
"type": "output_text",
"text": "",
"annotations": [],
}
)
return {
"id": message_item_id,
"type": "message",
"status": "completed",
"role": "assistant",
"content": content,
}
def _final_tool_item(state: Dict[str, Any]) -> Dict[str, Any]:
return {
"id": state["item_id"],
"type": "function_call",
"status": "completed",
"call_id": state["call_id"],
"name": state["name"],
"arguments": state["arguments"],
}
def _ensure_tool_state(index: int, tool_delta: Dict[str, Any]) -> tuple[Dict[str, Any], List[str]]:
function_delta = tool_delta.get("function") if isinstance(tool_delta.get("function"), dict) else {}
state = tool_call_states.get(index)
if state is not None:
if tool_delta.get("id"):
state["call_id"] = str(tool_delta.get("id") or state["call_id"])
if function_delta.get("name"):
state["name"] = str(function_delta.get("name") or state["name"])
return state, []
call_id = str(tool_delta.get("id") or _new_response_item_id("call")).strip()
name = str(function_delta.get("name") or "").strip()
item_id = _response_function_call_item_from_tool_call(
{"id": call_id, "function": {"name": name, "arguments": ""}}
)["id"]
state = {
"output_index": _next_output_index(),
"item_id": item_id,
"call_id": call_id,
"name": name,
"arguments": "",
"done": False,
}
tool_call_states[index] = state
return state, [
_pack_response_event(
"response.output_item.added",
{
"type": "response.output_item.added",
"response_id": response_id,
"output_index": state["output_index"],
"item": {
"id": state["item_id"],
"type": "function_call",
"status": "in_progress",
"call_id": state["call_id"],
"name": state["name"],
"arguments": "",
},
},
)
]
def _completed_output() -> List[Dict[str, Any]]:
output: List[Optional[Dict[str, Any]]] = [None] * max(next_output_index, 0)
for state in tool_call_states.values():
output[int(state["output_index"])] = _final_tool_item(state)
if message_output_index is not None:
output[int(message_output_index)] = _final_message_item()
return [item for item in output if isinstance(item, dict)]
def _completed_chat_payload() -> Dict[str, Any]:
message: Dict[str, Any] = {
"role": "assistant",
"content": "".join(collected_text_parts),
}
if collected_media:
message["media"] = _dedupe_media_items(collected_media)
if tool_call_states:
message["tool_calls"] = [
{
"id": state["call_id"],
"type": "function",
"function": {
"name": state["name"],
"arguments": state["arguments"],
},
}
for _index, state in sorted(tool_call_states.items())
]
return {
"model": body.model,
"choices": [
{
"message": message,
"finish_reason": finish_reason or "stop",
}
],
"usage": usage_payload,
"media": _dedupe_media_items(collected_media),
}
def _failed_response(payload: Dict[str, Any], error: Optional[Dict[str, Any]] = None) -> Dict[str, Any]:
return _build_responses_object(
body,
payload if isinstance(payload, dict) else {"choices": [], "usage": {}},
response_id=response_id,
created_at=created_at,
status="failed",
error=error or _responses_error_payload(payload),
)
def _complete_tool_events(state: Dict[str, Any]) -> List[str]:
if state.get("done"):
return []
state["done"] = True
return [
_pack_response_event(
"response.function_call_arguments.done",
{
"type": "response.function_call_arguments.done",
"response_id": response_id,
"item_id": state["item_id"],
"output_index": state["output_index"],
"arguments": state["arguments"],
"name": state["name"],
"call_id": state["call_id"],
},
),
_pack_response_event(
"response.output_item.done",
{
"type": "response.output_item.done",
"response_id": response_id,
"output_index": state["output_index"],
"item": _final_tool_item(state),
},
),
]
def _complete_message_events() -> List[str]:
nonlocal message_item_done
if message_output_index is None or message_item_done:
return []
message_item_done = True
text = "".join(collected_text_parts)
events: List[str] = []
if text_part_started:
events.extend(
[
_pack_response_event(
"response.output_text.done",
{
"type": "response.output_text.done",
"response_id": response_id,
"item_id": message_item_id,
"output_index": message_output_index,
"content_index": 0,
"text": text,
},
),
_pack_response_event(
"response.content_part.done",
{
"type": "response.content_part.done",
"response_id": response_id,
"item_id": message_item_id,
"output_index": message_output_index,
"content_index": 0,
"part": {
"type": "output_text",
"text": text,
"annotations": [],
},
},
),
]
)
content_index = 1 if text_part_started else 0
for media_item in _dedupe_media_items(collected_media):
media_part = _response_content_part_from_media_item(media_item, content_index)
if media_part is None:
continue
events.append(
_pack_response_event(
"response.content_part.added",
{
"type": "response.content_part.added",
"response_id": response_id,
"item_id": message_item_id,
"output_index": message_output_index,
"content_index": content_index,
"part": media_part,
},
)
)
events.append(
_pack_response_event(
"response.content_part.done",
{
"type": "response.content_part.done",
"response_id": response_id,
"item_id": message_item_id,
"output_index": message_output_index,
"content_index": content_index,
"part": media_part,
},
)
)
content_index += 1
events.append(
_pack_response_event(
"response.output_item.done",
{
"type": "response.output_item.done",
"response_id": response_id,
"output_index": message_output_index,
"item": _final_message_item(),
},
)
)
return events
async def _iter_backing_chunks(streaming_response: StreamingResponse):
iterator = streaming_response.body_iterator.__aiter__()
next_task = asyncio.create_task(iterator.__anext__())
try:
while True:
done, _pending = await asyncio.wait(
{next_task},
timeout=SSE_HEARTBEAT_INTERVAL,
return_when=asyncio.FIRST_COMPLETED,
)
if not done:
if await request.is_disconnected():
next_task.cancel()
try:
await next_task
except asyncio.CancelledError:
pass
return
yield ": keepalive\n\n"
continue
try:
chunk = next_task.result()
except StopAsyncIteration:
break
yield chunk
next_task = asyncio.create_task(iterator.__anext__())
finally:
if not next_task.done():
next_task.cancel()
try:
await next_task
except asyncio.CancelledError:
pass
close = getattr(iterator, "aclose", None)
if callable(close):
try:
await close()
except Exception:
pass
async def _emit_openai_payload_events(payloads: List[Dict[str, Any]]) -> AsyncIterator[str]:
nonlocal finish_reason, usage_payload
for payload in payloads:
if "error" in payload:
yield _pack_response_event("response.failed", _failed_response(payload))
return
if isinstance(payload.get("usage"), dict):
usage_payload = dict(payload.get("usage") or {})
choice = _first_choice(payload)
if not choice:
continue
if choice.get("finish_reason") is not None:
finish_reason = str(choice.get("finish_reason") or "")
delta = choice.get("delta") if isinstance(choice.get("delta"), dict) else {}
collected_media.extend(_extract_chunk_media_items(payload))
content_text = _extract_delta_content_text(delta.get("content"))
if content_text:
collected_text_parts.append(content_text)
for event_chunk in _ensure_text_part():
yield event_chunk
yield _pack_response_event(
"response.output_text.delta",
{
"type": "response.output_text.delta",
"response_id": response_id,
"item_id": message_item_id,
"output_index": message_output_index,
"content_index": 0,
"delta": content_text,
},
)
tool_call_deltas = delta.get("tool_calls")
if not isinstance(tool_call_deltas, list):
continue
for fallback_index, tool_delta in enumerate(tool_call_deltas):
if not isinstance(tool_delta, dict):
continue
try:
tool_index = int(tool_delta.get("index", fallback_index) or fallback_index)
except Exception:
tool_index = fallback_index
state, added_events = _ensure_tool_state(tool_index, tool_delta)
for event_chunk in added_events:
yield event_chunk
function_delta = tool_delta.get("function") if isinstance(tool_delta.get("function"), dict) else {}
if function_delta.get("name"):
state["name"] = str(function_delta.get("name") or state["name"])
arguments_delta = function_delta.get("arguments")
if arguments_delta:
text_delta = str(arguments_delta)
state["arguments"] = str(state.get("arguments") or "") + text_delta
yield _pack_response_event(
"response.function_call_arguments.delta",
{
"type": "response.function_call_arguments.delta",
"response_id": response_id,
"item_id": state["item_id"],
"output_index": state["output_index"],
"delta": text_delta,
},
)
yield _pack_response_event(
"response.created",
{
"id": response_id,
"object": "response",
"created_at": created_at,
"status": "in_progress",
"error": None,
"incomplete_details": None,
"instructions": body.instructions,
"model": body.model,
"output": [],
"tools": body.tools or [],
"text": _build_responses_text_payload(body),
**_responses_request_settings(body),
},
)
try:
chat_response = await chat_completions(
request=request,
body=chat_body,
authenticated=authenticated,
)
if isinstance(chat_response, JSONResponse):
yield _pack_response_event(
"response.failed",
_failed_response(_decode_json_response(chat_response)),
)
return
if not isinstance(chat_response, StarletteStreamingResponse):
raise RuntimeError("responses_backing_request_unexpected_response_type")
async for raw_chunk in _iter_backing_chunks(chat_response):
if await request.is_disconnected():
return
chunk = _decode_stream_chunk(raw_chunk)
if not chunk:
continue
if chunk.lstrip().startswith(":") and "data:" not in chunk:
yield chunk
continue
async for event_chunk in _emit_openai_payload_events(_iter_buffered_openai_sse_payloads(chunk)):
yield event_chunk
if event_chunk.startswith("event: response.failed"):
return
decoder_tail = _decode_stream_chunk(b"", final=True)
if decoder_tail:
async for event_chunk in _emit_openai_payload_events(_iter_buffered_openai_sse_payloads(decoder_tail)):
yield event_chunk
if event_chunk.startswith("event: response.failed"):
return
async for event_chunk in _emit_openai_payload_events(_flush_buffered_openai_sse_payloads()):
yield event_chunk
if event_chunk.startswith("event: response.failed"):
return
if collected_media and message_output_index is None:
for event_chunk in _ensure_message_item():
yield event_chunk
for _index, state in sorted(tool_call_states.items()):
for event_chunk in _complete_tool_events(state):
yield event_chunk
for event_chunk in _complete_message_events():
yield event_chunk
chat_payload = _completed_chat_payload()
response_status, incomplete_details, terminal_event = _responses_completion_status_from_chat_payload(chat_payload)
completed = _build_responses_object(
body,
chat_payload,
response_id=response_id,
created_at=created_at,
status=response_status,
error=None,
incomplete_details=incomplete_details,
)
completed["output"] = _completed_output()
_store_responses_state(
response_id,
chat_body.messages,
chat_payload,
enabled=body.store is not False,
principal=_responses_principal_from_request(request),
)
yield _pack_response_event(terminal_event, completed)
except Exception as e:
yield _pack_response_event(
"response.failed",
_failed_response(
{"choices": [], "usage": {}},
error={
"message": str(e),
"type": "execution_error",
"code": "responses_backing_request_failed",
},
),
)
# ================= response_format 转化 =================
DEFAULT_RESPONSE_FORMAT_HINTS = {
"json_object": "\n\n[系统指令:请以 JSON 格式输出你的回复。确保输出是有效的 JSON 对象,不要包含 ```json 代码块标记或任何其他非 JSON 文字。]",
"json_schema": "\n\n[系统指令:请严格按照以下 JSON Schema 格式输出你的回复,确保输出是有效的 JSON,不要包含代码块标记:\n{schema}]",
"text": ""
}
def _get_response_format_hint(format_type: str) -> str:
"""获取指定格式类型的提示词模板"""
format_type = str(format_type or "text").strip().lower() or "text"
try:
from app.services.config_engine import config_engine
hints = config_engine.global_config.get("response_format_hints")
if hints and isinstance(hints, dict) and format_type in hints:
return hints[format_type]
except Exception:
pass
return DEFAULT_RESPONSE_FORMAT_HINTS.get(format_type, "")
def _apply_response_format(messages: list, response_format: dict) -> list:
"""将 response_format 转化为提示词并追加到最后一条用户消息"""
if not isinstance(response_format, dict) or not response_format:
return messages
format_type = str(response_format.get("type") or "text").strip().lower() or "text"
hint_template = _get_response_format_hint(format_type)
if not hint_template:
return messages
hint = hint_template
if format_type == "json_schema":
json_schema = response_format.get("json_schema", {})
schema_content = (
json_schema.get("schema", json_schema)
if isinstance(json_schema, dict)
else json_schema
)
try:
schema_str = json.dumps(schema_content, ensure_ascii=False, indent=2)
hint = hint_template.replace("{schema}", schema_str)
except Exception:
hint = hint_template.replace("{schema}", str(schema_content))
import copy
new_messages = copy.deepcopy(messages)
for i in range(len(new_messages) - 1, -1, -1):
msg = new_messages[i]
if msg.get("role") == "user":
content = msg.get("content", "")
if isinstance(content, str):
msg["content"] = content + hint
break
elif isinstance(content, list):
for j in range(len(content) - 1, -1, -1):
item = content[j]
if isinstance(item, dict) and item.get("type") == "text":
item["text"] = item.get("text", "") + hint
break
else:
content.append({"type": "text", "text": hint})
break
return new_messages
def _should_include_stream_usage(body: ChatRequest) -> bool:
stream_options = body.stream_options
return isinstance(stream_options, dict) and bool(stream_options.get("include_usage"))
def _pack_stream_usage_chunk(model: str, ctx: Optional[RequestContext] = None) -> str:
# 修复(8):usage 不再硬编码 0。request_manager 已在 ctx.monitor 上维护估算值:
# prompt_tokens 由 record_request_input 写入,response_tokens 由
# capture_response_chunk/capture_response_payload 累计,此处低成本读取填入
prompt_tokens = 0
completion_tokens = 0
if ctx is not None:
try:
prompt_tokens = max(0, int(ctx.monitor.get("prompt_tokens") or 0))
completion_tokens = max(0, int(ctx.monitor.get("response_tokens") or 0))
except (TypeError, ValueError, AttributeError):
prompt_tokens = 0
completion_tokens = 0
data = {
"id": f"chatcmpl-usage-{int(time.time() * 1000)}",
"object": "chat.completion.chunk",
"created": int(time.time()),
"model": model,
"choices": [],
"usage": {
"prompt_tokens": prompt_tokens,
"completion_tokens": completion_tokens,
"total_tokens": prompt_tokens + completion_tokens,
},
}
return f"data: {json.dumps(data, ensure_ascii=False)}\n\n"
def _maybe_pack_stream_usage_chunk(
body: ChatRequest,
ctx: Optional[RequestContext] = None,
) -> Optional[str]:
if not _should_include_stream_usage(body):
return None
return _pack_stream_usage_chunk(body.model, ctx)
def _split_sse_done_frame(chunk: Any) -> tuple[str, bool]:
if not isinstance(chunk, str) or not sse_chunk_has_done(chunk):
return str(chunk or ""), False
frames = chunk.replace("\r\n", "\n").replace("\r", "\n").split("\n\n")
kept_frames = []
had_done = False
for frame in frames:
if not frame:
continue
if sse_frame_data_text(frame).strip() == "[DONE]":
had_done = True
continue
kept_frames.append(f"{frame}\n\n")
return "".join(kept_frames), had_done
def _iter_stream_chunks_with_optional_usage(
body: ChatRequest,
chunks,
ctx: Optional[RequestContext] = None,
):
usage_emitted = False
for chunk in chunks:
emit_chunk, chunk_had_done = _split_sse_done_frame(chunk)
if emit_chunk:
yield emit_chunk
if chunk_had_done:
if not usage_emitted:
# 修复(8):透传 ctx 以便 usage 携带已有的 token 估算
usage_chunk = _maybe_pack_stream_usage_chunk(body, ctx)
if usage_chunk:
usage_emitted = True
yield usage_chunk
yield _pack_done()
# ================= 认证依赖 =================
async def verify_auth(
authorization: Optional[str] = Header(None),
x_api_key: Optional[str] = Header(None, alias="X-API-Key"),
) -> bool:
"""验证对外服务 API 的 Bearer Token 或 X-API-Key。"""
return await verify_service_auth(
authorization=authorization,
x_api_key=x_api_key,
)
# ================= 核心聊天 API =================
@router.post("/v1/chat/completions")
async def chat_completions(
request: Request,
body: ChatRequest,
authenticated: bool = Depends(verify_auth)
):
"""
OpenAI 兼容的聊天补全接口
"""
client_fp = cancel_storm_guard.get_client_fingerprint(request)
await cancel_storm_guard.maybe_backoff(client_fp)
logger.info(
f"[CHAT:ENTRY] 收到聊天补全请求: model={body.model!r}, preset={body.preset_name!r}, "
f"stream={body.stream}, messages_count={len(body.messages)}"
)
from app.utils.attachments import normalize_messages, AttachmentError
try:
body.messages = normalize_messages(body.messages)
except AttachmentError as exc:
raise HTTPException(status_code=400, detail=exc.detail()) from exc
if isinstance(body.response_format, dict) and body.response_format:
format_type = str(body.response_format.get("type") or "text").strip().lower() or "text"
if format_type != "text":
logger.debug(f"检测到 response_format.type={format_type},转化为提示词")
body.messages = _apply_response_format(body.messages, body.response_format)
setattr(body, "_response_format_applied", True)
catalog_tab_index: Optional[int] = None
try:
browser = get_browser(auto_connect=False)
# get_tabs_with_index 可能触发同步 CDP 扫描(阻塞网络 I/O),
# 不能在事件循环线程直接调用,否则浏览器卡顿时全服务假死。
tabs = await asyncio.to_thread(browser.tab_pool.get_tabs_with_index)
tab_models = collect_route_domain_models(tabs)
# 阶段 1:先尝试标签页完全精确匹配(暂不检查通用 ID 和前缀模糊,以便 Arena Catalog 优先于通用 ID 和前缀)
route_info = inspect_model_route(
body.model,
tabs,
allow_prefix=False,
allow_generic=False,
models=tab_models,
)
route_domain = str(route_info.get("route_domain") or "")
if route_info.get("match_type") == "none":
from app.services.config_engine import config_engine
# 阶段 2:通用动态模型目录匹配(只在标签页当前绑定的预设内匹配,不跨预设自动切换)
catalog_match_result = await asyncio.to_thread(
catalog_router.match_route,
browser,
config_engine,
tabs,
body.model,
body.preset_name,
) if browser else None
if catalog_match_result:
route_domain = str(catalog_match_result.get("route_domain") or "")
catalog_tab_index = catalog_match_result.get("catalog_tab_index")
route_info.update({
"route_domain": route_domain,
"route_type": catalog_match_result.get("route_type", "model_catalog"),
"model_name": catalog_match_result.get("model_name") or str(body.model or ""),
"matched_id": catalog_match_result.get("matched_id") or str(body.model or ""),
"match_type": catalog_match_result.get("match_type", "catalog"),
"preset_name": catalog_match_result.get("preset_name"),
"available_model_ids": catalog_match_result.get("available_model_ids") or [],
})
else:
logger.debug(
f"[CATALOG_ROUTER:NOMATCH] 当前标签页预设目录不包含请求模型: requested_model={body.model!r}"
)
# 阶段 3:若动态目录仍未命中,检查通用模型 ID(Generic)与标签页前缀模糊匹配(Prefix)兜底
if route_info.get("match_type") == "none":
fallback_info = inspect_model_route(
body.model,
tabs,
allow_prefix=True,
allow_generic=True,
models=tab_models,
)
if fallback_info.get("match_type") in {"generic", "prefix"}:
route_info = fallback_info
route_domain = str(route_info.get("route_domain") or "")
except Exception as e:
logger.debug(f"模型路由解析失败(已忽略): {e}")
route_info = {
"normalized_model": str(body.model or "").strip().lower(),
"route_domain": "",
"matched_id": "",
"match_type": "error",
"available_model_ids": [],
}
route_domain = ""
if route_info.get("route_type") == "model_name" and route_info.get("model_name"):
model_name = str(route_info.get("model_name") or "")
logger.info(
"模型显示名称路由命中: "
f"model={body.model!r}, normalized={route_info.get('normalized_model')!r}, "
f"matched_id={route_info.get('matched_id')!r}, match_type={route_info.get('match_type')}, "
f"model_name={model_name}, available={route_info.get('available_model_ids')}"
)
from app.api import tab_routes as tab_routes_api
route_body = tab_routes_api.ChatRequest(**body.model_dump())
if bool(getattr(body, "_response_format_applied", False)):
setattr(route_body, "_response_format_applied", True)
return await tab_routes_api.chat_with_exposed_model_name(
model_name=model_name,
request=request,
body=route_body,
authenticated=authenticated,
)
if route_domain:
logger.info(
"模型路由命中: "
f"model={body.model!r}, normalized={route_info.get('normalized_model')!r}, "
f"matched_id={route_info.get('matched_id')!r}, match_type={route_info.get('match_type')}, "
f"route_domain={route_domain}, available={route_info.get('available_model_ids')}"
)
from app.api import tab_routes as tab_routes_api
route_body = tab_routes_api.ChatRequest(**body.model_dump())
if bool(getattr(body, "_response_format_applied", False)):
setattr(route_body, "_response_format_applied", True)
return await tab_routes_api.chat_with_route_domain(
route_domain=route_domain,
request=request,
body=route_body,
tab_index=(catalog_tab_index if route_info.get("match_type") == "catalog" else None),
selector=None,
preset_name=(route_info.get("preset_name") or body.preset_name),
authenticated=authenticated,
)
elif route_info.get("match_type") == "none" and route_info.get("normalized_model"):
logger.debug(
"模型路由未命中: "
f"model={body.model!r}, normalized={route_info.get('normalized_model')!r}, "
f"available={route_info.get('available_model_ids')}"
)
ctx = request_manager.create_request(client_fp=client_fp)
try:
raw_input_len = sum(len(str(msg.get("content") or "")) for msg in body.messages if isinstance(msg, dict))
logger.info(f"[DIAG] 接收到的原始请求 messages 总字符长度: {raw_input_len} 字符, 消息数: {len(body.messages)}")
except Exception as e:
logger.debug(f"[DIAG] 估算原始请求长度失败: {e}")
# 修复(5):create_request 之后、生命周期函数(其内部 finally 才会 finish_request)
# 接管之前若抛异常,主动 finish_request 清理,避免 ctx 永久滞留 QUEUED 状态
try:
# 修复(6):record_request_input 内部对完整消息执行重正则/大文本处理,
# 用 to_thread 移到线程池执行,避免阻塞事件循环;调用顺序保持不变
await asyncio.to_thread(
request_manager.record_request_input,
ctx,
body.model_dump(),
endpoint="/v1/chat/completions",
route_domain=route_domain,
preset_name=body.preset_name,
)
with logger.context(ctx.request_id):
logger.info("开始")
try:
logger.debug(
"[chat] 请求消息摘要: "
f"{summarize_messages_for_debug(body.messages)}"
)
except Exception as e:
logger.debug(f"[chat] 请求消息摘要生成失败: {e}")
is_tool_calling = has_tool_calling_request(
messages=body.messages,
tools=body.tools,
functions=body.functions,
)
except asyncio.CancelledError:
ctx.request_cancel("coroutine_cancelled")
with request_finalization(ctx):
pass
raise
except Exception:
request_manager.finish_request(ctx, success=False)
raise
with logger.context(ctx.request_id):
if is_tool_calling:
if body.stream:
return StreamingResponse(
_stream_tool_calling_with_lifecycle(request, body, ctx),
media_type="text/event-stream",
headers={
"Cache-Control": "no-cache, no-transform",
"Connection": "keep-alive",
"X-Accel-Buffering": "no"
},
request_context=ctx,
)
return await _non_stream_tool_calling_with_lifecycle(request, body, ctx)
if body.stream:
return StreamingResponse(
_stream_with_lifecycle(request, body, ctx),
media_type="text/event-stream",
headers={
"Cache-Control": "no-cache, no-transform",
"Connection": "keep-alive",
"X-Accel-Buffering": "no"
},
request_context=ctx,
)
else:
return await _non_stream_with_lifecycle(request, body, ctx)
@router.post("/v1/responses")
async def create_response(
request: Request,
body: ResponsesRequest,
authenticated: bool = Depends(verify_auth)
):
"""OpenAI Responses API 兼容入口。"""
principal = _responses_principal_from_request(request)
chat_body = _responses_request_to_chat_request(body, stream=bool(body.stream), principal=principal)
# R2-2:标注协议,执行入口据此给 RequestContext 打标签并按协议记录指标
job = job_for(PROTOCOL_OPENAI_RESPONSES, model=body.model, stream=bool(body.stream))
if body.stream:
return StreamingResponse(
with_chat_job(job, _stream_responses_compat(
request=request,
body=body,
chat_body=chat_body,
authenticated=authenticated,
)),
media_type="text/event-stream",
headers={
"Cache-Control": "no-cache, no-transform",
"Connection": "keep-alive",
"X-Accel-Buffering": "no",
},
)
try:
with use_chat_job(job):
status_code, payload = await _run_chat_completion_final(
request=request,
body=chat_body,
authenticated=authenticated,
)
except Exception as e:
failed = _build_responses_object(
body,
{"choices": [], "usage": {}},
status="failed",
error={
"message": str(e),
"type": "execution_error",
"code": "responses_backing_request_failed",
},
)
return JSONResponse(content=failed, status_code=500)
if status_code >= 400 or "error" in payload:
failed = _build_responses_object(
body,
payload,
status="failed",
error=_responses_error_payload(payload),
)
return JSONResponse(content=failed, status_code=status_code)
response_status, incomplete_details, _terminal_event = _responses_completion_status_from_chat_payload(payload)
response_obj = _build_responses_object(
body,
payload,
status=response_status,
error=None,
incomplete_details=incomplete_details,
)
_store_responses_state(
str(response_obj.get("id") or ""),
chat_body.messages,
payload,
enabled=body.store is not False,
principal=principal,
)
return JSONResponse(content=response_obj)
async def _stream_with_lifecycle(
request: Request,
body: ChatRequest,
ctx: RequestContext
):
"""流式响应 + 完整生命周期管理"""
request_context_token = _request_context.set(ctx.request_id)
disconnect_task = None
worker_thread = None
chunk_queue: Optional[queue.Queue] = None
try:
request_manager.start_request(ctx)
disconnect_task = asyncio.create_task(
watch_client_disconnect(request, ctx, check_interval=0.3)
)
browser = get_browser(auto_connect=False)
chunk_queue = queue.Queue(maxsize=100)
def worker():
gen = None
chunk_counter = 0
try:
gen = browser.execute_workflow(
body.messages,
stream=True,
task_id=ctx.request_id,
stop_checker=ctx.should_stop,
requested_model=body.model,
**({"workflow_variables": body.workflow_variables} if body.workflow_variables else {}),
)
for chunk in gen:
chunk_counter += 1
if ctx.should_stop():
cancel_reason = str(ctx.cancel_reason or "unknown")
if cancel_reason in {"cleanup", "client_disconnected", "coroutine_cancelled"}:
logger.debug(f"工作线程检测到停止: {cancel_reason}")
else:
logger.info(f"工作线程检测到取消: {cancel_reason}")
break
if not _put_worker_queue_item(chunk_queue, ctx, chunk):
logger.debug("工作线程停止入队,结束流式生产")
break
except Exception as e:
logger.error(f"工作线程异常: {e}")
_put_worker_queue_item(chunk_queue, ctx, ("ERROR", str(e)), final=True)
finally:
if gen is not None:
try:
gen.close()
except Exception as e:
logger.debug(f"关闭工作流生成器失败(忽略): {e}")
_put_worker_queue_item(chunk_queue, ctx, None, final=True)
logger.debug("工作线程结束")
worker_thread = threading.Thread(
target=run_in_request_context,
args=(ctx, worker),
daemon=True,
)
# 供 TabRecovery 判定旧 worker 是否退出(setattr 规避 dataclass 字段限制)
setattr(ctx, "worker_thread", worker_thread)
worker_thread.start()
# Keep the connection alive while the browser workflow is acquiring a
# tab or waiting for its first upstream chunk.
keepalive_id = f"chatcmpl-{ctx.request_id}"
initial_keepalive = SSEFormatter.pack_keepalive(
model=body.model,
completion_id=keepalive_id,
)
request_manager.capture_response_chunk(ctx, initial_keepalive)
yield initial_keepalive
last_sse_emit_at = time.monotonic()
request_started_at = time.monotonic()
max_execute_time_sec = get_max_request_execute_time_sec()
done_emitted = False
client_disconnected = False
stop_state = build_stop_sequence_stream_state(
body.stop,
single_choice=_is_single_choice_request(body),
upstream_single_choice=True,
)
while True:
if mark_request_hard_timeout(
ctx,
request_started_at,
max_execute_time_sec,
label="chat_stream",
):
request_manager.capture_error(ctx, "请求执行超过最大绝对超时", code="absolute_request_timeout")
ctx.mark_failed("absolute_request_timeout")
yield _pack_error_done("请求执行超过最大绝对超时,已强制中断", "absolute_request_timeout")
done_emitted = True
break
if await request.is_disconnected():
logger.debug("客户端断开")
ctx.request_cancel("client_disconnected")
client_disconnected = True
break
try:
chunk = await wait_worker_queue_item(
chunk_queue,
timeout=STREAM_QUEUE_POLL_TIMEOUT,
)
except queue.Empty:
# 兜底:worker 的结束哨兵(None)在队列持续打满时可能被
# put_worker_queue_item 丢弃(final 宽限超时/取消时立即丢弃)。
# 若 worker 线程已退出且队列已排空,说明流已结束,直接收尾,
# 避免空转到 300 秒绝对超时并把已完成的响应误报为错误。
if not worker_thread.is_alive() and chunk_queue.empty():
logger.warning("工作线程已退出且队列为空(结束标记疑似丢失),提前收尾流式消费")
break
if time.monotonic() - last_sse_emit_at >= SSE_HEARTBEAT_INTERVAL:
yield SSEFormatter.pack_keepalive(
model=body.model,
completion_id=keepalive_id,
)
last_sse_emit_at = time.monotonic()
continue
if chunk is None:
logger.debug("收到结束标记")
break
if isinstance(chunk, tuple) and chunk[0] == "ERROR":
logger.error(f"错误: {chunk[1]}")
request_manager.capture_error(ctx, chunk[1], code="worker_error")
ctx.mark_failed(chunk[1])
yield _pack_error_done(f"执行错误: {chunk[1]}", "internal_error")
break
# 🔍 探针:记录发送给客户端的 chunk
has_images = '"images"' in chunk if isinstance(chunk, str) else False
if has_images:
logger.info(f"[SEND] 发送包含图片的 chunk 给客户端")
outgoing_chunks = filter_openai_stop_sse_chunk(chunk, stop_state, body.model)
for outgoing_chunk in outgoing_chunks:
emit_chunk, chunk_had_done = _split_sse_done_frame(outgoing_chunk)
if emit_chunk:
request_manager.capture_response_chunk(ctx, emit_chunk)
yield emit_chunk
last_sse_emit_at = time.monotonic()
error_message = _extract_stream_error_message(emit_chunk or outgoing_chunk)
if error_message:
logger.error(f"流式响应返回错误事件: {error_message}")
error_code = next((str(payload["error"].get("code") or "stream_error")
for payload in iter_openai_sse_payloads(emit_chunk or outgoing_chunk)
if isinstance(payload.get("error"), dict)), "stream_error")
request_manager.capture_error(ctx, error_message, code=error_code)
ctx.mark_failed(error_code)
done_chunk = _pack_done()
request_manager.capture_response_chunk(ctx, done_chunk)
yield done_chunk
done_emitted = True
break
if chunk_had_done:
# 修复(8):透传 ctx 以便 usage 携带已有的 token 估算
usage_chunk = _maybe_pack_stream_usage_chunk(body, ctx)
if usage_chunk:
request_manager.capture_response_chunk(ctx, usage_chunk)
yield usage_chunk
done_chunk = _pack_done()
request_manager.capture_response_chunk(ctx, done_chunk)
yield done_chunk
done_emitted = True
ctx.request_cancel("stream_done")
ctx.mark_completed()
if done_emitted:
break
if stop_state.stopped:
done_emitted = True
ctx.request_cancel("stop_sequence")
ctx.mark_completed()
break
if ctx.status == RequestStatus.FAILED:
break
await asyncio.sleep(0)
# 修复(3):外部取消(如手动终止)后不再补发 usage+[DONE] 伪装成正常完成;
# stream_done/stop_sequence/audio_media_fast_return 是正常完成路径设置的取消原因,
# 仍走原有 flush 收尾
cancelled_midway = (
ctx.should_stop()
and str(ctx.cancel_reason or "") not in {
"stream_done",
"stop_sequence",
"audio_media_fast_return",
}
)
if (
not client_disconnected
and not done_emitted
and ctx.status != RequestStatus.FAILED
):
if cancelled_midway:
# 明确告知客户端响应因取消而不完整,避免半截回复被当成完整响应
error_meta = resolve_error_metadata(ctx)
is_manual = _is_manual_terminate(ctx)
if error_meta and not error_meta.retryable:
cancel_code = error_meta.code
cancel_message = error_meta.message
request_manager.capture_error(ctx, cancel_message, code=cancel_code)
cancel_chunk = _pack_error(
cancel_message,
cancel_code,
error_type=error_meta.error_type,
status_code=error_meta.status_code,
retryable=error_meta.retryable,
)
else:
cancel_code = "manual_terminate" if is_manual else "request_cancelled"
cancel_message = "请求已被取消,响应不完整"
cancel_chunk = _pack_error(
cancel_message,
cancel_code,
error_type="execution_error",
status_code=None,
retryable=None,
)
request_manager.capture_response_chunk(ctx, cancel_chunk)
yield cancel_chunk
done_chunk = _pack_done()
request_manager.capture_response_chunk(ctx, done_chunk)
yield done_chunk
else:
for tail_chunk in flush_openai_stop_state(stop_state, body.model):
request_manager.capture_response_chunk(ctx, tail_chunk)
yield tail_chunk
usage_chunk = _maybe_pack_stream_usage_chunk(body, ctx)
if usage_chunk:
request_manager.capture_response_chunk(ctx, usage_chunk)
yield usage_chunk
done_chunk = _pack_done()
request_manager.capture_response_chunk(ctx, done_chunk)
yield done_chunk
if not ctx.should_stop() and ctx.status == RequestStatus.RUNNING:
ctx.mark_completed()
except GeneratorExit:
if not ctx.is_terminal():
ctx.request_cancel("stream_closed")
raise
except asyncio.CancelledError:
logger.debug("协程取消")
ctx.request_cancel("coroutine_cancelled")
raise
except Exception as e:
logger.error(f"异常: {e}")
request_manager.capture_error(ctx, e, code="internal_error")
ctx.mark_failed(str(getattr(e, "code", None) or e))
yield _pack_error(f"执行错误: {str(e)}", "internal_error")
yield _pack_done()
finally:
with request_finalization(ctx, request_context_token):
if worker_thread and worker_thread.is_alive():
await cleanup_worker_thread_after_request(
worker_thread,
ctx,
completed=ctx.status == RequestStatus.COMPLETED,
cancel_reason="cleanup",
join_timeout=5.0,
retire_reason="worker_cleanup_timeout",
completed_join_timeout=0.5,
)
if chunk_queue is not None:
try:
while not chunk_queue.empty():
chunk_queue.get_nowait()
except Exception:
pass
if disconnect_task:
disconnect_task.cancel()
try:
await disconnect_task
except asyncio.CancelledError:
pass
async def _non_stream_with_lifecycle(
request: Request,
body: ChatRequest,
ctx: RequestContext
) -> JSONResponse:
"""非流式响应 + 生命周期管理"""
collected_content = []
collected_media = []
error_data = None
sse_buffer = ""
def _iter_buffered_stream_payloads(chunk: str) -> List[Dict[str, Any]]:
nonlocal sse_buffer
if not chunk:
return []
sse_buffer += chunk.replace("\r\n", "\n").replace("\r", "\n")
payloads: List[Dict[str, Any]] = []
while "\n\n" in sse_buffer:
frame, sse_buffer = sse_buffer.split("\n\n", 1)
if not frame.strip():
continue
payloads.extend(iter_openai_sse_payloads(frame + "\n\n"))
return payloads
def _flush_buffered_stream_payloads() -> List[Dict[str, Any]]:
nonlocal sse_buffer
tail = sse_buffer
sse_buffer = ""
if not tail.strip():
return []
return iter_openai_sse_payloads(tail + "\n\n")
def _consume_stream_payload(data: Dict[str, Any]) -> bool:
nonlocal error_data
if "error" in data:
error_data = data
return False
collected_media.extend(_extract_chunk_media_items(data))
if "choices" in data and data["choices"]:
delta = data["choices"][0].get("delta", {})
content = delta.get("content", "")
content_text = _extract_delta_content_text(content)
if content_text:
collected_content.append(content_text)
return True
# 修复(7):error 时 break 提前退出 async for,内部生成器的 finally(含 finish_request、
# worker 清理)会被推迟到 GC 才执行;用 aclosing 保证退出本块时同步 aclose
async with contextlib.aclosing(_stream_with_lifecycle(request, body, ctx)) as stream_gen:
async for chunk in stream_gen:
if isinstance(chunk, str):
for data in _iter_buffered_stream_payloads(chunk):
if not _consume_stream_payload(data):
break
if error_data:
break
if not error_data:
for data in _flush_buffered_stream_payloads():
if not _consume_stream_payload(data):
break
# 优先解析错误响应(如 400 安全违规、422 提示词拒绝、429 限流等)
error_meta = resolve_error_metadata(error_data) or resolve_error_metadata(ctx)
if error_meta:
return build_error_response(error_meta)
if _is_manual_terminate(ctx):
return _manual_terminate_response()
if error_data:
fallback_meta = resolve_error_metadata(error_data)
if fallback_meta:
return build_error_response(fallback_meta)
return JSONResponse(content=error_data, status_code=500)
full_content = sanitize_response_text("".join(collected_content), stop_sequences=body.stop)
response = SSEFormatter.pack_non_stream(
full_content,
model=body.model,
media=_dedupe_media_items(collected_media),
)
request_manager.capture_response_payload(ctx, response)
return JSONResponse(content=response)
def _execute_browser_non_stream_messages(
browser,
messages: List[Dict[str, Any]],
request_id: str,
stop_checker=None,
requested_model: Optional[str] = None,
workflow_variables: Optional[Dict[str, Any]] = None,
) -> Dict[str, Any]:
payload = None
for chunk in browser.execute_workflow(
messages,
stream=False,
task_id=request_id,
stop_checker=stop_checker,
allow_media_postprocess=get_tool_calling_allow_media_postprocess(),
requested_model=requested_model,
**({"workflow_variables": workflow_variables} if workflow_variables else {}),
):
payload = chunk
if not payload:
raise RuntimeError("empty_browser_response")
data = decode_browser_non_stream_payload(payload)
if "error" in data:
error = data.get("error") or {}
from app.services.error_metadata import BrowserExecutionError
raise BrowserExecutionError(error)
return data
def _extract_assistant_content(response: Dict[str, Any]) -> str:
try:
return extract_tool_calling_assistant_content(response)
except Exception:
return ""
async def _run_tool_calling_async(
browser,
body: ChatRequest,
request_id: str,
stop_checker=None,
worker_state: Optional[Dict[str, Any]] = None,
) -> Dict[str, Any]:
legacy_function_call = bool(body.functions) and not bool(body.tools)
tools, tool_choice = normalize_tool_request(
tools=body.tools,
tool_choice=body.tool_choice,
functions=body.functions,
function_call=body.function_call,
)
tracked_worker_state = worker_state if isinstance(worker_state, dict) else {}
async def _round_executor(browser_messages: List[Dict[str, str]]) -> str:
worker_fn = lambda: _extract_assistant_content(
_execute_browser_non_stream_messages(
browser=browser,
messages=browser_messages,
request_id=request_id,
stop_checker=stop_checker,
requested_model=body.model,
**({"workflow_variables": body.workflow_variables} if body.workflow_variables else {}),
)
)
if isinstance(tracked_worker_state.get("ctx"), RequestContext):
return await _run_tracked_tool_calling_worker(
worker_fn,
ctx=tracked_worker_state["ctx"],
worker_state=tracked_worker_state,
label=f"{request_id[:8]}-round",
)
return await asyncio.to_thread(worker_fn)
parsed = await complete_tool_calling_roundtrip_async(
messages=body.messages,
tools=tools,
tool_choice=tool_choice,
parallel_tool_calls=(
False if legacy_function_call else body.parallel_tool_calls
),
round_executor=_round_executor,
stop_checker=stop_checker,
)
if not parsed.get("tool_calls"):
parsed = dict(parsed)
parsed["content"] = apply_stop_sequences_to_text(
str(parsed.get("content") or ""),
body.stop,
)
return build_tool_completion_response(
body.model,
parsed,
legacy_function_call=legacy_function_call,
)
async def _complete_tool_calling_with_lifecycle(
request: Request,
body: ChatRequest,
ctx: RequestContext,
) -> Dict[str, Any]:
disconnect_task = None
worker_state: Dict[str, Any] = {"thread": None, "label": None, "ctx": ctx}
try:
disconnect_task = asyncio.create_task(
watch_client_disconnect(request, ctx, check_interval=0.3)
)
browser = get_browser(auto_connect=False)
request_manager.start_request(ctx)
response = await _run_tool_calling_async(
browser,
body,
ctx.request_id,
ctx.should_stop,
worker_state=worker_state,
)
if _is_manual_terminate(ctx):
raise _ToolCallingExecutionCancelled(_get_tool_calling_cancel_reason(ctx))
request_manager.capture_response_payload(ctx, response)
if not ctx.should_stop() and ctx.status == RequestStatus.RUNNING:
ctx.mark_completed()
return response
except _ToolCallingExecutionCancelled:
cancel_reason = _get_tool_calling_cancel_reason(ctx)
if cancel_reason == "absolute_request_timeout":
request_manager.capture_error(ctx, "请求执行超过最大绝对超时", code="absolute_request_timeout")
ctx.mark_failed("absolute_request_timeout")
raise RuntimeError("absolute_request_timeout")
if not ctx.should_stop():
ctx.request_cancel(cancel_reason or "tool_calling_cancelled")
if _is_manual_terminate(ctx):
raise
raise asyncio.CancelledError()
except RuntimeError as e:
if _is_manual_terminate(ctx):
raise _ToolCallingExecutionCancelled(
_get_tool_calling_cancel_reason(ctx)
) from e
if str(e) == "tool_calling_cancelled" and ctx.should_stop():
raise asyncio.CancelledError()
logger.error(f"tool_calling_failed: {e}")
request_manager.capture_error(ctx, e, code=getattr(e, "code", "tool_calling_failed"))
ctx.mark_failed(str(getattr(e, "code", None) or e))
raise
except asyncio.CancelledError:
ctx.request_cancel("coroutine_cancelled")
raise asyncio.CancelledError()
except Exception as e:
logger.error(f"tool_calling_failed: {e}")
request_manager.capture_error(ctx, e, code=getattr(e, "code", "tool_calling_failed"))
ctx.mark_failed(str(getattr(e, "code", None) or e))
raise
finally:
with request_finalization(ctx):
if disconnect_task:
disconnect_task.cancel()
try:
await disconnect_task
except asyncio.CancelledError:
pass
worker_thread = worker_state.get("thread")
if isinstance(worker_thread, threading.Thread) and worker_thread.is_alive():
await cleanup_worker_thread_after_request(
worker_thread,
ctx,
completed=ctx.status == RequestStatus.COMPLETED,
cancel_reason="cleanup",
join_timeout=5.0,
retire_reason="worker_cleanup_timeout",
)
worker_state["thread"] = None
worker_state["label"] = None
async def _non_stream_tool_calling_with_lifecycle(
request: Request,
body: ChatRequest,
ctx: RequestContext,
) -> JSONResponse:
try:
response = await _complete_tool_calling_with_lifecycle(request, body, ctx)
return JSONResponse(content=response)
except _ToolCallingExecutionCancelled:
return _manual_terminate_response()
except asyncio.CancelledError:
if _is_manual_terminate(ctx):
return _manual_terminate_response()
if _is_genuine_task_cancellation():
raise
return _request_cancelled_response()
except Exception as e:
if _is_manual_terminate(ctx):
return _manual_terminate_response()
message, code = _format_tool_calling_error(e)
ctx.mark_failed(message)
request_manager.capture_error(ctx, message, code=code)
return JSONResponse(
content={
"error": {
"message": message,
"type": "execution_error",
"code": code,
}
},
status_code=500,
)
async def _stream_tool_calling_with_lifecycle(
request: Request,
body: ChatRequest,
ctx: RequestContext
):
response_task = None
keepalive_id = f"chatcmpl-{ctx.request_id}"
try:
yield SSEFormatter.pack_keepalive(
model=body.model,
completion_id=keepalive_id,
)
response_task = asyncio.create_task(
_complete_tool_calling_with_lifecycle(request, body, ctx)
)
while True:
try:
response = await asyncio.wait_for(
asyncio.shield(response_task),
timeout=SSE_HEARTBEAT_INTERVAL,
)
break
except asyncio.TimeoutError:
yield SSEFormatter.pack_keepalive(
model=body.model,
completion_id=keepalive_id,
)
message = response.get("choices", [{}])[0].get("message", {}) or {}
legacy_function_call = bool(body.functions) and not bool(body.tools)
response_tool_calls = message.get("tool_calls") or []
if legacy_function_call and not response_tool_calls and isinstance(message.get("function_call"), dict):
response_tool_calls = [
{"type": "function", "function": message["function_call"]}
]
parsed = {
"content": message.get("content"),
"tool_calls": response_tool_calls,
}
for chunk in _iter_stream_chunks_with_optional_usage(
body,
iter_tool_stream_chunks(
body.model,
parsed,
legacy_function_call=legacy_function_call,
),
ctx=ctx,
):
if await request.is_disconnected():
ctx.request_cancel("client_disconnected")
break
yield chunk
await asyncio.sleep(0)
except GeneratorExit:
if not ctx.is_terminal():
ctx.request_cancel("stream_closed")
raise
except asyncio.CancelledError:
if _is_genuine_task_cancellation():
raise
cancel_code = "manual_terminate" if _is_manual_terminate(ctx) else "request_cancelled"
yield _pack_error("请求已被取消", cancel_code)
yield _pack_done()
except Exception as e:
message, code = _format_tool_calling_error(e)
ctx.mark_failed(message)
request_manager.capture_error(ctx, message, code=code)
yield _pack_error(message, code)
yield _pack_done()
finally:
with request_finalization(ctx):
if response_task is not None and not response_task.done():
response_task.cancel()
try:
await response_task
except asyncio.CancelledError:
pass
def _pack_error(
message: str,
code: str = "error",
*,
error_type: str = "execution_error",
status_code: Optional[int] = None,
retryable: Optional[bool] = None,
param: Optional[str] = None,
extra: Optional[Dict[str, Any]] = None,
) -> str:
return SSEFormatter.pack_error(
message=message,
error_type=error_type,
code=code,
status_code=status_code,
retryable=retryable,
param=param,
extra=extra,
)
def _pack_done() -> str:
return "data: [DONE]\n\n"
def _pack_error_done(message: str, code: str = "error") -> str:
return f"{_pack_error(message, code)}{_pack_done()}"
def _collect_model_entries() -> List[Dict[str, Any]]:
entries: List[Dict[str, Any]] = []
seen_model_ids = set()
def _append_entry(model_id: Any, owned_by: Any = "universal-web-api", display_name: Any = "") -> None:
clean_id = str(model_id or "").strip()
normalized_id = clean_id.lower()
if not clean_id or normalized_id in seen_model_ids:
return
seen_model_ids.add(normalized_id)
entries.append(
{
"id": clean_id,
"object": "model",
"type": "model",
"created": MODEL_LIST_CREATED,
"owned_by": str(owned_by or "universal-web-api"),
"display_name": str(display_name or clean_id),
}
)
try:
browser = get_browser(auto_connect=False)
tabs = browser.tab_pool.get_tabs_with_index()
from app.services.config_engine import config_engine
catalog_router.collect_models_for_catalog(
browser=browser,
config_engine=config_engine,
tabs=tabs,
append_entry=_append_entry,
created=MODEL_LIST_CREATED,
)
except Exception as e:
logger.debug(f"构建模型列表失败(已忽略): {e}")
if not entries:
_append_entry("web-browser")
return entries
def _build_anthropic_models_response(entries: List[Dict[str, Any]]) -> Dict[str, Any]:
data = []
for item in entries:
model_id = str(item.get("id") or "").strip()
if not model_id:
continue
data.append(
{
"type": "model",
"id": model_id,
"display_name": str(item.get("display_name") or model_id),
"created_at": _format_rfc3339_timestamp(item.get("created")),
}
)
first_id = data[0]["id"] if data else None
last_id = data[-1]["id"] if data else None
return {
"data": data,
"has_more": False,
"first_id": first_id,
"last_id": last_id,
}
def _verify_models_auth(
authorization: Optional[str],
x_api_key: Optional[str],
) -> None:
verify_service_token(authorization=authorization, x_api_key=x_api_key)
# ================= 模型列表 =================
@router.get("/models")
@router.get("/model")
@router.get("/v1/v1/models")
@router.get("/v1/v1/model")
@router.get("/v1/models")
@router.get("/v1/model")
async def list_models(
authorization: Optional[str] = Header(None),
x_api_key: Optional[str] = Header(None, alias="x-api-key"),
anthropic_version: Optional[str] = Header(None, alias="anthropic-version"),
):
_verify_models_auth(authorization, x_api_key)
data = await asyncio.to_thread(_collect_model_entries)
if anthropic_version:
return _build_anthropic_models_response(data)
return {
"object": "list",
"data": data,
}
@router.get("/api/pool/status")
async def get_pool_status(authenticated: bool = Depends(verify_dashboard_auth)):
"""获取标签页池状态"""
try:
browser = get_browser(auto_connect=False)
return browser.get_pool_status()
except Exception as e:
return {"error": str(e), "initialized": False}