Initial commit

This commit is contained in:
Alex committed 2026-11-02 22:54:14 +08:00
commit 57fc3525eb
7311 files changed
+1291398

No files matched your search

+7
View File
@@ -0,0 +1,7 @@
"""
app/api - API 路由模块
"""
from app.api.routes import router
__all__ = ['router']
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
+81
View File
@@ -0,0 +1,81 @@
"""R1-3:站点适配器在线更新接口。"""
from __future__ import annotations
from typing import List, Optional
import requests
from fastapi import APIRouter, Depends, HTTPException
from pydantic import BaseModel, Field
from app.api.deps import verify_auth
from app.core.config import get_logger
from app.utils.remote_resource import UnsafeRemoteResourceError
logger = get_logger("API.ADAPTER")
router = APIRouter()
class ApplyAdapterUpdatesRequest(BaseModel):
sites: Optional[List[str]] = Field(default=None, description="要更新的站点;为空时应用全部安全更新")
include_conflicts: bool = Field(default=False, description="是否同时处理本地改过的站点(按本地优先合并)")
branch: str = Field(default="main", max_length=100)
def _run(action):
try:
return action()
except (requests.RequestException, UnsafeRemoteResourceError, OSError) as exc:
raise HTTPException(status_code=503, detail=f"无法获取官方适配器清单:{exc}") from exc
except ValueError as exc:
raise HTTPException(status_code=502, detail=str(exc)) from exc
@router.get("/api/adapters/updates")
async def check_adapter_updates(branch: str = "main", authenticated: bool = Depends(verify_auth)):
from app.services.adapter_updates import default_updater
return _run(lambda: default_updater(branch).check())
@router.post("/api/adapters/updates/apply")
async def apply_adapter_updates(body: ApplyAdapterUpdatesRequest, authenticated: bool = Depends(verify_auth)):
from app.services.adapter_updates import default_updater
return _run(lambda: default_updater(body.branch).apply(body.sites, include_conflicts=body.include_conflicts))
@router.get("/api/adapters/health")
async def adapter_health(site: str, preset: Optional[str] = None, authenticated: bool = Depends(verify_auth)):
"""R1-6:在浏览器里该站点的空闲标签页上巡检选择器(只查 DOM,不输入、不点击、不发送)。"""
import asyncio
from app.services.adapter_health import check_site_health
try:
return await asyncio.to_thread(check_site_health, site, preset)
except KeyError as exc:
raise HTTPException(status_code=404, detail=str(exc)) from exc
except Exception as exc: # broad-except: 浏览器未连接、驱动异常等一律转换为 503
logger.warning(f"适配器巡检失败: {exc}")
raise HTTPException(status_code=503, detail=f"巡检失败:{exc}") from exc
@router.get("/api/adapters/lint")
async def adapter_lint(site: str, authenticated: bool = Depends(verify_auth)):
"""R1-7:静态检查站点各预设选择器的脆弱度(不访问浏览器,不修改配置)。"""
from app.services.config_engine import config_engine
from app.services.selector_quality import lint_selector
config = config_engine.sites.get(site)
if not isinstance(config, dict):
raise HTTPException(status_code=404, detail=f"未知站点: {site}")
presets = {}
for name, preset in (config.get("presets") or {}).items():
rows = [
{"key": key, **lint_selector(selector)}
for key, selector in ((preset or {}).get("selectors") or {}).items()
if isinstance(selector, str) and selector.strip()
]
presets[name] = sorted(rows, key=lambda row: row["score"])
return {"site": site, "presets": presets}
File diff suppressed because it is too large. Load diff
+188
View File
@@ -0,0 +1,188 @@
"""Local integration endpoints for the project-controlled browser."""
from __future__ import annotations
import ipaddress
from typing import Any, Dict
from urllib.parse import urlparse
from typing import Optional
from fastapi import APIRouter, Header, HTTPException, Request
from pydantic import BaseModel, Field
from app.core import get_browser
from app.core.config import AppConfig, get_logger
from app.core.http_security import is_trusted_local_request
from app.utils.browser_profile_identity import resolve_tab_browser_profile
from app.core.driver import browser_driver, driver_for_tab
logger = get_logger("API.BROWSER")
router = APIRouter(prefix="/api/browser", tags=["browser"])
class OpenProfileUrlRequest(BaseModel):
url: str = Field(min_length=1, max_length=8192)
profile: Dict[str, Any] = Field(default_factory=dict)
def _is_loopback(host: str) -> bool:
try:
return ipaddress.ip_address(str(host or "").split("%", 1)[0]).is_loopback
except ValueError:
return str(host or "").casefold() == "localhost"
_INTERNAL_HOST_SUFFIXES = (
"localhost",
".localhost",
".local",
".internal",
".lan",
".home.arpa",
)
_METADATA_HOSTS = frozenset({"metadata.google.internal", "metadata", "instance-data"})
def _valid_web_url(url: str) -> str:
value = str(url or "").strip()
parsed = urlparse(value)
if parsed.scheme not in {"http", "https"} or not parsed.netloc:
raise HTTPException(status_code=400, detail="只允许打开 http/https 链接")
# S4:受控浏览器带着用户登录态,不替调用方打开本机/内网地址或带凭据的 URL
if parsed.username is not None or parsed.password is not None:
raise HTTPException(status_code=400, detail="链接不能包含用户名/密码")
host = str(parsed.hostname or "").strip().rstrip(".").casefold()
if not host or host in _METADATA_HOSTS or any(
host == suffix.lstrip(".") or host.endswith(suffix)
for suffix in _INTERNAL_HOST_SUFFIXES
):
raise HTTPException(status_code=400, detail="不允许打开本机/内部地址")
try:
literal = ipaddress.ip_address(host.split("%", 1)[0])
except ValueError:
literal = None
if literal is not None and not literal.is_global:
raise HTTPException(status_code=400, detail="不允许打开本机/内网 IP 地址")
return value
def verify_open_profile_url_auth(
request: Request,
authorization: Optional[str] = Header(None),
) -> bool:
"""S4:打开受控浏览器链接的鉴权。
- 带了 Authorization 且控制面板认证已启用:按管理令牌校验(错误令牌 401);
- 否则仅接受「直接来自本机回环、没有任何反代/隧道转发头」的请求
(经 start.py handoff 代理时以其注入的真实客户端地址为准)——
Link Drawer 不保存面板密钥,仍可在本机使用;
- 其余:认证已启用 → 401,未启用 → 403。
"""
from app.api.deps import verify_dashboard_token
dashboard_auth = AppConfig.is_dashboard_auth_enabled()
if dashboard_auth and str(authorization or "").strip():
verify_dashboard_token(authorization=authorization)
return True
client_host = request.client.host if request.client else None
if is_trusted_local_request(client_host, request.headers):
return True
if dashboard_auth:
raise HTTPException(
status_code=401,
detail="需要控制面板认证令牌",
headers={"WWW-Authenticate": "Bearer"},
)
raise HTTPException(status_code=403, detail="此接口仅允许本机直接调用")
def _target_info(tab: Any) -> Dict[str, Any]:
try:
result = driver_for_tab(tab).run_cdp("Target.getTargetInfo") or {}
info = result.get("targetInfo") if isinstance(result, dict) else {}
return info if isinstance(info, dict) else {}
except Exception:
return {}
def _identity_matches(actual: Dict[str, Any], expected: Dict[str, Any]) -> bool:
for key in ("profile_directory", "profile_path"):
wanted = str(expected.get(key) or "").strip().casefold()
if wanted:
return str(actual.get(key) or "").strip().casefold() == wanted
wanted_name = str(expected.get("name") or "").strip().casefold()
return bool(wanted_name and str(actual.get("name") or "").strip().casefold() == wanted_name)
def _find_profile_tab(browser: Any, profile: Dict[str, Any]) -> tuple[Any, str]:
tabs = list(browser.get_tabs() or [])
source_tab_id = str(profile.get("source_tab_id") or "").strip()
expected_context = str(profile.get("browser_context_id") or "").strip()
for tab in tabs:
info = _target_info(tab)
tab_id = str(info.get("targetId") or getattr(tab, "tab_id", "") or "").strip()
context_id = str(info.get("browserContextId") or "").strip()
if source_tab_id and tab_id == source_tab_id:
return tab, context_id
if expected_context and context_id == expected_context:
return tab, context_id
for tab in tabs:
identity = resolve_tab_browser_profile(tab)
if _identity_matches(identity, profile):
info = _target_info(tab)
return tab, str(info.get("browserContextId") or identity.get("browser_context_id") or "").strip()
raise HTTPException(status_code=404, detail="受控浏览器中未找到对应用户目录")
def open_url_in_profile(url: str, profile: Dict[str, Any]) -> Dict[str, Any]:
if not isinstance(profile, dict) or not any(
str(profile.get(key) or "").strip()
for key in ("profile_directory", "profile_path", "name", "browser_context_id")
):
raise HTTPException(status_code=400, detail="缺少浏览器用户目录标识")
browser = get_browser(auto_connect=False)
health = browser.health_check()
if not isinstance(health, dict) or not health.get("connected"):
raise HTTPException(status_code=503, detail="项目受控浏览器未连接")
source_tab, context_id = _find_profile_tab(browser, profile)
handle = browser.get_browser_handle()
try:
kwargs: Dict[str, Any] = {"url": url}
if context_id:
kwargs["browserContextId"] = context_id
created = browser_driver(handle).run_cdp("Target.createTarget", **kwargs) or {}
target_id = str(created.get("targetId") or "").strip() if isinstance(created, dict) else ""
if target_id:
try:
browser_driver(handle).run_cdp("Target.activateTarget", targetId=target_id)
except Exception:
pass
return {"success": True, "targetId": target_id, "browserContextId": context_id}
except Exception as error:
# The default Chrome profile may not expose a context id. Opening from one
# of its own pages keeps the new tab in that exact profile.
try:
script = "window.open(arguments[0], '_blank'); return true;"
driver_for_tab(source_tab).run_js(script, url)
return {"success": True, "targetId": "", "browserContextId": context_id}
except Exception as fallback_error:
logger.warning(f"打开用户目录链接失败: {error}; fallback={fallback_error}")
raise HTTPException(status_code=502, detail="无法在对应用户目录中打开链接") from fallback_error
# Link Drawer stores only routing metadata, not the dashboard secret. Without a
# token only direct (non-proxied, non-tunneled) loopback callers are accepted.
@router.post("/open-profile-url")
def open_profile_url(
payload: OpenProfileUrlRequest,
request: Request,
authorization: Optional[str] = Header(None),
):
verify_open_profile_url_auth(request, authorization)
return open_url_in_profile(_valid_web_url(payload.url), payload.profile)
+3299
View File
File diff suppressed because it is too large. Load diff
+882
View File
@@ -0,0 +1,882 @@
"""
app/api/cmd_routes.py - 命令系统 API 路由
职责:
- 命令 CRUD 接口
- 元信息查询
- 手动触发(调试用)
"""
import base64
import binascii
import json
import time
import uuid
from pathlib import Path
from typing import Optional, List
from fastapi import APIRouter, HTTPException, Depends
from pydantic import BaseModel, Field
from app.core.config import get_logger
from app.api.deps import verify_dashboard_auth as verify_auth
from app.services.command_engine import command_engine
from app.services.command_result_store import clear_command_results, list_command_results
from app.services.request_manager import request_manager, RequestStatus
logger = get_logger("API.CMD")
router = APIRouter(tags=["commands"])
_PROJECT_ROOT = Path(__file__).resolve().parents[2]
_COMMAND_IMAGE_DIR = _PROJECT_ROOT / "temp" / "command_images"
_MAX_COMMAND_IMAGE_BYTES = 25 * 1024 * 1024
SECRET_PLACEHOLDER = "__SECRET_REDACTED__"
NON_SENSITIVE_KEY_HINTS = {
"claude_required_token",
}
SENSITIVE_KEY_HINTS = {
"access_token",
"authorization",
"auth_token",
"api_key",
"apikey",
"client_credentials",
"credential",
"credentials",
"credentials_json",
"csrf",
"github_token",
"password",
"passwd",
"private_key",
"private_pem",
"refresh_token",
"secret",
"service_account",
"service_account_json",
"session",
"token",
"x_api_key",
"x_github_token",
}
SENSITIVE_KEY_SUFFIXES = (
"_access_token",
"_auth_token",
"_token",
"_api_key",
"_apikey",
"_credential",
"_credentials",
"_credentials_json",
"_github_token",
"_password",
"_passwd",
"_private_key",
"_private_pem",
"_secret",
"_service_account",
"_service_account_json",
)
def _normalize_secret_key(key: object) -> str:
return str(key or "").strip().lower().replace("-", "_")
def _is_command_secret_key(key: object) -> bool:
normalized = _normalize_secret_key(key)
if normalized in NON_SENSITIVE_KEY_HINTS:
return False
return normalized in SENSITIVE_KEY_HINTS or normalized.endswith(SENSITIVE_KEY_SUFFIXES)
def _try_parse_json_text(value: object):
if not isinstance(value, str):
return None
stripped = value.strip()
if not stripped or stripped[:1] not in "{[":
return None
try:
return json.loads(stripped)
except (TypeError, ValueError, RecursionError):
return None
def _restore_list_item_key(item: object) -> str:
if not isinstance(item, dict):
return ""
for key in ("id", "action_id", "command_id", "name"):
value = str(item.get(key) or "").strip()
if value:
return f"{key}:{value}"
return ""
def _redact_command_secrets(data):
if isinstance(data, dict):
result = {}
for key, value in data.items():
if _is_command_secret_key(key) and value not in (None, ""):
result[key] = SECRET_PLACEHOLDER
else:
result[key] = _redact_command_secrets(value)
return result
if isinstance(data, list):
return [_redact_command_secrets(item) for item in data]
parsed = _try_parse_json_text(data)
if isinstance(parsed, (dict, list)):
redacted = _redact_command_secrets(parsed)
return json.dumps(redacted, ensure_ascii=False, indent=2)
return data
def _restore_secret_placeholders(updates, existing):
if isinstance(updates, dict) and isinstance(existing, dict):
result = {}
for key, value in updates.items():
existing_value = existing.get(key)
if _is_command_secret_key(key) and value == SECRET_PLACEHOLDER:
result[key] = existing_value
else:
result[key] = _restore_secret_placeholders(value, existing_value)
return result
if isinstance(updates, list) and isinstance(existing, list):
result = []
existing_by_key = {
key: value
for value in existing
for key in [_restore_list_item_key(value)]
if key
}
for idx, value in enumerate(updates):
item_key = _restore_list_item_key(value)
if item_key and item_key in existing_by_key:
existing_value = existing_by_key[item_key]
elif item_key:
existing_value = None
else:
existing_value = existing[idx] if idx < len(existing) else None
result.append(_restore_secret_placeholders(value, existing_value))
return result
parsed_updates = _try_parse_json_text(updates)
parsed_existing = _try_parse_json_text(existing)
if isinstance(parsed_updates, (dict, list)) and isinstance(parsed_existing, type(parsed_updates)):
restored = _restore_secret_placeholders(parsed_updates, parsed_existing)
return json.dumps(restored, ensure_ascii=False, indent=2)
return updates
def _find_unresolved_secret_placeholders(data, path: str = "") -> List[str]:
found: List[str] = []
if isinstance(data, dict):
for key, value in data.items():
key_path = str(key or "")
next_path = f"{path}.{key_path}" if path else key_path
if _is_command_secret_key(key) and value == SECRET_PLACEHOLDER:
found.append(next_path)
continue
found.extend(_find_unresolved_secret_placeholders(value, next_path))
return found
if isinstance(data, list):
for index, value in enumerate(data):
next_path = f"{path}[{index}]" if path else f"[{index}]"
found.extend(_find_unresolved_secret_placeholders(value, next_path))
return found
parsed = _try_parse_json_text(data)
if isinstance(parsed, (dict, list)):
json_path = f"{path}<json>" if path else "<json>"
found.extend(_find_unresolved_secret_placeholders(parsed, json_path))
return found
def _reject_unresolved_secret_placeholders(data) -> None:
unresolved = _find_unresolved_secret_placeholders(data)
if not unresolved:
return
preview = ", ".join(unresolved[:5])
if len(unresolved) > 5:
preview = f"{preview}, ... (+{len(unresolved) - 5})"
raise HTTPException(
status_code=400,
detail=f"unresolved_secret_placeholder: {preview}",
)
def _raise_command_save_failed() -> None:
raise HTTPException(status_code=500, detail="命令保存失败")
def _finish_manual_command_request(ctx, success: bool) -> None:
try:
request_manager.finish_request(ctx, success=success)
except Exception as exc:
logger.debug(f"manual command request finish failed: {exc}")
def _release_manual_command_session(
pool,
session,
*,
expected_task_id: str,
rollback_request_count: bool = False,
) -> None:
try:
kwargs = {
"check_triggers": False,
"expected_task_id": expected_task_id,
}
if rollback_request_count:
kwargs["rollback_request_count"] = True
pool.release(session.id, **kwargs)
except Exception as release_error:
logger.debug(f"manual command release failed: {release_error}")
def _coerce_tab_index(value) -> Optional[int]:
try:
return int(value)
except (TypeError, ValueError):
return None
def _manual_command_session_route_excluded(pool, session) -> bool:
checker = getattr(pool, "_is_session_excluded_from_dynamic_routing", None)
if callable(checker):
try:
return bool(checker(session))
except Exception as exc:
logger.debug(f"manual command route-exclusion session check failed: {exc}")
current_url = ""
try:
current_url, _domain = session.get_cached_route_snapshot()
except Exception:
current_url = str(getattr(session, "url", "") or "").strip()
if not current_url:
try:
info = session.get_info(use_cached_url=True)
if isinstance(info, dict):
current_url = str(info.get("url") or "").strip()
except Exception:
current_url = ""
if not current_url:
return False
url_checker = getattr(pool, "is_url_excluded", None)
if callable(url_checker):
try:
return bool(url_checker(current_url))
except Exception as exc:
logger.debug(f"manual command route-exclusion URL check failed: {exc}")
return False
def _manual_command_tab_item_route_excluded(pool, item) -> bool:
if not isinstance(item, dict):
return False
if item.get("route_excluded"):
return True
current_url = str(item.get("url") or item.get("current_url") or "").strip()
if not current_url:
return False
url_checker = getattr(pool, "is_url_excluded", None)
if callable(url_checker):
try:
return bool(url_checker(current_url))
except Exception as exc:
logger.debug(f"manual command route-exclusion status check failed: {exc}")
return False
def _idle_tabs_for_manual_command(pool) -> List[dict]:
items: List[dict] = []
snapshot_failed = False
if hasattr(pool, "get_sessions_snapshot"):
try:
sessions = pool.get_sessions_snapshot()
except Exception:
sessions = []
snapshot_failed = True
for session in sessions or []:
if getattr(getattr(session, "status", None), "value", "") != "idle":
continue
tab_index = _coerce_tab_index(getattr(session, "persistent_index", None))
if tab_index is None:
continue
if _manual_command_session_route_excluded(pool, session):
continue
items.append({"tab_index": tab_index, "session": session})
if not hasattr(pool, "get_sessions_snapshot") or snapshot_failed:
status = pool.get_status()
for item in status.get("tabs", []) if isinstance(status, dict) else []:
if item.get("status") != "idle":
continue
tab_index = _coerce_tab_index(item.get("persistent_index"))
if tab_index is None:
continue
if _manual_command_tab_item_route_excluded(pool, item):
continue
items.append({"tab_index": tab_index, "session": None})
items.sort(key=lambda item: item["tab_index"])
return items
def _manual_command_matches_scope(cmd, session) -> bool:
return command_engine._matches_scope(cmd, session)
# ================= 请求模型 =================
class CommandCreateRequest(BaseModel):
name: str = Field(default="新命令", max_length=100)
enabled: bool = Field(default=True)
log_enabled: bool = Field(default=True)
log_level: str = Field(default="GLOBAL")
mode: str = Field(default="simple")
stop_on_error: bool = Field(default=False)
trigger: dict = Field(default_factory=lambda: {
"type": "request_count", "value": 10,
"command_id": "",
"action_ref": "",
"match_rule": "equals",
"expected_value": "",
"match_mode": "keyword",
"status_codes": "403,429,500,502,503,504",
"abort_on_match": True,
"scope": "all", "domain": "", "tab_index": None,
"priority": 2,
"stable_for_sec": 0,
"check_while_busy_workflow": True
})
actions: list = Field(default_factory=lambda: [
{"type": "clear_cookies"},
{"type": "refresh_page"},
])
group_name: str = Field(default="")
script: str = Field(default="")
script_lang: str = Field(default="javascript")
advanced_ui: dict = Field(default_factory=dict)
class CommandUpdateRequest(BaseModel):
name: Optional[str] = None
enabled: Optional[bool] = None
log_enabled: Optional[bool] = None
log_level: Optional[str] = None
mode: Optional[str] = None
stop_on_error: Optional[bool] = None
trigger: Optional[dict] = None
actions: Optional[list] = None
group_name: Optional[str] = None
script: Optional[str] = None
script_lang: Optional[str] = None
advanced_ui: Optional[dict] = None
class CommandReorderRequest(BaseModel):
command_ids: List[str]
class CommandGroupAssignRequest(BaseModel):
command_ids: List[str]
group_name: str = Field(default="", max_length=100)
class CommandBatchEnabledRequest(BaseModel):
command_ids: List[str]
enabled: bool = Field(default=True)
class CommandGroupEnabledRequest(BaseModel):
enabled: bool = Field(default=True)
class CommandGroupRenameRequest(BaseModel):
new_group_name: str = Field(default="", max_length=100)
class CommandGroupExecuteRequest(BaseModel):
include_disabled: bool = Field(default=False)
acquire_policy: str = Field(default="inherit_session")
class CommandImageUploadRequest(BaseModel):
filename: str = Field(default="image.png", max_length=255)
data_url: str = Field(min_length=1)
# ================= 路由 =================
@router.post("/api/commands/assets/image")
async def upload_command_image(
request: CommandImageUploadRequest,
authenticated: bool = Depends(verify_auth),
):
encoded = str(request.data_url or "")
if "," in encoded:
encoded = encoded.split(",", 1)[1]
try:
payload = base64.b64decode(encoded, validate=True)
except (ValueError, binascii.Error) as error:
raise HTTPException(status_code=400, detail="图片数据不是有效的 Base64") from error
if not payload or len(payload) > _MAX_COMMAND_IMAGE_BYTES:
raise HTTPException(status_code=400, detail="图片必须小于 25 MB")
try:
from PIL import Image
from io import BytesIO
with Image.open(BytesIO(payload)) as image:
image.verify()
image_format = str(image.format or "").lower()
except Exception as error:
raise HTTPException(status_code=400, detail="上传内容不是有效图片") from error
suffixes = {"png": ".png", "jpeg": ".jpg", "webp": ".webp", "gif": ".gif"}
suffix = suffixes.get(image_format)
if not suffix:
raise HTTPException(status_code=400, detail=f"不支持的图片格式: {image_format or 'unknown'}")
_COMMAND_IMAGE_DIR.mkdir(parents=True, exist_ok=True)
destination = _COMMAND_IMAGE_DIR / f"{uuid.uuid4().hex}{suffix}"
destination.write_bytes(payload)
return {"path": str(destination.resolve()), "filename": Path(request.filename).name, "size": len(payload)}
@router.get("/api/commands")
async def list_commands(authenticated: bool = Depends(verify_auth)):
commands = command_engine.list_commands()
return {"commands": _redact_command_secrets(commands), "count": len(commands)}
@router.get("/api/commands/meta")
async def get_meta(authenticated: bool = Depends(verify_auth)):
return {
"trigger_types": command_engine.get_trigger_types(),
"action_types": command_engine.get_action_types(),
}
@router.get("/api/commands/states")
async def get_trigger_states(authenticated: bool = Depends(verify_auth)):
return {"states": command_engine.get_trigger_states()}
@router.get("/api/commands/{command_id}/results")
async def get_persistent_command_results(command_id: str, authenticated: bool = Depends(verify_auth)):
if not command_engine.get_command(command_id):
raise HTTPException(status_code=404, detail="命令不存在")
records = list_command_results(command_id)
return {"records": records, "count": len(records)}
@router.delete("/api/commands/{command_id}/results")
async def delete_persistent_command_results(
command_id: str,
rule_id: str = "",
authenticated: bool = Depends(verify_auth),
):
if not command_engine.get_command(command_id):
raise HTTPException(status_code=404, detail="命令不存在")
return {"success": True, "removed": clear_command_results(command_id, rule_id=rule_id)}
@router.get("/api/commands/{command_id}")
async def get_command(command_id: str, authenticated: bool = Depends(verify_auth)):
cmd = command_engine.get_command(command_id)
if not cmd:
raise HTTPException(status_code=404, detail="命令不存在")
return _redact_command_secrets(cmd)
@router.post("/api/commands")
async def create_command(
body: CommandCreateRequest,
authenticated: bool = Depends(verify_auth)
):
cmd_data = body.model_dump()
_reject_unresolved_secret_placeholders(cmd_data)
cmd = command_engine.add_command(cmd_data)
if not cmd:
_raise_command_save_failed()
return {"success": True, "command": _redact_command_secrets(cmd)}
@router.post("/api/commands/{command_id}/duplicate")
async def duplicate_command(command_id: str, authenticated: bool = Depends(verify_auth)):
existing = command_engine.get_command_config(command_id)
if not existing:
raise HTTPException(status_code=404, detail="命令不存在")
duplicated = command_engine.duplicate_command(command_id)
if not duplicated:
_raise_command_save_failed()
return {"success": True, "command": _redact_command_secrets(duplicated)}
@router.put("/api/commands/reorder")
async def reorder_commands(
body: CommandReorderRequest,
authenticated: bool = Depends(verify_auth)
):
success = command_engine.reorder_commands(body.command_ids)
if not success:
_raise_command_save_failed()
return {"success": success}
@router.get("/api/command-groups")
async def list_command_groups(authenticated: bool = Depends(verify_auth)):
groups = command_engine.list_command_groups()
return {"groups": groups, "count": len(groups)}
@router.post("/api/command-groups/{group_name}/duplicate")
async def duplicate_command_group(group_name: str, authenticated: bool = Depends(verify_auth)):
normalized_name = (group_name or "").strip()
existing_names = {item.get("name") for item in command_engine.list_command_groups()}
if not normalized_name or normalized_name not in existing_names:
raise HTTPException(status_code=404, detail="命令组不存在")
result = command_engine.duplicate_group(normalized_name)
if not result:
_raise_command_save_failed()
return {
"success": True,
"group_name": result["group_name"],
"count": result["count"],
"commands": _redact_command_secrets(result["commands"]),
}
@router.put("/api/command-groups")
async def assign_command_group(
body: CommandGroupAssignRequest,
authenticated: bool = Depends(verify_auth)
):
updated = command_engine.set_commands_group(body.command_ids, body.group_name)
if updated < 0:
_raise_command_save_failed()
return {"success": True, "updated": updated, "group_name": (body.group_name or "").strip()}
@router.put("/api/commands/enabled")
async def update_commands_enabled(
body: CommandBatchEnabledRequest,
authenticated: bool = Depends(verify_auth)
):
updated = command_engine.set_commands_enabled(body.command_ids, body.enabled)
if updated < 0:
_raise_command_save_failed()
return {"success": True, "updated": updated, "enabled": body.enabled}
@router.put("/api/command-groups/{group_name}/enabled")
async def update_command_group_enabled(
group_name: str,
body: CommandGroupEnabledRequest,
authenticated: bool = Depends(verify_auth)
):
normalized_name = (group_name or "").strip()
updated = command_engine.set_group_enabled(normalized_name, body.enabled)
if updated < 0:
_raise_command_save_failed()
return {
"success": True,
"updated": updated,
"group_name": normalized_name,
"enabled": body.enabled,
}
@router.put("/api/command-groups/{group_name}/rename")
async def rename_command_group(
group_name: str,
body: CommandGroupRenameRequest,
authenticated: bool = Depends(verify_auth)
):
source_name = (group_name or "").strip()
target_name = (body.new_group_name or "").strip()
if not source_name or not target_name:
raise HTTPException(status_code=400, detail="命令组名称不能为空")
if source_name == target_name:
return {"success": True, "updated": 0, "group_name": source_name, "new_group_name": target_name}
existing_names = {item.get("name") for item in command_engine.list_command_groups()}
if target_name in existing_names:
raise HTTPException(status_code=400, detail=f"命令组已存在:{target_name}")
updated = command_engine.rename_group(source_name, target_name)
if updated < 0:
_raise_command_save_failed()
return {
"success": True,
"updated": updated,
"group_name": source_name,
"new_group_name": target_name,
}
@router.delete("/api/command-groups/{group_name}")
async def disband_command_group(
group_name: str,
authenticated: bool = Depends(verify_auth)
):
updated = command_engine.disband_group(group_name)
if updated < 0:
_raise_command_save_failed()
return {"success": True, "updated": updated, "group_name": group_name}
@router.put("/api/commands/{command_id}")
async def update_command(
command_id: str,
body: CommandUpdateRequest,
authenticated: bool = Depends(verify_auth)
):
updates = body.model_dump(exclude_none=True)
existing = command_engine.get_command_config(command_id)
if not existing:
raise HTTPException(status_code=404, detail="命令不存在")
updates = _restore_secret_placeholders(updates, existing)
_reject_unresolved_secret_placeholders(updates)
cmd = command_engine.update_command(command_id, updates)
if not cmd:
_raise_command_save_failed()
return {"success": True, "command": _redact_command_secrets(cmd)}
@router.delete("/api/commands/{command_id}")
async def delete_command(command_id: str, authenticated: bool = Depends(verify_auth)):
existing = command_engine.get_command_config(command_id)
if not existing:
raise HTTPException(status_code=404, detail="命令不存在")
success = command_engine.delete_command(command_id)
if not success:
_raise_command_save_failed()
return {"success": True}
@router.post("/api/commands/{command_id}/test")
async def test_command(command_id: str, authenticated: bool = Depends(verify_auth)):
"""Manual trigger command test on all idle tabs that match command scope."""
cmd = command_engine.get_command(command_id)
if not cmd:
raise HTTPException(status_code=404, detail="command_not_found")
try:
from app.core.browser import get_browser
browser = get_browser(auto_connect=False)
pool = browser.tab_pool
idle_tabs = _idle_tabs_for_manual_command(pool)
if not idle_tabs:
raise HTTPException(status_code=409, detail="no_idle_tabs")
scheduled_tabs: List[int] = []
skipped_tabs: List[int] = []
failed_tabs: List[dict] = []
def _run_command_in_background(target_session, target_ctx):
try:
setattr(target_session, "_command_request_id", target_ctx.request_id)
request_manager.start_request(target_ctx, tab_id=target_session.id)
with command_engine._command_logging_context(cmd):
execution_result = command_engine._execute_command(cmd, target_session)
result_text = str((execution_result or {}).get("result") or "").strip()
if result_text:
request_manager.update_request_metadata(
target_ctx.request_id,
response_text=result_text,
has_response_text=True,
)
if not target_ctx.should_stop() and target_ctx.status == RequestStatus.RUNNING:
target_ctx.mark_completed()
except Exception as e:
logger.error(f"manual command test worker failed(tab={getattr(target_session, 'persistent_index', '?')}): {e}")
target_ctx.mark_failed(str(e))
finally:
try:
_finish_manual_command_request(
target_ctx,
success=(target_ctx.status == RequestStatus.COMPLETED),
)
finally:
setattr(target_session, "_command_request_id", None)
_release_manual_command_session(
pool,
target_session,
expected_task_id=target_ctx.request_id,
)
for tab_info in idle_tabs:
tab_index = tab_info["tab_index"]
candidate_session = tab_info.get("session")
ctx = None
session = None
try:
# Respect command scope even in manual test mode.
if candidate_session is not None and not _manual_command_matches_scope(cmd, candidate_session):
skipped_tabs.append(tab_index)
continue
ctx = request_manager.create_request()
session = pool.acquire_by_index(tab_index, ctx.request_id, timeout=3)
if not session:
ctx.mark_failed("acquire_failed")
_finish_manual_command_request(ctx, success=False)
failed_tabs.append({"tab_index": tab_index, "error": "acquire_failed"})
continue
if not _manual_command_matches_scope(cmd, session):
skipped_tabs.append(tab_index)
_release_manual_command_session(
pool,
session,
expected_task_id=ctx.request_id,
rollback_request_count=True,
)
_finish_manual_command_request(ctx, success=False)
continue
command_engine.submit_background_task(_run_command_in_background, session, ctx)
scheduled_tabs.append(tab_index)
except Exception as e:
if ctx is not None and not ctx.is_terminal():
ctx.mark_failed(str(e))
_finish_manual_command_request(ctx, success=False)
if session is not None and getattr(getattr(session, "status", None), "value", "") == "busy":
_release_manual_command_session(
pool,
session,
expected_task_id=ctx.request_id,
rollback_request_count=True,
)
failed_tabs.append({"tab_index": tab_index, "error": str(e)})
if not scheduled_tabs:
if skipped_tabs and not failed_tabs:
raise HTTPException(
status_code=409,
detail=f"no_idle_tabs_match_scope, skipped={skipped_tabs}",
)
raise HTTPException(
status_code=500,
detail=f"command_test_failed, failed={failed_tabs}, skipped={skipped_tabs}",
)
return {
"success": True,
"message": f"command scheduled on tabs: {scheduled_tabs}",
"executed_tabs": scheduled_tabs,
"skipped_tabs": skipped_tabs,
"failed_tabs": failed_tabs,
"executed_count": len(scheduled_tabs),
}
except HTTPException:
raise
except Exception as e:
logger.error(f"manual command test failed: {e}")
raise HTTPException(status_code=500, detail=str(e))
@router.post("/api/command-groups/{group_name}/execute")
async def execute_command_group(
group_name: str,
body: Optional[CommandGroupExecuteRequest] = None,
authenticated: bool = Depends(verify_auth)
):
normalized_name = (group_name or "").strip()
if not normalized_name:
raise HTTPException(status_code=400, detail="命令组名称不能为空")
include_disabled = bool(body.include_disabled) if body else False
acquire_policy = str(body.acquire_policy or "inherit_session").strip() if body else "inherit_session"
try:
from app.core.browser import get_browser
browser = get_browser(auto_connect=False)
pool = browser.tab_pool
idle_tabs = _idle_tabs_for_manual_command(pool)
if not idle_tabs:
raise HTTPException(status_code=409, detail="没有空闲标签页可用于执行命令组")
skipped_tabs: List[dict] = []
acquire_failures: List[int] = []
for tab_info in idle_tabs:
tab_index = tab_info["tab_index"]
group_task_id = f"group_test_{normalized_name}_{tab_index}_{time.time_ns()}"
session = pool.acquire_by_index(tab_index, group_task_id, timeout=5)
if not session:
acquire_failures.append(tab_index)
continue
try:
plan = command_engine.preview_command_group(
group_name=normalized_name,
session=session,
include_disabled=include_disabled,
)
if int(plan.get("runnable_count", 0) or 0) <= 0:
skipped_tabs.append({
"tab_index": tab_index,
"runnable_count": plan.get("runnable_count", 0),
"scope_skipped": plan.get("scope_skipped", 0),
})
continue
effective_acquire_policy = (
"inherit_session" if acquire_policy == "require_acquire" else acquire_policy
)
result = command_engine.execute_command_group(
group_name=normalized_name,
session=session,
include_disabled=include_disabled,
acquire_policy=effective_acquire_policy,
prepared_plan=plan,
)
if not result.get("ok") and not result.get("partial_ok"):
raise HTTPException(status_code=400, detail=result.get("error", "命令组执行失败"))
result["requested_acquire_policy"] = acquire_policy
message = f"命令组已在标签页 #{tab_index} 执行"
if result.get("partial_ok") and not result.get("ok"):
message += "(部分命令被跳过或执行失败)"
return {
"success": True,
"message": message,
"tab_index": tab_index,
**result,
}
finally:
_release_manual_command_session(
pool,
session,
expected_task_id=group_task_id,
)
detail = {
"error": "no_idle_tabs_match_group_scope",
"skipped_tabs": skipped_tabs,
"acquire_failures": acquire_failures,
}
raise HTTPException(status_code=409, detail=detail)
except HTTPException:
raise
except Exception as e:
logger.error(f"执行命令组失败: {e}")
raise HTTPException(status_code=500, detail=str(e))
+604
View File
@@ -0,0 +1,604 @@
"""Official main-branch compare helpers for config routes."""
from __future__ import annotations
import hashlib
import json
import os
import re
import tempfile
import threading
from datetime import datetime, timezone
from pathlib import Path
from typing import Optional, Dict, Any
from urllib.parse import quote
import requests
from fastapi import HTTPException
from app.api.config_route_models import _normalize_preset_config_payload
from app.services.config_engine import config_engine, ConfigConstants
from app.utils.remote_resource import (
UnsafeRemoteResourceError,
get_public_remote_resource,
)
DEFAULT_OFFICIAL_REPO = "lumingya/universal-web-api"
OFFICIAL_CONFIG_CACHE_DIR = (
Path(getattr(ConfigConstants, "_PROJECT_ROOT", "") or os.getcwd())
/ "temp"
/ "official_config_cache"
)
OFFICIAL_CONFIG_CACHE_FILE = "sites-main.json"
OFFICIAL_CONFIG_META_FILE = "sites-main.meta.json"
MAX_OFFICIAL_CONFIG_BYTES = 16 * 1024 * 1024
_OFFICIAL_REPO_PATTERN = re.compile(r"^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$")
_OFFICIAL_CONFIG_CACHE_LOCK = threading.Lock()
def _utc_now_text() -> str:
return datetime.now(timezone.utc).isoformat(timespec="seconds").replace("+00:00", "Z")
def _official_config_relative_path() -> str:
project_root = getattr(ConfigConstants, "_PROJECT_ROOT", "") or os.getcwd()
return os.path.relpath(ConfigConstants.CONFIG_FILE, project_root).replace("\\", "/")
def _resolve_official_repo() -> str:
repo = str(os.getenv("GITHUB_REPO", DEFAULT_OFFICIAL_REPO) or "").strip()
if not _OFFICIAL_REPO_PATTERN.fullmatch(repo):
raise ValueError("GITHUB_REPO 格式无效,应为 owner/repo")
return repo
def _official_config_url(repo: str, branch_name: str, relative_path: str) -> str:
encoded_branch = quote(str(branch_name or "main").strip() or "main", safe="")
encoded_path = quote(relative_path, safe="/")
return f"https://raw.githubusercontent.com/{repo}/{encoded_branch}/{encoded_path}"
def _cache_paths() -> tuple[Path, Path]:
return (
OFFICIAL_CONFIG_CACHE_DIR / OFFICIAL_CONFIG_CACHE_FILE,
OFFICIAL_CONFIG_CACHE_DIR / OFFICIAL_CONFIG_META_FILE,
)
def _read_json_object(path: Path) -> Optional[Dict[str, Any]]:
try:
payload = json.loads(path.read_text(encoding="utf-8"))
except (OSError, UnicodeDecodeError, json.JSONDecodeError):
return None
return payload if isinstance(payload, dict) else None
def _atomic_write_json(path: Path, payload: Dict[str, Any]) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
fd, temp_name = tempfile.mkstemp(prefix=f".{path.name}.", suffix=".tmp", dir=path.parent)
temp_path = Path(temp_name)
try:
with os.fdopen(fd, "w", encoding="utf-8", newline="\n") as handle:
fd = -1
json.dump(payload, handle, ensure_ascii=False, indent=2)
handle.write("\n")
handle.flush()
os.fsync(handle.fileno())
os.replace(temp_path, path)
finally:
if fd >= 0:
os.close(fd)
try:
temp_path.unlink(missing_ok=True)
except OSError:
pass
def _read_official_config_cache(
repo: str,
branch_name: str,
relative_path: str,
) -> Optional[Dict[str, Any]]:
config_path, meta_path = _cache_paths()
payload = _read_json_object(config_path)
metadata = _read_json_object(meta_path)
if payload is None or metadata is None:
return None
if (
metadata.get("repository") != repo
or metadata.get("branch") != branch_name
or metadata.get("path") != relative_path
):
return None
return {"payload": payload, "metadata": metadata}
def _parse_official_config_bytes(content: bytes, relative_path: str) -> Dict[str, Any]:
if len(content) > MAX_OFFICIAL_CONFIG_BYTES:
raise ValueError("官方配置文件超过大小限制")
try:
payload = json.loads(content.decode("utf-8"))
except (UnicodeDecodeError, json.JSONDecodeError) as exc:
raise ValueError(f"官方 {relative_path} 不是合法 UTF-8 JSON: {exc}") from exc
if not isinstance(payload, dict):
raise ValueError(f"官方 {relative_path} 顶层必须是对象")
return payload
def _read_limited_response(response: requests.Response) -> bytes:
content_length = str(response.headers.get("Content-Length") or "").strip()
if content_length:
try:
declared_length = int(content_length)
except ValueError:
declared_length = 0
if declared_length > MAX_OFFICIAL_CONFIG_BYTES:
raise ValueError("官方配置文件超过大小限制")
chunks = []
total = 0
for chunk in response.iter_content(chunk_size=64 * 1024):
if not chunk:
continue
total += len(chunk)
if total > MAX_OFFICIAL_CONFIG_BYTES:
raise ValueError("官方配置文件超过大小限制")
chunks.append(chunk)
return b"".join(chunks)
def _source_payload(
metadata: Dict[str, Any],
*,
status: str,
stale: bool,
warning: str = "",
) -> Dict[str, Any]:
return {
"repository": str(metadata.get("repository") or ""),
"branch": str(metadata.get("branch") or "main"),
"path": str(metadata.get("path") or "config/sites.json"),
"url": str(metadata.get("url") or ""),
"status": status,
"stale": bool(stale),
"fetched_at": str(metadata.get("fetched_at") or ""),
"checked_at": str(metadata.get("checked_at") or ""),
"warning": warning,
}
def _build_sites_payload(payload: Dict[str, Any], source: Dict[str, Any]) -> Dict[str, Any]:
return {
"path": source["path"],
"sites": {
key: value
for key, value in payload.items()
if not str(key).startswith("_")
},
"source": source,
}
def _fetch_failure_text(exc: Exception) -> str:
if isinstance(exc, requests.Timeout):
return "连接官方仓库超时"
if isinstance(exc, requests.HTTPError) and exc.response is not None:
return f"官方仓库返回 HTTP {exc.response.status_code}"
if isinstance(exc, UnsafeRemoteResourceError):
return "官方仓库地址未通过网络安全校验"
return str(exc).strip() or exc.__class__.__name__
OFFICIAL_INDEX_RELATIVE_PATH = "config/sites/index.json"
_OFFICIAL_SITE_FILE_PATTERN = re.compile(r"^[A-Za-z0-9._%~-]+\.json$")
def _fetch_official_bytes(url: str, headers: Dict[str, str]):
"""返回 (response, content);304 时 content 为 None。调用方负责关闭 response。"""
response = get_public_remote_resource(url, headers=headers, timeout=(5, 15), stream=True)
if response.status_code == 304:
return response, None
response.raise_for_status()
return response, _read_limited_response(response)
def _assemble_official_sites_from_index(repo: str, branch: str, index_bytes: bytes) -> Dict[str, Any]:
"""R1-2:按 index.json 拉取每个站点文件,校验 file_sha256 后拼回旧 sites.json 结构。"""
try:
index = json.loads(index_bytes.decode("utf-8-sig"))
except (UnicodeDecodeError, json.JSONDecodeError) as exc:
raise ValueError(f"官方 index.json 不是有效 JSON:{exc}") from exc
entries = index.get("sites") if isinstance(index, dict) else None
if not isinstance(entries, dict) or not entries:
raise ValueError("官方 index.json 缺少 sites 清单")
payload: Dict[str, Any] = {}
total = 0
for site, entry in entries.items():
file_name = str((entry or {}).get("file") or "")
expected = str((entry or {}).get("file_sha256") or "").lower()
if not _OFFICIAL_SITE_FILE_PATTERN.fullmatch(file_name) or not re.fullmatch(r"[0-9a-f]{64}", expected):
raise ValueError(f"官方 index.json 中站点 {site} 的条目无效")
url = _official_config_url(repo, branch, f"config/sites/{file_name}")
response = None
try:
response, content = _fetch_official_bytes(
url, {"Accept": "application/json", "User-Agent": "Universal-Web-API-Config-Compare/1.0"}
)
finally:
if response is not None:
response.close()
if content is None:
raise ValueError(f"官方站点文件 {file_name} 返回了意外的 304")
total += len(content)
if total > MAX_OFFICIAL_CONFIG_BYTES:
raise ValueError("官方配置文件超过大小限制")
if hashlib.sha256(content).hexdigest() != expected:
raise ValueError(f"官方站点文件 {file_name} 的 sha256 与 index.json 不一致")
envelope = json.loads(content.decode("utf-8-sig"))
if not isinstance(envelope, dict) or envelope.get("site") != site or not isinstance(envelope.get("config"), dict):
raise ValueError(f"官方站点文件 {file_name} 的内容与 index.json 不符")
payload[site] = envelope["config"]
return payload
def _load_official_sites_config(branch_name: str = "main") -> Dict[str, Any]:
"""实时校验并读取官方分支配置,网络失败时回退到最近一次有效缓存。
R1-2 起官方配置位于 config/sites/(index.json + 每站点一个文件);该分支还没有 index.json 时
(例如尚未合并拆分的旧 main),回退读取旧的单文件 config/sites.json。
"""
branch = str(branch_name or "main").strip() or "main"
legacy_path = _official_config_relative_path()
checked_at = _utc_now_text()
try:
repo = _resolve_official_repo()
except ValueError as exc:
raise HTTPException(status_code=500, detail=str(exc)) from exc
with _OFFICIAL_CONFIG_CACHE_LOCK:
cached = _read_official_config_cache(repo, branch, OFFICIAL_INDEX_RELATIVE_PATH) or _read_official_config_cache(
repo, branch, legacy_path
)
cached_meta = dict(cached["metadata"]) if cached else {}
base_headers = {
"Accept": "application/json",
"User-Agent": "Universal-Web-API-Config-Compare/1.0",
}
def conditional_headers(path: str) -> Dict[str, str]:
headers = dict(base_headers)
if cached_meta.get("path") == path:
if cached_meta.get("etag"):
headers["If-None-Match"] = str(cached_meta["etag"])
if cached_meta.get("last_modified"):
headers["If-Modified-Since"] = str(cached_meta["last_modified"])
return headers
response = None
try:
relative_path = OFFICIAL_INDEX_RELATIVE_PATH
url = _official_config_url(repo, branch, relative_path)
try:
response, content = _fetch_official_bytes(url, conditional_headers(relative_path))
except requests.HTTPError as http_exc:
status = getattr(getattr(http_exc, "response", None), "status_code", None)
if status != 404:
raise
if response is not None:
response.close()
response = None
relative_path = legacy_path
url = _official_config_url(repo, branch, relative_path)
response, content = _fetch_official_bytes(url, conditional_headers(relative_path))
if content is None and cached:
cached_meta["checked_at"] = checked_at
_, meta_path = _cache_paths()
_atomic_write_json(meta_path, cached_meta)
source = _source_payload(
cached_meta,
status="validated_cache",
stale=False,
)
return _build_sites_payload(cached["payload"], source)
if content is None:
raise ValueError("官方配置返回 304,但本地没有缓存")
if relative_path == OFFICIAL_INDEX_RELATIVE_PATH:
payload = _assemble_official_sites_from_index(repo, branch, content)
else:
payload = _parse_official_config_bytes(content, relative_path)
metadata = {
"repository": repo,
"branch": branch,
"path": relative_path,
"url": url,
"etag": str(response.headers.get("ETag") or ""),
"last_modified": str(response.headers.get("Last-Modified") or ""),
"fetched_at": checked_at,
"checked_at": checked_at,
}
config_path, meta_path = _cache_paths()
_atomic_write_json(config_path, payload)
_atomic_write_json(meta_path, metadata)
source = _source_payload(metadata, status="remote", stale=False)
return _build_sites_payload(payload, source)
except (
OSError,
ValueError,
requests.RequestException,
UnsafeRemoteResourceError,
) as exc:
failure = _fetch_failure_text(exc)
if cached:
cached_meta["checked_at"] = checked_at
source = _source_payload(
cached_meta,
status="cache_fallback",
stale=True,
warning=f"获取官方最新配置失败,当前使用本地缓存:{failure}",
)
return _build_sites_payload(cached["payload"], source)
raise HTTPException(
status_code=503,
detail=f"无法获取官方配置,且本地没有可用缓存:{failure}",
) from exc
finally:
if response is not None:
response.close()
def _resolve_branch_preset_config(
site_config: Dict[str, Any],
requested_preset_name: Optional[str] = None,
) -> Dict[str, Any]:
"""从分支中的站点配置里解析最合适的预设。"""
if not isinstance(site_config, dict):
raise HTTPException(status_code=500, detail="站点配置格式无效")
presets = site_config.get("presets")
if not isinstance(presets, dict) or not presets:
return {
"preset_name": str(requested_preset_name or "主预设").strip() or "主预设",
"config": _normalize_preset_config_payload(site_config),
"match_mode": "legacy_flat",
}
requested = str(requested_preset_name or "").strip()
if requested:
resolved = config_engine._resolve_preset_alias_key(requested, presets)
if resolved in presets:
return {
"preset_name": resolved,
"config": _normalize_preset_config_payload(presets[resolved]),
"match_mode": "exact",
}
default_preset = str(site_config.get("default_preset") or "").strip()
if default_preset in presets:
return {
"preset_name": default_preset,
"config": _normalize_preset_config_payload(presets[default_preset]),
"match_mode": "default",
}
if "主预设" in presets:
return {
"preset_name": "主预设",
"config": _normalize_preset_config_payload(presets["主预设"]),
"match_mode": "main_preset",
}
first_key = next(iter(presets))
return {
"preset_name": first_key,
"config": _normalize_preset_config_payload(presets[first_key]),
"match_mode": "first",
}
_PRESET_COMPARE_FIELD_ORDER = [
"selectors",
"workflow",
"stream_config",
"image_extraction",
"file_paste",
"prompt_padding",
"stealth",
"extractor_id",
"extractor_verified",
]
_PRESET_COMPARE_FIELD_LABELS = {
"selectors": "选择器",
"workflow": "工作流",
"stream_config": "流式配置",
"image_extraction": "图片提取",
"file_paste": "文件粘贴",
"prompt_padding": "开头注入",
"stealth": "低熵模式",
"extractor_id": "提取器",
"extractor_verified": "提取器验证",
}
def _stable_compare_dump(value: Any) -> str:
return json.dumps(value, ensure_ascii=False, sort_keys=True)
def _extract_site_presets_for_compare(site_config: Dict[str, Any]) -> Dict[str, Dict[str, Any]]:
if not isinstance(site_config, dict):
return {}
presets = site_config.get("presets")
if isinstance(presets, dict) and presets:
normalized = {}
for preset_name, preset_config in presets.items():
if not isinstance(preset_config, dict):
continue
normalized[str(preset_name)] = _normalize_preset_config_payload(preset_config)
if normalized:
return normalized
try:
fallback_name = str(site_config.get("default_preset") or "主预设").strip() or "主预设"
return {
fallback_name: _normalize_preset_config_payload(site_config)
}
except HTTPException:
return {}
def _get_preset_compare_keys(local_config: Dict[str, Any], main_config: Dict[str, Any]) -> list[str]:
remaining = set(local_config.keys()) | set(main_config.keys())
ordered = []
for key in _PRESET_COMPARE_FIELD_ORDER:
if key in remaining:
ordered.append(key)
remaining.remove(key)
ordered.extend(sorted(remaining, key=lambda item: str(item)))
return ordered
def _collect_preset_different_fields(local_config: Dict[str, Any], main_config: Dict[str, Any]) -> list[str]:
different_fields = []
for key in _get_preset_compare_keys(local_config, main_config):
local_has = key in local_config
main_has = key in main_config
if not local_has or not main_has:
different_fields.append(key)
continue
if _stable_compare_dump(local_config[key]) != _stable_compare_dump(main_config[key]):
different_fields.append(key)
return different_fields
def _build_main_branch_compare_summary() -> Dict[str, Any]:
config_engine.refresh_if_changed()
branch_payload = _load_official_sites_config("main")
local_sites = {
key: value
for key, value in config_engine.sites.items()
if not str(key).startswith("_") and isinstance(value, dict)
}
main_sites = branch_payload["sites"]
items = []
counts = {
"same": 0,
"different": 0,
"local_only_preset": 0,
"local_only_site": 0,
"main_only_preset": 0,
"main_only_site": 0,
}
for domain in sorted(local_sites.keys(), key=lambda item: str(item)):
local_site = local_sites[domain]
local_presets = _extract_site_presets_for_compare(local_site)
main_site = main_sites.get(domain)
main_presets = _extract_site_presets_for_compare(main_site) if isinstance(main_site, dict) else {}
matched_main_presets = set()
for local_preset_name in sorted(local_presets.keys(), key=lambda item: str(item)):
local_preset_config = local_presets[local_preset_name]
item = {
"domain": domain,
"local_preset_name": local_preset_name,
"main_preset_name": "",
"local_exists": True,
"main_exists": bool(main_site),
"match_mode": "",
"different_fields": [],
"different_field_labels": [],
"difference_count": 0,
"detail_available": True,
"summary_text": "",
"status": "same",
}
if not main_site:
item["status"] = "local_only_site"
item["difference_count"] = 1
item["summary_text"] = "main 分支中没有这个站点"
counts["local_only_site"] += 1
items.append(item)
continue
resolved_main_preset_name = config_engine._resolve_preset_alias_key(local_preset_name, main_presets)
if resolved_main_preset_name not in main_presets:
item["status"] = "local_only_preset"
item["difference_count"] = 1
item["summary_text"] = "main 分支中没有同名预设"
counts["local_only_preset"] += 1
items.append(item)
continue
matched_main_presets.add(resolved_main_preset_name)
main_preset_config = main_presets[resolved_main_preset_name]
different_fields = _collect_preset_different_fields(local_preset_config, main_preset_config)
item["main_preset_name"] = resolved_main_preset_name
item["match_mode"] = "exact" if resolved_main_preset_name == local_preset_name else "alias"
item["different_fields"] = different_fields
item["different_field_labels"] = [
_PRESET_COMPARE_FIELD_LABELS.get(field, field)
for field in different_fields
]
item["difference_count"] = len(different_fields)
if different_fields:
item["status"] = "different"
item["summary_text"] = f"{len(different_fields)} 项字段与官方预设不同"
counts["different"] += 1
else:
item["status"] = "same"
item["summary_text"] = "与官方预设一致"
counts["same"] += 1
items.append(item)
for main_preset_name in sorted(main_presets.keys(), key=lambda item: str(item)):
if main_preset_name in matched_main_presets:
continue
counts["main_only_preset"] += 1
for domain in sorted(set(main_sites.keys()) - set(local_sites.keys()), key=lambda item: str(item)):
main_presets = _extract_site_presets_for_compare(main_sites.get(domain))
counts["main_only_site"] += max(1, len(main_presets))
status_priority = {
"different": 0,
"local_only_preset": 1,
"local_only_site": 2,
"same": 3,
}
items.sort(
key=lambda item: (
status_priority.get(str(item.get("status") or ""), 99),
str(item.get("domain") or ""),
str(item.get("local_preset_name") or ""),
)
)
return {
"branch": "main",
"path": branch_payload["path"],
"source": branch_payload["source"],
"counts": counts,
"items": items,
}
# ================= 认证依赖 =================
__all__ = [
'_load_official_sites_config',
'_resolve_branch_preset_config',
'_build_main_branch_compare_summary',
]
+220
View File
@@ -0,0 +1,220 @@
"""Request models and preset normalization helpers for config routes."""
import copy
from typing import Optional, Dict, Any, List
from fastapi import HTTPException
from pydantic import BaseModel, Field
from app.models.schemas import ADVANCED_FIELDS, PRESET_ADVANCED_FIELDS, SITE_ADVANCED_FIELDS
from app.services.arena_direct_models import normalize_model_catalog_config
class ConfigUpdateRequest(BaseModel):
"""配置更新请求"""
config: Dict[str, Any] = Field(...)
class SiteAdvancedConfigRequest(BaseModel):
"""站点级高级配置更新请求。"""
preset_name: Optional[str] = Field(default=None)
advanced: Optional[Dict[str, Any]] = Field(default=None)
independent_cookies: bool = Field(default=False)
independent_cookies_auto_takeover: bool = Field(default=False)
input_box_stability_wait_enabled: bool = Field(default=False)
input_box_stability_wait_after_new_chat_only: bool = Field(default=True)
input_box_stability_wait_timeout: float = Field(default=1.5, ge=0.1, le=10.0)
url_transition_wait_on_new_chat: bool = Field(default=False)
url_transition_wait_patterns: List[str] = Field(default_factory=list)
send_confirmation_check_enabled: bool = Field(default=False)
send_confirmation_check_timeout: float = Field(default=1.5, ge=0.1, le=10.0)
skip_new_chat_on_retry: bool = Field(default=False)
class ModelCatalogUpdateRequest(BaseModel):
preset_name: Optional[str] = None
catalog: Optional[Dict[str, Any]] = None
class PresetConfigUpdateRequest(BaseModel):
"""单个预设完整配置更新请求。"""
preset_name: Optional[str] = Field(default=None)
config: Dict[str, Any] = Field(...)
replace: bool = Field(default=False)
def _get_model_fields_set(model: BaseModel) -> set:
fields_set = getattr(model, "model_fields_set", None)
if fields_set is None:
fields_set = getattr(model, "__fields_set__", set())
return set(fields_set or set())
def _extract_site_advanced_update_payload(
body: SiteAdvancedConfigRequest,
*,
preset_scope: bool = False,
allow_inherited_site_fields: bool = False,
) -> Dict[str, Any]:
"""提取高级配置更新 payload,兼容扁平字段与 GET 返回的 advanced 包裹形状。"""
provided_fields = _get_model_fields_set(body)
payload: Dict[str, Any] = {}
nested_advanced = body.advanced
if "advanced" in provided_fields and nested_advanced is not None:
if not isinstance(nested_advanced, dict) or isinstance(nested_advanced, list):
raise HTTPException(status_code=400, detail="advanced 必须是对象")
if preset_scope:
invalid_nested_site_fields = [
key for key in SITE_ADVANCED_FIELDS
if key in nested_advanced
]
if invalid_nested_site_fields and not allow_inherited_site_fields:
joined = ", ".join(sorted(invalid_nested_site_fields))
raise HTTPException(status_code=400, detail=f"预设级高级配置不能包含站点级字段: {joined}")
allowed_nested_fields = PRESET_ADVANCED_FIELDS if preset_scope else ADVANCED_FIELDS
payload.update({
key: copy.deepcopy(value)
for key, value in nested_advanced.items()
if key in allowed_nested_fields
})
if preset_scope:
invalid_site_fields = [
key for key in SITE_ADVANCED_FIELDS
if key in provided_fields
]
if invalid_site_fields:
joined = ", ".join(sorted(invalid_site_fields))
raise HTTPException(
status_code=400,
detail=f"预设级高级配置不能包含站点级字段: {joined}",
)
flat_payload = {
"independent_cookies": bool(body.independent_cookies),
"independent_cookies_auto_takeover": bool(body.independent_cookies_auto_takeover),
"input_box_stability_wait_enabled": bool(body.input_box_stability_wait_enabled),
"input_box_stability_wait_after_new_chat_only": bool(body.input_box_stability_wait_after_new_chat_only),
"input_box_stability_wait_timeout": float(body.input_box_stability_wait_timeout),
"url_transition_wait_on_new_chat": bool(body.url_transition_wait_on_new_chat),
"url_transition_wait_patterns": [
str(pattern or "").strip()
for pattern in (body.url_transition_wait_patterns or [])
if str(pattern or "").strip()
],
"send_confirmation_check_enabled": bool(body.send_confirmation_check_enabled),
"send_confirmation_check_timeout": float(body.send_confirmation_check_timeout),
"skip_new_chat_on_retry": bool(body.skip_new_chat_on_retry),
}
payload.update({
key: value
for key, value in flat_payload.items()
if key in provided_fields
})
return payload
def _normalize_preset_config_payload(
payload: Dict[str, Any],
domain: Optional[str] = None,
) -> Dict[str, Any]:
"""校验并规范化单个预设配置对象。"""
if not isinstance(payload, dict) or isinstance(payload, list):
raise HTTPException(status_code=400, detail="config 必须是对象")
reserved_site_fields = {"presets", "default_preset"}
invalid_fields = [key for key in reserved_site_fields if key in payload]
if invalid_fields:
joined = ", ".join(invalid_fields)
raise HTTPException(
status_code=400,
detail=f"这里只接受单个预设配置对象,不能包含站点级字段: {joined}",
)
normalized = copy.deepcopy(payload)
selectors = normalized.get("selectors")
if selectors is None:
normalized["selectors"] = {}
elif not isinstance(selectors, dict) or isinstance(selectors, list):
raise HTTPException(status_code=400, detail="selectors 必须是对象")
workflow = normalized.get("workflow")
if workflow is None:
normalized["workflow"] = []
elif not isinstance(workflow, list):
raise HTTPException(status_code=400, detail="workflow 必须是数组")
from app.core.workflow.flow_runtime import has_control_flow, validate_workflow, FlowValidationError
if has_control_flow(normalized.get("workflow", [])):
try:
validate_workflow(normalized["workflow"])
except FlowValidationError as exc:
raise HTTPException(status_code=400, detail=str(exc)) from exc
advanced = normalized.get("advanced")
if (
advanced is not None
and not isinstance(advanced, dict)
or isinstance(advanced, list)
):
raise HTTPException(status_code=400, detail="advanced 必须是对象")
if isinstance(advanced, dict):
invalid_advanced_fields = [
key for key in advanced.keys()
if key in SITE_ADVANCED_FIELDS
]
if invalid_advanced_fields:
joined = ", ".join(invalid_advanced_fields)
raise HTTPException(
status_code=400,
detail=f"预设 advanced 不能包含站点级字段: {joined}",
)
model_catalog = normalized.get("model_catalog")
if model_catalog is not None:
if not isinstance(model_catalog, dict) or isinstance(model_catalog, list):
raise HTTPException(status_code=400, detail="model_catalog 必须是对象")
if domain is not None:
from app.utils.site_url import route_domain_matches
if route_domain_matches("arena.ai", domain):
normalized.pop("model_catalog", None)
normalized["stealth"] = bool(normalized.get("stealth", False))
return normalized
def _merge_preset_config_payload(
base_payload: Dict[str, Any],
payload: Dict[str, Any],
) -> Dict[str, Any]:
"""将局部预设配置叠加到现有预设上,保留未显式提供的字段。"""
if not isinstance(base_payload, dict):
base_payload = {}
if not isinstance(payload, dict) or isinstance(payload, list):
raise HTTPException(status_code=400, detail="config 必须是对象")
return _merge_dict_patch(base_payload, payload)
def _merge_dict_patch(base_payload: Dict[str, Any], payload: Dict[str, Any]) -> Dict[str, Any]:
merged = dict(base_payload)
for key, value in payload.items():
existing = base_payload.get(key)
if isinstance(existing, dict) and isinstance(value, dict):
merged[key] = _merge_dict_patch(existing, value)
continue
merged[key] = copy.deepcopy(value)
return merged
__all__ = [
'ConfigUpdateRequest',
'SiteAdvancedConfigRequest',
'PresetConfigUpdateRequest',
'_extract_site_advanced_update_payload',
'_get_model_fields_set',
'_normalize_preset_config_payload',
'_merge_preset_config_payload',
]
File diff suppressed because it is too large. Load diff
+592
View File
@@ -0,0 +1,592 @@
"""Workflow-editor helpers for config routes."""
import copy
import json
import time
from typing import Optional, Dict, Any, Callable, get_args
from fastapi import HTTPException
from app.core.config import get_logger, WorkflowCancelledError
from app.core.page_lifecycle import BACKGROUND_WAKE_CDP_TIMEOUT, BACKGROUND_WAKE_JS_TIMEOUT
from app.models.schemas import ActionType
from app.core.workflow.flow_runtime import (FlowProgram, FlowValidationError, validate_workflow, has_control_flow, capture_page_state, validate_inputs)
from app.services.config_engine import config_engine
from app.utils.site_url import extract_remote_site_domain
from app.core.driver import driver_for_tab
logger = get_logger('API.CONFIG.WORKFLOW')
# 修复:保存时校验动作名。合法集合直接取自 schemas.ActionType;
# 运行期 executor 的 dispatch 用原始 action 精确匹配大写字面量、不做归一化,故此处也要求精确大写。
VALID_WORKFLOW_ACTIONS = frozenset(get_args(ActionType))
def _notify_workflow_editor_action_result(tab, action_id: str, success: bool, message: str, trace=None) -> None:
"""将测试结果回推给已注入的可视化编辑器页面。"""
try:
driver_for_tab(tab).run_js(
"""
return (function(actionId, ok, text, trace) {
if (window.WorkflowEditor && typeof window.WorkflowEditor.handleBackendResult === 'function') {
window.WorkflowEditor.handleBackendResult(actionId, ok, text, trace);
return true;
}
return false;
})(arguments[0], arguments[1], arguments[2], arguments[3]);
""",
str(action_id or ""),
bool(success),
str(message or ""),
list(trace or [])[:500],
)
except Exception as e:
logger.debug(f"回推编辑器测试结果失败(忽略): {e}")
def _notify_workflow_editor_action_status(tab, action_id: str, phase: str, message: str) -> None:
"""将测试中间状态回推给已注入的可视化编辑器页面。"""
try:
driver_for_tab(tab).run_js(
"""
return (function(actionId, phaseName, text) {
if (window.WorkflowEditor && typeof window.WorkflowEditor.handleBackendStatus === 'function') {
window.WorkflowEditor.handleBackendStatus(actionId, phaseName, text);
return true;
}
return false;
})(arguments[0], arguments[1], arguments[2]);
""",
str(action_id or ""),
str(phase or ""),
str(message or ""),
)
except Exception as e:
logger.debug(f"回推编辑器测试状态失败(忽略): {e}")
def _wake_workflow_editor_test_tab(session) -> None:
"""在测试前尽量唤醒后台标签页,避免页面被冻结导致点击无效。"""
if session is None:
return
try:
if hasattr(session, "activate"):
session.activate()
except Exception:
pass
focus_emulation_set = False
try:
driver_for_tab(session.tab).run_cdp(
"Emulation.setFocusEmulationEnabled",
enabled=True,
_timeout=BACKGROUND_WAKE_CDP_TIMEOUT,
)
focus_emulation_set = True
except Exception:
pass
try:
driver_for_tab(session.tab).run_cdp(
"Page.setWebLifecycleState",
state="active",
_timeout=BACKGROUND_WAKE_CDP_TIMEOUT,
)
except Exception:
pass
try:
driver_for_tab(session.tab).run_js("return document.readyState || '';", timeout=BACKGROUND_WAKE_JS_TIMEOUT)
except Exception:
pass
finally:
if focus_emulation_set:
try:
driver_for_tab(session.tab).run_cdp(
"Emulation.setFocusEmulationEnabled",
enabled=False,
_timeout=BACKGROUND_WAKE_CDP_TIMEOUT,
)
except Exception:
pass
def _execute_workflow_editor_test_payload(
browser_instance,
data: Dict[str, Any],
progress_callback: Optional[Callable[[str, str], None]] = None
) -> Dict[str, Any]:
"""复用真实执行器执行可视化编辑器测试。"""
from app.core.workflow.executor import WorkflowExecutor
action_labels = {
"CLICK": "点击元素",
"SELECT_MODEL": "选择请求模型",
"COORD_CLICK": "坐标点击",
"COORD_SCROLL": "模拟滑动",
"FILL_INPUT": "填入内容",
"STREAM_WAIT": "流式等待",
# 修复:补上缺失的两项,否则测试进度提示会退化成显示英文原始 action 名
"STREAM_OUTPUT": "流式等待",
"WAIT": "等待",
"KEY_PRESS": "按键",
"JS_EXEC": "执行脚本",
"PAGE_FETCH": "页面直发",
"READONLY_HINT": "只读提示",
}
domain = str(data.get("domain") or "").strip()
tab_id = str(data.get("tab_id") or "").strip()
workflow = data.get("workflow") or []
selectors = data.get("selectors") or {}
preset_name = str(data.get("preset_name") or "").strip() or None
prompt_text = str(data.get("prompt") or "")
stealth = bool(data.get("stealth", False))
stream_config = dict(data.get("stream_config") or {})
image_config = data.get("image_extraction") or {}
file_paste_config = data.get("file_paste") or {}
if not domain:
raise HTTPException(status_code=400, detail="缺少 domain")
if not isinstance(workflow, list) or not workflow:
raise HTTPException(status_code=400, detail="workflow 必须是非空数组")
if not isinstance(selectors, dict):
raise HTTPException(status_code=400, detail="selectors 必须是对象")
try:
validate_workflow(workflow)
validate_inputs(data.get("workflow_variables", {}))
except FlowValidationError as exc:
raise HTTPException(status_code=400, detail=str(exc)) from exc
if not browser_instance.get_browser_handle():
raise HTTPException(status_code=503, detail="浏览器未连接")
task_id = f"workflow_editor_test_{time.time_ns()}"
session = None
executor = None
flow_program = None
started_at = time.time()
executed = 0
try:
if tab_id and getattr(browser_instance, "tab_pool", None) is not None:
session = browser_instance.tab_pool.acquire_by_raw_tab_id(
tab_id,
task_id,
timeout=5,
count_request=False,
)
tab = session.tab if session is not None else (
browser_instance.get_tab(tab_id) if tab_id else browser_instance.get_latest_tab()
)
logger.debug(
"[WFE_TEST] tab resolved "
f"session={getattr(session, 'id', None)!r} "
f"raw_tab_id={getattr(tab, 'tab_id', None)!r} "
f"url={str(getattr(tab, 'url', '') or '')[:160]!r}"
)
test_stream_config = dict(stream_config or {})
test_network_config = dict(test_stream_config.get("network") or {})
test_stream_config["hard_timeout"] = min(
float(test_stream_config.get("hard_timeout", 45) or 45),
45.0,
)
test_network_config["first_response_timeout"] = min(
float(test_network_config.get("first_response_timeout", 12) or 12),
12.0,
)
test_network_config["silence_threshold"] = max(
0.5,
min(float(test_network_config.get("silence_threshold", 3) or 3), 30.0),
)
test_stream_config["network"] = test_network_config
logger.debug(
"[WFE_TEST] start "
f"domain={domain!r} tab_id={tab_id!r} preset={preset_name!r} "
f"steps={len(workflow)} selectors={len(selectors)} "
f"hard_timeout={test_stream_config['hard_timeout']:.1f}s "
f"first_response_timeout={test_network_config['first_response_timeout']:.1f}s "
f"silence_threshold={test_network_config['silence_threshold']:.1f}s"
)
if progress_callback is None and data.get("visual_feedback") is True:
progress_callback = lambda phase, message: _notify_workflow_editor_action_status(tab, "", phase, message)
if progress_callback:
progress_callback("running", f"本地控制台已接管,准备执行 {len(workflow)} 个动作")
_wake_workflow_editor_test_tab(session)
url = str(getattr(tab, "url", "") or "")
actual_domain = extract_remote_site_domain(url)
if not actual_domain:
raise HTTPException(status_code=400, detail="当前页面不是可解析的网站")
if actual_domain != domain:
raise HTTPException(
status_code=400,
detail=f"域名不匹配:当前页面是 {actual_domain},测试目标是 {domain}"
)
resolved_site_config = config_engine.get_site_config(
domain,
html_content=lambda: getattr(tab, "html", ""),
preset_name=preset_name,
) or {}
site_advanced_config = config_engine.get_site_advanced_config(
domain,
preset_name=preset_name,
)
extractor = config_engine.get_site_extractor(domain, preset_name=preset_name)
cancellation = {"checked_at": 0.0, "cancelled": False}
def editor_should_stop():
if time.time() - started_at > 45:
return True
if data.get("visual_feedback") is True and time.monotonic() - cancellation["checked_at"] >= 0.2:
cancellation["checked_at"] = time.monotonic()
try:
cancellation["cancelled"] = cancellation["cancelled"] or driver_for_tab(tab).run_js(
"return window.__WORKFLOW_EDITOR_TEST_CANCELLED__ === true || (arguments[0] && window.__WORKFLOW_EDITOR_TEST_RUN_ID__ !== arguments[0]);",
str(data.get("visual_run_id") or "")[:128], timeout=0.5,
) is True
except Exception:
pass # Browser failure still respects the overall time budget.
return cancellation["cancelled"]
executor = WorkflowExecutor(
tab=tab,
stealth_mode=stealth,
should_stop_checker=editor_should_stop,
extractor=extractor,
image_config=image_config,
stream_config=test_stream_config or resolved_site_config.get("stream_config") or {},
file_paste_config=file_paste_config,
site_advanced_config=site_advanced_config,
selectors=selectors,
session=session,
)
executed = 0
context = {
"prompt": prompt_text,
"images": [],
}
# 修复:此前不透传 model/model_catalog,导致 SELECT_MODEL 步骤在编辑器测试里必然 early return,
# 但仍计入 executed_steps,测试显示“成功”而实际未执行。
requested_model = str((data or {}).get("model") or "").strip()
if requested_model:
context["model"] = requested_model
model_catalog = (data or {}).get("model_catalog")
if isinstance(model_catalog, dict) and model_catalog:
context["model_catalog"] = model_catalog
skipped_select_model = 0
flow_program = None
if has_control_flow(workflow) or data.get("workflow_variables"):
flow_program = FlowProgram(
workflow, context=context, inputs=data.get("workflow_variables"),
capture=lambda spec, target: capture_page_state(executor, spec, target, flow_program.variables),
stop_checker=executor._check_cancelled, deadline_seconds=45,
)
workflow = flow_program.steps
with executor.workflow_execution_scope():
step_index = 0
while step_index < len(workflow):
if flow_program is not None:
step_index = flow_program.next_leaf(step_index)
if step_index >= len(workflow): break
try:
step, flow_selector = flow_program.prepare_leaf(step_index, selectors)
except Exception as exc:
resumed = flow_program.recover(exc, min(step_index, len(workflow) - 1))
if resumed is None: raise
step_index = resumed
continue
else:
step = workflow[step_index]
action = str(step.get("action") or "").strip()
target_key = str(step.get("target") or "")
optional = bool(step.get("optional", False))
value = step.get("value")
execution_policy = step.get("execution")
selector = flow_selector if flow_program is not None else selectors.get(target_key, "")
current_index = step_index + 1
logger.debug(
"[WFE_TEST] step "
f"index={current_index} action={action!r} target={target_key!r} "
f"selector={selector!r} optional={optional}"
)
if not action:
raise HTTPException(status_code=400, detail="workflow 中存在缺少 action 的步骤")
# 修复:未指定模型时 SELECT_MODEL 必然 early return(executor_actions.py:1284-1289),
# 此前仍会计入 executed_steps,让"已测试 N 个步骤"虚高、掩盖空跑。
# 注意:这是 while 循环,跳过前必须自行推进 step_index,否则死循环。
if action == "SELECT_MODEL" and not requested_model:
skipped_select_model += 1
if progress_callback:
progress_callback(
"step",
f"跳过 {current_index}/{len(workflow)} · 选择请求模型(未指定测试模型名)"
)
step_index += 1
continue
if progress_callback:
# Ephemeral page feedback uses resolved targets; it is NOT
# persisted in the privacy-preserving workflow trace.
phase = "step"
if data.get("visual_feedback") is True:
phase = "step:" + json.dumps({
"path": step.get("_path", f"root.{step_index}"),
"selector": selector,
"coordinates": value if action in {"COORD_CLICK", "COORD_SCROLL"} and isinstance(value, dict) else None,
}, ensure_ascii=False)
progress_callback(
phase,
f"执行 {current_index}/{len(workflow)} · {action_labels.get(action, action)}"
)
if data.get("visual_feedback") is True:
# Give the browser one visible frame before performing
# the action. Only editor tests opt into this pacing.
until = time.monotonic() + 0.25
while time.monotonic() < until and not executor._check_cancelled():
time.sleep(0.02)
if flow_program is None and action == "FILL_INPUT" and value is not None:
context["prompt"] = str(value)
step_events = executor.execute_step(
action=action,
selector=selector,
target_key=target_key,
value=value,
optional=optional,
context=flow_program.leaf_context(step) if flow_program is not None else context,
execution=execution_policy,
)
if flow_program is not None:
step_events = flow_program.events(step_events, step_index)
try:
for _ in step_events:
pass
except Exception as exc:
if flow_program is None: raise
resumed = flow_program.recover(exc, step_index)
if resumed is None: raise
step_index = resumed
continue
if executor._check_cancelled():
raise WorkflowCancelledError("workflow_cancelled")
executed += 1
if (
action == "PAGE_FETCH"
and hasattr(executor, "consume_last_request_transport_sent")
and executor.consume_last_request_transport_sent()
):
step_index = executor._consume_request_transport_followup_steps(
workflow,
step_index,
)
step_index += 1
logger.debug(
"[WFE_TEST] done "
f"domain={domain!r} tab_id={tab_id or str(getattr(tab, 'tab_id', '') or '')!r} "
f"executed_steps={executed} duration={time.time() - started_at:.2f}s"
)
# 修复:把被跳过的 SELECT_MODEL 步骤如实告知前端,避免“已测试 N 个步骤”掩盖空跑
message = f"已测试 {executed} 个步骤"
if skipped_select_model:
message += f"(其中 {skipped_select_model} 个选择模型步骤已跳过:未指定测试模型名)"
return {
"success": True,
"message": message,
"domain": domain,
"tab_id": tab_id or str(getattr(tab, "tab_id", "") or ""),
"preset_name": preset_name or config_engine.get_default_preset(domain) or "主预设",
"executed_steps": executed,
"trace": flow_program.trace if flow_program is not None else [],
"skipped_select_model_steps": skipped_select_model,
"_tab_ref": tab,
}
except Exception as exc:
if data.get("visual_feedback") is True and isinstance(exc, WorkflowCancelledError):
return {"success": False, "cancelled": True, "message": "测试已停止(取消或达到时间预算),已执行的网页操作不会撤销。",
"trace": flow_program.trace if flow_program is not None else [],
"executed_steps": executed, "_tab_ref": tab}
if flow_program is None:
raise
return {"success": False, "message": str(exc), "trace": flow_program.trace,
"executed_steps": executed, "_tab_ref": tab}
finally:
if executor is not None:
try:
executor.cleanup_after_workflow()
except Exception as e:
logger.debug(f"[WFE_TEST] executor cleanup skipped: {e}")
if session is not None and getattr(browser_instance, "tab_pool", None) is not None:
logger.debug(f"[WFE_TEST] release session={session.id!r}")
browser_instance.tab_pool.release(
session.id,
check_triggers=False,
expected_task_id=task_id,
)
def _save_site_workflow_payload(domain: str, data: Dict[str, Any]) -> Dict[str, Any]:
"""保存工作流配置,供直接 API 和桥接模式复用。"""
new_workflow = data.get("workflow")
new_selectors = data.get("selectors")
preset_name = data.get("preset_name")
if new_workflow is None:
raise HTTPException(status_code=400, detail="缺少 workflow 字段")
if not isinstance(new_workflow, list):
raise HTTPException(status_code=400, detail="workflow 必须是数组")
# 修复:此前不校验动作名,拼错(如 STREAM_WIAT)或小写(click)会静默入库,
# 运行期只在 debug 日志里记一行“未知动作”,表现为工作流跑完但什么都没发生。
for i, step in enumerate(new_workflow):
if not isinstance(step, dict):
raise HTTPException(status_code=400, detail=f"第 {i + 1} 步必须是对象")
action = step.get("action")
if not isinstance(action, str) or action not in VALID_WORKFLOW_ACTIONS:
raise HTTPException(
status_code=400,
detail=(
f"第 {i + 1} 步的动作类型无效: {action}"
f"(动作名必须大写,可选值:{'、'.join(sorted(VALID_WORKFLOW_ACTIONS))})"
),
)
try:
validate_workflow(new_workflow)
except FlowValidationError as exc:
raise HTTPException(status_code=400, detail=str(exc)) from exc
if new_selectors is not None and (
not isinstance(new_selectors, dict) or isinstance(new_selectors, list)
):
raise HTTPException(status_code=400, detail="selectors 必须是对象")
config_engine.refresh_if_changed()
if domain not in config_engine.sites:
raise HTTPException(status_code=404, detail=f"站点不存在: {domain}")
site = config_engine.sites[domain]
presets = site.get("presets", {})
resolved_preset_name = None
if preset_name:
resolved_preset_name = config_engine._resolve_preset_alias_key(preset_name, presets)
if resolved_preset_name not in presets:
raise HTTPException(status_code=404, detail=f"预设不存在: {preset_name}")
preset_data = config_engine._get_site_data(domain, resolved_preset_name)
if preset_data is None:
raise HTTPException(status_code=404, detail="站点或预设不存在")
workflow_existed = "workflow" in preset_data
previous_workflow = copy.deepcopy(preset_data.get("workflow"))
selectors_existed = "selectors" in preset_data
previous_selectors = copy.deepcopy(preset_data.get("selectors"))
preset_data["workflow"] = new_workflow
if new_selectors is not None:
preset_data["selectors"] = new_selectors
def rollback_workflow_memory() -> None:
if workflow_existed:
preset_data["workflow"] = previous_workflow
else:
preset_data.pop("workflow", None)
if new_selectors is not None:
if selectors_existed:
preset_data["selectors"] = previous_selectors
else:
preset_data.pop("selectors", None)
try:
success = config_engine.save_config()
except Exception as exc:
rollback_workflow_memory()
raise HTTPException(status_code=500, detail=f"保存配置文件失败: {exc}") from exc
if not success:
rollback_workflow_memory()
raise HTTPException(status_code=500, detail="保存配置文件失败")
used_preset = (
resolved_preset_name
or config_engine.get_default_preset(domain)
or "主预设"
)
logger.info(f"站点 {domain} [{used_preset}] 工作流已更新: {len(new_workflow)} 个步骤")
return {
"status": "success",
"message": f"工作流已保存",
"domain": domain,
"preset_name": used_preset,
"steps_count": len(new_workflow)
}
def _scan_workflow_scripts() -> Dict[str, Any]:
"""扫描可用工作流脚本列表"""
from app.core.workflow.script_loader import script_loader
scripts = script_loader.scan_available_scripts()
return {"scripts": scripts}
def _load_workflow_script_content(target_path: str) -> Dict[str, Any]:
"""加载指定脚本的内容与元数据(用于前端预览)"""
from app.core.workflow.script_loader import script_loader
from app.core.config import WorkflowError
if not target_path:
raise HTTPException(status_code=400, detail="缺少脚本路径参数")
try:
resolved_path = script_loader.resolve_script_path(target_path)
content = script_loader.load_script_content(resolved_path)
stat = resolved_path.stat()
mtime = int(stat.st_mtime)
desc = script_loader.extract_script_description(content, resolved_path.name)
rel_path = resolved_path.relative_to(script_loader.base_dir).as_posix()
return {
"path": rel_path,
"uri": f"file://{rel_path}",
"name": resolved_path.name,
"description": desc,
"content": content,
"mtime": mtime,
}
except WorkflowError as e:
raise HTTPException(status_code=400, detail=str(e))
except Exception as e:
logger.error(f"读取脚本内容失败 ({target_path!r}): {e}")
raise HTTPException(status_code=500, detail=f"读取脚本失败: {e}")
# ================= 请求模型 =================
__all__ = [
'_notify_workflow_editor_action_result',
'_notify_workflow_editor_action_status',
'_execute_workflow_editor_test_payload',
'_save_site_workflow_payload',
'_scan_workflow_scripts',
'_load_workflow_script_content',
]
+131
View File
@@ -0,0 +1,131 @@
"""
app/api/deps.py - API 共享依赖
"""
import hmac
from typing import Iterable, Optional
from fastapi import Header, HTTPException, Request
from app.core.config import AppConfig
from app.core.http_security import is_trusted_local_request
def extract_authorization_token(authorization: Optional[str]) -> str:
"""Extract a token from Authorization, accepting raw tokens and Bearer schemes."""
raw = str(authorization or "").strip()
if raw.lower().startswith("bearer "):
return raw[7:].strip()
return raw
def _auth_candidates(
authorization: Optional[str],
x_api_key: Optional[str] = None,
) -> list[str]:
candidates: list[str] = []
if isinstance(authorization, str) and authorization.strip():
raw = authorization.strip()
candidates.append(raw)
extracted = extract_authorization_token(raw)
if extracted and extracted != raw:
candidates.append(extracted)
if isinstance(x_api_key, str) and x_api_key.strip():
candidates.append(x_api_key.strip())
return candidates
def _verify_token_candidates(
*,
enabled: bool,
token_value: str,
candidates: Iterable[str],
) -> None:
if not enabled:
return
expected = str(token_value or "").strip()
if not expected:
raise HTTPException(status_code=500, detail="服务配置错误")
normalized = [str(item or "").strip() for item in candidates if str(item or "").strip()]
if not normalized:
raise HTTPException(
status_code=401,
detail="未提供认证令牌",
headers={"WWW-Authenticate": "Bearer"},
)
# 修复6: str 版 compare_digest 遇到非 ASCII 令牌会抛 TypeError 导致 500,统一按 UTF-8 编码后比较
expected_bytes = expected.encode("utf-8")
if not any(
hmac.compare_digest(expected_bytes, candidate.encode("utf-8"))
for candidate in normalized
):
raise HTTPException(
status_code=401,
detail="认证令牌无效",
headers={"WWW-Authenticate": "Bearer"},
)
def verify_service_token(
authorization: Optional[str] = None,
x_api_key: Optional[str] = None,
) -> None:
_verify_token_candidates(
enabled=AppConfig.is_auth_enabled(),
token_value=AppConfig.get_auth_token(),
candidates=_auth_candidates(authorization, x_api_key),
)
def verify_dashboard_token(authorization: Optional[str] = None) -> None:
_verify_token_candidates(
enabled=AppConfig.is_dashboard_auth_enabled(),
token_value=AppConfig.get_dashboard_auth_token(),
candidates=_auth_candidates(authorization),
)
async def verify_service_auth(
authorization: Optional[str] = Header(None),
x_api_key: Optional[str] = Header(None, alias="X-API-Key"),
) -> bool:
"""验证对外服务 API 的 Bearer Token 或 X-API-Key。"""
verify_service_token(authorization=authorization, x_api_key=x_api_key)
return True
async def verify_dashboard_auth(authorization: Optional[str] = Header(None)) -> bool:
"""验证控制面板管理接口的访问密钥。"""
verify_dashboard_token(authorization=authorization)
return True
async def verify_auth(authorization: Optional[str] = Header(None)) -> bool:
"""向后兼容:默认用于控制面板管理接口。"""
return await verify_dashboard_auth(authorization)
async def verify_sensitive_admin_auth(
request: Request,
authorization: Optional[str] = Header(None),
) -> bool:
"""高敏感管理接口(完整备份导出/导入、打开受控浏览器等)的鉴权。
- 控制面板认证已启用:必须提供有效令牌(与 verify_dashboard_auth 相同);
- 未启用:不再无条件放行,只接受「直接来自本机回环、未经反代/隧道转发」的请求,
其余一律 403,并提示启用 DASHBOARD_AUTH_*。(审查条目 S13 / S4)
"""
if AppConfig.is_dashboard_auth_enabled():
verify_dashboard_token(authorization=authorization)
return True
client_host = request.client.host if request.client else None
if is_trusted_local_request(client_host, request.headers):
return True
raise HTTPException(
status_code=403,
detail="该接口仅允许本机直接访问;远程访问请先启用控制面板认证(DASHBOARD_AUTH_ENABLED / DASHBOARD_AUTH_TOKEN)",
)
+22
View File
@@ -0,0 +1,22 @@
"""R2-7:Prometheus 指标接口。
与 /health 的详细信息同样的访问规则(S7):严格本机直连,或携带有效的 AUTH_TOKEN / DASHBOARD_AUTH_TOKEN;
其余请求返回 403,避免对外暴露路由与负载信息。
"""
from __future__ import annotations
from fastapi import APIRouter, HTTPException, Request
from fastapi.responses import PlainTextResponse
from app.services.metrics import render_prometheus
from app.utils.diagnostics_access import request_is_privileged
router = APIRouter()
@router.get("/metrics", response_class=PlainTextResponse)
async def metrics(request: Request):
if not request_is_privileged(request):
raise HTTPException(status_code=403, detail="metrics 仅限本机直连或携带有效令牌访问")
return PlainTextResponse(render_prometheus(), media_type="text/plain; version=0.0.4; charset=utf-8")
+891
View File
@@ -0,0 +1,891 @@
from __future__ import annotations
import json
from dataclasses import dataclass, field
from typing import Any, Dict, List, Optional
from app.core.config import SSEFormatter, get_logger
from app.services.sse_utils import (
extract_sse_error_message,
iter_sse_payloads,
sse_frame_data_text,
)
logger = get_logger("API.OPENAI_STOP")
@dataclass
class StopSequenceStreamState:
sequences: List[str]
single_choice: bool = False
upstream_single_choice: bool = False
max_sequence_len: int = 0
tail: str = ""
stopped: bool = False
stopped_choice_keys: set[str] = field(default_factory=set)
seen_choice_keys: set[str] = field(default_factory=set)
choice_tails: Dict[str, str] = field(default_factory=dict)
choice_pending_parts: Dict[str, List[Any]] = field(default_factory=dict)
pending_parts: List[Any] = field(default_factory=list)
sse_frame_buffer: str = ""
MAX_STOP_SSE_FRAME_BUFFER_CHARS = 262144
def _cap_sse_frame_buffer(buffered: str) -> str:
# 修复7: 单个未完成 SSE 帧超过缓冲上限时整帧丢弃并告警;
# 原先从头截断保留尾部片段,帧完成后会拼出坏 JSON 转发给客户端
if len(buffered) > MAX_STOP_SSE_FRAME_BUFFER_CHARS:
logger.warning(
f"SSE 帧超过缓冲上限 {MAX_STOP_SSE_FRAME_BUFFER_CHARS} 字符"
f"(实际 {len(buffered)}),该帧过大已被丢弃"
)
return ""
return buffered
def normalize_openai_stop_sequences(value: Any) -> List[str]:
if value is None:
return []
if isinstance(value, str):
return [value] if value else []
if isinstance(value, (list, tuple)):
sequences: List[str] = []
for item in value:
if item is None:
continue
text = str(item)
if text:
sequences.append(text)
return sequences
text = str(value)
return [text] if text else []
def build_stop_sequence_stream_state(
value: Any,
*,
single_choice: bool = False,
upstream_single_choice: bool = False,
) -> StopSequenceStreamState:
sequences = normalize_openai_stop_sequences(value)
return StopSequenceStreamState(
sequences=sequences,
single_choice=bool(single_choice),
upstream_single_choice=bool(upstream_single_choice),
max_sequence_len=max((len(sequence) for sequence in sequences), default=0),
)
def find_first_stop_sequence(text: str, stop_sequences: List[str]) -> Optional[tuple[int, str]]:
first_match: Optional[tuple[int, str]] = None
for sequence in stop_sequences:
position = text.find(sequence)
if position < 0:
continue
if first_match is None or position < first_match[0]:
first_match = (position, sequence)
return first_match
def split_stream_text_for_stop_sequences(
text_buffer: str,
stop_sequences: List[str],
max_sequence_len: int = 0,
) -> tuple[str, str, Optional[str]]:
if not stop_sequences:
return text_buffer, "", None
match = find_first_stop_sequence(text_buffer, stop_sequences)
if match is not None:
position, sequence = match
return text_buffer[:position], "", sequence
max_len = max_sequence_len or max(len(sequence) for sequence in stop_sequences)
keep_len = min(max_len - 1, len(text_buffer))
if keep_len <= 0:
return text_buffer, "", None
emit_len = len(text_buffer) - keep_len
return text_buffer[:emit_len], text_buffer[emit_len:], None
def apply_stop_sequences_to_text(content: str, stop_sequences: Any) -> str:
sequences = normalize_openai_stop_sequences(stop_sequences)
if not sequences:
return content
match = find_first_stop_sequence(content or "", sequences)
if match is None:
return content
position, _sequence = match
return (content or "")[:position]
def sse_chunk_has_done(chunk: Any) -> bool:
if not isinstance(chunk, str):
return False
if "[DONE]" not in chunk:
return False
frames = chunk.replace("\r\n", "\n").replace("\r", "\n").split("\n\n")
for frame in frames:
if sse_frame_data_text(frame).strip() == "[DONE]":
return True
return False
def iter_openai_sse_payloads(chunk: Any) -> List[Dict[str, Any]]:
return iter_sse_payloads(chunk)
def extract_openai_sse_error_message(chunk: Any) -> str:
return extract_sse_error_message(chunk)
def _pack_sse_payload(payload: Dict[str, Any]) -> str:
return f"data: {json.dumps(payload, ensure_ascii=False)}\n\n"
def _choice_tail_key(choice: Dict[str, Any], fallback_index: int) -> str:
index = choice.get("index")
if index is None:
index = fallback_index
return str(index)
def _get_choice_tail(state: StopSequenceStreamState, key: str) -> str:
return state.choice_tails.get(key, "")
def _set_choice_tail(state: StopSequenceStreamState, key: str, value: str) -> None:
if value:
state.choice_tails[key] = value
else:
state.choice_tails.pop(key, None)
if key == "0":
state.tail = value
def _get_choice_pending_parts(state: StopSequenceStreamState, key: str) -> List[Any]:
pending_parts = state.choice_pending_parts.get(key)
if pending_parts is None and key == "0":
pending_parts = state.pending_parts
return list(pending_parts or [])
def _set_choice_pending_parts(
state: StopSequenceStreamState,
key: str,
value: List[Any],
) -> None:
if value:
state.choice_pending_parts[key] = list(value)
else:
state.choice_pending_parts.pop(key, None)
if key == "0":
state.pending_parts = list(value or [])
def _choice_index_value(key: Any) -> Any:
if isinstance(key, str) and key.isdigit():
try:
return int(key)
except ValueError:
return key
return key
def _mark_seen_choices(payload: Dict[str, Any], state: StopSequenceStreamState) -> None:
choices = payload.get("choices")
if not isinstance(choices, list):
return
for index, choice in enumerate(choices):
if isinstance(choice, dict):
state.seen_choice_keys.add(_choice_tail_key(choice, index))
def _all_seen_choices_stopped(state: StopSequenceStreamState) -> bool:
if state.single_choice or state.upstream_single_choice:
return bool(state.seen_choice_keys) and state.seen_choice_keys.issubset(state.stopped_choice_keys)
return len(state.seen_choice_keys) > 1 and state.seen_choice_keys.issubset(state.stopped_choice_keys)
def _finish_when_all_seen_choices_stopped(
chunks: List[str],
state: StopSequenceStreamState,
) -> List[str]:
if not state.stopped and _all_seen_choices_stopped(state):
state.stopped = True
chunks.append("data: [DONE]\n\n")
return chunks
def _replace_first_delta_content(payload: Dict[str, Any], content: str) -> Dict[str, Any]:
choices = payload.get("choices")
if not isinstance(choices, list) or not choices:
return payload
first = choices[0] if isinstance(choices[0], dict) else {}
delta = first.get("delta") if isinstance(first.get("delta"), dict) else {}
next_payload = dict(payload)
next_choices = list(choices)
next_first = dict(first)
next_delta = dict(delta)
next_delta["content"] = content
next_first["delta"] = next_delta
next_choices[0] = next_first
next_payload["choices"] = next_choices
return next_payload
def _replace_first_delta_content_value(payload: Dict[str, Any], content: Any) -> Dict[str, Any]:
choices = payload.get("choices")
if not isinstance(choices, list) or not choices:
return payload
first = choices[0] if isinstance(choices[0], dict) else {}
delta = first.get("delta") if isinstance(first.get("delta"), dict) else {}
next_payload = dict(payload)
next_choices = list(choices)
next_first = dict(first)
next_delta = dict(delta)
next_delta["content"] = content
next_first["delta"] = next_delta
next_choices[0] = next_first
next_payload["choices"] = next_choices
return next_payload
def _replace_first_finish_reason(payload: Dict[str, Any], finish_reason: Any) -> Dict[str, Any]:
choices = payload.get("choices")
if not isinstance(choices, list) or not choices:
return payload
first = choices[0] if isinstance(choices[0], dict) else {}
next_payload = dict(payload)
next_choices = list(choices)
next_first = dict(first)
next_first["finish_reason"] = finish_reason
next_choices[0] = next_first
next_payload["choices"] = next_choices
return next_payload
def _empty_choice_for_stopped_stream(choice: Dict[str, Any], fallback_index: int) -> Dict[str, Any]:
index = choice.get("index")
if index is None:
index = fallback_index
return {
"index": _choice_index_value(index),
"delta": {},
}
def _choice_has_stream_output(choice: Any) -> bool:
if not isinstance(choice, dict):
return False
if choice.get("finish_reason") is not None:
return True
if choice.get("media") is not None:
return True
delta = choice.get("delta") if isinstance(choice.get("delta"), dict) else {}
for key, value in delta.items():
if key != "content":
return True
if value not in ("", [], None):
return True
return False
def _sanitize_non_primary_choices_for_stop(
payload: Dict[str, Any],
state: StopSequenceStreamState,
) -> tuple[Dict[str, Any], set[str], bool]:
choices = payload.get("choices")
if not isinstance(choices, list) or len(choices) <= 1:
return payload, set(), False
next_choices = list(choices)
matched_choice_keys: set[str] = set()
changed = False
for index in range(1, len(next_choices)):
choice = next_choices[index]
if not isinstance(choice, dict):
continue
choice_key = _choice_tail_key(choice, index)
if choice_key in state.stopped_choice_keys:
next_choices[index] = _empty_choice_for_stopped_stream(choice, index)
changed = True
continue
delta = choice.get("delta") if isinstance(choice.get("delta"), dict) else {}
content = delta.get("content")
finish_reason = choice.get("finish_reason")
next_content = content
matched: Optional[str] = None
if isinstance(content, str) and content:
text_buffer = _get_choice_tail(state, choice_key) + content
if finish_reason:
match = find_first_stop_sequence(text_buffer, state.sequences)
if match is None:
next_content = text_buffer
_set_choice_tail(state, choice_key, "")
else:
position, matched = match
next_content = text_buffer[:position]
_set_choice_tail(state, choice_key, "")
else:
emit_text, next_tail, matched = split_stream_text_for_stop_sequences(
text_buffer,
state.sequences,
state.max_sequence_len,
)
next_content = emit_text
_set_choice_tail(state, choice_key, "" if matched is not None else next_tail)
if next_content != content or matched is not None:
changed = True
elif isinstance(content, list):
local_state = StopSequenceStreamState(
sequences=state.sequences,
tail=_get_choice_tail(state, choice_key),
pending_parts=_get_choice_pending_parts(state, choice_key),
)
next_content, matched = _filter_content_parts_for_stop(
content,
local_state,
flush_tail=bool(finish_reason),
)
_set_choice_tail(
state,
choice_key,
"" if matched is not None else local_state.tail,
)
_set_choice_pending_parts(
state,
choice_key,
[] if matched is not None else local_state.pending_parts,
)
if next_content != content or matched is not None:
changed = True
elif finish_reason and _get_choice_tail(state, choice_key):
next_content = _get_choice_tail(state, choice_key)
_set_choice_tail(state, choice_key, "")
changed = True
if matched is None and next_content is content:
continue
if (
matched is None
and (next_content == "" or next_content == [])
and not any(key != "content" for key in delta)
):
next_choices[index] = _empty_choice_for_stopped_stream(choice, index)
changed = True
continue
next_delta = dict(delta)
if next_content is not None:
next_delta["content"] = next_content
next_choice = dict(choice)
next_choice["delta"] = next_delta
if matched is not None:
next_choice["finish_reason"] = "stop"
state.stopped_choice_keys.add(choice_key)
_set_choice_tail(state, choice_key, "")
_set_choice_pending_parts(state, choice_key, [])
next_choices[index] = next_choice
if matched is not None:
matched_choice_keys.add(choice_key)
if not changed:
return payload, set(), any(
_choice_has_stream_output(choice)
for choice in next_choices[1:]
)
next_payload = dict(payload)
next_payload["choices"] = next_choices
has_output = any(
_choice_has_stream_output(choice)
for choice in next_choices[1:]
)
return next_payload, matched_choice_keys, has_output
def _payload_has_non_content_delta(payload: Dict[str, Any]) -> bool:
if payload.get("media") is not None:
return True
choices = payload.get("choices")
if not isinstance(choices, list) or not choices:
return False
first = choices[0] if isinstance(choices[0], dict) else {}
delta = first.get("delta") if isinstance(first.get("delta"), dict) else {}
return any(key != "content" for key in delta)
def _pack_choice_tail_chunk(content: str, model: str, choice_index: Any = 0) -> str:
payload = json.loads(SSEFormatter.pack_chunk(content, model=model).removeprefix("data: ").strip())
payload["choices"][0]["index"] = _choice_index_value(choice_index)
return _pack_sse_payload(payload)
def _pack_choice_content_parts_chunk(
content: List[Any],
model: str,
choice_index: Any = 0,
) -> str:
payload = json.loads(SSEFormatter.pack_chunk("", model=model).removeprefix("data: ").strip())
payload["choices"][0]["index"] = _choice_index_value(choice_index)
payload["choices"][0]["delta"]["content"] = content
return _pack_sse_payload(payload)
def _pack_finish_chunk(model: str, choice_index: Any = 0) -> str:
finish_chunk = SSEFormatter.pack_finish(model=model)
parts = finish_chunk.split("\n\n")
if not parts:
return finish_chunk
data_line = parts[0]
if not data_line.startswith("data: "):
return finish_chunk
try:
payload = json.loads(data_line[len("data: "):])
except json.JSONDecodeError:
return finish_chunk
choices = payload.get("choices")
if isinstance(choices, list) and choices and isinstance(choices[0], dict):
choices[0]["index"] = _choice_index_value(choice_index)
return f"{_pack_sse_payload(payload)}data: [DONE]\n\n"
def _pack_choice_finish_chunk(model: str, choice_index: Any = 0) -> str:
finish_chunk = _pack_finish_chunk(model=model, choice_index=choice_index)
frames = finish_chunk.replace("\r\n", "\n").replace("\r", "\n").split("\n\n")
kept_frames = [
f"{frame}\n\n"
for frame in frames
if frame and sse_frame_data_text(frame).strip() != "[DONE]"
]
return "".join(kept_frames)
def _pack_stop_finish_chunk(
state: StopSequenceStreamState,
model: str,
choice_index: Any = 0,
) -> str:
finish_chunk = _pack_choice_finish_chunk(model=model, choice_index=choice_index)
if (state.single_choice or state.upstream_single_choice) and str(choice_index) == "0":
return f"{finish_chunk}data: [DONE]\n\n"
return finish_chunk
def _flush_tail_chunk(
state: StopSequenceStreamState,
model: str,
choice_key: Optional[str] = None,
) -> List[str]:
if choice_key is None:
choice_keys = set(state.choice_tails.keys()) | set(state.choice_pending_parts.keys())
if choice_keys:
chunks: List[str] = []
for key in sorted(
choice_keys,
key=lambda value: (
0,
int(value),
) if str(value).isdigit() else (1, str(value)),
):
pending_parts = _get_choice_pending_parts(state, key)
if pending_parts:
chunks.append(_pack_choice_content_parts_chunk(pending_parts, model=model, choice_index=key))
_set_choice_pending_parts(state, key, [])
_set_choice_tail(state, key, "")
continue
tail = _get_choice_tail(state, key)
if not tail:
continue
chunks.append(_pack_choice_tail_chunk(tail, model=model, choice_index=key))
state.choice_tails.clear()
state.choice_pending_parts.clear()
state.tail = ""
state.pending_parts = []
return chunks
choice_key = "0"
pending_parts = _get_choice_pending_parts(state, choice_key)
if pending_parts:
_set_choice_pending_parts(state, choice_key, [])
_set_choice_tail(state, choice_key, "")
return [_pack_choice_content_parts_chunk(pending_parts, model=model, choice_index=choice_key)]
tail = _get_choice_tail(state, choice_key) or (state.tail if choice_key == "0" else "")
if not tail:
return []
_set_choice_tail(state, choice_key, "")
return [_pack_choice_tail_chunk(tail, model=model, choice_index=choice_key)]
def _filter_openai_stop_payload(
payload: Dict[str, Any],
state: StopSequenceStreamState,
model: str,
) -> List[str]:
if state.stopped:
return []
choices = payload.get("choices")
if not isinstance(choices, list) or not choices:
return [_pack_sse_payload(payload)]
_mark_seen_choices(payload, state)
payload, secondary_matched_keys, has_secondary_output = _sanitize_non_primary_choices_for_stop(payload, state)
choices = payload.get("choices")
def append_secondary_finish(chunks: List[str]) -> List[str]:
return _finish_when_all_seen_choices_stopped(chunks, state)
first = choices[0] if isinstance(choices[0], dict) else {}
first_key = _choice_tail_key(first, 0)
if first_key in state.stopped_choice_keys:
next_choices: List[Any] = []
if isinstance(first, dict):
next_choices.append(_empty_choice_for_stopped_stream(first, 0))
for index, choice in enumerate(choices[1:], start=1):
if not isinstance(choice, dict):
continue
choice_key = _choice_tail_key(choice, index)
if (
choice_key in state.stopped_choice_keys
and choice_key not in secondary_matched_keys
):
next_choices.append(_empty_choice_for_stopped_stream(choice, index))
else:
next_choices.append(choice)
if not any(_choice_has_stream_output(choice) for choice in next_choices):
return append_secondary_finish([])
next_payload = dict(payload)
next_payload["choices"] = next_choices
return append_secondary_finish([_pack_sse_payload(next_payload)])
delta = first.get("delta") if isinstance(first.get("delta"), dict) else {}
content = delta.get("content")
finish_reason = first.get("finish_reason")
def maybe_stop_single_choice_stream(choice_key: str) -> None:
if (state.single_choice or state.upstream_single_choice) and choice_key == "0":
state.stopped = True
if isinstance(content, str) and content:
text_buffer = _get_choice_tail(state, first_key) + content
if finish_reason:
match = find_first_stop_sequence(text_buffer, state.sequences)
if match is None:
_set_choice_tail(state, first_key, "")
return append_secondary_finish([
_pack_sse_payload(_replace_first_delta_content(payload, text_buffer))
])
position, _sequence = match
state.stopped_choice_keys.add(first_key)
_set_choice_tail(state, first_key, "")
_set_choice_pending_parts(state, first_key, [])
maybe_stop_single_choice_stream(first_key)
chunks = []
if position > 0 or _payload_has_non_content_delta(payload):
content_payload = _replace_first_delta_content(payload, text_buffer[:position])
chunks.append(_pack_sse_payload(_replace_first_finish_reason(content_payload, None)))
chunks.append(_pack_stop_finish_chunk(state, model=model, choice_index=first_key))
return chunks
emit_text, next_tail, matched = split_stream_text_for_stop_sequences(
text_buffer,
state.sequences,
state.max_sequence_len,
)
_set_choice_tail(state, first_key, "" if matched is not None else next_tail)
chunks: List[str] = []
if (
emit_text
or _payload_has_non_content_delta(payload)
or bool(secondary_matched_keys)
or has_secondary_output
):
chunks.append(_pack_sse_payload(_replace_first_delta_content(payload, emit_text)))
if matched is not None:
state.stopped_choice_keys.add(first_key)
_set_choice_tail(state, first_key, "")
_set_choice_pending_parts(state, first_key, [])
maybe_stop_single_choice_stream(first_key)
chunks.append(_pack_stop_finish_chunk(state, model=model, choice_index=first_key))
return append_secondary_finish(chunks)
if isinstance(content, list):
local_state = StopSequenceStreamState(
sequences=state.sequences,
max_sequence_len=state.max_sequence_len,
tail=_get_choice_tail(state, first_key),
pending_parts=_get_choice_pending_parts(state, first_key),
)
filtered_parts, matched = _filter_content_parts_for_stop(
content,
local_state,
flush_tail=bool(finish_reason),
)
_set_choice_tail(
state,
first_key,
"" if matched is not None else local_state.tail,
)
_set_choice_pending_parts(
state,
first_key,
[] if matched is not None else local_state.pending_parts,
)
chunks: List[str] = []
if (
filtered_parts
or _payload_has_non_content_delta(payload)
or finish_reason
or bool(secondary_matched_keys)
or has_secondary_output
):
content_payload = _replace_first_delta_content_value(payload, filtered_parts)
chunks.append(_pack_sse_payload(_replace_first_finish_reason(content_payload, None if matched else finish_reason)))
if matched is not None:
state.stopped_choice_keys.add(first_key)
_set_choice_tail(state, first_key, "")
_set_choice_pending_parts(state, first_key, [])
maybe_stop_single_choice_stream(first_key)
chunks.append(_pack_stop_finish_chunk(state, model=model, choice_index=first_key))
return append_secondary_finish(chunks)
chunks = []
if finish_reason:
chunks.extend(_flush_tail_chunk(state, model, choice_key=first_key))
chunks.append(_pack_sse_payload(payload))
return append_secondary_finish(chunks)
_CONTENT_TEXT_PART_TYPES = {"text", "input_text", "output_text"}
def _is_text_content_part(part: Any) -> bool:
if not isinstance(part, dict):
return False
part_type = str(part.get("type") or "").strip().lower()
return part_type in _CONTENT_TEXT_PART_TYPES
def _content_part_text(part: Any) -> str:
if not _is_text_content_part(part):
return ""
return str(part.get("text") or "")
def _content_parts_text(parts: List[Any]) -> str:
return "".join(_content_part_text(part) for part in parts)
def _split_content_parts_by_text_length(
parts: List[Any],
emit_text_len: int,
) -> tuple[List[Any], List[Any]]:
emitted: List[Any] = []
pending: List[Any] = []
emitted_text_len = 0
boundary_reached = False
for part in parts:
if not _is_text_content_part(part):
if boundary_reached:
pending.append(part)
else:
emitted.append(part)
continue
text = _content_part_text(part)
if boundary_reached or emitted_text_len >= emit_text_len:
boundary_reached = True
pending.append(part)
continue
remaining = emit_text_len - emitted_text_len
if len(text) <= remaining:
emitted.append(part)
emitted_text_len += len(text)
if emitted_text_len >= emit_text_len:
boundary_reached = True
continue
emitted_part = dict(part)
emitted_part["text"] = text[:remaining]
emitted.append(emitted_part)
pending_part = dict(part)
pending_part["text"] = text[remaining:]
pending.append(pending_part)
emitted_text_len = emit_text_len
boundary_reached = True
return emitted, pending
def _filter_content_parts_for_stop(
content: List[Any],
state: StopSequenceStreamState,
*,
flush_tail: bool = False,
) -> tuple[List[Any], Optional[str]]:
pending_prefix = list(state.pending_parts or [])
if not pending_prefix and state.tail:
pending_prefix = [{"type": "text", "text": state.tail}]
candidate_parts = pending_prefix + list(content)
text_buffer = _content_parts_text(candidate_parts)
if flush_tail:
match = find_first_stop_sequence(text_buffer, state.sequences)
if match is not None:
position, matched = match
filtered_parts, _pending_parts = _split_content_parts_by_text_length(
candidate_parts,
position,
)
state.tail = ""
state.pending_parts = []
return filtered_parts, matched
state.tail = ""
state.pending_parts = []
return candidate_parts, None
emit_text, next_tail, matched = split_stream_text_for_stop_sequences(
text_buffer,
state.sequences,
state.max_sequence_len,
)
if matched is not None:
filtered_parts, _pending_parts = _split_content_parts_by_text_length(
candidate_parts,
len(emit_text),
)
state.tail = ""
state.pending_parts = []
return filtered_parts, matched
filtered_parts, pending_parts = _split_content_parts_by_text_length(
candidate_parts,
len(emit_text),
)
state.tail = next_tail
state.pending_parts = pending_parts
return filtered_parts, None
def filter_openai_stop_sse_chunk(
chunk: Any,
state: StopSequenceStreamState,
model: str,
) -> List[str]:
if state.stopped:
return []
if not isinstance(chunk, str):
return [chunk]
if not state.sequences:
return _normalize_done_sse_frames(chunk, state, model)
combined = f"{state.sse_frame_buffer}{chunk}".replace("\r\n", "\n").replace("\r", "\n")
if "\n\n" not in combined:
# 修复7: 超限帧直接丢弃,不再保留截断片段
state.sse_frame_buffer = _cap_sse_frame_buffer(combined)
return []
frames = combined.split("\n\n")
tail = frames[-1]
# 修复7: 超限帧直接丢弃,不再保留截断片段
state.sse_frame_buffer = _cap_sse_frame_buffer(tail) if tail else ""
output: List[str] = []
for frame in frames[:-1]:
frame = frame.strip("\n")
if not frame:
continue
if state.stopped:
break
data_text = sse_frame_data_text(frame)
if frame.startswith(":") and not data_text:
output.append(f"{frame}\n\n")
continue
if not data_text:
output.append(f"{frame}\n\n")
continue
if data_text.strip() == "[DONE]":
output.extend(_flush_tail_chunk(state, model))
output.append("data: [DONE]\n\n")
continue
try:
payload = json.loads(data_text)
except json.JSONDecodeError:
output.append(f"{frame}\n\n")
continue
if isinstance(payload, dict):
output.extend(_filter_openai_stop_payload(payload, state, model))
else:
output.append(f"{frame}\n\n")
return output
def flush_openai_stop_state(
state: StopSequenceStreamState,
model: str,
) -> List[str]:
"""Emit any text held back for cross-chunk stop matching."""
if state.stopped:
return []
output: List[str] = []
if state.sse_frame_buffer:
buffered = state.sse_frame_buffer
state.sse_frame_buffer = ""
output.extend(filter_openai_stop_sse_chunk(f"{buffered}\n\n", state, model))
if state.stopped:
return output
output.extend(_flush_tail_chunk(state, model))
return output
def _normalize_done_sse_frames(
chunk: str,
state: StopSequenceStreamState,
model: str,
) -> List[str]:
combined = f"{state.sse_frame_buffer}{chunk}".replace("\r\n", "\n").replace("\r", "\n")
if "\n\n" not in combined:
# 修复7: 超限帧直接丢弃,不再保留截断片段
state.sse_frame_buffer = _cap_sse_frame_buffer(combined)
return []
frames = combined.split("\n\n")
tail = frames[-1]
# 修复7: 超限帧直接丢弃,不再保留截断片段
state.sse_frame_buffer = _cap_sse_frame_buffer(tail) if tail else ""
output: List[str] = []
for frame in frames[:-1]:
frame = frame.strip("\n")
if not frame:
continue
if sse_frame_data_text(frame).strip() == "[DONE]":
output.extend(_flush_tail_chunk(state, model))
output.append("data: [DONE]\n\n")
continue
output.append(f"{frame}\n\n")
return output
+127
View File
@@ -0,0 +1,127 @@
"""Machine-readable provider capabilities and runtime status endpoints."""
from __future__ import annotations
import asyncio
import time
from typing import Any, Dict, List
from fastapi import APIRouter, Depends
from app.api.deps import verify_service_auth
from app.core import get_browser
from app.core.config import get_logger
from app.services.request_manager import request_manager
logger = get_logger("API.PROVIDER")
router = APIRouter()
_API_PATHS = [
"/v1/chat/completions",
"/v1/responses",
"/v1/models",
"/v1/messages",
"/v1/messages/count_tokens",
]
def _model_entries() -> List[Dict[str, Any]]:
# Keep model discovery identical to /v1/models, including dynamic Arena models.
from app.api.chat import _collect_model_entries
return _collect_model_entries()
def _safe_tab_summary(tab: Any) -> Dict[str, Any]:
return {
"index": int(tab.get("persistent_index") or 0),
"id": str(tab.get("id") or ""),
"status": str(tab.get("status") or "unknown"),
"route_domain": str(tab.get("route_domain") or ""),
"preset_name": str(tab.get("preset_name") or ""),
"model": str(tab.get("model_name") or tab.get("exposed_model_name") or ""),
"isolated_profile": bool(tab.get("is_isolated_context")),
}
def _browser_snapshot() -> tuple[Dict[str, Any], Dict[str, Any], List[Dict[str, Any]]]:
try:
browser = get_browser(auto_connect=False)
health = browser.health_check()
pool = browser.get_pool_status()
tabs = pool.get("tabs") if isinstance(pool, dict) else []
safe_tabs = [_safe_tab_summary(tab) for tab in tabs if isinstance(tab, dict)]
return (
health if isinstance(health, dict) else {"connected": False},
pool if isinstance(pool, dict) else {"initialized": False},
safe_tabs,
)
except Exception as exc:
logger.debug(f"Provider status browser snapshot failed: {exc}")
return {"connected": False, "error": "browser_unavailable"}, {"initialized": False}, []
@router.get("/v1/provider/capabilities")
async def provider_capabilities(
authenticated: bool = Depends(verify_service_auth),
):
"""Describe the local bridge features clients can safely discover."""
models = await asyncio.to_thread(_model_entries)
health, pool, _tabs = await asyncio.to_thread(_browser_snapshot)
return {
"object": "provider.capabilities",
"provider": "universal-web-api",
"api_paths": _API_PATHS,
"protocols": {
"openai_chat_completions": True,
"openai_responses": True,
"anthropic_messages": True,
"anthropic_count_tokens": True,
},
"features": {
"streaming": True,
"tool_calling": True,
"multimodal_input": True,
"media_output": True,
"tab_pool": bool(pool.get("initialized", True)),
"new_session": False,
"reasoning_effort": False,
},
"models": models,
"readiness": {
"browser_connected": bool(health.get("connected")),
"idle_tabs": int(pool.get("idle") or 0),
"busy_tabs": int(pool.get("busy") or 0),
},
"generated_at": int(time.time()),
}
@router.get("/v1/provider/status")
async def provider_status(
authenticated: bool = Depends(verify_service_auth),
):
"""Return a secret-free runtime snapshot for clients and diagnostics."""
health, pool, tabs = await asyncio.to_thread(_browser_snapshot)
requests = request_manager.get_status()
return {
"object": "provider.status",
"provider": "universal-web-api",
"ready": bool(health.get("connected")) and int(pool.get("idle") or 0) > 0,
"browser": {
"connected": bool(health.get("connected")),
"status": str(health.get("status") or "unhealthy"),
},
"pool": {
"initialized": bool(pool.get("initialized", True)),
"total": int(pool.get("total") or 0),
"idle": int(pool.get("idle") or 0),
"busy": int(pool.get("busy") or 0),
"max_tabs": int(pool.get("max_tabs") or 0),
"allocation_mode": str(pool.get("allocation_mode") or ""),
"tabs": tabs,
},
"requests": requests,
"generated_at": int(time.time()),
}
+36
View File
@@ -0,0 +1,36 @@
"""
app/api/routes.py - API 路由聚合入口
职责:
- 聚合所有子路由模块
- 保持向后兼容(外部仍使用 from app.api.routes import router)
"""
from fastapi import APIRouter
# 导入子路由
from app.api.chat import router as chat_router
from app.api.anthropic_routes import router as anthropic_router
from app.api.config_routes import router as config_router
from app.api.system import router as system_router
from app.api.tab_routes import router as tab_router
from app.api.cmd_routes import router as cmd_router # 🆕 命令系统
from app.api.browser_routes import router as browser_router
from app.api.provider import router as provider_router
from app.api.adapter_routes import router as adapter_router # R1-3 站点适配器在线更新
from app.api.metrics_routes import router as metrics_router # R2-7 Prometheus 指标
# 创建主路由器
router = APIRouter()
# 聚合所有子路由
router.include_router(chat_router)
router.include_router(anthropic_router)
router.include_router(config_router)
router.include_router(system_router)
router.include_router(tab_router)
router.include_router(cmd_router) # 🆕
router.include_router(browser_router)
router.include_router(provider_router)
router.include_router(adapter_router)
router.include_router(metrics_router)
+47
View File
@@ -0,0 +1,47 @@
"""Close request generators promptly, including disconnects during ASGI send()."""
import anyio
from starlette.responses import StreamingResponse as StarletteStreamingResponse
from app.services.request_lifecycle import request_finalization
class RequestStreamingResponse(StarletteStreamingResponse):
def __init__(self, *args, request_context=None, **kwargs):
super().__init__(*args, **kwargs)
self.request_context = request_context
self._body_closed = False
async def _close_body(self):
if self._body_closed:
return
try:
close = getattr(self.body_iterator, 'aclose', None)
if close is not None:
with anyio.CancelScope(shield=True):
await close()
finally:
self._body_closed = True
# An unstarted async generator has no active finally block. Cover
# disconnects during headers/before its first iteration as well.
ctx = self.request_context
if ctx is not None:
if not ctx.is_terminal():
ctx.request_cancel('stream_closed')
with request_finalization(ctx):
pass
async def stream_response(self, send):
try:
await super().stream_response(send)
finally:
# A disconnect while sending a yielded chunk leaves the generator
# suspended. Cancelling the response task alone does not close it.
# Close in the iterator's original context; never wait for GC.
await self._close_body()
async def __call__(self, scope, receive, send):
try:
await super().__call__(scope, receive, send)
finally:
# Also cover cancellation before the stream task was scheduled.
await self._close_body()
+2662
View File
File diff suppressed because it is too large. Load diff
File diff suppressed because it is too large. Load diff