Initial commit

This commit is contained in:
Alex committed 2026-11-02 22:54:14 +08:00
commit 57fc3525eb
7311 files changed
+1291398

No files matched your search

+35
View File
@@ -0,0 +1,35 @@
"""R2-6:进程模型(实验性,默认关闭)。
``UWAPI_WORKER_MODE=inproc``(默认):与以前完全一样,单进程持有浏览器。
``UWAPI_WORKER_MODE=process``:
- **worker 进程**(``UWAPI_WORKER_ROLE=worker``)运行完整应用,持有浏览器、标签页池与命令调度,
只监听 127.0.0.1,另外提供带令牌鉴权的 ``/internal/worker/*`` 接口;
- **API 进程**对外服务:协议接口(/v1、/url、/tab 等)在本进程完成协议转换、鉴权与计量,
执行时经 ``RemoteBrowserProxy`` 交给 worker;所有 ``/api/*`` 面板请求整体转发给 worker;
worker 由 API 进程启动、监控并在退出时关闭;意外退出后自动以指数退避重启。重启窗口内的在途请求仍可能失败,但 API 进程保持可用。
"""
from __future__ import annotations
import os
MODE_ENV = "UWAPI_WORKER_MODE"
ROLE_ENV = "UWAPI_WORKER_ROLE"
URL_ENV = "UWAPI_WORKER_URL"
TOKEN_ENV = "UWAPI_WORKER_TOKEN"
PORT_ENV = "UWAPI_WORKER_PORT"
TOKEN_HEADER = "X-UWAPI-Worker-Token"
def worker_mode() -> str:
mode = str(os.getenv(MODE_ENV, "inproc") or "inproc").strip().lower()
return mode if mode in ("inproc", "process") else "inproc"
def is_worker_role() -> bool:
return str(os.getenv(ROLE_ENV, "") or "").strip().lower() == "worker"
def is_api_process() -> bool:
"""process 模式下的对外 API 进程(不持有浏览器)。"""
return worker_mode() == "process" and not is_worker_role()
Binary file not shown.
Binary file not shown.
+142
View File
@@ -0,0 +1,142 @@
"""R2-6:API 进程侧的远程浏览器代理(process 模式下由 get_browser() 返回)。
只覆盖协议接口执行所需的能力:五个 execute_workflow* 方法、标签页池中参数与返回值可 JSON 化的方法与配置属性、
get_pool_status / health_check。其余能力由 worker 通过面板接口提供(API 进程会把 /api/* 整体转发给 worker),
在 API 进程里访问时抛出 WorkerModeUnsupported,给出明确原因。
"""
from __future__ import annotations
import json
import os
import threading
from typing import Any, Dict, Iterator, Optional
import requests
from app.core.config import SSEFormatter, get_logger
from app.worker import TOKEN_ENV, TOKEN_HEADER, URL_ENV
from app.worker.protocol import BROWSER_METHODS, TAB_POOL_ATTRS, TAB_POOL_METHODS
logger = get_logger("WORKER_CLIENT")
class WorkerUnavailable(RuntimeError):
"""worker 未配置、未就绪或已退出。"""
class WorkerModeUnsupported(RuntimeError):
"""process 模式下 API 进程不持有浏览器,该能力由 worker 通过面板接口提供。"""
def _endpoint(path: str) -> str:
base = str(os.getenv(URL_ENV, "") or "").rstrip("/")
if not base:
raise WorkerUnavailable("process 模式下未配置 UWAPI_WORKER_URL(worker 未启动)")
return base + path
def _headers() -> Dict[str, str]:
return {TOKEN_HEADER: str(os.getenv(TOKEN_ENV, "") or "")}
def _rpc(path: str, payload: Optional[Dict[str, Any]] = None, timeout: float = 30.0) -> Any:
try:
if payload is None:
response = requests.get(_endpoint(path), headers=_headers(), timeout=timeout)
else:
response = requests.post(_endpoint(path), json=payload, headers=_headers(), timeout=timeout)
except requests.RequestException as exc:
raise WorkerUnavailable(f"无法连接 worker:{exc}") from exc
if response.status_code != 200:
raise WorkerUnavailable(f"worker 返回 {response.status_code}:{response.text[:200]}")
return response.json()
class RemoteTabPool:
def __getattr__(self, name: str) -> Any:
if name in TAB_POOL_ATTRS:
return _rpc("/internal/worker/tab_pool_attrs").get(name)
if name in TAB_POOL_METHODS:
return lambda *args, **kwargs: _rpc(
"/internal/worker/tab_pool", {"method": name, "args": list(args), "kwargs": kwargs}
)["result"]
raise WorkerModeUnsupported(f"process 模式下无法在 API 进程中访问标签页池的 {name}(由 worker 提供)")
class RemoteBrowserProxy:
"""与 BrowserCore 同名的执行入口;执行结果以浏览器层产出的 SSE 文本块原样返回。"""
is_remote = True
def __init__(self) -> None:
self.tab_pool = RemoteTabPool()
def _execute(self, method: str, kwargs: Dict[str, Any]) -> Iterator[str]:
stop_checker = kwargs.pop("stop_checker", None)
payload = {"method": method, "kwargs": kwargs}
try:
with requests.post(_endpoint("/internal/worker/execute"), json=payload, headers=_headers(),
stream=True, timeout=(5, 30)) as response:
if response.status_code != 200:
yield SSEFormatter.pack_error(f"worker 拒绝执行({response.status_code}):{response.text[:200]}")
return
for line in response.iter_lines():
if stop_checker is not None and stop_checker():
break # 关闭连接即通知 worker 停止
if not line:
continue
item = json.loads(line)
if "chunk" in item:
yield item["chunk"]
elif "error" in item:
yield SSEFormatter.pack_error(f"worker 执行失败:{item['error']}")
except (requests.RequestException, WorkerUnavailable, ValueError) as exc:
logger.warning(f"worker 执行请求失败: {exc}")
yield SSEFormatter.pack_error(f"worker 不可用:{exc}")
def execute_workflow(self, messages, **kwargs):
return self._execute("execute_workflow", {"messages": messages, **kwargs})
def execute_workflow_for_tab_index(self, tab_index, messages, **kwargs):
return self._execute("execute_workflow_for_tab_index", {"tab_index": tab_index, "messages": messages, **kwargs})
def execute_workflow_for_route_domain(self, route_domain, messages, **kwargs):
return self._execute("execute_workflow_for_route_domain",
{"route_domain": route_domain, "messages": messages, **kwargs})
def execute_workflow_for_route_group(self, group_id, messages, **kwargs):
return self._execute("execute_workflow_for_route_group", {"group_id": group_id, "messages": messages, **kwargs})
def execute_workflow_for_exact_url(self, exact_url, messages, **kwargs):
return self._execute("execute_workflow_for_exact_url", {"exact_url": exact_url, "messages": messages, **kwargs})
def get_pool_status(self) -> Any:
return _rpc("/internal/worker/browser", {"method": "get_pool_status"})["result"]
def health_check(self) -> Dict[str, Any]:
try:
result = _rpc("/internal/worker/browser", {"method": "health_check"}, timeout=10.0)["result"]
except WorkerUnavailable as exc:
return {"connected": False, "error": str(exc), "worker": "unavailable"}
return {**(result or {}), "worker": "process"}
def __getattr__(self, name: str) -> Any:
if name in BROWSER_METHODS:
return lambda: _rpc("/internal/worker/browser", {"method": name})["result"]
raise WorkerModeUnsupported(
f"process 模式下 API 进程不持有浏览器,无法访问 {name};该功能由 worker 提供(面板的 /api 请求会自动转发给 worker)"
)
_REMOTE: Optional[RemoteBrowserProxy] = None
_REMOTE_LOCK = threading.Lock()
def get_remote_browser() -> RemoteBrowserProxy:
global _REMOTE
if _REMOTE is None:
with _REMOTE_LOCK:
if _REMOTE is None:
_REMOTE = RemoteBrowserProxy()
return _REMOTE
+76
View File
@@ -0,0 +1,76 @@
"""R2-6:process 模式下 API 进程把 /api/* 面板请求整体转发给 worker(纯 ASGI,流式透传响应)。
worker 看到的连接都来自本机回环地址,为了不让外部请求被当成“本机直连”而绕过按来源判断的限制,
转发时带上 UWAPI_PROXY_SECRET 对应的密钥头与 X-Forwarded-For:http_security 只在“回环来源 + 密钥匹配”时
才采信转发头,worker 因而看到真实的客户端地址。
"""
from __future__ import annotations
import os
from app.worker import URL_ENV, is_api_process
PROXY_SECRET_ENV = "UWAPI_PROXY_SECRET"
PROXY_SECRET_HEADER = b"x-uwa-proxy-secret"
_HOP_BY_HOP = {b"host", b"content-length", b"connection", b"keep-alive", b"transfer-encoding", b"upgrade",
b"x-forwarded-for", PROXY_SECRET_HEADER}
_RESPONSE_SKIP = {"content-length", "transfer-encoding", "connection", "keep-alive"}
class WorkerForwardingMiddleware:
def __init__(self, app):
self.app = app
self._client = None
async def __call__(self, scope, receive, send):
if scope.get("type") != "http" or not str(scope.get("path", "")).startswith("/api/") or not is_api_process():
await self.app(scope, receive, send)
return
base = str(os.getenv(URL_ENV, "") or "").rstrip("/")
if not base:
await self._reply(send, 503, b'{"detail": "worker \\u672a\\u5c31\\u7eea"}')
return
import httpx
if self._client is None:
self._client = httpx.AsyncClient(timeout=httpx.Timeout(10.0, read=None))
body = bytearray()
while True:
message = await receive()
body.extend(message.get("body", b""))
if not message.get("more_body"):
break
headers = [(k, v) for k, v in scope.get("headers") or [] if k.lower() not in _HOP_BY_HOP]
client_host = (scope.get("client") or ("", 0))[0]
if client_host:
headers.append((b"x-forwarded-for", client_host.encode("latin-1")))
secret = str(os.getenv(PROXY_SECRET_ENV, "") or "")
if secret:
headers.append((PROXY_SECRET_HEADER, secret.encode("latin-1")))
query = scope.get("query_string", b"").decode("latin-1")
url = base + scope["path"] + (f"?{query}" if query else "")
try:
request = self._client.build_request(scope.get("method", "GET"), url, headers=headers, content=bytes(body))
response = await self._client.send(request, stream=True)
except httpx.HTTPError as exc:
await self._reply(send, 503, f'{{"detail": "worker unavailable: {type(exc).__name__}"}}'.encode())
return
try:
await send({
"type": "http.response.start",
"status": response.status_code,
"headers": [(k.encode("latin-1"), v.encode("latin-1")) for k, v in response.headers.items()
if k.lower() not in _RESPONSE_SKIP],
})
async for chunk in response.aiter_raw():
await send({"type": "http.response.body", "body": chunk, "more_body": True})
await send({"type": "http.response.body", "body": b"", "more_body": False})
finally:
await response.aclose()
@staticmethod
async def _reply(send, status: int, body: bytes) -> None:
await send({"type": "http.response.start", "status": status,
"headers": [(b"content-type", b"application/json")]})
await send({"type": "http.response.body", "body": body})
+38
View File
@@ -0,0 +1,38 @@
"""R2-6:API 进程与 worker 之间的协议约定。
执行接口的响应是 NDJSON:每行一个 JSON 对象——
``{"chunk": "<浏览器层产出的一段 SSE 文本>"}``、``{"hb": 1}``(心跳,约每秒一次,
让 API 侧在长时间没有数据时也能及时检查停止请求)、``{"error": "..."}``。
停止信号通过断开连接传递:API 侧关闭响应,worker 检测到断开后置位 stop_checker。
"""
from __future__ import annotations
EXECUTE_METHODS = frozenset({
"execute_workflow",
"execute_workflow_for_tab_index",
"execute_workflow_for_route_domain",
"execute_workflow_for_route_group",
"execute_workflow_for_exact_url",
})
# 执行参数中允许跨进程传递的键(stop_checker 为回调,改由断开连接传递)
EXECUTE_KWARGS = frozenset({
"messages", "stream", "task_id", "preset_name", "workflow_priority", "allow_media_postprocess",
"requested_model", "workflow_variables", "allocation_mode", "resolved_tab_index",
"tab_index", "route_domain", "group_id", "exact_url",
})
# 标签页池中参数与返回值均可 JSON 化、允许远程调用的方法
TAB_POOL_METHODS = frozenset({
"get_tabs_with_index", "set_tab_model_name", "apply_runtime_config", "terminate_by_index",
"set_tab_preset", "get_tab_preset", "get_route_groups_snapshot", "get_status", "get_watchdog_summary",
})
# 标签页池中允许远程读取的配置属性
TAB_POOL_ATTRS = ("allocation_mode", "excluded_urls", "preserve_error_tabs", "auto_remember_url_presets", "route_groups")
# 浏览器对象上允许远程调用的方法
BROWSER_METHODS = frozenset({"get_pool_status", "health_check"})
HEARTBEAT_SEC = 1.0
+135
View File
@@ -0,0 +1,135 @@
"""R2-6:worker 进程提供给 API 进程的内部接口(只在 UWAPI_WORKER_ROLE=worker 时注册)。"""
from __future__ import annotations
import asyncio
import hmac
import json
import os
import threading
from typing import Any, Dict
from fastapi import APIRouter, HTTPException, Request
from fastapi.encoders import jsonable_encoder
from fastapi.responses import StreamingResponse
from app.core.config import get_logger
from app.worker import TOKEN_ENV, TOKEN_HEADER
from app.worker.protocol import (
BROWSER_METHODS,
EXECUTE_KWARGS,
EXECUTE_METHODS,
HEARTBEAT_SEC,
TAB_POOL_ATTRS,
TAB_POOL_METHODS,
)
logger = get_logger("WORKER")
router = APIRouter()
_LOOPBACK = {"127.0.0.1", "::1", "localhost"}
def _authorize(request: Request) -> None:
expected = str(os.getenv(TOKEN_ENV, "") or "")
supplied = str(request.headers.get(TOKEN_HEADER, "") or "")
client = getattr(request.client, "host", "") if request.client else ""
if not expected or not hmac.compare_digest(expected, supplied) or client not in _LOOPBACK:
raise HTTPException(status_code=403, detail="worker 内部接口仅限本机 API 进程访问")
def _browser():
from app.core.browser import get_browser
return get_browser(auto_connect=False)
@router.get("/internal/worker/health")
async def worker_health(request: Request) -> Dict[str, Any]:
_authorize(request)
return {"ok": True, "role": "worker", "pid": os.getpid()}
@router.post("/internal/worker/execute")
async def worker_execute(request: Request):
_authorize(request)
body = await request.json()
method = str(body.get("method") or "")
kwargs = body.get("kwargs") or {}
if method not in EXECUTE_METHODS or not isinstance(kwargs, dict) or set(kwargs) - EXECUTE_KWARGS:
raise HTTPException(status_code=400, detail="不支持的执行方法或参数")
stop = threading.Event()
loop = asyncio.get_running_loop()
queue: asyncio.Queue = asyncio.Queue()
target = getattr(_browser(), method)
def pump() -> None:
generator = None
try:
generator = target(**kwargs, stop_checker=stop.is_set)
for chunk in generator:
loop.call_soon_threadsafe(queue.put_nowait, ("chunk", chunk))
if stop.is_set():
break
except Exception as exc: # broad-except: 执行错误原样作为 error 行传回 API 进程
logger.warning(f"worker 执行失败: {exc}")
loop.call_soon_threadsafe(queue.put_nowait, ("error", str(exc)))
finally:
if generator is not None:
try:
generator.close() # 在迭代它的同一线程里关闭,触发工作流的清理
except Exception: # broad-except: 收尾操作
pass
loop.call_soon_threadsafe(queue.put_nowait, ("end", None))
async def stream():
thread = threading.Thread(target=pump, name=f"worker-exec-{kwargs.get('task_id') or ''}", daemon=True)
thread.start()
try:
while True:
try:
kind, value = await asyncio.wait_for(queue.get(), timeout=HEARTBEAT_SEC)
except asyncio.TimeoutError:
yield b'{"hb": 1}\n'
continue
if kind == "end":
break
line = {"chunk": value} if kind == "chunk" else {"error": value}
yield (json.dumps(line, ensure_ascii=False) + "\n").encode("utf-8")
finally:
stop.set() # 正常结束或 API 进程断开连接:都通知工作流停止
return StreamingResponse(stream(), media_type="application/x-ndjson")
@router.post("/internal/worker/tab_pool")
async def worker_tab_pool(request: Request):
_authorize(request)
body = await request.json()
method = str(body.get("method") or "")
if method not in TAB_POOL_METHODS:
raise HTTPException(status_code=400, detail=f"不允许远程调用的标签页池方法: {method}")
pool = _browser().tab_pool
result = await asyncio.to_thread(getattr(pool, method), *(body.get("args") or []), **(body.get("kwargs") or {}))
return {"result": jsonable_encoder(result)}
@router.get("/internal/worker/tab_pool_attrs")
async def worker_tab_pool_attrs(request: Request):
_authorize(request)
pool = _browser().tab_pool
attrs = {name: getattr(pool, name, None) for name in TAB_POOL_ATTRS if name != "route_groups"}
getter = getattr(pool, "get_route_groups_snapshot", None)
attrs["route_groups"] = getter() if callable(getter) else getattr(pool, "route_groups", None)
return jsonable_encoder(attrs)
@router.post("/internal/worker/browser")
async def worker_browser(request: Request):
_authorize(request)
body = await request.json()
method = str(body.get("method") or "")
if method not in BROWSER_METHODS:
raise HTTPException(status_code=400, detail=f"不允许远程调用的浏览器方法: {method}")
result = await asyncio.to_thread(getattr(_browser(), method))
return {"result": jsonable_encoder(result)}
+184
View File
@@ -0,0 +1,184 @@
"""R2-6:API 进程负责启动、监控与关闭 worker。"""
from __future__ import annotations
import os
import secrets
import socket
import subprocess
import sys
import threading
import time
from pathlib import Path
from typing import Optional
import requests
from app.core.config import get_logger
from app.worker import MODE_ENV, PORT_ENV, ROLE_ENV, TOKEN_ENV, TOKEN_HEADER, URL_ENV
from app.worker.forwarding import PROXY_SECRET_ENV
logger = get_logger("WORKER_SUP")
PROJECT_ROOT = Path(__file__).resolve().parents[2]
def _free_port() -> int:
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
sock.bind(("127.0.0.1", 0))
return int(sock.getsockname()[1])
class WorkerSupervisor:
def __init__(self) -> None:
self.process: Optional[subprocess.Popen] = None
self.url: Optional[str] = None
self._lifecycle_lock = threading.RLock()
self._shutdown = threading.Event()
self._monitor_thread: Optional[threading.Thread] = None
def start(self, ready_timeout: float = 90.0) -> str:
"""启动 worker 并等待就绪,返回其地址;同时设置本进程的 UWAPI_WORKER_URL / TOKEN / 代理密钥。"""
monitor = self._monitor_thread
if monitor is None or not monitor.is_alive():
self._shutdown.clear()
with self._lifecycle_lock:
if self.process is not None and self.process.poll() is None and self.url:
return self.url
port = int(os.getenv(PORT_ENV) or 0) or _free_port()
token = secrets.token_urlsafe(32)
proxy_secret = os.getenv(PROXY_SECRET_ENV) or secrets.token_urlsafe(32)
env = dict(os.environ)
env.update({
MODE_ENV: "process", ROLE_ENV: "worker", TOKEN_ENV: token, PROXY_SECRET_ENV: proxy_secret,
"APP_HOST": "127.0.0.1", "APP_PORT": str(port), "UWAPI_PUBLIC_BIND_HOST": "127.0.0.1",
})
env.pop(URL_ENV, None)
cmd = [sys.executable, "-m", "uvicorn", "main:app", "--host", "127.0.0.1", "--port", str(port),
"--log-level", "warning"]
logger.info(f"启动浏览器 worker 进程(127.0.0.1:{port})")
process = subprocess.Popen(cmd, cwd=str(PROJECT_ROOT), env=env)
self.process = process
url = f"http://127.0.0.1:{port}"
deadline = time.monotonic() + ready_timeout
while time.monotonic() < deadline:
if self._shutdown.is_set():
self._terminate_process(process, timeout=3)
raise RuntimeError("worker 启动被取消(服务正在关闭)")
if process.poll() is not None:
raise RuntimeError(f"worker 进程启动失败(退出码 {process.returncode})")
try:
response = requests.get(url + "/internal/worker/health", headers={TOKEN_HEADER: token}, timeout=2)
if response.status_code == 200:
health = response.json()
if isinstance(health, dict) and health.get("role") == "worker":
break
except (requests.RequestException, ValueError):
pass
time.sleep(0.5)
else:
self._terminate_process(process, timeout=3)
raise RuntimeError(f"worker 进程 {ready_timeout:.0f} 秒内未就绪")
os.environ[URL_ENV] = url
os.environ[TOKEN_ENV] = token
os.environ[PROXY_SECRET_ENV] = proxy_secret
self.url = url
worker_pid = health.get("pid")
logger.info(f"浏览器 worker 已就绪:{url}(worker pid {worker_pid},启动器 pid {process.pid})")
return url
def start_monitoring(
self,
check_interval: float = 1.0,
ready_timeout: float = 90.0,
initial_retry_delay: float = 1.0,
max_retry_delay: float = 30.0,
) -> None:
"""监控子进程;意外退出时以指数退避重启,关闭 API 时停止监控。"""
if check_interval <= 0 or initial_retry_delay <= 0 or max_retry_delay < initial_retry_delay:
raise ValueError("worker 监控间隔与退避时长必须为正数,且最大退避不能小于初始值")
with self._lifecycle_lock:
if self.process is None:
raise RuntimeError("worker 尚未启动,不能启动监控")
if self._monitor_thread is not None and self._monitor_thread.is_alive():
return
self._shutdown.clear()
self._monitor_thread = threading.Thread(
target=self._monitor,
args=(check_interval, ready_timeout, initial_retry_delay, max_retry_delay),
name="worker-supervisor",
daemon=True,
)
self._monitor_thread.start()
def _monitor(
self,
check_interval: float,
ready_timeout: float,
initial_retry_delay: float,
max_retry_delay: float,
) -> None:
retry_delay = 0.0
while not self._shutdown.wait(check_interval):
process = self.process
if process is None or process.poll() is None:
retry_delay = 0.0
continue
logger.error(f"浏览器 worker 意外退出(exit={process.returncode}),准备自动重启")
while not self._shutdown.is_set():
if retry_delay and self._shutdown.wait(retry_delay):
return
try:
self.start(ready_timeout=ready_timeout)
except (OSError, RuntimeError, ValueError, subprocess.SubprocessError) as exc:
if self._shutdown.is_set():
return
logger.error(f"浏览器 worker 自动重启失败,将退避重试:{exc}")
retry_delay = min(initial_retry_delay if not retry_delay else retry_delay * 2,
max_retry_delay)
else:
logger.info("浏览器 worker 已由监控器自动重启")
retry_delay = 0.0
break
@staticmethod
def _terminate_process(process: Optional[subprocess.Popen], timeout: float) -> None:
if process is None or process.poll() is not None:
return
try:
process.terminate()
except OSError:
if process.poll() is None:
raise
return
try:
process.wait(timeout=timeout)
except subprocess.TimeoutExpired:
if process.poll() is None:
process.kill()
process.wait(timeout=5)
def alive(self) -> bool:
with self._lifecycle_lock:
return self.process is not None and self.process.poll() is None
def stop(self, timeout: float = 15.0) -> None:
self._shutdown.set()
monitor = self._monitor_thread
if monitor is not None and monitor is not threading.current_thread():
monitor.join(timeout=5)
with self._lifecycle_lock:
process = self.process
self._terminate_process(process, timeout=timeout)
if self.process is process:
self.process = None
self.url = None
if monitor is not None and not monitor.is_alive():
self._monitor_thread = None
if process is not None:
logger.info("浏览器 worker 已停止")
supervisor = WorkerSupervisor()