Initial commit
This commit is contained in:
commit
57fc3525eb
7311 files changed
+1291398
No files matched your search
@@ -0,0 +1,35 @@
|
||||
"""R2-6:进程模型(实验性,默认关闭)。
|
||||
|
||||
``UWAPI_WORKER_MODE=inproc``(默认):与以前完全一样,单进程持有浏览器。
|
||||
``UWAPI_WORKER_MODE=process``:
|
||||
- **worker 进程**(``UWAPI_WORKER_ROLE=worker``)运行完整应用,持有浏览器、标签页池与命令调度,
|
||||
只监听 127.0.0.1,另外提供带令牌鉴权的 ``/internal/worker/*`` 接口;
|
||||
- **API 进程**对外服务:协议接口(/v1、/url、/tab 等)在本进程完成协议转换、鉴权与计量,
|
||||
执行时经 ``RemoteBrowserProxy`` 交给 worker;所有 ``/api/*`` 面板请求整体转发给 worker;
|
||||
worker 由 API 进程启动、监控并在退出时关闭;意外退出后自动以指数退避重启。重启窗口内的在途请求仍可能失败,但 API 进程保持可用。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
|
||||
MODE_ENV = "UWAPI_WORKER_MODE"
|
||||
ROLE_ENV = "UWAPI_WORKER_ROLE"
|
||||
URL_ENV = "UWAPI_WORKER_URL"
|
||||
TOKEN_ENV = "UWAPI_WORKER_TOKEN"
|
||||
PORT_ENV = "UWAPI_WORKER_PORT"
|
||||
TOKEN_HEADER = "X-UWAPI-Worker-Token"
|
||||
|
||||
|
||||
def worker_mode() -> str:
|
||||
mode = str(os.getenv(MODE_ENV, "inproc") or "inproc").strip().lower()
|
||||
return mode if mode in ("inproc", "process") else "inproc"
|
||||
|
||||
|
||||
def is_worker_role() -> bool:
|
||||
return str(os.getenv(ROLE_ENV, "") or "").strip().lower() == "worker"
|
||||
|
||||
|
||||
def is_api_process() -> bool:
|
||||
"""process 模式下的对外 API 进程(不持有浏览器)。"""
|
||||
return worker_mode() == "process" and not is_worker_role()
|
||||
Binary file not shown.
Binary file not shown.
@@ -0,0 +1,142 @@
|
||||
"""R2-6:API 进程侧的远程浏览器代理(process 模式下由 get_browser() 返回)。
|
||||
|
||||
只覆盖协议接口执行所需的能力:五个 execute_workflow* 方法、标签页池中参数与返回值可 JSON 化的方法与配置属性、
|
||||
get_pool_status / health_check。其余能力由 worker 通过面板接口提供(API 进程会把 /api/* 整体转发给 worker),
|
||||
在 API 进程里访问时抛出 WorkerModeUnsupported,给出明确原因。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import threading
|
||||
from typing import Any, Dict, Iterator, Optional
|
||||
|
||||
import requests
|
||||
|
||||
from app.core.config import SSEFormatter, get_logger
|
||||
from app.worker import TOKEN_ENV, TOKEN_HEADER, URL_ENV
|
||||
from app.worker.protocol import BROWSER_METHODS, TAB_POOL_ATTRS, TAB_POOL_METHODS
|
||||
|
||||
logger = get_logger("WORKER_CLIENT")
|
||||
|
||||
|
||||
class WorkerUnavailable(RuntimeError):
|
||||
"""worker 未配置、未就绪或已退出。"""
|
||||
|
||||
|
||||
class WorkerModeUnsupported(RuntimeError):
|
||||
"""process 模式下 API 进程不持有浏览器,该能力由 worker 通过面板接口提供。"""
|
||||
|
||||
|
||||
def _endpoint(path: str) -> str:
|
||||
base = str(os.getenv(URL_ENV, "") or "").rstrip("/")
|
||||
if not base:
|
||||
raise WorkerUnavailable("process 模式下未配置 UWAPI_WORKER_URL(worker 未启动)")
|
||||
return base + path
|
||||
|
||||
|
||||
def _headers() -> Dict[str, str]:
|
||||
return {TOKEN_HEADER: str(os.getenv(TOKEN_ENV, "") or "")}
|
||||
|
||||
|
||||
def _rpc(path: str, payload: Optional[Dict[str, Any]] = None, timeout: float = 30.0) -> Any:
|
||||
try:
|
||||
if payload is None:
|
||||
response = requests.get(_endpoint(path), headers=_headers(), timeout=timeout)
|
||||
else:
|
||||
response = requests.post(_endpoint(path), json=payload, headers=_headers(), timeout=timeout)
|
||||
except requests.RequestException as exc:
|
||||
raise WorkerUnavailable(f"无法连接 worker:{exc}") from exc
|
||||
if response.status_code != 200:
|
||||
raise WorkerUnavailable(f"worker 返回 {response.status_code}:{response.text[:200]}")
|
||||
return response.json()
|
||||
|
||||
|
||||
class RemoteTabPool:
|
||||
def __getattr__(self, name: str) -> Any:
|
||||
if name in TAB_POOL_ATTRS:
|
||||
return _rpc("/internal/worker/tab_pool_attrs").get(name)
|
||||
if name in TAB_POOL_METHODS:
|
||||
return lambda *args, **kwargs: _rpc(
|
||||
"/internal/worker/tab_pool", {"method": name, "args": list(args), "kwargs": kwargs}
|
||||
)["result"]
|
||||
raise WorkerModeUnsupported(f"process 模式下无法在 API 进程中访问标签页池的 {name}(由 worker 提供)")
|
||||
|
||||
|
||||
class RemoteBrowserProxy:
|
||||
"""与 BrowserCore 同名的执行入口;执行结果以浏览器层产出的 SSE 文本块原样返回。"""
|
||||
|
||||
is_remote = True
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.tab_pool = RemoteTabPool()
|
||||
|
||||
def _execute(self, method: str, kwargs: Dict[str, Any]) -> Iterator[str]:
|
||||
stop_checker = kwargs.pop("stop_checker", None)
|
||||
payload = {"method": method, "kwargs": kwargs}
|
||||
try:
|
||||
with requests.post(_endpoint("/internal/worker/execute"), json=payload, headers=_headers(),
|
||||
stream=True, timeout=(5, 30)) as response:
|
||||
if response.status_code != 200:
|
||||
yield SSEFormatter.pack_error(f"worker 拒绝执行({response.status_code}):{response.text[:200]}")
|
||||
return
|
||||
for line in response.iter_lines():
|
||||
if stop_checker is not None and stop_checker():
|
||||
break # 关闭连接即通知 worker 停止
|
||||
if not line:
|
||||
continue
|
||||
item = json.loads(line)
|
||||
if "chunk" in item:
|
||||
yield item["chunk"]
|
||||
elif "error" in item:
|
||||
yield SSEFormatter.pack_error(f"worker 执行失败:{item['error']}")
|
||||
except (requests.RequestException, WorkerUnavailable, ValueError) as exc:
|
||||
logger.warning(f"worker 执行请求失败: {exc}")
|
||||
yield SSEFormatter.pack_error(f"worker 不可用:{exc}")
|
||||
|
||||
def execute_workflow(self, messages, **kwargs):
|
||||
return self._execute("execute_workflow", {"messages": messages, **kwargs})
|
||||
|
||||
def execute_workflow_for_tab_index(self, tab_index, messages, **kwargs):
|
||||
return self._execute("execute_workflow_for_tab_index", {"tab_index": tab_index, "messages": messages, **kwargs})
|
||||
|
||||
def execute_workflow_for_route_domain(self, route_domain, messages, **kwargs):
|
||||
return self._execute("execute_workflow_for_route_domain",
|
||||
{"route_domain": route_domain, "messages": messages, **kwargs})
|
||||
|
||||
def execute_workflow_for_route_group(self, group_id, messages, **kwargs):
|
||||
return self._execute("execute_workflow_for_route_group", {"group_id": group_id, "messages": messages, **kwargs})
|
||||
|
||||
def execute_workflow_for_exact_url(self, exact_url, messages, **kwargs):
|
||||
return self._execute("execute_workflow_for_exact_url", {"exact_url": exact_url, "messages": messages, **kwargs})
|
||||
|
||||
def get_pool_status(self) -> Any:
|
||||
return _rpc("/internal/worker/browser", {"method": "get_pool_status"})["result"]
|
||||
|
||||
def health_check(self) -> Dict[str, Any]:
|
||||
try:
|
||||
result = _rpc("/internal/worker/browser", {"method": "health_check"}, timeout=10.0)["result"]
|
||||
except WorkerUnavailable as exc:
|
||||
return {"connected": False, "error": str(exc), "worker": "unavailable"}
|
||||
return {**(result or {}), "worker": "process"}
|
||||
|
||||
def __getattr__(self, name: str) -> Any:
|
||||
if name in BROWSER_METHODS:
|
||||
return lambda: _rpc("/internal/worker/browser", {"method": name})["result"]
|
||||
raise WorkerModeUnsupported(
|
||||
f"process 模式下 API 进程不持有浏览器,无法访问 {name};该功能由 worker 提供(面板的 /api 请求会自动转发给 worker)"
|
||||
)
|
||||
|
||||
|
||||
_REMOTE: Optional[RemoteBrowserProxy] = None
|
||||
_REMOTE_LOCK = threading.Lock()
|
||||
|
||||
|
||||
def get_remote_browser() -> RemoteBrowserProxy:
|
||||
global _REMOTE
|
||||
if _REMOTE is None:
|
||||
with _REMOTE_LOCK:
|
||||
if _REMOTE is None:
|
||||
_REMOTE = RemoteBrowserProxy()
|
||||
return _REMOTE
|
||||
@@ -0,0 +1,76 @@
|
||||
"""R2-6:process 模式下 API 进程把 /api/* 面板请求整体转发给 worker(纯 ASGI,流式透传响应)。
|
||||
|
||||
worker 看到的连接都来自本机回环地址,为了不让外部请求被当成“本机直连”而绕过按来源判断的限制,
|
||||
转发时带上 UWAPI_PROXY_SECRET 对应的密钥头与 X-Forwarded-For:http_security 只在“回环来源 + 密钥匹配”时
|
||||
才采信转发头,worker 因而看到真实的客户端地址。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
|
||||
from app.worker import URL_ENV, is_api_process
|
||||
|
||||
PROXY_SECRET_ENV = "UWAPI_PROXY_SECRET"
|
||||
PROXY_SECRET_HEADER = b"x-uwa-proxy-secret"
|
||||
_HOP_BY_HOP = {b"host", b"content-length", b"connection", b"keep-alive", b"transfer-encoding", b"upgrade",
|
||||
b"x-forwarded-for", PROXY_SECRET_HEADER}
|
||||
_RESPONSE_SKIP = {"content-length", "transfer-encoding", "connection", "keep-alive"}
|
||||
|
||||
|
||||
class WorkerForwardingMiddleware:
|
||||
def __init__(self, app):
|
||||
self.app = app
|
||||
self._client = None
|
||||
|
||||
async def __call__(self, scope, receive, send):
|
||||
if scope.get("type") != "http" or not str(scope.get("path", "")).startswith("/api/") or not is_api_process():
|
||||
await self.app(scope, receive, send)
|
||||
return
|
||||
base = str(os.getenv(URL_ENV, "") or "").rstrip("/")
|
||||
if not base:
|
||||
await self._reply(send, 503, b'{"detail": "worker \\u672a\\u5c31\\u7eea"}')
|
||||
return
|
||||
import httpx
|
||||
|
||||
if self._client is None:
|
||||
self._client = httpx.AsyncClient(timeout=httpx.Timeout(10.0, read=None))
|
||||
body = bytearray()
|
||||
while True:
|
||||
message = await receive()
|
||||
body.extend(message.get("body", b""))
|
||||
if not message.get("more_body"):
|
||||
break
|
||||
headers = [(k, v) for k, v in scope.get("headers") or [] if k.lower() not in _HOP_BY_HOP]
|
||||
client_host = (scope.get("client") or ("", 0))[0]
|
||||
if client_host:
|
||||
headers.append((b"x-forwarded-for", client_host.encode("latin-1")))
|
||||
secret = str(os.getenv(PROXY_SECRET_ENV, "") or "")
|
||||
if secret:
|
||||
headers.append((PROXY_SECRET_HEADER, secret.encode("latin-1")))
|
||||
query = scope.get("query_string", b"").decode("latin-1")
|
||||
url = base + scope["path"] + (f"?{query}" if query else "")
|
||||
try:
|
||||
request = self._client.build_request(scope.get("method", "GET"), url, headers=headers, content=bytes(body))
|
||||
response = await self._client.send(request, stream=True)
|
||||
except httpx.HTTPError as exc:
|
||||
await self._reply(send, 503, f'{{"detail": "worker unavailable: {type(exc).__name__}"}}'.encode())
|
||||
return
|
||||
try:
|
||||
await send({
|
||||
"type": "http.response.start",
|
||||
"status": response.status_code,
|
||||
"headers": [(k.encode("latin-1"), v.encode("latin-1")) for k, v in response.headers.items()
|
||||
if k.lower() not in _RESPONSE_SKIP],
|
||||
})
|
||||
async for chunk in response.aiter_raw():
|
||||
await send({"type": "http.response.body", "body": chunk, "more_body": True})
|
||||
await send({"type": "http.response.body", "body": b"", "more_body": False})
|
||||
finally:
|
||||
await response.aclose()
|
||||
|
||||
@staticmethod
|
||||
async def _reply(send, status: int, body: bytes) -> None:
|
||||
await send({"type": "http.response.start", "status": status,
|
||||
"headers": [(b"content-type", b"application/json")]})
|
||||
await send({"type": "http.response.body", "body": body})
|
||||
@@ -0,0 +1,38 @@
|
||||
"""R2-6:API 进程与 worker 之间的协议约定。
|
||||
|
||||
执行接口的响应是 NDJSON:每行一个 JSON 对象——
|
||||
``{"chunk": "<浏览器层产出的一段 SSE 文本>"}``、``{"hb": 1}``(心跳,约每秒一次,
|
||||
让 API 侧在长时间没有数据时也能及时检查停止请求)、``{"error": "..."}``。
|
||||
停止信号通过断开连接传递:API 侧关闭响应,worker 检测到断开后置位 stop_checker。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
EXECUTE_METHODS = frozenset({
|
||||
"execute_workflow",
|
||||
"execute_workflow_for_tab_index",
|
||||
"execute_workflow_for_route_domain",
|
||||
"execute_workflow_for_route_group",
|
||||
"execute_workflow_for_exact_url",
|
||||
})
|
||||
|
||||
# 执行参数中允许跨进程传递的键(stop_checker 为回调,改由断开连接传递)
|
||||
EXECUTE_KWARGS = frozenset({
|
||||
"messages", "stream", "task_id", "preset_name", "workflow_priority", "allow_media_postprocess",
|
||||
"requested_model", "workflow_variables", "allocation_mode", "resolved_tab_index",
|
||||
"tab_index", "route_domain", "group_id", "exact_url",
|
||||
})
|
||||
|
||||
# 标签页池中参数与返回值均可 JSON 化、允许远程调用的方法
|
||||
TAB_POOL_METHODS = frozenset({
|
||||
"get_tabs_with_index", "set_tab_model_name", "apply_runtime_config", "terminate_by_index",
|
||||
"set_tab_preset", "get_tab_preset", "get_route_groups_snapshot", "get_status", "get_watchdog_summary",
|
||||
})
|
||||
|
||||
# 标签页池中允许远程读取的配置属性
|
||||
TAB_POOL_ATTRS = ("allocation_mode", "excluded_urls", "preserve_error_tabs", "auto_remember_url_presets", "route_groups")
|
||||
|
||||
# 浏览器对象上允许远程调用的方法
|
||||
BROWSER_METHODS = frozenset({"get_pool_status", "health_check"})
|
||||
|
||||
HEARTBEAT_SEC = 1.0
|
||||
@@ -0,0 +1,135 @@
|
||||
"""R2-6:worker 进程提供给 API 进程的内部接口(只在 UWAPI_WORKER_ROLE=worker 时注册)。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import hmac
|
||||
import json
|
||||
import os
|
||||
import threading
|
||||
from typing import Any, Dict
|
||||
|
||||
from fastapi import APIRouter, HTTPException, Request
|
||||
from fastapi.encoders import jsonable_encoder
|
||||
from fastapi.responses import StreamingResponse
|
||||
|
||||
from app.core.config import get_logger
|
||||
from app.worker import TOKEN_ENV, TOKEN_HEADER
|
||||
from app.worker.protocol import (
|
||||
BROWSER_METHODS,
|
||||
EXECUTE_KWARGS,
|
||||
EXECUTE_METHODS,
|
||||
HEARTBEAT_SEC,
|
||||
TAB_POOL_ATTRS,
|
||||
TAB_POOL_METHODS,
|
||||
)
|
||||
|
||||
logger = get_logger("WORKER")
|
||||
router = APIRouter()
|
||||
_LOOPBACK = {"127.0.0.1", "::1", "localhost"}
|
||||
|
||||
|
||||
def _authorize(request: Request) -> None:
|
||||
expected = str(os.getenv(TOKEN_ENV, "") or "")
|
||||
supplied = str(request.headers.get(TOKEN_HEADER, "") or "")
|
||||
client = getattr(request.client, "host", "") if request.client else ""
|
||||
if not expected or not hmac.compare_digest(expected, supplied) or client not in _LOOPBACK:
|
||||
raise HTTPException(status_code=403, detail="worker 内部接口仅限本机 API 进程访问")
|
||||
|
||||
|
||||
def _browser():
|
||||
from app.core.browser import get_browser
|
||||
|
||||
return get_browser(auto_connect=False)
|
||||
|
||||
|
||||
@router.get("/internal/worker/health")
|
||||
async def worker_health(request: Request) -> Dict[str, Any]:
|
||||
_authorize(request)
|
||||
return {"ok": True, "role": "worker", "pid": os.getpid()}
|
||||
|
||||
|
||||
@router.post("/internal/worker/execute")
|
||||
async def worker_execute(request: Request):
|
||||
_authorize(request)
|
||||
body = await request.json()
|
||||
method = str(body.get("method") or "")
|
||||
kwargs = body.get("kwargs") or {}
|
||||
if method not in EXECUTE_METHODS or not isinstance(kwargs, dict) or set(kwargs) - EXECUTE_KWARGS:
|
||||
raise HTTPException(status_code=400, detail="不支持的执行方法或参数")
|
||||
|
||||
stop = threading.Event()
|
||||
loop = asyncio.get_running_loop()
|
||||
queue: asyncio.Queue = asyncio.Queue()
|
||||
target = getattr(_browser(), method)
|
||||
|
||||
def pump() -> None:
|
||||
generator = None
|
||||
try:
|
||||
generator = target(**kwargs, stop_checker=stop.is_set)
|
||||
for chunk in generator:
|
||||
loop.call_soon_threadsafe(queue.put_nowait, ("chunk", chunk))
|
||||
if stop.is_set():
|
||||
break
|
||||
except Exception as exc: # broad-except: 执行错误原样作为 error 行传回 API 进程
|
||||
logger.warning(f"worker 执行失败: {exc}")
|
||||
loop.call_soon_threadsafe(queue.put_nowait, ("error", str(exc)))
|
||||
finally:
|
||||
if generator is not None:
|
||||
try:
|
||||
generator.close() # 在迭代它的同一线程里关闭,触发工作流的清理
|
||||
except Exception: # broad-except: 收尾操作
|
||||
pass
|
||||
loop.call_soon_threadsafe(queue.put_nowait, ("end", None))
|
||||
|
||||
async def stream():
|
||||
thread = threading.Thread(target=pump, name=f"worker-exec-{kwargs.get('task_id') or ''}", daemon=True)
|
||||
thread.start()
|
||||
try:
|
||||
while True:
|
||||
try:
|
||||
kind, value = await asyncio.wait_for(queue.get(), timeout=HEARTBEAT_SEC)
|
||||
except asyncio.TimeoutError:
|
||||
yield b'{"hb": 1}\n'
|
||||
continue
|
||||
if kind == "end":
|
||||
break
|
||||
line = {"chunk": value} if kind == "chunk" else {"error": value}
|
||||
yield (json.dumps(line, ensure_ascii=False) + "\n").encode("utf-8")
|
||||
finally:
|
||||
stop.set() # 正常结束或 API 进程断开连接:都通知工作流停止
|
||||
|
||||
return StreamingResponse(stream(), media_type="application/x-ndjson")
|
||||
|
||||
|
||||
@router.post("/internal/worker/tab_pool")
|
||||
async def worker_tab_pool(request: Request):
|
||||
_authorize(request)
|
||||
body = await request.json()
|
||||
method = str(body.get("method") or "")
|
||||
if method not in TAB_POOL_METHODS:
|
||||
raise HTTPException(status_code=400, detail=f"不允许远程调用的标签页池方法: {method}")
|
||||
pool = _browser().tab_pool
|
||||
result = await asyncio.to_thread(getattr(pool, method), *(body.get("args") or []), **(body.get("kwargs") or {}))
|
||||
return {"result": jsonable_encoder(result)}
|
||||
|
||||
|
||||
@router.get("/internal/worker/tab_pool_attrs")
|
||||
async def worker_tab_pool_attrs(request: Request):
|
||||
_authorize(request)
|
||||
pool = _browser().tab_pool
|
||||
attrs = {name: getattr(pool, name, None) for name in TAB_POOL_ATTRS if name != "route_groups"}
|
||||
getter = getattr(pool, "get_route_groups_snapshot", None)
|
||||
attrs["route_groups"] = getter() if callable(getter) else getattr(pool, "route_groups", None)
|
||||
return jsonable_encoder(attrs)
|
||||
|
||||
|
||||
@router.post("/internal/worker/browser")
|
||||
async def worker_browser(request: Request):
|
||||
_authorize(request)
|
||||
body = await request.json()
|
||||
method = str(body.get("method") or "")
|
||||
if method not in BROWSER_METHODS:
|
||||
raise HTTPException(status_code=400, detail=f"不允许远程调用的浏览器方法: {method}")
|
||||
result = await asyncio.to_thread(getattr(_browser(), method))
|
||||
return {"result": jsonable_encoder(result)}
|
||||
@@ -0,0 +1,184 @@
|
||||
"""R2-6:API 进程负责启动、监控与关闭 worker。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import secrets
|
||||
import socket
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
import requests
|
||||
|
||||
from app.core.config import get_logger
|
||||
from app.worker import MODE_ENV, PORT_ENV, ROLE_ENV, TOKEN_ENV, TOKEN_HEADER, URL_ENV
|
||||
from app.worker.forwarding import PROXY_SECRET_ENV
|
||||
|
||||
logger = get_logger("WORKER_SUP")
|
||||
PROJECT_ROOT = Path(__file__).resolve().parents[2]
|
||||
|
||||
|
||||
def _free_port() -> int:
|
||||
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
|
||||
sock.bind(("127.0.0.1", 0))
|
||||
return int(sock.getsockname()[1])
|
||||
|
||||
|
||||
class WorkerSupervisor:
|
||||
def __init__(self) -> None:
|
||||
self.process: Optional[subprocess.Popen] = None
|
||||
self.url: Optional[str] = None
|
||||
self._lifecycle_lock = threading.RLock()
|
||||
self._shutdown = threading.Event()
|
||||
self._monitor_thread: Optional[threading.Thread] = None
|
||||
|
||||
def start(self, ready_timeout: float = 90.0) -> str:
|
||||
"""启动 worker 并等待就绪,返回其地址;同时设置本进程的 UWAPI_WORKER_URL / TOKEN / 代理密钥。"""
|
||||
monitor = self._monitor_thread
|
||||
if monitor is None or not monitor.is_alive():
|
||||
self._shutdown.clear()
|
||||
|
||||
with self._lifecycle_lock:
|
||||
if self.process is not None and self.process.poll() is None and self.url:
|
||||
return self.url
|
||||
|
||||
port = int(os.getenv(PORT_ENV) or 0) or _free_port()
|
||||
token = secrets.token_urlsafe(32)
|
||||
proxy_secret = os.getenv(PROXY_SECRET_ENV) or secrets.token_urlsafe(32)
|
||||
env = dict(os.environ)
|
||||
env.update({
|
||||
MODE_ENV: "process", ROLE_ENV: "worker", TOKEN_ENV: token, PROXY_SECRET_ENV: proxy_secret,
|
||||
"APP_HOST": "127.0.0.1", "APP_PORT": str(port), "UWAPI_PUBLIC_BIND_HOST": "127.0.0.1",
|
||||
})
|
||||
env.pop(URL_ENV, None)
|
||||
cmd = [sys.executable, "-m", "uvicorn", "main:app", "--host", "127.0.0.1", "--port", str(port),
|
||||
"--log-level", "warning"]
|
||||
logger.info(f"启动浏览器 worker 进程(127.0.0.1:{port})")
|
||||
process = subprocess.Popen(cmd, cwd=str(PROJECT_ROOT), env=env)
|
||||
self.process = process
|
||||
url = f"http://127.0.0.1:{port}"
|
||||
deadline = time.monotonic() + ready_timeout
|
||||
while time.monotonic() < deadline:
|
||||
if self._shutdown.is_set():
|
||||
self._terminate_process(process, timeout=3)
|
||||
raise RuntimeError("worker 启动被取消(服务正在关闭)")
|
||||
if process.poll() is not None:
|
||||
raise RuntimeError(f"worker 进程启动失败(退出码 {process.returncode})")
|
||||
try:
|
||||
response = requests.get(url + "/internal/worker/health", headers={TOKEN_HEADER: token}, timeout=2)
|
||||
if response.status_code == 200:
|
||||
health = response.json()
|
||||
if isinstance(health, dict) and health.get("role") == "worker":
|
||||
break
|
||||
except (requests.RequestException, ValueError):
|
||||
pass
|
||||
time.sleep(0.5)
|
||||
else:
|
||||
self._terminate_process(process, timeout=3)
|
||||
raise RuntimeError(f"worker 进程 {ready_timeout:.0f} 秒内未就绪")
|
||||
os.environ[URL_ENV] = url
|
||||
os.environ[TOKEN_ENV] = token
|
||||
os.environ[PROXY_SECRET_ENV] = proxy_secret
|
||||
self.url = url
|
||||
worker_pid = health.get("pid")
|
||||
logger.info(f"浏览器 worker 已就绪:{url}(worker pid {worker_pid},启动器 pid {process.pid})")
|
||||
return url
|
||||
|
||||
def start_monitoring(
|
||||
self,
|
||||
check_interval: float = 1.0,
|
||||
ready_timeout: float = 90.0,
|
||||
initial_retry_delay: float = 1.0,
|
||||
max_retry_delay: float = 30.0,
|
||||
) -> None:
|
||||
"""监控子进程;意外退出时以指数退避重启,关闭 API 时停止监控。"""
|
||||
if check_interval <= 0 or initial_retry_delay <= 0 or max_retry_delay < initial_retry_delay:
|
||||
raise ValueError("worker 监控间隔与退避时长必须为正数,且最大退避不能小于初始值")
|
||||
with self._lifecycle_lock:
|
||||
if self.process is None:
|
||||
raise RuntimeError("worker 尚未启动,不能启动监控")
|
||||
if self._monitor_thread is not None and self._monitor_thread.is_alive():
|
||||
return
|
||||
self._shutdown.clear()
|
||||
self._monitor_thread = threading.Thread(
|
||||
target=self._monitor,
|
||||
args=(check_interval, ready_timeout, initial_retry_delay, max_retry_delay),
|
||||
name="worker-supervisor",
|
||||
daemon=True,
|
||||
)
|
||||
self._monitor_thread.start()
|
||||
|
||||
def _monitor(
|
||||
self,
|
||||
check_interval: float,
|
||||
ready_timeout: float,
|
||||
initial_retry_delay: float,
|
||||
max_retry_delay: float,
|
||||
) -> None:
|
||||
retry_delay = 0.0
|
||||
while not self._shutdown.wait(check_interval):
|
||||
process = self.process
|
||||
if process is None or process.poll() is None:
|
||||
retry_delay = 0.0
|
||||
continue
|
||||
|
||||
logger.error(f"浏览器 worker 意外退出(exit={process.returncode}),准备自动重启")
|
||||
while not self._shutdown.is_set():
|
||||
if retry_delay and self._shutdown.wait(retry_delay):
|
||||
return
|
||||
try:
|
||||
self.start(ready_timeout=ready_timeout)
|
||||
except (OSError, RuntimeError, ValueError, subprocess.SubprocessError) as exc:
|
||||
if self._shutdown.is_set():
|
||||
return
|
||||
logger.error(f"浏览器 worker 自动重启失败,将退避重试:{exc}")
|
||||
retry_delay = min(initial_retry_delay if not retry_delay else retry_delay * 2,
|
||||
max_retry_delay)
|
||||
else:
|
||||
logger.info("浏览器 worker 已由监控器自动重启")
|
||||
retry_delay = 0.0
|
||||
break
|
||||
|
||||
@staticmethod
|
||||
def _terminate_process(process: Optional[subprocess.Popen], timeout: float) -> None:
|
||||
if process is None or process.poll() is not None:
|
||||
return
|
||||
try:
|
||||
process.terminate()
|
||||
except OSError:
|
||||
if process.poll() is None:
|
||||
raise
|
||||
return
|
||||
try:
|
||||
process.wait(timeout=timeout)
|
||||
except subprocess.TimeoutExpired:
|
||||
if process.poll() is None:
|
||||
process.kill()
|
||||
process.wait(timeout=5)
|
||||
|
||||
def alive(self) -> bool:
|
||||
with self._lifecycle_lock:
|
||||
return self.process is not None and self.process.poll() is None
|
||||
|
||||
def stop(self, timeout: float = 15.0) -> None:
|
||||
self._shutdown.set()
|
||||
monitor = self._monitor_thread
|
||||
if monitor is not None and monitor is not threading.current_thread():
|
||||
monitor.join(timeout=5)
|
||||
with self._lifecycle_lock:
|
||||
process = self.process
|
||||
self._terminate_process(process, timeout=timeout)
|
||||
if self.process is process:
|
||||
self.process = None
|
||||
self.url = None
|
||||
if monitor is not None and not monitor.is_alive():
|
||||
self._monitor_thread = None
|
||||
if process is not None:
|
||||
logger.info("浏览器 worker 已停止")
|
||||
|
||||
|
||||
supervisor = WorkerSupervisor()
|
||||
Reference in new issue
Block a user