560 lines
19 KiB
Python
560 lines
19 KiB
Python
"""
|
|
Helpers for request execution lifetime, worker cleanup, and tab retirement.
|
|
"""
|
|
|
|
import asyncio
|
|
import anyio
|
|
from contextlib import contextmanager
|
|
import os
|
|
import queue
|
|
import threading
|
|
import time
|
|
from typing import Any, Callable, Dict, Optional
|
|
|
|
from app.core.config import BrowserConstants, _request_context, get_logger
|
|
from app.services.request_manager import RequestContext, RequestStatus, request_manager
|
|
|
|
|
|
logger = get_logger("REQUEST.LIFECYCLE")
|
|
|
|
DEFAULT_MAX_REQUEST_EXECUTE_TIME_SEC = 300.0
|
|
MIN_MAX_REQUEST_EXECUTE_TIME_SEC = 5.0
|
|
WORKER_QUEUE_FINAL_GRACE_SEC = 5.0
|
|
WORKER_QUEUE_MAX_PUT_BLOCK_SEC = 0.1
|
|
WORKER_QUEUE_CANCEL_PUT_BLOCK_SEC = 0.01
|
|
WORKER_QUEUE_WAIT_BACKOFF_FACTOR = 1.8
|
|
COMPLETED_WORKER_FAST_JOIN_SEC = 0.5
|
|
COMPLETED_WORKER_RETIRE_GRACE_SEC = 30.0
|
|
|
|
|
|
@contextmanager
|
|
def request_finalization(ctx: RequestContext, context_token=None):
|
|
"""Shield only bounded cleanup, never browser execution, from ASGI cancellation.
|
|
|
|
Synchronous finalization is guaranteed even if cleanup raises or a second raw
|
|
asyncio Task.cancel() interrupts it. A request is not retried here.
|
|
"""
|
|
try:
|
|
with anyio.CancelScope(shield=True):
|
|
yield
|
|
finally:
|
|
try:
|
|
with ctx._lock:
|
|
already_finalized = getattr(ctx, '_lifecycle_finalized', False)
|
|
ctx._lifecycle_finalized = True
|
|
if not already_finalized:
|
|
request_manager.finish_request(ctx, success=(ctx.status == RequestStatus.COMPLETED))
|
|
finally:
|
|
if context_token is not None:
|
|
try:
|
|
_request_context.reset(context_token)
|
|
except (ValueError, RuntimeError):
|
|
# Explicit close usually runs in the original response context;
|
|
# generator finalization during shutdown may use another one.
|
|
logger.debug("Request context already released or finalized in another context")
|
|
|
|
|
|
class TrackedWorkerExecutionCancelled(Exception):
|
|
"""Raised when a tracked blocking worker keeps running after cancellation."""
|
|
|
|
|
|
def _coerce_max_request_execute_time_sec(raw: Any, default: float, source: str) -> float:
|
|
try:
|
|
value = float(raw)
|
|
except (TypeError, ValueError, OverflowError):
|
|
logger.debug(f"Invalid {source} MAX_REQUEST_EXECUTE_TIME_SEC={raw!r}, using default {default}s")
|
|
return max(MIN_MAX_REQUEST_EXECUTE_TIME_SEC, float(default))
|
|
|
|
if value <= 0:
|
|
return 0.0
|
|
return max(MIN_MAX_REQUEST_EXECUTE_TIME_SEC, value)
|
|
|
|
|
|
def get_max_request_execute_time_sec(default: float = DEFAULT_MAX_REQUEST_EXECUTE_TIME_SEC) -> float:
|
|
"""Return the request hard timeout; set MAX_REQUEST_EXECUTE_TIME_SEC<=0 to disable."""
|
|
env_raw = os.getenv("MAX_REQUEST_EXECUTE_TIME_SEC")
|
|
if env_raw is not None and str(env_raw).strip() != "":
|
|
return _coerce_max_request_execute_time_sec(env_raw, default, "env")
|
|
|
|
try:
|
|
config_raw = BrowserConstants.get("MAX_REQUEST_EXECUTE_TIME_SEC")
|
|
except Exception as e:
|
|
logger.debug(f"Read browser config MAX_REQUEST_EXECUTE_TIME_SEC failed: {e}")
|
|
config_raw = default
|
|
|
|
if config_raw is None or str(config_raw).strip() == "":
|
|
config_raw = default
|
|
return _coerce_max_request_execute_time_sec(config_raw, default, "browser_config")
|
|
|
|
|
|
def mark_request_hard_timeout(
|
|
ctx: RequestContext,
|
|
started_at: float,
|
|
max_execute_time_sec: Optional[float],
|
|
*,
|
|
label: str = "",
|
|
) -> bool:
|
|
"""Cancel ctx when an application-level hard timeout is reached."""
|
|
try:
|
|
max_seconds = (
|
|
get_max_request_execute_time_sec()
|
|
if max_execute_time_sec is None
|
|
else float(max_execute_time_sec)
|
|
)
|
|
except Exception:
|
|
max_seconds = DEFAULT_MAX_REQUEST_EXECUTE_TIME_SEC
|
|
|
|
if max_seconds <= 0:
|
|
return False
|
|
ctx_started_at = getattr(ctx, "started_at_monotonic", None)
|
|
if ctx_started_at is None:
|
|
ctx_started_at = float(started_at or time.monotonic())
|
|
elapsed = time.monotonic() - ctx_started_at
|
|
if elapsed < max_seconds:
|
|
return False
|
|
|
|
if not ctx.should_stop():
|
|
logger.warning(
|
|
f"[{ctx.request_id}] request hard timeout reached "
|
|
f"(elapsed={elapsed:.1f}s, limit={max_seconds:.1f}s, {label or '-'})"
|
|
)
|
|
ctx.request_cancel("absolute_request_timeout")
|
|
return True
|
|
|
|
|
|
def put_worker_queue_item(
|
|
chunk_queue: queue.Queue,
|
|
ctx: RequestContext,
|
|
item: Any,
|
|
*,
|
|
final: bool = False,
|
|
poll_timeout: float = 0.5,
|
|
) -> bool:
|
|
"""Put worker output with bounded backpressure and cancellation awareness."""
|
|
deadline = time.monotonic() + WORKER_QUEUE_FINAL_GRACE_SEC if final else None
|
|
|
|
while final or not ctx.should_stop():
|
|
try:
|
|
chunk_queue.put(
|
|
item,
|
|
timeout=_coerce_worker_queue_put_timeout(
|
|
poll_timeout,
|
|
cancelled=ctx.should_stop(),
|
|
),
|
|
)
|
|
return True
|
|
except queue.Full:
|
|
if final:
|
|
if ctx.should_stop() or (deadline is not None and time.monotonic() >= deadline):
|
|
return False
|
|
continue
|
|
if ctx.should_stop():
|
|
return False
|
|
|
|
return False
|
|
|
|
|
|
def _coerce_worker_queue_put_timeout(raw_timeout: Any, *, cancelled: bool = False) -> float:
|
|
try:
|
|
value = float(raw_timeout)
|
|
except (TypeError, ValueError, OverflowError):
|
|
value = 0.5
|
|
if cancelled:
|
|
return WORKER_QUEUE_CANCEL_PUT_BLOCK_SEC
|
|
return max(
|
|
WORKER_QUEUE_CANCEL_PUT_BLOCK_SEC,
|
|
min(value if value > 0 else 0.5, WORKER_QUEUE_MAX_PUT_BLOCK_SEC),
|
|
)
|
|
|
|
|
|
async def wait_worker_queue_item(
|
|
chunk_queue: queue.Queue,
|
|
*,
|
|
timeout: float = 0.5,
|
|
poll_interval: float = 0.01,
|
|
max_poll_interval: float = 0.05,
|
|
) -> Any:
|
|
"""Wait for a worker queue item without occupying the default executor."""
|
|
try:
|
|
timeout_value = float(timeout)
|
|
except (TypeError, ValueError, OverflowError):
|
|
timeout_value = 0.5
|
|
timeout_value = max(0.0, timeout_value)
|
|
|
|
try:
|
|
interval = float(poll_interval)
|
|
except (TypeError, ValueError, OverflowError):
|
|
interval = 0.01
|
|
interval = max(0.001, min(interval, 0.05))
|
|
|
|
try:
|
|
max_interval = float(max_poll_interval)
|
|
except (TypeError, ValueError, OverflowError):
|
|
max_interval = 0.05
|
|
max_interval = max(interval, min(max_interval, 0.05))
|
|
|
|
deadline = time.monotonic() + timeout_value
|
|
current_interval = min(interval, timeout_value) if timeout_value > 0 else interval
|
|
while True:
|
|
try:
|
|
return chunk_queue.get_nowait()
|
|
except queue.Empty:
|
|
remaining = deadline - time.monotonic()
|
|
if remaining <= 0:
|
|
raise
|
|
sleep_for = min(current_interval, remaining)
|
|
await asyncio.sleep(sleep_for)
|
|
current_interval = _next_worker_queue_wait_interval(
|
|
current_interval,
|
|
max_interval,
|
|
remaining,
|
|
)
|
|
|
|
|
|
def _next_worker_queue_wait_interval(current: float, max_interval: float, remaining: float) -> float:
|
|
try:
|
|
current_value = float(current)
|
|
except (TypeError, ValueError, OverflowError):
|
|
current_value = 0.001
|
|
try:
|
|
max_value = float(max_interval)
|
|
except (TypeError, ValueError, OverflowError):
|
|
max_value = 0.05
|
|
try:
|
|
remaining_value = float(remaining)
|
|
except (TypeError, ValueError, OverflowError):
|
|
remaining_value = max_value
|
|
|
|
max_value = max(0.001, min(max_value, 0.05))
|
|
current_value = max(0.001, current_value)
|
|
remaining_value = max(0.0, remaining_value)
|
|
|
|
next_value = current_value * WORKER_QUEUE_WAIT_BACKOFF_FACTOR
|
|
return max(0.001, min(next_value, max_value, remaining_value or max_value))
|
|
|
|
|
|
def _set_worker_future_result(
|
|
future: "asyncio.Future[Any]",
|
|
*,
|
|
result: Any = None,
|
|
error: Optional[BaseException] = None,
|
|
) -> None:
|
|
if future.done():
|
|
return
|
|
if error is not None:
|
|
future.set_exception(error)
|
|
return
|
|
future.set_result(result)
|
|
|
|
|
|
def _consume_worker_future_exception(future: "asyncio.Future[Any]") -> None:
|
|
"""Retrieve late worker failures after the awaiting request has been cancelled."""
|
|
if future.cancelled():
|
|
return
|
|
try:
|
|
future.exception()
|
|
except (asyncio.CancelledError, Exception):
|
|
pass
|
|
|
|
|
|
def _get_cancel_reason(ctx: RequestContext, fallback: str = "worker_cancelled") -> str:
|
|
reason = str(getattr(ctx, "cancel_reason", "") or "").strip()
|
|
return reason or fallback
|
|
|
|
|
|
def run_in_request_context(ctx: RequestContext, worker_fn: Callable[[], Any]) -> Any:
|
|
"""Run a callable with the request id attached to all logs it emits."""
|
|
token = _request_context.set(str(getattr(ctx, "request_id", "") or "SYSTEM"))
|
|
try:
|
|
return worker_fn()
|
|
finally:
|
|
_request_context.reset(token)
|
|
|
|
|
|
async def run_tracked_blocking_call(
|
|
worker_fn: Callable[[], Any],
|
|
*,
|
|
ctx: RequestContext,
|
|
worker_state: Dict[str, Any],
|
|
label: str,
|
|
poll_timeout: float = 0.5,
|
|
max_execute_time_sec: Optional[float] = None,
|
|
) -> Any:
|
|
"""Run a blocking callable on a daemon thread and keep it visible for cleanup."""
|
|
loop = asyncio.get_running_loop()
|
|
result_future: "asyncio.Future[Any]" = loop.create_future()
|
|
started_at = time.monotonic()
|
|
|
|
def worker() -> None:
|
|
try:
|
|
result = run_in_request_context(ctx, worker_fn)
|
|
except Exception as exc:
|
|
try:
|
|
loop.call_soon_threadsafe(
|
|
lambda exc=exc: _set_worker_future_result(result_future, error=exc)
|
|
)
|
|
except RuntimeError:
|
|
pass
|
|
else:
|
|
try:
|
|
loop.call_soon_threadsafe(
|
|
lambda result=result: _set_worker_future_result(result_future, result=result)
|
|
)
|
|
except RuntimeError:
|
|
pass
|
|
|
|
worker_thread = threading.Thread(
|
|
target=worker,
|
|
daemon=True,
|
|
name=f"tracked-worker-{label}",
|
|
)
|
|
worker_state["thread"] = worker_thread
|
|
worker_state["label"] = label
|
|
ctx.worker_thread = worker_thread
|
|
|
|
def _clear_finished_worker_state() -> None:
|
|
if worker_state.get("thread") is worker_thread:
|
|
worker_state["thread"] = None
|
|
worker_state["label"] = None
|
|
|
|
def _handle_finished_worker(future: "asyncio.Future[Any]") -> None:
|
|
_clear_finished_worker_state()
|
|
_consume_worker_future_exception(future)
|
|
|
|
result_future.add_done_callback(_handle_finished_worker)
|
|
worker_thread.start()
|
|
|
|
try:
|
|
while True:
|
|
if mark_request_hard_timeout(
|
|
ctx,
|
|
started_at,
|
|
max_execute_time_sec,
|
|
label=f"worker={label}",
|
|
):
|
|
raise TrackedWorkerExecutionCancelled("absolute_request_timeout")
|
|
if ctx.should_stop():
|
|
raise TrackedWorkerExecutionCancelled(_get_cancel_reason(ctx))
|
|
try:
|
|
return await asyncio.wait_for(
|
|
asyncio.shield(result_future),
|
|
timeout=max(0.05, float(poll_timeout or 0.5)),
|
|
)
|
|
except asyncio.TimeoutError:
|
|
continue
|
|
finally:
|
|
if result_future.done():
|
|
_clear_finished_worker_state()
|
|
|
|
|
|
def _resolve_raw_tab_id(pool: Any, session: Any) -> str:
|
|
raw_tab_id = ""
|
|
if pool is not None:
|
|
try:
|
|
persistent_index = int(getattr(session, "persistent_index", 0) or 0)
|
|
for candidate_raw_id, candidate_index in getattr(pool, "_raw_id_to_persistent", {}).items():
|
|
if int(candidate_index or 0) == persistent_index:
|
|
raw_tab_id = str(candidate_raw_id or "").strip()
|
|
break
|
|
except Exception:
|
|
raw_tab_id = ""
|
|
if raw_tab_id:
|
|
return raw_tab_id
|
|
|
|
try:
|
|
return str(getattr(getattr(session, "tab", None), "tab_id", "") or "").strip()
|
|
except Exception:
|
|
return ""
|
|
|
|
|
|
def retire_bound_tab_after_worker_leak(ctx: RequestContext, reason: str) -> None:
|
|
"""Mark the tab bound to ctx as unhealthy without closing its browser target."""
|
|
tab_id = str(getattr(ctx, "tab_id", "") or "").strip()
|
|
if not tab_id:
|
|
return
|
|
|
|
try:
|
|
from app.core import get_browser
|
|
|
|
browser = get_browser(auto_connect=False)
|
|
pool = getattr(browser, "_tab_pool", None)
|
|
session = getattr(pool, "_tabs", {}).get(tab_id) if pool is not None else None
|
|
if session is None:
|
|
return
|
|
|
|
current_task = str(getattr(session, "current_task_id", "") or "").strip()
|
|
bound_request_id = str(getattr(session, "_bound_request_id", "") or "").strip()
|
|
if current_task and current_task != ctx.request_id:
|
|
logger.warning(
|
|
f"[{tab_id}] leaked worker found but tab has another task; skip retire "
|
|
f"(request={ctx.request_id}, current_task={current_task})"
|
|
)
|
|
return
|
|
if bound_request_id and bound_request_id != ctx.request_id:
|
|
logger.warning(
|
|
f"[{tab_id}] leaked worker found but bound request changed; skip retire "
|
|
f"(request={ctx.request_id}, bound={bound_request_id})"
|
|
)
|
|
return
|
|
|
|
if hasattr(session, "mark_error"):
|
|
session.mark_error(reason)
|
|
|
|
raw_tab_id = _resolve_raw_tab_id(pool, session)
|
|
|
|
if pool is not None:
|
|
try:
|
|
monitor = getattr(pool, "_global_network_monitor", None)
|
|
if monitor is not None and hasattr(monitor, "request_stop_for_session"):
|
|
monitor.request_stop_for_session(tab_id, reason=reason, detach=True)
|
|
except Exception as e:
|
|
logger.debug(f"[{tab_id}] stop leaked tab global monitor failed: {e}")
|
|
|
|
if pool is not None and hasattr(pool, "_condition"):
|
|
try:
|
|
with pool._condition:
|
|
pool._condition.notify_all()
|
|
except Exception:
|
|
pass
|
|
|
|
logger.warning(
|
|
f"[{tab_id}] worker did not exit in time; marked tab ERROR "
|
|
f"(request={ctx.request_id}, reason={reason}, raw={raw_tab_id or '-'}, "
|
|
"browser_tab_retained=True)"
|
|
)
|
|
except Exception as e:
|
|
logger.debug(f"retire leaked worker tab failed: {e}")
|
|
|
|
|
|
def cleanup_worker_thread(
|
|
worker_thread: Optional[threading.Thread],
|
|
ctx: RequestContext,
|
|
*,
|
|
cancel_reason: str = "cleanup",
|
|
join_timeout: float = 5.0,
|
|
retire_reason: str = "worker_cleanup_timeout",
|
|
) -> bool:
|
|
"""Cancel, join, and retire a worker thread that outlives its request."""
|
|
if not isinstance(worker_thread, threading.Thread) or not worker_thread.is_alive():
|
|
return False
|
|
|
|
ctx.request_cancel(cancel_reason)
|
|
worker_thread.join(timeout=max(0.0, float(join_timeout or 0.0)))
|
|
if worker_thread.is_alive():
|
|
retire_bound_tab_after_worker_leak(ctx, retire_reason)
|
|
return True
|
|
return False
|
|
|
|
|
|
def schedule_completed_worker_retire_check(
|
|
worker_thread: threading.Thread,
|
|
ctx: RequestContext,
|
|
reason: str,
|
|
*,
|
|
delay_sec: float = COMPLETED_WORKER_RETIRE_GRACE_SEC,
|
|
) -> None:
|
|
"""Watch a completed request's slow worker cleanup without blocking response finalization."""
|
|
try:
|
|
delay = max(0.1, float(delay_sec or COMPLETED_WORKER_RETIRE_GRACE_SEC))
|
|
except Exception:
|
|
delay = COMPLETED_WORKER_RETIRE_GRACE_SEC
|
|
|
|
def _wait_and_retire_if_still_alive() -> None:
|
|
started_at = time.monotonic()
|
|
worker_thread.join(timeout=delay)
|
|
elapsed = time.monotonic() - started_at
|
|
if worker_thread.is_alive():
|
|
logger.warning(
|
|
f"[{ctx.request_id}] completed request worker still alive after cleanup grace; "
|
|
f"retiring bound tab (tab={ctx.tab_id or '-'}, reason={reason}, "
|
|
f"grace={delay:.1f}s)"
|
|
)
|
|
retire_bound_tab_after_worker_leak(ctx, reason)
|
|
return
|
|
logger.debug(
|
|
f"[{ctx.request_id}] completed request worker cleanup finished in background "
|
|
f"(elapsed={elapsed:.2f}s, tab={ctx.tab_id or '-'})"
|
|
)
|
|
|
|
threading.Thread(
|
|
target=_wait_and_retire_if_still_alive,
|
|
daemon=True,
|
|
name=f"completed-worker-cleanup-{ctx.request_id[:8]}",
|
|
).start()
|
|
|
|
|
|
async def _wait_for_worker_exit(worker_thread: threading.Thread, timeout: float) -> None:
|
|
"""Bound cleanup wall time without queueing joins in the shared executor.
|
|
|
|
A saturated default executor must not postpone cleanup or starve the next
|
|
request's route discovery. is_alive() is nonblocking; cancellation is still
|
|
allowed here, and callers shield only this bounded cleanup window.
|
|
"""
|
|
deadline = time.monotonic() + max(0.0, timeout)
|
|
while worker_thread.is_alive():
|
|
remaining = deadline - time.monotonic()
|
|
if remaining <= 0:
|
|
break
|
|
await asyncio.sleep(min(.02, remaining))
|
|
|
|
|
|
async def cleanup_worker_thread_after_request(
|
|
worker_thread: Optional[threading.Thread],
|
|
ctx: RequestContext,
|
|
*,
|
|
completed: bool = False,
|
|
cancel_reason: str = "cleanup",
|
|
join_timeout: float = 5.0,
|
|
retire_reason: str = "worker_cleanup_timeout",
|
|
completed_join_timeout: float = COMPLETED_WORKER_FAST_JOIN_SEC,
|
|
completed_retire_delay: float = COMPLETED_WORKER_RETIRE_GRACE_SEC,
|
|
) -> bool:
|
|
"""Cleanup a request worker, deferring leak retirement for already-completed requests.
|
|
|
|
Returns True when the worker was still alive after the immediate cleanup window.
|
|
"""
|
|
if not isinstance(worker_thread, threading.Thread) or not worker_thread.is_alive():
|
|
return False
|
|
|
|
if completed:
|
|
try:
|
|
fast_join = max(0.0, float(completed_join_timeout or 0.0))
|
|
except Exception:
|
|
fast_join = COMPLETED_WORKER_FAST_JOIN_SEC
|
|
try:
|
|
await _wait_for_worker_exit(worker_thread, fast_join)
|
|
except asyncio.CancelledError:
|
|
if worker_thread.is_alive():
|
|
schedule_completed_worker_retire_check(worker_thread, ctx, retire_reason, delay_sec=completed_retire_delay)
|
|
raise
|
|
if worker_thread.is_alive():
|
|
logger.warning(
|
|
f"[{ctx.request_id}] completed request worker cleanup is still running; "
|
|
f"defer tab retirement check (tab={ctx.tab_id or '-'}, "
|
|
f"fast_join={fast_join:.1f}s, grace={completed_retire_delay:.1f}s)"
|
|
)
|
|
schedule_completed_worker_retire_check(
|
|
worker_thread,
|
|
ctx,
|
|
retire_reason,
|
|
delay_sec=completed_retire_delay,
|
|
)
|
|
return True
|
|
return False
|
|
|
|
ctx.request_cancel(cancel_reason)
|
|
try:
|
|
timeout = max(0.0, float(join_timeout or 0.0))
|
|
except Exception:
|
|
timeout = 5.0
|
|
try:
|
|
await _wait_for_worker_exit(worker_thread, timeout)
|
|
except asyncio.CancelledError:
|
|
if worker_thread.is_alive():
|
|
schedule_completed_worker_retire_check(worker_thread, ctx, retire_reason, delay_sec=timeout)
|
|
raise
|
|
if worker_thread.is_alive():
|
|
retire_bound_tab_after_worker_leak(ctx, retire_reason)
|
|
return True
|
|
return False
|