Initial commit

This commit is contained in:
Alex committed 2026-11-02 22:54:14 +08:00
commit 57fc3525eb
7311 files changed
+1291398

No files matched your search

+265
View File
@@ -0,0 +1,265 @@
"""
app/core/cdp_hygiene.py - DrissionPage 运行期 CDP 资源卫生补丁
背景(见 docs/performance_refactor.md):
- DrissionPage 的 ``run_js`` 始终以 ``returnByValue=False`` 调用,返回对象/数组时
会在渲染进程里生成 RemoteObject,并且从不 ``Runtime.releaseObject``。
V8 inspector 对这些对象持强引用,直到整页导航或 CDP 会话断开——已经从页面上
移除的旧 DOM 会被一直钉在内存里,强制 GC 也回收不掉。
- ``Network.enable`` 不带缓冲上限,渲染进程会为每个 XHR/Fetch 响应保留副本。
本模块只做**运行期**的 monkeypatch(不改写磁盘上的库源码),且全部幂等:
1. ``parse_js_result`` 包装:非节点的对象/数组结果在取回后立即 ``releaseObject``。
节点结果(会被包装成 ChromiumElement 继续使用)保持不变。
2. ``Driver.run`` 包装:``Network.enable`` 未显式给出缓冲上限时自动补上
``maxTotalBufferSize`` / ``maxResourceBufferSize``(可用环境变量调整或关闭)。
3. 提供 ``run_js_json``:在页面内 ``JSON.stringify`` 后只返回一个字符串,
既不生成 RemoteObject,也省掉 DrissionPage 额外的一次 ``JSON.stringify`` 往返。
环境变量:
- ``CDP_HYGIENE_ENABLED``(默认 true):总开关。
- ``CDP_RELEASE_JS_OBJECTS``(默认 true):是否释放 run_js 的对象结果。
- ``CDP_NETWORK_MAX_TOTAL_BUFFER_MB``(默认 32,0 = 不注入)
- ``CDP_NETWORK_MAX_RESOURCE_BUFFER_MB``(默认 16,0 = 不注入)
"""
from __future__ import annotations
import json
import os
import threading
from typing import Any, Dict, Optional
from app.core.config import logger
from app.core.driver import as_element
_INSTALL_LOCK = threading.Lock()
_INSTALLED = False
_STATS_LOCK = threading.Lock()
_STATS: Dict[str, int] = {
"released_js_objects": 0,
"release_errors": 0,
"network_enable_limited": 0,
}
_RELEASE_SKIP_CLASSES = frozenset({"HTMLDocument"})
def _env_bool(name: str, default: bool) -> bool:
raw = os.getenv(name)
if raw is None:
return default
return str(raw).strip().lower() in {"1", "true", "yes", "y", "on"}
def _env_mb(name: str, default: float) -> int:
raw = os.getenv(name)
try:
value = float(raw) if raw not in (None, "") else float(default)
except (TypeError, ValueError):
value = float(default)
if value <= 0:
return 0
return int(value * 1024 * 1024)
def _bump(key: str, amount: int = 1) -> None:
with _STATS_LOCK:
_STATS[key] = int(_STATS.get(key, 0) or 0) + amount
def get_stats() -> Dict[str, Any]:
with _STATS_LOCK:
data = dict(_STATS)
data["installed"] = _INSTALLED
return data
def network_enable_limits() -> Dict[str, int]:
"""Buffer limits injected into ``Network.enable`` (empty dict = disabled)."""
limits: Dict[str, int] = {}
total = _env_mb("CDP_NETWORK_MAX_TOTAL_BUFFER_MB", 32)
resource = _env_mb("CDP_NETWORK_MAX_RESOURCE_BUFFER_MB", 16)
if total > 0:
limits["maxTotalBufferSize"] = total
if resource > 0:
if total > 0:
resource = min(resource, total)
limits["maxResourceBufferSize"] = resource
return limits
def _should_release(result: Any) -> Optional[str]:
if not isinstance(result, dict):
return None
if result.get("type") != "object":
return None
object_id = result.get("objectId")
if not object_id:
return None
subtype = result.get("subtype")
if subtype in ("node", "null"):
# 节点会被包装成 ChromiumElement 继续使用,不能释放
return None
if result.get("className") in _RELEASE_SKIP_CLASSES:
return None
return str(object_id)
def _release_object(page: Any, object_id: str) -> None:
try:
driver = getattr(page, "driver", None) or getattr(page, "_driver", None)
if driver is None:
return
driver.run("Runtime.releaseObject", objectId=object_id, _timeout=2)
_bump("released_js_objects")
except Exception:
_bump("release_errors")
def _patch_parse_js_result() -> bool:
try:
from app.core.driver.drission_internals import chromium_element_module
ce = chromium_element_module()
except Exception as exc: # pragma: no cover - DrissionPage missing
logger.debug(f"[CDP_HYGIENE] DrissionPage 不可用,跳过 run_js 补丁: {exc}")
return False
original = getattr(ce, "parse_js_result", None)
if original is None:
return False
if getattr(original, "_uwapi_release_wrapper", False):
return True
def parse_js_result(page, ele, result, end_time): # noqa: D401 - same signature
object_id = _should_release(result)
try:
return original(page, ele, result, end_time)
finally:
if object_id:
_release_object(page, object_id)
parse_js_result._uwapi_release_wrapper = True # type: ignore[attr-defined]
parse_js_result._uwapi_original = original # type: ignore[attr-defined]
ce.parse_js_result = parse_js_result
return True
def _patch_network_enable() -> bool:
try:
from app.core.driver.drission_internals import driver_class
Driver = driver_class()
except Exception as exc: # pragma: no cover - DrissionPage missing
logger.debug(f"[CDP_HYGIENE] DrissionPage Driver 不可用,跳过 Network.enable 补丁: {exc}")
return False
original = Driver.run
if getattr(original, "_uwapi_network_limit_wrapper", False):
return True
# 签名与各版本原版保持透传:4.0.x–4.1.0.x 是 run(self, _method, **kwargs),kwargs 原样进 params;
# 4.1.1.x 才有独立的 sessionId 形参。不能在这里声明 sessionId=None 再转发,否则老版本会把
# "sessionId": null 塞进每条 CDP 命令的 params。
def run(self, _method, *args, **kwargs):
if _method == "Network.enable":
limits = network_enable_limits()
injected = False
for key, value in limits.items():
if key not in kwargs:
kwargs[key] = value
injected = True
if injected:
_bump("network_enable_limited")
return original(self, _method, *args, **kwargs)
run._uwapi_network_limit_wrapper = True # type: ignore[attr-defined]
run._uwapi_original = original # type: ignore[attr-defined]
Driver.run = run
return True
def install() -> bool:
"""Install all runtime patches once. Safe to call repeatedly."""
global _INSTALLED
if _INSTALLED:
return True
if not _env_bool("CDP_HYGIENE_ENABLED", True):
return False
with _INSTALL_LOCK:
if _INSTALLED:
return True
applied = []
if _env_bool("CDP_RELEASE_JS_OBJECTS", True) and _patch_parse_js_result():
applied.append("release_js_objects")
if network_enable_limits() and _patch_network_enable():
applied.append("network_buffer_limits")
_INSTALLED = True
if applied:
logger.debug(f"[CDP_HYGIENE] 已安装运行期补丁: {', '.join(applied)}")
return True
# ---------------------------------------------------------------------------
# JSON 化的 run_js:只返回原始字符串,不生成 RemoteObject
# ---------------------------------------------------------------------------
_JSON_WRAPPER_TEMPLATE = (
"function(){"
"const __uwapiResult = (function(){\n%s\n}).apply(this, arguments);"
"if (__uwapiResult && typeof __uwapiResult.then === 'function') {"
"return __uwapiResult.then(function(v){ return JSON.stringify(v === undefined ? null : v); });"
"}"
"return JSON.stringify(__uwapiResult === undefined ? null : __uwapiResult);"
"}"
)
def wrap_js_json(script_body: str) -> str:
"""Wrap a ``run_js`` body so that its return value is JSON-stringified in page."""
return _JSON_WRAPPER_TEMPLATE % str(script_body or "")
def decode_js_json(raw: Any, default: Any = None) -> Any:
"""Decode a JSON string produced by :func:`wrap_js_json`.
Test doubles and legacy call sites may still hand back already-decoded
Python objects; those are returned unchanged.
"""
if raw is None:
return default
if isinstance(raw, (dict, list, bool, int, float)):
return raw
if isinstance(raw, (bytes, bytearray)):
raw = raw.decode("utf-8", errors="replace")
if isinstance(raw, str):
text = raw.strip()
if not text:
return default
try:
return json.loads(text)
except (TypeError, ValueError):
return default
return default
def run_js_json(target: Any, script_body: str, *args: Any, timeout: Optional[float] = None, default: Any = None) -> Any:
"""Run ``script_body`` on a tab/element and return the decoded JSON value."""
wrapped = wrap_js_json(script_body)
if timeout is None:
raw = as_element(target).run_js(wrapped, *args)
else:
raw = as_element(target).run_js(wrapped, *args, timeout=timeout)
return decode_js_json(raw, default)
__all__ = [
"decode_js_json",
"get_stats",
"install",
"network_enable_limits",
"run_js_json",
"wrap_js_json",
]