Initial commit
This commit is contained in:
commit
57fc3525eb
7311 files changed
+1291398
No files matched your search
@@ -0,0 +1,196 @@
|
||||
"""
|
||||
app/core/config_parts - 配置与基础设施子模块包
|
||||
"""
|
||||
from .env_config import (
|
||||
PROJECT_ROOT,
|
||||
DEFAULT_LOG_DIR,
|
||||
atomic_write_json,
|
||||
_replace_file_with_retry,
|
||||
classproperty,
|
||||
load_dotenv,
|
||||
AppConfig,
|
||||
app_config,
|
||||
get_tutorial_selectors_url,
|
||||
)
|
||||
from .browser_constants import (
|
||||
BrowserConstants,
|
||||
_browser_constant_bool,
|
||||
_browser_constant_int,
|
||||
_BrowserConstantEnabledFilter,
|
||||
)
|
||||
from .exceptions import (
|
||||
BrowserError,
|
||||
BrowserConnectionError,
|
||||
ElementNotFoundError,
|
||||
WorkflowError,
|
||||
WorkflowCancelledError,
|
||||
ConfigurationError,
|
||||
)
|
||||
from .sse_formatter import SSEFormatter
|
||||
from .message_validator import MessageValidator
|
||||
from .log_collector import LogCollector, log_collector
|
||||
from .log_redaction import (
|
||||
_SENSITIVE_KEY_HINTS,
|
||||
_REDACTED_TEXT,
|
||||
_redact_data_uri_for_log,
|
||||
_redact_long_base64_for_log,
|
||||
_BASE64_LOG_CHARS,
|
||||
_SENSITIVE_TEXT_SCAN_HINT_RE,
|
||||
_SENSITIVE_TEXT_PRECHECK_MIN_CHARS,
|
||||
_SENSITIVE_TEXT_LARGE_OMIT_THRESHOLD,
|
||||
_SENSITIVE_TEXT_LARGE_EDGE_CHARS,
|
||||
_has_long_base64_candidate,
|
||||
_redact_long_base64_runs_for_log,
|
||||
_should_scan_sensitive_text,
|
||||
_SENSITIVE_TEXT_PATTERNS,
|
||||
_SENSITIVE_BEARER_PATTERN,
|
||||
_SENSITIVE_STANDALONE_TOKEN_PATTERNS,
|
||||
_SENSITIVE_HEADER_PATTERN,
|
||||
_SENSITIVE_PARAM_PATTERNS,
|
||||
_sanitize_standalone_sensitive_tokens,
|
||||
_is_sensitive_key,
|
||||
_sanitize_sensitive_text,
|
||||
sanitize_sensitive_data,
|
||||
)
|
||||
from .log_formatters import (
|
||||
_truncate_long_message,
|
||||
_get_log_display_limit,
|
||||
_record_request_id,
|
||||
_REQUEST_SHORT_ID_PATTERN,
|
||||
_request_display_tag,
|
||||
_record_request_tag,
|
||||
_compact_logger_name_impl,
|
||||
_compact_logger_name,
|
||||
_LOGGER_DISPLAY_WIDTH,
|
||||
_record_logger_name,
|
||||
_record_kind,
|
||||
_LEVEL_BADGES,
|
||||
_record_level_badge,
|
||||
_replace_log_tag,
|
||||
_normalize_log_display_expression,
|
||||
_record_display_message,
|
||||
_format_log_display_parts,
|
||||
_join_log_display_parts,
|
||||
_format_log_display_line,
|
||||
_WebLogHandler,
|
||||
_web_log_handler,
|
||||
_enable_windows_ansi,
|
||||
_should_use_console_color,
|
||||
_ConsoleColorFormatter,
|
||||
_FileLogFormatter,
|
||||
_DisplayLogFormatter,
|
||||
_is_windows_file_lock_error,
|
||||
_SafeRotatingFileHandler,
|
||||
_get_positive_int_env,
|
||||
_resolve_log_dir,
|
||||
get_log_file_path,
|
||||
get_shared_file_log_handler,
|
||||
_shared_file_log_handler,
|
||||
_shared_file_log_handler_lock,
|
||||
)
|
||||
from .cute_translator import (
|
||||
_REQUEST_FINISH_PATTERN,
|
||||
_TAB_START_INDEX_PATTERN,
|
||||
_TAB_START_ROUTE_PATTERN,
|
||||
_CHUNKED_LONG_TEXT_PATTERN,
|
||||
_CHUNKED_DONE_PATTERN,
|
||||
_CHUNKED_SHORT_TEXT_PATTERN,
|
||||
_CHUNKED_FIRST_BLOCK_PATTERN,
|
||||
_CHUNKED_PROGRESS_PATTERN,
|
||||
_VERIFY_OK_EXACT_PATTERN,
|
||||
_SEND_SUCCESS_RETRY_PATTERN,
|
||||
_FILE_PASTE_DONE_PATTERN,
|
||||
_CLIPBOARD_OK_PATTERN,
|
||||
_FILE_PASTE_UPLOAD_SIGNAL_PATTERN,
|
||||
_FILE_PASTE_WEAK_SIGNAL_PATTERN,
|
||||
_FILE_PASTE_INPUT_SIGNAL_PATTERN,
|
||||
_FILE_PASTE_INPUT_UPLOADED_PATTERN,
|
||||
_FILE_PASTE_HINT_PATTERN,
|
||||
_FILE_PASTE_TEMP_FILE_PATTERN,
|
||||
_STEALTH_SEND_RETRY_PATTERN,
|
||||
_STEALTH_SEND_RETRY_SIMPLE_PATTERN,
|
||||
_STEALTH_WARMUP_DONE_PATTERN,
|
||||
_STEALTH_REVIEW_DELAY_PATTERN,
|
||||
_STEALTH_CLIPBOARD_PASTE_NOCLICK_PATTERN,
|
||||
_STEALTH_CLIPBOARD_PASTE_PATTERN,
|
||||
_STEALTH_RANDOM_PAUSE_PATTERN,
|
||||
_STEALTH_PRE_SEND_HESITATE_PATTERN,
|
||||
_STEALTH_HUMAN_CLICK_PATTERN,
|
||||
_SEND_ATTACHMENT_WAIT_PATTERN,
|
||||
_SEND_ATTACHMENT_READY_PATTERN,
|
||||
_SEND_ATTACHMENT_SETTLE_PATTERN,
|
||||
_SEND_RETRY_ACTION_PATTERN,
|
||||
_SEND_RETRY_WINDOW_PATTERN,
|
||||
_bool_phrase,
|
||||
_split_suppressed_suffix,
|
||||
_add_suppressed_marker,
|
||||
_restore_suppressed_hint,
|
||||
_apply_cute_rules,
|
||||
_cute_stage_title,
|
||||
_CUTE_EXTRA_RULES,
|
||||
_ERROR_KEYWORD_HINTS,
|
||||
_cuteify_error_message,
|
||||
_cuteify_info_message,
|
||||
_cuteify_warning_message,
|
||||
_cuteify_debug_message,
|
||||
)
|
||||
from .secure_logger import (
|
||||
_request_context,
|
||||
_command_log_context,
|
||||
_logger_setup_lock,
|
||||
_logger_registry_lock,
|
||||
_logger_registry,
|
||||
SecureLogger,
|
||||
command_log_context,
|
||||
get_logger,
|
||||
logger,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
# 应用配置
|
||||
'AppConfig',
|
||||
'app_config',
|
||||
'get_tutorial_selectors_url',
|
||||
'load_dotenv',
|
||||
'PROJECT_ROOT',
|
||||
'DEFAULT_LOG_DIR',
|
||||
'atomic_write_json',
|
||||
'_replace_file_with_retry',
|
||||
'classproperty',
|
||||
|
||||
# 浏览器常量
|
||||
'BrowserConstants',
|
||||
'_browser_constant_bool',
|
||||
'_browser_constant_int',
|
||||
'_BrowserConstantEnabledFilter',
|
||||
|
||||
# 日志系统与收集器
|
||||
'SecureLogger',
|
||||
'logger',
|
||||
'get_logger',
|
||||
'log_collector',
|
||||
'LogCollector',
|
||||
'_request_context',
|
||||
'_command_log_context',
|
||||
'command_log_context',
|
||||
'get_log_file_path',
|
||||
'get_shared_file_log_handler',
|
||||
'_shared_file_log_handler',
|
||||
'_shared_file_log_handler_lock',
|
||||
'_logger_setup_lock',
|
||||
'_logger_registry_lock',
|
||||
'_logger_registry',
|
||||
'sanitize_sensitive_data',
|
||||
|
||||
# 异常
|
||||
'BrowserError',
|
||||
'BrowserConnectionError',
|
||||
'ElementNotFoundError',
|
||||
'WorkflowError',
|
||||
'WorkflowCancelledError',
|
||||
'ConfigurationError',
|
||||
|
||||
# SSE 与消息工具
|
||||
'SSEFormatter',
|
||||
'MessageValidator',
|
||||
]
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -0,0 +1,315 @@
|
||||
"""
|
||||
app/core/config_parts/browser_constants.py - 浏览器常量配置与过滤器模块
|
||||
"""
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional
|
||||
from .env_config import AppConfig, PROJECT_ROOT
|
||||
|
||||
|
||||
class BrowserConstants:
|
||||
"""浏览器相关常量(从 JSON 文件加载,支持热重载)"""
|
||||
|
||||
# ===== 配置缓存 =====
|
||||
_config: Optional[Dict] = None
|
||||
_config_file = Path("config/browser_config.json")
|
||||
|
||||
# ===== 默认值字典 =====
|
||||
_DEFAULTS = {
|
||||
'DEFAULT_PORT': 9222,
|
||||
'CONNECTION_TIMEOUT': 10,
|
||||
'MAX_REQUEST_EXECUTE_TIME_SEC': 300.0,
|
||||
'STEALTH_DELAY_MIN': 0.03,
|
||||
'STEALTH_DELAY_MAX': 0.1,
|
||||
'ACTION_DELAY_MIN': 0.06,
|
||||
'ACTION_DELAY_MAX': 0.14,
|
||||
'STEALTH_PAUSE_PROBABILITY': 0.0,
|
||||
'STEALTH_PAUSE_EXTRA_MAX': 0.15,
|
||||
'STEALTH_KEY_DOWN_UP_MIN': 0.015,
|
||||
'STEALTH_KEY_DOWN_UP_MAX': 0.04,
|
||||
'STEALTH_KEY_BETWEEN_MIN': 0.02,
|
||||
'STEALTH_KEY_BETWEEN_MAX': 0.06,
|
||||
'STEALTH_PASTE_SETTLE_MIN': 0.12,
|
||||
'STEALTH_PASTE_SETTLE_MAX': 0.25,
|
||||
'STEALTH_SKIP_PASTE_VERIFY': True,
|
||||
'STEALTH_SEND_IMAGE_WAIT': 8.0,
|
||||
'STEALTH_SEND_IMAGE_RETRY_INTERVAL': 1.2,
|
||||
'STEALTH_MOUSE_WARMUP_ENABLED': False,
|
||||
'STEALTH_CLICK_STRATEGY': 'auto',
|
||||
'STEALTH_DOM_CLICK_TARGETS': ['new_chat_btn', 'input_box', 'send_btn'],
|
||||
'PAGE_INTERACTION_THROTTLE_ENABLED': True,
|
||||
'PAGE_INTERACTION_MAX_CONCURRENT': 3,
|
||||
'PAGE_INTERACTION_MAX_WAIT': 20.0,
|
||||
'PAGE_INTERACTION_MIN_INTERVAL': 0.25,
|
||||
'PAGE_INTERACTION_READY_TIMEOUT': 1.5,
|
||||
'PAGE_INTERACTION_STABLE_SAMPLES': 2,
|
||||
'PAGE_INTERACTION_SAMPLE_INTERVAL': 0.12,
|
||||
'PAGE_INTERACTION_RECT_TOLERANCE': 3,
|
||||
'COORD_CLICK_READY_TIMEOUT': 0.9,
|
||||
'COORD_CLICK_STABLE_SAMPLES': 2,
|
||||
'COORD_CLICK_SAMPLE_INTERVAL': 0.08,
|
||||
'COORD_CLICK_RECT_TOLERANCE': 3,
|
||||
'COORD_CLICK_EDGE_INSET': 4,
|
||||
'COORD_CLICK_RETRY_OFFSETS': [[0, 0], [4, 0], [-4, 0], [0, 4], [0, -4], [7, 3], [-7, 3]],
|
||||
'WORKFLOW_WAKE_TAB_BEFORE_INTERACTION': True,
|
||||
'WORKFLOW_FOCUS_EMULATION_ON_INTERACTION': True,
|
||||
'DEFAULT_ELEMENT_TIMEOUT': 3,
|
||||
'FALLBACK_ELEMENT_TIMEOUT': 1,
|
||||
'ELEMENT_CACHE_MAX_AGE': 5.0,
|
||||
'LOG_INFO_CUTE_MODE': True,
|
||||
'LOG_DEBUG_CUTE_MODE': True,
|
||||
'LOG_CONSOLE_ENABLED': True,
|
||||
'LOG_FILE_ENABLED': True,
|
||||
'LOG_WEB_COLLECTOR_ENABLED': True,
|
||||
'LOG_WEB_MAX_RECORDS': 5000,
|
||||
'STREAM_CHECK_INTERVAL_MIN': 0.1,
|
||||
'STREAM_CHECK_INTERVAL_MAX': 1.0,
|
||||
'STREAM_CHECK_INTERVAL_DEFAULT': 0.3,
|
||||
'STREAM_SILENCE_THRESHOLD': 6.0,
|
||||
'STREAM_MAX_TIMEOUT': 600,
|
||||
'STREAM_INITIAL_WAIT': 180,
|
||||
'STREAM_CONTENT_SHRINK_TOLERANCE': 3,
|
||||
'STREAM_STABLE_COUNT_THRESHOLD': 5,
|
||||
'STREAM_SILENCE_THRESHOLD_FALLBACK': 10.0,
|
||||
'MAX_MESSAGE_LENGTH': 100000,
|
||||
'MAX_MESSAGES_COUNT': 100,
|
||||
'TEXT_INPUT_CHUNK_SIZE': 30000,
|
||||
'STREAM_USER_MSG_WAIT': 1.5,
|
||||
'STREAM_PRE_BASELINE_DELAY': 0.3,
|
||||
'GLOBAL_NETWORK_INTERCEPTION_ENABLED': False,
|
||||
'GLOBAL_NETWORK_INTERCEPTION_LISTEN_PATTERN': 'http',
|
||||
'GLOBAL_NETWORK_INTERCEPTION_WAIT_TIMEOUT': 0.5,
|
||||
'GLOBAL_NETWORK_INTERCEPTION_RETRY_DELAY': 1.0,
|
||||
'GLOBAL_NETWORK_INTERCEPTION_RES_TYPES': 'XHR,Fetch,EventSource,Document',
|
||||
'NETWORK_DEBUG_CAPTURE_ENABLED': False,
|
||||
'NETWORK_DEBUG_CAPTURE_MAX_BODY_CHARS': 50000,
|
||||
'NETWORK_DEBUG_CAPTURE_MAX_FILES_PER_REQUEST': 3,
|
||||
'NETWORK_DEBUG_CAPTURE_PARSER_FILTER': '',
|
||||
'NETWORK_DEBUG_CAPTURE_MAX_TOTAL_MB': 50,
|
||||
'NETWORK_DEBUG_CAPTURE_RETENTION_HOURS': 24,
|
||||
'CONVERSATION_TIMEOUT_THRESHOLD': 0.0,
|
||||
'FORCE_NEW_CONVERSATION': False,
|
||||
'REQUEST_MONITOR_ENABLED': True,
|
||||
'REQUEST_MONITOR_MAX_RECORDS': 200,
|
||||
'REQUEST_MONITOR_DETAIL_ENABLED': True,
|
||||
'REQUEST_MONITOR_SAVE_TO_FILE': True,
|
||||
'REQUEST_MONITOR_MAX_CAPTURED_RESPONSE_CHARS': 30000,
|
||||
'DASHBOARD_LOG_POLL_INTERVAL_MS': 1000,
|
||||
'DASHBOARD_LOG_BACKGROUND_POLL_INTERVAL_MS': 5000,
|
||||
'DASHBOARD_REQUEST_HISTORY_POLL_INTERVAL_MS': 3000,
|
||||
'DASHBOARD_SYSTEM_STATS_ENABLED': True,
|
||||
'DASHBOARD_SYSTEM_STATS_POLL_INTERVAL_MS': 3000,
|
||||
'ATTACHMENT_READY_IDLE_TIMEOUT': 8.0,
|
||||
'ATTACHMENT_READY_HARD_MAX_WAIT': 90.0,
|
||||
}
|
||||
|
||||
# ===== 类属性(会被配置文件覆盖)=====
|
||||
|
||||
# 连接配置
|
||||
DEFAULT_PORT = 9222
|
||||
CONNECTION_TIMEOUT = 10
|
||||
MAX_REQUEST_EXECUTE_TIME_SEC = 300.0
|
||||
|
||||
# 延迟配置
|
||||
STEALTH_DELAY_MIN = 0.03
|
||||
STEALTH_DELAY_MAX = 0.1
|
||||
ACTION_DELAY_MIN = 0.06
|
||||
ACTION_DELAY_MAX = 0.14
|
||||
STEALTH_PAUSE_PROBABILITY = 0.0
|
||||
STEALTH_PAUSE_EXTRA_MAX = 0.15
|
||||
STEALTH_KEY_DOWN_UP_MIN = 0.015
|
||||
STEALTH_KEY_DOWN_UP_MAX = 0.04
|
||||
STEALTH_KEY_BETWEEN_MIN = 0.02
|
||||
STEALTH_KEY_BETWEEN_MAX = 0.06
|
||||
STEALTH_PASTE_SETTLE_MIN = 0.12
|
||||
STEALTH_PASTE_SETTLE_MAX = 0.25
|
||||
STEALTH_SKIP_PASTE_VERIFY = True
|
||||
STEALTH_SEND_IMAGE_WAIT = 8.0
|
||||
STEALTH_SEND_IMAGE_RETRY_INTERVAL = 1.2
|
||||
STEALTH_MOUSE_WARMUP_ENABLED = False
|
||||
STEALTH_CLICK_STRATEGY = "auto"
|
||||
STEALTH_DOM_CLICK_TARGETS = ["new_chat_btn", "input_box", "send_btn"]
|
||||
PAGE_INTERACTION_THROTTLE_ENABLED = True
|
||||
PAGE_INTERACTION_MAX_CONCURRENT = 3
|
||||
PAGE_INTERACTION_MAX_WAIT = 20.0
|
||||
PAGE_INTERACTION_MIN_INTERVAL = 0.25
|
||||
PAGE_INTERACTION_READY_TIMEOUT = 1.5
|
||||
PAGE_INTERACTION_STABLE_SAMPLES = 2
|
||||
PAGE_INTERACTION_SAMPLE_INTERVAL = 0.12
|
||||
PAGE_INTERACTION_RECT_TOLERANCE = 3
|
||||
WORKFLOW_WAKE_TAB_BEFORE_INTERACTION = True
|
||||
WORKFLOW_FOCUS_EMULATION_ON_INTERACTION = True
|
||||
|
||||
# 元素查找
|
||||
DEFAULT_ELEMENT_TIMEOUT = 3
|
||||
FALLBACK_ELEMENT_TIMEOUT = 1
|
||||
ELEMENT_CACHE_MAX_AGE = 5.0
|
||||
|
||||
# 日志
|
||||
LOG_INFO_CUTE_MODE = True
|
||||
LOG_DEBUG_CUTE_MODE = True
|
||||
LOG_CONSOLE_ENABLED = True
|
||||
LOG_FILE_ENABLED = True
|
||||
LOG_WEB_COLLECTOR_ENABLED = True
|
||||
LOG_WEB_MAX_RECORDS = 5000
|
||||
|
||||
# 流式监控
|
||||
STREAM_CHECK_INTERVAL_MIN = 0.1
|
||||
STREAM_CHECK_INTERVAL_MAX = 1.0
|
||||
STREAM_CHECK_INTERVAL_DEFAULT = 0.3
|
||||
|
||||
STREAM_SILENCE_THRESHOLD = 6.0
|
||||
STREAM_MAX_TIMEOUT = 600
|
||||
STREAM_INITIAL_WAIT = 180
|
||||
|
||||
# 流式监控增强配置
|
||||
STREAM_CONTENT_SHRINK_TOLERANCE = 3
|
||||
|
||||
STREAM_STABLE_COUNT_THRESHOLD = 5
|
||||
STREAM_SILENCE_THRESHOLD_FALLBACK = 10.0
|
||||
|
||||
# 输入验证
|
||||
MAX_MESSAGE_LENGTH = 100000
|
||||
MAX_MESSAGES_COUNT = 100
|
||||
|
||||
# 附件/图片上传就绪判定
|
||||
ATTACHMENT_READY_IDLE_TIMEOUT = 8.0
|
||||
ATTACHMENT_READY_HARD_MAX_WAIT = 90.0
|
||||
|
||||
# 文本输入
|
||||
TEXT_INPUT_CHUNK_SIZE = 30000
|
||||
|
||||
# 两阶段 baseline 配置
|
||||
STREAM_USER_MSG_WAIT = 1.5
|
||||
STREAM_PRE_BASELINE_DELAY = 0.3
|
||||
|
||||
# 全局常驻网络监听(仅事件上报)
|
||||
GLOBAL_NETWORK_INTERCEPTION_ENABLED = False
|
||||
GLOBAL_NETWORK_INTERCEPTION_LISTEN_PATTERN = "http"
|
||||
GLOBAL_NETWORK_INTERCEPTION_WAIT_TIMEOUT = 0.5
|
||||
GLOBAL_NETWORK_INTERCEPTION_RETRY_DELAY = 1.0
|
||||
# 仅捕获的资源类型(逗号分隔;"*" 表示全部)。P0-4:默认只抓 API 类请求
|
||||
GLOBAL_NETWORK_INTERCEPTION_RES_TYPES = "XHR,Fetch,EventSource,Document"
|
||||
|
||||
# 网络解析调试捕获
|
||||
NETWORK_DEBUG_CAPTURE_ENABLED = False
|
||||
NETWORK_DEBUG_CAPTURE_MAX_BODY_CHARS = 50000
|
||||
NETWORK_DEBUG_CAPTURE_MAX_FILES_PER_REQUEST = 3
|
||||
NETWORK_DEBUG_CAPTURE_PARSER_FILTER = ""
|
||||
NETWORK_DEBUG_CAPTURE_MAX_TOTAL_MB = 50
|
||||
NETWORK_DEBUG_CAPTURE_RETENTION_HOURS = 24
|
||||
|
||||
# 对话会话控制
|
||||
CONVERSATION_TIMEOUT_THRESHOLD = 0.0
|
||||
FORCE_NEW_CONVERSATION = False
|
||||
|
||||
# 低资源运行
|
||||
REQUEST_MONITOR_ENABLED = True
|
||||
REQUEST_MONITOR_MAX_RECORDS = 200
|
||||
REQUEST_MONITOR_DETAIL_ENABLED = True
|
||||
REQUEST_MONITOR_SAVE_TO_FILE = True
|
||||
REQUEST_MONITOR_MAX_CAPTURED_RESPONSE_CHARS = 30000
|
||||
DASHBOARD_LOG_POLL_INTERVAL_MS = 1000
|
||||
DASHBOARD_LOG_BACKGROUND_POLL_INTERVAL_MS = 5000
|
||||
DASHBOARD_REQUEST_HISTORY_POLL_INTERVAL_MS = 3000
|
||||
DASHBOARD_SYSTEM_STATS_ENABLED = True
|
||||
DASHBOARD_SYSTEM_STATS_POLL_INTERVAL_MS = 3000
|
||||
|
||||
@classmethod
|
||||
def _load_config(cls):
|
||||
"""从文件加载配置"""
|
||||
if cls._config_file.exists():
|
||||
try:
|
||||
with open(cls._config_file, 'r', encoding='utf-8') as f:
|
||||
cls._config = json.load(f)
|
||||
return
|
||||
except Exception as e:
|
||||
print(f"[BrowserConstants] 加载配置失败: {e}")
|
||||
|
||||
# 加载失败或文件不存在,使用默认值
|
||||
cls._config = cls._DEFAULTS.copy()
|
||||
|
||||
@classmethod
|
||||
def _apply_to_class_attrs(cls):
|
||||
"""将配置值应用到类属性(兼容旧代码直接访问类属性的方式)"""
|
||||
if cls._config is None:
|
||||
cls._load_config()
|
||||
|
||||
for key, value in cls._config.items():
|
||||
if hasattr(cls, key):
|
||||
setattr(cls, key, value)
|
||||
|
||||
# 同步环境变量中的浏览器端口
|
||||
env_port = AppConfig.get_browser_port()
|
||||
if env_port:
|
||||
cls.DEFAULT_PORT = env_port
|
||||
|
||||
@classmethod
|
||||
def get(cls, key: str):
|
||||
"""获取配置值(支持动态加载)"""
|
||||
if cls._config is None:
|
||||
cls._load_config()
|
||||
|
||||
return cls._config.get(key, cls._DEFAULTS.get(key))
|
||||
|
||||
@classmethod
|
||||
def get_defaults(cls) -> Dict:
|
||||
"""获取所有默认值"""
|
||||
return cls._DEFAULTS.copy()
|
||||
|
||||
@classmethod
|
||||
def reload(cls):
|
||||
"""重新加载配置(热重载)"""
|
||||
cls._config = None
|
||||
cls._load_config()
|
||||
cls._apply_to_class_attrs()
|
||||
|
||||
|
||||
def _browser_constant_bool(key: str, default: bool = True) -> bool:
|
||||
value = BrowserConstants.get(key)
|
||||
if value is None:
|
||||
return bool(default)
|
||||
if isinstance(value, bool):
|
||||
return value
|
||||
if isinstance(value, (int, float)):
|
||||
return bool(value)
|
||||
text = str(value).strip().lower()
|
||||
if text in {"1", "true", "yes", "y", "on"}:
|
||||
return True
|
||||
if text in {"0", "false", "no", "n", "off"}:
|
||||
return False
|
||||
return bool(default)
|
||||
|
||||
|
||||
def _browser_constant_int(
|
||||
key: str,
|
||||
default: int,
|
||||
*,
|
||||
min_value: int = 0,
|
||||
max_value: Optional[int] = None,
|
||||
) -> int:
|
||||
try:
|
||||
value = int(BrowserConstants.get(key))
|
||||
except Exception:
|
||||
value = int(default)
|
||||
value = max(int(min_value), value)
|
||||
if max_value is not None:
|
||||
value = min(int(max_value), value)
|
||||
return value
|
||||
|
||||
|
||||
class _BrowserConstantEnabledFilter(logging.Filter):
|
||||
def __init__(self, key: str, default: bool = True):
|
||||
super().__init__()
|
||||
self.key = key
|
||||
self.default = default
|
||||
|
||||
def filter(self, record: logging.LogRecord) -> bool:
|
||||
return _browser_constant_bool(self.key, self.default)
|
||||
|
||||
|
||||
# 模块加载时执行自初始化
|
||||
BrowserConstants._load_config()
|
||||
BrowserConstants._apply_to_class_attrs()
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,364 @@
|
||||
"""
|
||||
app/core/config_parts/env_config.py - 环境变量与基础文件工具模块
|
||||
"""
|
||||
import os
|
||||
import json
|
||||
import tempfile
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
PROJECT_ROOT = Path(__file__).resolve().parents[3]
|
||||
DEFAULT_LOG_DIR = PROJECT_ROOT / "logs"
|
||||
|
||||
|
||||
def atomic_write_json(path: str | Path, payload: Any) -> None:
|
||||
"""Atomically write JSON to disk using a same-directory temporary file."""
|
||||
target = Path(path)
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
fd = None
|
||||
tmp_path: Optional[Path] = None
|
||||
|
||||
try:
|
||||
fd, tmp_name = tempfile.mkstemp(
|
||||
prefix=f".{target.name}.",
|
||||
suffix=".tmp",
|
||||
dir=str(target.parent),
|
||||
)
|
||||
tmp_path = Path(tmp_name)
|
||||
with os.fdopen(fd, "w", encoding="utf-8", newline="\n") as f:
|
||||
fd = None
|
||||
json.dump(payload, f, indent=2, ensure_ascii=False)
|
||||
f.write("\n")
|
||||
f.flush()
|
||||
os.fsync(f.fileno())
|
||||
_replace_file_with_retry(tmp_path, target)
|
||||
except Exception:
|
||||
if fd is not None:
|
||||
try:
|
||||
os.close(fd)
|
||||
except Exception:
|
||||
pass
|
||||
if tmp_path is not None:
|
||||
try:
|
||||
tmp_path.unlink(missing_ok=True)
|
||||
except Exception:
|
||||
pass
|
||||
raise
|
||||
|
||||
|
||||
def _replace_file_with_retry(source: str | Path, dest: str | Path) -> None:
|
||||
"""Replace a file, retrying transient Windows sharing violations."""
|
||||
source_path = Path(source)
|
||||
dest_path = Path(dest)
|
||||
attempts = 3 if os.name == "nt" else 1
|
||||
delay = 0.02
|
||||
for attempt in range(attempts):
|
||||
try:
|
||||
os.replace(source_path, dest_path)
|
||||
return
|
||||
except PermissionError:
|
||||
if attempt >= attempts - 1:
|
||||
raise
|
||||
time.sleep(delay)
|
||||
delay *= 2
|
||||
|
||||
|
||||
class classproperty:
|
||||
"""Allow config access via both `AppConfig.X` and `app_config.X`."""
|
||||
|
||||
def __init__(self, fget):
|
||||
self.fget = fget
|
||||
|
||||
def __get__(self, obj, owner=None):
|
||||
return self.fget(owner)
|
||||
|
||||
|
||||
def load_dotenv(env_file: str = ".env", override: bool = False):
|
||||
"""
|
||||
手动加载 .env 文件(不依赖 python-dotenv)
|
||||
"""
|
||||
env_path = Path(env_file)
|
||||
if not env_path.exists():
|
||||
return
|
||||
|
||||
try:
|
||||
with open(env_path, 'r', encoding='utf-8') as f:
|
||||
for line in f:
|
||||
line = line.strip()
|
||||
if not line or line.startswith('#'):
|
||||
continue
|
||||
if '=' in line:
|
||||
key, _, value = line.partition('=')
|
||||
key = key.strip()
|
||||
value = value.strip()
|
||||
if (value.startswith('"') and value.endswith('"')) or \
|
||||
(value.startswith("'") and value.endswith("'")):
|
||||
value = value[1:-1]
|
||||
if key:
|
||||
if override or key not in os.environ:
|
||||
os.environ[key] = value
|
||||
except Exception as e:
|
||||
print(f"[Config] 加载 .env 失败: {e}")
|
||||
|
||||
|
||||
load_dotenv(override=os.getenv("UWAPI_DOTENV_OVERRIDE", "").lower() in ("1", "true", "yes", "on"))
|
||||
|
||||
|
||||
class AppConfig:
|
||||
"""应用配置(从环境变量读取)"""
|
||||
|
||||
@staticmethod
|
||||
def _env_bool(name: str, default: bool = False) -> bool:
|
||||
value = os.getenv(name)
|
||||
if value is None or str(value).strip() == "":
|
||||
return bool(default)
|
||||
return str(value).strip().lower() in ("true", "1", "yes", "on")
|
||||
|
||||
@staticmethod
|
||||
def _env_bool_secure(name: str, default: bool = False) -> bool:
|
||||
"""安全开关专用布尔解析(H1):值存在但不是合法布尔字面量时 fail-closed 返回 True。
|
||||
|
||||
兼容逻辑:若值为 3.0.0 模板自带的占位符 ``your-secret-here``,视为未设置(回退 default),
|
||||
避免因无令牌 fail-closed 开启认证导致接口与控制面板 500。
|
||||
"""
|
||||
value = os.getenv(name)
|
||||
if value is None or str(value).strip() == "":
|
||||
return bool(default)
|
||||
lowered = str(value).strip().lower()
|
||||
if lowered == "your-secret-here":
|
||||
return bool(default)
|
||||
if lowered in ("true", "1", "yes", "on"):
|
||||
return True
|
||||
if lowered in ("false", "0", "no", "off"):
|
||||
return False
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
def _env_int(name: str, default: int) -> int:
|
||||
"""安全读取整数环境变量:空值/非法值一律回落默认值,避免服务启动期崩溃。"""
|
||||
raw = os.getenv(name)
|
||||
if raw is None or str(raw).strip() == "":
|
||||
return int(default)
|
||||
try:
|
||||
return int(str(raw).strip())
|
||||
except (TypeError, ValueError):
|
||||
return int(default)
|
||||
|
||||
@staticmethod
|
||||
def _env_float(name: str, default: float) -> float:
|
||||
"""安全读取浮点环境变量:空值/非法值一律回落默认值,避免服务启动期崩溃。"""
|
||||
raw = os.getenv(name)
|
||||
if raw is None or str(raw).strip() == "":
|
||||
return float(default)
|
||||
try:
|
||||
return float(str(raw).strip())
|
||||
except (TypeError, ValueError):
|
||||
return float(default)
|
||||
|
||||
# ===== 服务配置 =====
|
||||
@staticmethod
|
||||
def get_host() -> str:
|
||||
return os.getenv("APP_HOST", "127.0.0.1")
|
||||
|
||||
@staticmethod
|
||||
def get_port() -> int:
|
||||
return AppConfig._env_int("APP_PORT", 8199)
|
||||
|
||||
@staticmethod
|
||||
def is_debug() -> bool:
|
||||
return os.getenv("APP_DEBUG", "false").lower() in ("true", "1", "yes")
|
||||
|
||||
@staticmethod
|
||||
def get_log_level() -> str:
|
||||
return os.getenv("LOG_LEVEL", "INFO").upper()
|
||||
|
||||
# ===== 认证配置 =====
|
||||
@staticmethod
|
||||
def is_auth_enabled() -> bool:
|
||||
return AppConfig._env_bool_secure("AUTH_ENABLED", False)
|
||||
|
||||
@staticmethod
|
||||
def get_auth_token() -> str:
|
||||
return os.getenv("AUTH_TOKEN", "").strip()
|
||||
|
||||
@staticmethod
|
||||
def is_dashboard_auth_enabled() -> bool:
|
||||
value = os.getenv("DASHBOARD_AUTH_ENABLED")
|
||||
if value is None or str(value).strip() == "":
|
||||
return AppConfig.is_auth_enabled()
|
||||
return AppConfig._env_bool_secure("DASHBOARD_AUTH_ENABLED", False)
|
||||
|
||||
@staticmethod
|
||||
def get_dashboard_auth_token() -> str:
|
||||
token = os.getenv("DASHBOARD_AUTH_TOKEN", "").strip()
|
||||
return token or AppConfig.get_auth_token()
|
||||
|
||||
# ===== CORS 配置 =====
|
||||
@staticmethod
|
||||
def is_cors_enabled() -> bool:
|
||||
return os.getenv("CORS_ENABLED", "true").lower() in ("true", "1", "yes")
|
||||
|
||||
@staticmethod
|
||||
def get_cors_origins() -> List[str]:
|
||||
"""S1:默认不放行任何跨源来源。
|
||||
|
||||
控制面板与 API 同源,本来就不需要 CORS;非浏览器客户端(SDK、curl、酒馆后端)
|
||||
不受 CORS 影响。需要浏览器跨源调用时请显式列出来源,``*`` 仍可显式配置但不推荐。
|
||||
"""
|
||||
from app.core.http_security import parse_cors_origins
|
||||
|
||||
return parse_cors_origins(os.getenv("CORS_ORIGINS", ""))
|
||||
|
||||
# ===== 浏览器配置 =====
|
||||
@staticmethod
|
||||
def get_browser_port() -> int:
|
||||
return AppConfig._env_int("BROWSER_PORT", 9222)
|
||||
|
||||
@staticmethod
|
||||
def is_auto_open_browser_enabled() -> bool:
|
||||
return AppConfig._env_bool("AUTO_OPEN_BROWSER", True)
|
||||
|
||||
# ===== Dashboard 配置 =====
|
||||
@staticmethod
|
||||
def is_dashboard_enabled() -> bool:
|
||||
return os.getenv("DASHBOARD_ENABLED", "true").lower() in ("true", "1", "yes")
|
||||
|
||||
@staticmethod
|
||||
def get_dashboard_file() -> str:
|
||||
return os.getenv("DASHBOARD_FILE", "static/index.html")
|
||||
|
||||
# ===== 定时服务重启守护 =====
|
||||
@staticmethod
|
||||
def is_scheduled_restart_enabled() -> bool:
|
||||
return AppConfig._env_bool("SCHEDULED_RESTART_ENABLED", False)
|
||||
|
||||
@staticmethod
|
||||
def get_scheduled_restart_interval_seconds() -> int:
|
||||
return max(60, AppConfig._env_int("SCHEDULED_RESTART_INTERVAL_SECONDS", 10800))
|
||||
|
||||
@staticmethod
|
||||
def get_scheduled_restart_drain_timeout_seconds() -> int:
|
||||
return max(0, AppConfig._env_int("SCHEDULED_RESTART_DRAIN_TIMEOUT_SECONDS", 1800))
|
||||
|
||||
@staticmethod
|
||||
def get_scheduled_restart_tab_state_policy() -> str:
|
||||
"""Reserved for future tab-state cleanup; preserve is currently the only policy."""
|
||||
return os.getenv("SCHEDULED_RESTART_TAB_STATE_POLICY", "preserve").strip().lower() or "preserve"
|
||||
|
||||
# ===== AI 分析配置 =====
|
||||
@staticmethod
|
||||
def get_helper_api_key() -> str:
|
||||
return os.getenv("HELPER_API_KEY", "")
|
||||
|
||||
@staticmethod
|
||||
def get_helper_base_url() -> str:
|
||||
return os.getenv("HELPER_BASE_URL", "")
|
||||
|
||||
@staticmethod
|
||||
def get_helper_model() -> str:
|
||||
return os.getenv("HELPER_MODEL", "gpt-4")
|
||||
|
||||
@staticmethod
|
||||
def get_helper_api_provider() -> str:
|
||||
return os.getenv("HELPER_API_PROVIDER", "auto").lower()
|
||||
|
||||
@staticmethod
|
||||
def get_max_html_chars() -> int:
|
||||
return AppConfig._env_int("MAX_HTML_CHARS", 120000)
|
||||
|
||||
@staticmethod
|
||||
def get_canvas_image_max_size() -> int:
|
||||
try:
|
||||
value = AppConfig._env_int("CANVAS_IMAGE_MAX_SIZE", 1024)
|
||||
except Exception:
|
||||
value = 1024
|
||||
return max(1, value)
|
||||
|
||||
# ===== 错误标签页 AI 恢复配置 =====
|
||||
@staticmethod
|
||||
def is_tab_recovery_enabled() -> bool:
|
||||
return AppConfig._env_bool("TAB_RECOVERY_ENABLED", False)
|
||||
|
||||
@staticmethod
|
||||
def get_tab_recovery_api_url() -> str:
|
||||
return os.getenv("TAB_RECOVERY_API_URL", "").strip()
|
||||
|
||||
@staticmethod
|
||||
def get_tab_recovery_api_key() -> str:
|
||||
return os.getenv("TAB_RECOVERY_API_KEY", "").strip()
|
||||
|
||||
@staticmethod
|
||||
def get_tab_recovery_model() -> str:
|
||||
return os.getenv("TAB_RECOVERY_MODEL", "gpt-4o").strip() or "gpt-4o"
|
||||
|
||||
@staticmethod
|
||||
def get_tab_recovery_max_attempts() -> int:
|
||||
try:
|
||||
value = AppConfig._env_int("TAB_RECOVERY_MAX_ATTEMPTS", 1)
|
||||
except Exception:
|
||||
value = 1
|
||||
return max(0, value)
|
||||
|
||||
@staticmethod
|
||||
def get_tab_recovery_timeout_sec() -> float:
|
||||
try:
|
||||
value = AppConfig._env_float("TAB_RECOVERY_TIMEOUT_SEC", 120.0)
|
||||
except Exception:
|
||||
value = 120.0
|
||||
return max(1.0, value)
|
||||
|
||||
@staticmethod
|
||||
def get_tab_recovery_worker_exit_wait_sec() -> float:
|
||||
try:
|
||||
value = AppConfig._env_float("TAB_RECOVERY_WORKER_EXIT_WAIT_SEC", 600.0)
|
||||
except Exception:
|
||||
value = 600.0
|
||||
return max(0.0, value)
|
||||
|
||||
@staticmethod
|
||||
def is_tab_recovery_refresh_on_unknown() -> bool:
|
||||
return AppConfig._env_bool("TAB_RECOVERY_REFRESH_ON_UNKNOWN", True)
|
||||
|
||||
# ===== 配置文件路径 =====
|
||||
@staticmethod
|
||||
def get_sites_config_file() -> str:
|
||||
return os.getenv("SITES_CONFIG_FILE", "config/sites.json")
|
||||
|
||||
# ===== 便捷属性(支持类/实例两种访问方式)=====
|
||||
@classproperty
|
||||
def HOST(cls) -> str:
|
||||
return cls.get_host()
|
||||
|
||||
@classproperty
|
||||
def PORT(cls) -> int:
|
||||
return cls.get_port()
|
||||
|
||||
@classproperty
|
||||
def DEBUG(cls) -> bool:
|
||||
return cls.is_debug()
|
||||
|
||||
@classproperty
|
||||
def LOG_LEVEL(cls) -> str:
|
||||
return cls.get_log_level()
|
||||
|
||||
@classproperty
|
||||
def AUTH_TOKEN(cls) -> str:
|
||||
return cls.get_auth_token()
|
||||
|
||||
@classproperty
|
||||
def DASHBOARD_AUTH_TOKEN(cls) -> str:
|
||||
return cls.get_dashboard_auth_token()
|
||||
|
||||
|
||||
# 创建全局配置实例
|
||||
app_config = AppConfig()
|
||||
|
||||
|
||||
def get_tutorial_selectors_url() -> str:
|
||||
"""返回本地控制台教程中选择器配置章节的跳转链接。"""
|
||||
try:
|
||||
port = AppConfig.get_port()
|
||||
except Exception:
|
||||
port = 8199
|
||||
return f"http://127.0.0.1:{port}/static/tutorial/index.html#selectors"
|
||||
@@ -0,0 +1,33 @@
|
||||
"""
|
||||
app/core/config_parts/exceptions.py - 异常定义模块
|
||||
"""
|
||||
|
||||
|
||||
class BrowserError(Exception):
|
||||
"""浏览器相关错误基类"""
|
||||
pass
|
||||
|
||||
|
||||
class BrowserConnectionError(BrowserError):
|
||||
"""浏览器连接错误"""
|
||||
pass
|
||||
|
||||
|
||||
class ElementNotFoundError(BrowserError):
|
||||
"""元素未找到错误"""
|
||||
pass
|
||||
|
||||
|
||||
class WorkflowError(BrowserError):
|
||||
"""工作流执行错误"""
|
||||
pass
|
||||
|
||||
|
||||
class WorkflowCancelledError(WorkflowError):
|
||||
"""工作流被取消"""
|
||||
pass
|
||||
|
||||
|
||||
class ConfigurationError(BrowserError):
|
||||
"""配置错误"""
|
||||
pass
|
||||
@@ -0,0 +1,90 @@
|
||||
"""
|
||||
app/core/config_parts/log_collector.py - Web 前端日志收集器模块
|
||||
"""
|
||||
from collections import deque
|
||||
import threading
|
||||
import time
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
from .browser_constants import _browser_constant_bool, _browser_constant_int
|
||||
|
||||
|
||||
class LogCollector:
|
||||
"""收集日志用于前端展示"""
|
||||
|
||||
def __init__(self, max_logs: int = 5000):
|
||||
self.logs: deque = deque(maxlen=max_logs)
|
||||
self.lock = threading.Lock()
|
||||
self._next_seq = 1
|
||||
self._last_clear_seq = 0
|
||||
|
||||
def is_enabled(self) -> bool:
|
||||
return _browser_constant_bool("LOG_WEB_COLLECTOR_ENABLED", True)
|
||||
|
||||
def _target_max_logs(self) -> int:
|
||||
return _browser_constant_int("LOG_WEB_MAX_RECORDS", 5000, min_value=0, max_value=10000)
|
||||
|
||||
def _sync_limits_unlocked(self) -> None:
|
||||
target_max_logs = self._target_max_logs()
|
||||
if self.logs.maxlen == target_max_logs:
|
||||
return
|
||||
self.logs = deque(list(self.logs)[-target_max_logs:] if target_max_logs else [], maxlen=target_max_logs)
|
||||
|
||||
def add(self, entry: Dict[str, Any]):
|
||||
with self.lock:
|
||||
self._sync_limits_unlocked()
|
||||
if not self.is_enabled() or self.logs.maxlen == 0:
|
||||
if self.logs:
|
||||
self.logs.clear()
|
||||
return
|
||||
payload = dict(entry or {})
|
||||
payload["seq"] = self._next_seq
|
||||
payload.setdefault("timestamp", time.time())
|
||||
payload.setdefault("level", "INFO")
|
||||
payload.setdefault("kind", payload["level"])
|
||||
payload.setdefault("message", "")
|
||||
payload.setdefault("display_message", payload["message"])
|
||||
payload.setdefault("message_text", payload["message"])
|
||||
payload.setdefault("original_message_text", payload["message_text"])
|
||||
payload.setdefault("message_alias", "")
|
||||
payload.setdefault("logger", "")
|
||||
payload.setdefault("request_id", "SYSTEM")
|
||||
self.logs.append(payload)
|
||||
self._next_seq += 1
|
||||
|
||||
def get_recent(self, since: float = 0, after_seq: int = 0) -> Tuple[List[Dict[str, Any]], int, bool]:
|
||||
with self.lock:
|
||||
self._sync_limits_unlocked()
|
||||
if not self.is_enabled() or self.logs.maxlen == 0:
|
||||
if self.logs:
|
||||
self.logs.clear()
|
||||
return [], self._next_seq - 1, False
|
||||
cursor = max(0, int(after_seq or 0))
|
||||
cleared = bool(cursor and self._last_clear_seq and cursor <= self._last_clear_seq)
|
||||
if cursor > 0:
|
||||
latest_seq = self._next_seq - 1
|
||||
if cursor == latest_seq:
|
||||
return [], latest_seq, cleared
|
||||
if cursor > latest_seq:
|
||||
return list(self.logs), latest_seq, True
|
||||
|
||||
oldest_seq = int(self.logs[0].get("seq", 0) or 0) if self.logs else latest_seq
|
||||
if self.logs and cursor < oldest_seq:
|
||||
return list(self.logs), latest_seq, True
|
||||
|
||||
logs = [
|
||||
log for log in self.logs
|
||||
if int(log.get("seq", 0) or 0) > cursor
|
||||
]
|
||||
else:
|
||||
logs = [log for log in self.logs if float(log.get("timestamp", 0) or 0) > since]
|
||||
return logs, self._next_seq - 1, cleared
|
||||
|
||||
def clear(self):
|
||||
with self.lock:
|
||||
self.logs.clear()
|
||||
self._last_clear_seq = self._next_seq - 1
|
||||
self._next_seq += 1
|
||||
|
||||
|
||||
# 全局日志收集器实例
|
||||
log_collector = LogCollector()
|
||||
@@ -0,0 +1,652 @@
|
||||
"""
|
||||
app/core/config_parts/log_formatters.py - 日志格式化器与 Handler 管理模块
|
||||
"""
|
||||
import ctypes
|
||||
import datetime
|
||||
import logging
|
||||
from logging.handlers import RotatingFileHandler
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
from typing import Any, List, Optional, Tuple
|
||||
|
||||
from .env_config import PROJECT_ROOT, DEFAULT_LOG_DIR
|
||||
from .browser_constants import _BrowserConstantEnabledFilter
|
||||
from .log_collector import log_collector
|
||||
from .log_redaction import _sanitize_sensitive_text
|
||||
|
||||
_shared_file_log_handler: Optional[logging.Handler] = None
|
||||
_shared_file_log_handler_lock = threading.Lock()
|
||||
|
||||
|
||||
def _get_positive_int_env(name: str, default: int) -> int:
|
||||
raw = str(os.getenv(name, "") or "").strip()
|
||||
if not raw:
|
||||
return default
|
||||
try:
|
||||
value = int(raw)
|
||||
except (TypeError, ValueError, OverflowError):
|
||||
return default
|
||||
return value if value > 0 else default
|
||||
|
||||
|
||||
def _truncate_long_message(text: str, max_chars: int) -> tuple[str, bool]:
|
||||
raw_text = str(text or "")
|
||||
if max_chars <= 0 or len(raw_text) <= max_chars:
|
||||
return raw_text, False
|
||||
omitted = len(raw_text) - max_chars
|
||||
return (
|
||||
f"{raw_text[:max_chars]}... [truncated {omitted} chars; see app.log for full sanitized text]",
|
||||
True,
|
||||
)
|
||||
|
||||
|
||||
def _get_log_display_limit(env_name: str, default: int) -> int:
|
||||
return _get_positive_int_env(env_name, default)
|
||||
|
||||
|
||||
def _record_request_id(record: logging.LogRecord) -> str:
|
||||
return str(getattr(record, "codex_request_id", "") or "SYSTEM")
|
||||
|
||||
|
||||
_REQUEST_SHORT_ID_PATTERN = re.compile(r"^req-(\d+)$", re.IGNORECASE)
|
||||
|
||||
|
||||
def _request_display_tag(request_id: Any) -> str:
|
||||
raw = str(request_id or "").strip()
|
||||
if not raw or raw.upper() == "SYSTEM":
|
||||
return "SYSTEM"
|
||||
|
||||
match = _REQUEST_SHORT_ID_PATTERN.match(raw)
|
||||
if match:
|
||||
try:
|
||||
return f"#{int(match.group(1)):03d}"
|
||||
except Exception:
|
||||
return f"#{match.group(1)}"
|
||||
|
||||
return raw if len(raw) <= 8 else f"#{raw[-6:]}"
|
||||
|
||||
|
||||
def _record_request_tag(record: logging.LogRecord) -> str:
|
||||
return _request_display_tag(_record_request_id(record))
|
||||
|
||||
|
||||
def _compact_logger_name_impl(name: Any, max_chars: int = 16) -> str:
|
||||
raw = str(name or "").strip().upper()
|
||||
if not raw:
|
||||
return ""
|
||||
if len(raw) <= max_chars:
|
||||
return raw
|
||||
|
||||
def cap(label: str) -> str:
|
||||
label = str(label or "")
|
||||
if len(label) <= max_chars:
|
||||
return label
|
||||
if max_chars <= 3:
|
||||
return label[:max_chars]
|
||||
head = max(1, (max_chars - 1) // 2)
|
||||
tail = max(1, max_chars - head - 1)
|
||||
return f"{label[:head]}~{label[-tail:]}"
|
||||
|
||||
parts = [part for part in raw.split(".") if part]
|
||||
if len(parts) > 1:
|
||||
prefix = ".".join(part[:1] for part in parts[:-1] if part)
|
||||
tail = parts[-1]
|
||||
candidate = f"{prefix}.{tail}" if prefix else tail
|
||||
if len(candidate) <= max_chars:
|
||||
return candidate
|
||||
|
||||
tail_budget = max(3, max_chars - len(prefix) - (1 if prefix else 0))
|
||||
short_tail = tail[-tail_budget:]
|
||||
return cap(f"{prefix}.{short_tail}" if prefix else short_tail)
|
||||
|
||||
if max_chars <= 3:
|
||||
return raw[:max_chars]
|
||||
return cap(raw)
|
||||
|
||||
|
||||
def _compact_logger_name(name: Any, max_chars: int = 16) -> str:
|
||||
res = _compact_logger_name_impl(name, max_chars)
|
||||
if len(res) <= max_chars:
|
||||
return res
|
||||
if max_chars <= 3:
|
||||
return res[:max_chars]
|
||||
head = max(1, (max_chars - 1) // 2)
|
||||
tail = max(1, max_chars - head - 1)
|
||||
return f"{res[:head]}~{res[-tail:]}"
|
||||
|
||||
|
||||
_LOGGER_DISPLAY_WIDTH = 12
|
||||
|
||||
|
||||
def _record_logger_name(record: logging.LogRecord) -> str:
|
||||
return _compact_logger_name(
|
||||
getattr(record, "codex_logger_name", "") or getattr(record, "name", "") or "",
|
||||
max_chars=_LOGGER_DISPLAY_WIDTH,
|
||||
)
|
||||
|
||||
|
||||
def _record_kind(record: logging.LogRecord) -> str:
|
||||
return str(getattr(record, "codex_kind", "") or record.levelname or "INFO").upper()
|
||||
|
||||
|
||||
# 级别徽标:控制台/展示行使用的 3 字符级别标识
|
||||
_LEVEL_BADGES = {
|
||||
"DEBUG": "DBG",
|
||||
"INFO": "INF",
|
||||
"WARNING": "WRN",
|
||||
"WARN": "WRN",
|
||||
"ERROR": "ERR",
|
||||
"CRITICAL": "CRI",
|
||||
"SUCCESS": "SUC",
|
||||
"STREAM": "STM",
|
||||
"NETWORK": "NET",
|
||||
}
|
||||
|
||||
|
||||
def _record_level_badge(record: logging.LogRecord) -> str:
|
||||
kind = _record_kind(record)
|
||||
if kind in _LEVEL_BADGES:
|
||||
return _LEVEL_BADGES[kind]
|
||||
level = str(record.levelname or "").upper()
|
||||
return _LEVEL_BADGES.get(level, (kind[:3] or "LOG").ljust(3))
|
||||
|
||||
|
||||
def _replace_log_tag(text: str, source_tag: str, target_tag: str) -> str:
|
||||
if text.startswith(source_tag):
|
||||
rest = text[len(source_tag):].lstrip()
|
||||
return f"{target_tag} {rest}".rstrip()
|
||||
return text
|
||||
|
||||
|
||||
def _normalize_log_display_expression(logger_name: str, message: str) -> str:
|
||||
"""Normalize legacy log expressions for display without changing raw logs."""
|
||||
text = str(message or "")
|
||||
if not text:
|
||||
return text
|
||||
|
||||
input_tags = (
|
||||
"[FILE_PASTE]",
|
||||
"[CHUNKED_INPUT]",
|
||||
"[CLIPBOARD_OK]",
|
||||
"[VERIFY_OK]",
|
||||
"[VERIFY_FAIL]",
|
||||
"[VERIFY]",
|
||||
"[INPUT_SNAPSHOT]",
|
||||
"[STEALTH_VERIFY]",
|
||||
)
|
||||
for tag in input_tags:
|
||||
normalized = _replace_log_tag(text, tag, "[INPUT]")
|
||||
if normalized != text:
|
||||
return normalized
|
||||
|
||||
for tag in ("[NetworkMonitor]", "[NETWORK_MONITOR]"):
|
||||
normalized = _replace_log_tag(text, tag, "[MONIT]")
|
||||
if normalized != text:
|
||||
return normalized
|
||||
|
||||
for tag in ("[JS_EXEC]", "[CONTENT_PARSE]", "[PROBE]", "[IMAGE]", "[STEALTH_CLICK]", "[STEALTH]"):
|
||||
normalized = _replace_log_tag(text, tag, "[PAGE]")
|
||||
if normalized != text:
|
||||
return normalized
|
||||
|
||||
if text.startswith("[REQUEST_TRANSPORT]"):
|
||||
return _replace_log_tag(text, "[REQUEST_TRANSPORT]", "[ROUTE]")
|
||||
|
||||
if text.startswith("[Executor]"):
|
||||
rest = text[len("[Executor]"):].lstrip()
|
||||
target = "[MONIT]" if any(hint in rest for hint in ("抓流", "监听", "网络")) else "[PAGE]"
|
||||
return f"{target} {rest}".rstrip()
|
||||
|
||||
if text.startswith("[TabPool]"):
|
||||
return _replace_log_tag(text, "[TabPool]", "[POOL]")
|
||||
|
||||
tabpool_match = re.match(r"^TabPool\s*(?:→|->)\s*(.+)$", text, re.S)
|
||||
if tabpool_match:
|
||||
return f"[POOL] 标签页已被占用: tab_id={tabpool_match.group(1)}"
|
||||
|
||||
if text.startswith("TabPoolManager "):
|
||||
return f"[POOL] {text}"
|
||||
|
||||
wait_done_match = re.match(r"^等待结束\s*(?:→|->)\s*(.+)$", text, re.S)
|
||||
if wait_done_match:
|
||||
return f"[POOL] 标签页等待结束: {wait_done_match.group(1)}"
|
||||
|
||||
if text.startswith("排队等待 "):
|
||||
return f"[POOL] {text}"
|
||||
|
||||
if text.startswith("等待标签页 ") or text.startswith("等待域名路由 "):
|
||||
return f"[POOL] {text}"
|
||||
|
||||
assign_match = re.match(r"^标签页 (.+) 分配编号 #(\d+)$", text)
|
||||
if assign_match:
|
||||
session_id, index_no = assign_match.groups()
|
||||
return f"[POOL] 标签页分配编号: tab_id={session_id}, idx=#{index_no}"
|
||||
|
||||
if text.startswith("发送成功"):
|
||||
return f"[SEND] {text}"
|
||||
|
||||
if text == "浏览器连接成功" or text == "关闭浏览器连接":
|
||||
return f"[SYS] {text}"
|
||||
|
||||
if logger_name == "REQUEST" and text == "创建":
|
||||
return "[ROUTE] 请求上下文已创建"
|
||||
|
||||
if logger_name == "API.CHAT" and text == "开始":
|
||||
return "[ROUTE] 聊天补全请求开始处理"
|
||||
|
||||
return text
|
||||
|
||||
|
||||
def _resolve_pending_display_text(record: logging.LogRecord) -> None:
|
||||
"""SecureLogger 只标记 codex_display_pending;第一次渲染时才做喵化翻译并缓存。"""
|
||||
if not getattr(record, "codex_display_pending", False):
|
||||
return
|
||||
record.codex_display_pending = False
|
||||
original = str(getattr(record, "codex_original_message_text", "") or "")
|
||||
try:
|
||||
from .cute_translator import cuteify_display_message
|
||||
|
||||
record.codex_display_message_text = cuteify_display_message(
|
||||
str(getattr(record, "codex_kind", "") or record.levelname or ""),
|
||||
str(getattr(record, "codex_logger_name", "") or record.name or ""),
|
||||
original,
|
||||
)
|
||||
except Exception:
|
||||
record.codex_display_message_text = original
|
||||
|
||||
|
||||
def _record_display_message(record: logging.LogRecord) -> str:
|
||||
_resolve_pending_display_text(record)
|
||||
message = str(getattr(record, "codex_display_message_text", "") or "")
|
||||
if not message:
|
||||
message = str(record.getMessage() or "")
|
||||
original_message = str(getattr(record, "codex_original_message_text", "") or "")
|
||||
if message == original_message or not original_message:
|
||||
message = _normalize_log_display_expression(_record_logger_name(record), message)
|
||||
return _sanitize_sensitive_text(message)
|
||||
|
||||
|
||||
def _format_log_display_parts(
|
||||
record: logging.LogRecord,
|
||||
message: str,
|
||||
*,
|
||||
max_chars: int = 0,
|
||||
) -> tuple[list[str], str, bool]:
|
||||
"""构建展示行的各个片段。
|
||||
|
||||
返回 (prefix_parts, body, truncated):
|
||||
- prefix_parts = [时间, 级别徽标, 请求标签, 日志器名],便于控制台按片段分别上色;
|
||||
- body 已按前缀宽度做了多行缩进对齐。
|
||||
"""
|
||||
now = datetime.datetime.fromtimestamp(
|
||||
float(getattr(record, "created", time.time()) or time.time())
|
||||
).strftime("%H:%M:%S")
|
||||
badge = _record_level_badge(record)
|
||||
request_tag = _record_request_tag(record)
|
||||
logger_name = _record_logger_name(record)
|
||||
parts = [
|
||||
now,
|
||||
f"{badge:<3}",
|
||||
f"{request_tag:<8}",
|
||||
f"{logger_name:<{_LOGGER_DISPLAY_WIDTH}}",
|
||||
]
|
||||
prefix_width = sum(len(part) for part in parts) + 3 * len(parts)
|
||||
body, truncated = _truncate_long_message(str(message or ""), max_chars)
|
||||
body = body.replace("\n", "\n" + " " * prefix_width)
|
||||
return parts, body, truncated
|
||||
|
||||
|
||||
def _join_log_display_parts(parts: list[str], body: str) -> str:
|
||||
return " │ ".join(parts) + " │ " + body
|
||||
|
||||
|
||||
def _format_log_display_line(
|
||||
record: logging.LogRecord,
|
||||
message: str,
|
||||
*,
|
||||
max_chars: int = 0,
|
||||
) -> tuple[str, bool]:
|
||||
parts, body, truncated = _format_log_display_parts(record, message, max_chars=max_chars)
|
||||
return _join_log_display_parts(parts, body), truncated
|
||||
|
||||
|
||||
class _WebLogHandler(logging.Handler):
|
||||
"""将日志发送到 Web 收集器(内部类)"""
|
||||
|
||||
def emit(self, record):
|
||||
try:
|
||||
if not log_collector.is_enabled():
|
||||
return
|
||||
raw_message = _sanitize_sensitive_text(str(getattr(record, "codex_message", "") or ""))
|
||||
if not raw_message:
|
||||
raw_message = _sanitize_sensitive_text(str(record.getMessage() or ""))
|
||||
message_text = _record_display_message(record)
|
||||
original_message_text = _sanitize_sensitive_text(str(
|
||||
getattr(record, "codex_original_message_text", "") or raw_message
|
||||
))
|
||||
if not message_text:
|
||||
message_text = raw_message
|
||||
web_limit = _get_log_display_limit("LOG_WEB_MAX_CHARS", 2000)
|
||||
message_text, message_truncated = _truncate_long_message(message_text, web_limit)
|
||||
original_message_text, original_truncated = _truncate_long_message(
|
||||
original_message_text,
|
||||
web_limit,
|
||||
)
|
||||
msg, line_truncated = _format_log_display_line(
|
||||
record,
|
||||
message_text,
|
||||
)
|
||||
logger_name = _record_logger_name(record)
|
||||
request_id = _record_request_id(record)
|
||||
request_tag = _record_request_tag(record)
|
||||
kind = _record_kind(record)
|
||||
log_collector.add({
|
||||
"timestamp": float(getattr(record, "created", time.time()) or time.time()),
|
||||
"level": str(record.levelname or "INFO").upper(),
|
||||
"kind": kind,
|
||||
"message": msg,
|
||||
"display_message": msg,
|
||||
"message_text": message_text,
|
||||
"original_message_text": original_message_text,
|
||||
"message_alias": message_text if message_text != original_message_text else "",
|
||||
"logger": logger_name,
|
||||
"request_id": request_id,
|
||||
"request_tag": request_tag,
|
||||
"truncated": bool(message_truncated or original_truncated or line_truncated),
|
||||
})
|
||||
except Exception:
|
||||
self.handleError(record)
|
||||
|
||||
|
||||
def _enable_windows_ansi() -> bool:
|
||||
"""在 Windows 控制台中尽量启用 ANSI 颜色支持。"""
|
||||
if os.name != "nt":
|
||||
return True
|
||||
|
||||
try:
|
||||
kernel32 = ctypes.windll.kernel32
|
||||
handle = kernel32.GetStdHandle(-11) # STD_OUTPUT_HANDLE
|
||||
if handle in (0, -1):
|
||||
return False
|
||||
|
||||
mode = ctypes.c_uint()
|
||||
if kernel32.GetConsoleMode(handle, ctypes.byref(mode)) == 0:
|
||||
return False
|
||||
|
||||
ENABLE_VIRTUAL_TERMINAL_PROCESSING = 0x0004
|
||||
if mode.value & ENABLE_VIRTUAL_TERMINAL_PROCESSING:
|
||||
return True
|
||||
|
||||
return kernel32.SetConsoleMode(
|
||||
handle,
|
||||
mode.value | ENABLE_VIRTUAL_TERMINAL_PROCESSING
|
||||
) != 0
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def _should_use_console_color() -> bool:
|
||||
"""判断当前控制台是否应启用 ANSI 颜色。"""
|
||||
if os.environ.get("NO_COLOR"):
|
||||
return False
|
||||
|
||||
if os.name == "nt":
|
||||
if _enable_windows_ansi():
|
||||
return True
|
||||
|
||||
if os.environ.get("WT_SESSION"):
|
||||
return True
|
||||
if os.environ.get("ANSICON"):
|
||||
return True
|
||||
if os.environ.get("ConEmuANSI", "").upper() == "ON":
|
||||
return True
|
||||
if os.environ.get("TERM_PROGRAM") == "vscode":
|
||||
return True
|
||||
return False
|
||||
|
||||
return bool(getattr(sys.stdout, "isatty", lambda: False)())
|
||||
|
||||
|
||||
class _ConsoleColorFormatter(logging.Formatter):
|
||||
"""仅用于控制台输出的彩色格式化器。"""
|
||||
|
||||
RESET = "\033[0m"
|
||||
# 整行色调
|
||||
LINE_TONES = {
|
||||
"ERROR": "\033[31m",
|
||||
"WARN": "\033[33m",
|
||||
"KEY": "\033[94m",
|
||||
"SUCCESS": "\033[92m",
|
||||
"DEBUG": "\033[90m",
|
||||
}
|
||||
# 级别徽标色
|
||||
BADGE_TONES = {
|
||||
"INF": "\033[32m",
|
||||
"STM": "\033[36m",
|
||||
"NET": "\033[36m",
|
||||
"SUC": "\033[92m",
|
||||
"WRN": "\033[1;33m",
|
||||
"ERR": "\033[1;31m",
|
||||
"CRI": "\033[1;31m",
|
||||
"DBG": "\033[90m",
|
||||
}
|
||||
KEY_PATTERNS = (
|
||||
"[CMD] ▶ 执行:",
|
||||
"[CMD] 执行:",
|
||||
"[CMD] 开始执行工作流:",
|
||||
"[CMD] 触发命令:",
|
||||
"[CMD] 链式触发:",
|
||||
"[CMD] 条件分支触发:",
|
||||
"[CMD] 结果事件触发:",
|
||||
)
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self._use_color = _should_use_console_color()
|
||||
|
||||
def _resolve_tone(self, record: logging.LogRecord, message: str) -> Optional[str]:
|
||||
level = str(record.levelname or "").upper()
|
||||
if level in ("ERROR", "CRITICAL"):
|
||||
return "ERROR"
|
||||
if level == "WARNING":
|
||||
return "WARN"
|
||||
if level == "DEBUG":
|
||||
return "DEBUG"
|
||||
if any(pattern in message for pattern in self.KEY_PATTERNS):
|
||||
return "KEY"
|
||||
if _record_kind(record) == "SUCCESS":
|
||||
return "SUCCESS"
|
||||
return None
|
||||
|
||||
def format(self, record: logging.LogRecord) -> str:
|
||||
message = _record_display_message(record)
|
||||
console_limit = _get_log_display_limit("LOG_CONSOLE_MAX_CHARS", 1200)
|
||||
message, _ = _truncate_long_message(message, console_limit)
|
||||
parts, body, _ = _format_log_display_parts(record, message)
|
||||
formatted = _join_log_display_parts(parts, body)
|
||||
if record.exc_info:
|
||||
exc_text = self.formatException(record.exc_info)
|
||||
if exc_text:
|
||||
formatted = f"{formatted}\n{exc_text}"
|
||||
|
||||
if not self._use_color:
|
||||
return formatted
|
||||
|
||||
tone = self._resolve_tone(record, formatted)
|
||||
if tone:
|
||||
color = self.LINE_TONES.get(tone)
|
||||
if color:
|
||||
return f"{color}{formatted}{self.RESET}"
|
||||
return formatted
|
||||
|
||||
badge = parts[1]
|
||||
badge_color = self.BADGE_TONES.get(badge.strip())
|
||||
if not badge_color:
|
||||
return formatted
|
||||
colored_parts = list(parts)
|
||||
colored_parts[1] = f"{badge_color}{badge}{self.RESET}"
|
||||
colored = _join_log_display_parts(colored_parts, body)
|
||||
if record.exc_info:
|
||||
exc_text = self.formatException(record.exc_info)
|
||||
if exc_text:
|
||||
colored = f"{colored}\n{exc_text}"
|
||||
return colored
|
||||
|
||||
|
||||
class _FileLogFormatter(logging.Formatter):
|
||||
"""文件日志使用结构化字段,避免控制台前缀被再次包裹。"""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__(
|
||||
"%(asctime)s | %(levelname)s | %(request_tag)s | %(name)s | %(message)s",
|
||||
"%Y-%m-%d %H:%M:%S",
|
||||
)
|
||||
|
||||
def format(self, record: logging.LogRecord) -> str:
|
||||
original_message = str(
|
||||
getattr(record, "codex_original_message_text", "") or record.getMessage() or ""
|
||||
)
|
||||
message = _sanitize_sensitive_text(
|
||||
_normalize_log_display_expression(_record_logger_name(record), original_message)
|
||||
)
|
||||
prefix = (
|
||||
f"{self.formatTime(record, self.datefmt)} | "
|
||||
f"{record.levelname} | "
|
||||
f"{_record_request_tag(record)} | "
|
||||
f"{_record_logger_name(record) or record.name} | "
|
||||
)
|
||||
formatted = f"{prefix}{message.replace(chr(10), chr(10) + ' ' * len(prefix))}"
|
||||
if record.exc_info:
|
||||
exc_text = self.formatException(record.exc_info)
|
||||
if exc_text:
|
||||
formatted = f"{formatted}\n{exc_text}"
|
||||
return formatted
|
||||
|
||||
|
||||
class _DisplayLogFormatter(logging.Formatter):
|
||||
"""保留旧式单行展示前缀,供兼容 handler 使用。"""
|
||||
|
||||
def format(self, record: logging.LogRecord) -> str:
|
||||
message = _record_display_message(record)
|
||||
formatted, _ = _format_log_display_line(record, message)
|
||||
if record.exc_info:
|
||||
exc_text = self.formatException(record.exc_info)
|
||||
if exc_text:
|
||||
formatted = f"{formatted}\n{exc_text}"
|
||||
return formatted
|
||||
|
||||
|
||||
def _is_windows_file_lock_error(exc: BaseException) -> bool:
|
||||
"""Return True for transient Windows sharing violations during file rotation."""
|
||||
if os.name != "nt" or not isinstance(exc, OSError):
|
||||
return False
|
||||
winerror = getattr(exc, "winerror", None)
|
||||
if winerror in {32, 33}:
|
||||
return True
|
||||
return getattr(exc, "errno", None) in {32, 33}
|
||||
|
||||
|
||||
class _SafeRotatingFileHandler(RotatingFileHandler):
|
||||
"""Windows 上轮转目标被短暂占用时,降级为继续写当前日志。"""
|
||||
|
||||
_ROLLOVER_RETRY_DELAY_SECONDS = 5.0
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
self._rollover_retry_after = 0.0
|
||||
|
||||
def shouldRollover(self, record: logging.LogRecord) -> bool:
|
||||
if self._rollover_retry_after and time.monotonic() < self._rollover_retry_after:
|
||||
return False
|
||||
return super().shouldRollover(record)
|
||||
|
||||
def doRollover(self) -> None:
|
||||
try:
|
||||
super().doRollover()
|
||||
self._rollover_retry_after = 0.0
|
||||
except OSError as exc:
|
||||
if not _is_windows_file_lock_error(exc):
|
||||
raise
|
||||
self._defer_rollover()
|
||||
if self.stream is None and not self.delay:
|
||||
self.stream = self._open()
|
||||
|
||||
def handleError(self, record: logging.LogRecord) -> None:
|
||||
exc = sys.exc_info()[1]
|
||||
if _is_windows_file_lock_error(exc):
|
||||
self._defer_rollover()
|
||||
try:
|
||||
if self.stream is None and not self.delay:
|
||||
self.stream = self._open()
|
||||
logging.FileHandler.emit(self, record)
|
||||
except Exception:
|
||||
pass
|
||||
return
|
||||
super().handleError(record)
|
||||
|
||||
def _defer_rollover(self) -> None:
|
||||
self._rollover_retry_after = time.monotonic() + self._ROLLOVER_RETRY_DELAY_SECONDS
|
||||
|
||||
|
||||
# 创建全局 Web 日志处理器
|
||||
_web_log_handler = _WebLogHandler()
|
||||
_web_log_handler.setLevel(logging.DEBUG)
|
||||
_web_log_handler.setFormatter(_DisplayLogFormatter())
|
||||
_web_log_handler.addFilter(_BrowserConstantEnabledFilter("LOG_WEB_COLLECTOR_ENABLED", True))
|
||||
setattr(_web_log_handler, "_codex_secure_handler", "web")
|
||||
|
||||
|
||||
def _resolve_log_dir() -> Path:
|
||||
configured = str(os.getenv("LOG_DIR", "") or "").strip()
|
||||
if not configured:
|
||||
return DEFAULT_LOG_DIR
|
||||
candidate = Path(configured)
|
||||
if not candidate.is_absolute():
|
||||
candidate = PROJECT_ROOT / candidate
|
||||
return candidate.resolve()
|
||||
|
||||
|
||||
def get_log_file_path() -> Path:
|
||||
configured = str(os.getenv("LOG_FILE", "") or "").strip()
|
||||
if configured:
|
||||
candidate = Path(configured)
|
||||
if not candidate.is_absolute():
|
||||
candidate = PROJECT_ROOT / candidate
|
||||
return candidate.resolve()
|
||||
return _resolve_log_dir() / "app.log"
|
||||
|
||||
|
||||
def get_shared_file_log_handler() -> Optional[logging.Handler]:
|
||||
global _shared_file_log_handler
|
||||
|
||||
with _shared_file_log_handler_lock:
|
||||
if _shared_file_log_handler is not None:
|
||||
return _shared_file_log_handler
|
||||
|
||||
try:
|
||||
log_file = get_log_file_path()
|
||||
log_file.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
handler = _SafeRotatingFileHandler(
|
||||
log_file,
|
||||
maxBytes=_get_positive_int_env("LOG_MAX_BYTES", 5 * 1024 * 1024),
|
||||
backupCount=_get_positive_int_env("LOG_BACKUP_COUNT", 5),
|
||||
encoding="utf-8",
|
||||
)
|
||||
handler.setLevel(logging.DEBUG)
|
||||
handler.setFormatter(_FileLogFormatter())
|
||||
handler.addFilter(_BrowserConstantEnabledFilter("LOG_FILE_ENABLED", True))
|
||||
setattr(handler, "_codex_secure_handler", "file")
|
||||
_shared_file_log_handler = handler
|
||||
except Exception as e:
|
||||
try:
|
||||
print(f"[Config] failed to initialize file logging: {e}", file=sys.stderr)
|
||||
except Exception:
|
||||
pass
|
||||
_shared_file_log_handler = None
|
||||
|
||||
return _shared_file_log_handler
|
||||
@@ -0,0 +1,313 @@
|
||||
"""
|
||||
app/core/config_parts/log_redaction.py - 日志敏感信息过滤与脱敏模块
|
||||
"""
|
||||
import re
|
||||
from typing import Any, Match, Optional
|
||||
|
||||
_SENSITIVE_KEY_HINTS = (
|
||||
"authorization",
|
||||
"cookie",
|
||||
"set-cookie",
|
||||
"password",
|
||||
"passwd",
|
||||
"secret",
|
||||
"token",
|
||||
"api_key",
|
||||
"apikey",
|
||||
"access_key",
|
||||
"client_secret",
|
||||
"refresh_token",
|
||||
"session",
|
||||
"csrf",
|
||||
)
|
||||
_REDACTED_TEXT = "******"
|
||||
|
||||
|
||||
def _redact_data_uri_for_log(match: Match) -> str:
|
||||
media_type = str(match.group(1) or "media").lower()
|
||||
mime_suffix = str(match.group(2) or "octet-stream").lower()
|
||||
payload_len = len(str(match.group(3) or ""))
|
||||
return f"data:{media_type}/{mime_suffix};base64,[omitted {payload_len} chars]"
|
||||
|
||||
|
||||
def _redact_long_base64_for_log(match: Match) -> str:
|
||||
return f"[base64 omitted: {len(match.group(0))} chars]"
|
||||
|
||||
|
||||
_BASE64_LOG_CHARS = frozenset("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/=_-")
|
||||
_SENSITIVE_TEXT_SCAN_HINT_RE = re.compile(
|
||||
r"(?i)data:(?:image|audio|video)/|bearer\s+|authorization|set-cookie|cookie|"
|
||||
r"x[-_]api[-_]key|x[-_]github[-_]token|github[-_]token|"
|
||||
r"access_token|refresh_token|id_token|api_key|apikey|access_key|client_secret|"
|
||||
r"secret|password|passwd|token|session|csrf|sk-[A-Za-z0-9_-]{8,}|"
|
||||
r"eyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}|"
|
||||
r"https?://[^/\s:@]+:[^/\s@]+@"
|
||||
)
|
||||
_SENSITIVE_TEXT_PRECHECK_MIN_CHARS = 4096
|
||||
_SENSITIVE_TEXT_LARGE_OMIT_THRESHOLD = 512 * 1024
|
||||
_SENSITIVE_TEXT_LARGE_EDGE_CHARS = 4096
|
||||
|
||||
|
||||
def _has_long_base64_candidate(text: str, min_chars: int = 1024) -> bool:
|
||||
run_len = 0
|
||||
for char in text:
|
||||
if char in _BASE64_LOG_CHARS:
|
||||
run_len += 1
|
||||
if run_len >= min_chars:
|
||||
return True
|
||||
elif char in "\r\n" and run_len:
|
||||
continue
|
||||
else:
|
||||
run_len = 0
|
||||
return False
|
||||
|
||||
|
||||
def _redact_long_base64_runs_for_log(text: str, min_chars: int = 1024) -> str:
|
||||
raw_text = str(text or "")
|
||||
if not raw_text:
|
||||
return raw_text
|
||||
|
||||
parts = []
|
||||
copy_from = 0
|
||||
run_start = None
|
||||
run_payload_chars = 0
|
||||
|
||||
for index, char in enumerate(raw_text):
|
||||
if char in _BASE64_LOG_CHARS:
|
||||
if run_start is None:
|
||||
run_start = index
|
||||
run_payload_chars = 0
|
||||
run_payload_chars += 1
|
||||
continue
|
||||
|
||||
if char in "\r\n" and run_start is not None:
|
||||
continue
|
||||
|
||||
if run_start is not None:
|
||||
if run_payload_chars >= min_chars:
|
||||
parts.append(raw_text[copy_from:run_start])
|
||||
parts.append(f"[base64 omitted: {index - run_start} chars]")
|
||||
copy_from = index
|
||||
run_start = None
|
||||
run_payload_chars = 0
|
||||
|
||||
if run_start is not None and run_payload_chars >= min_chars:
|
||||
parts.append(raw_text[copy_from:run_start])
|
||||
parts.append(f"[base64 omitted: {len(raw_text) - run_start} chars]")
|
||||
copy_from = len(raw_text)
|
||||
|
||||
if not parts:
|
||||
return raw_text
|
||||
parts.append(raw_text[copy_from:])
|
||||
return "".join(parts)
|
||||
|
||||
|
||||
def _should_scan_sensitive_text(text: str) -> bool:
|
||||
if len(text) <= _SENSITIVE_TEXT_PRECHECK_MIN_CHARS:
|
||||
return True
|
||||
if _SENSITIVE_TEXT_SCAN_HINT_RE.search(text):
|
||||
return True
|
||||
return _has_long_base64_candidate(text)
|
||||
|
||||
|
||||
_SENSITIVE_TEXT_PATTERNS = (
|
||||
(
|
||||
re.compile(
|
||||
r"(?i)data:(image|audio|video)/([a-zA-Z0-9.+-]{1,100});base64,"
|
||||
r"([A-Za-z0-9+/=_\-\r\n]{64,})"
|
||||
),
|
||||
_redact_data_uri_for_log,
|
||||
),
|
||||
(
|
||||
re.compile(r"(?<![A-Za-z0-9+/=_-])[A-Za-z0-9+/=_-]{1024,}(?![A-Za-z0-9+/=_-])"),
|
||||
_redact_long_base64_for_log,
|
||||
),
|
||||
(
|
||||
re.compile(
|
||||
r"(?<![A-Za-z0-9+/=_-])"
|
||||
r"(?:[A-Za-z0-9+/=_-]{64,}[\r\n]+){7,}[A-Za-z0-9+/=_-]{64,}"
|
||||
r"(?![A-Za-z0-9+/=_-])"
|
||||
),
|
||||
_redact_long_base64_for_log,
|
||||
),
|
||||
(
|
||||
re.compile(r"(?i)\b(Bearer)\s+([A-Za-z0-9._~+/=-]{8,})"),
|
||||
r"\1 ******",
|
||||
),
|
||||
(
|
||||
re.compile(
|
||||
r"(?<![A-Za-z0-9_-])"
|
||||
r"(sk-(?:proj-)?[A-Za-z0-9_-]{16,})"
|
||||
r"(?![A-Za-z0-9_-])"
|
||||
),
|
||||
"sk-******",
|
||||
),
|
||||
(
|
||||
re.compile(
|
||||
r"(?<![A-Za-z0-9_-])"
|
||||
r"(eyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{6,})"
|
||||
r"(?![A-Za-z0-9_-])"
|
||||
),
|
||||
"[jwt omitted]",
|
||||
),
|
||||
(
|
||||
re.compile(r"(?i)(https?://[^/\s:@]+):([^/\s@]+)(@)"),
|
||||
r"\1:******\3",
|
||||
),
|
||||
(
|
||||
re.compile(
|
||||
r"(?i)\b(Authorization|Cookie|Set-Cookie|X[-_]Api[-_]Key|"
|
||||
r"X[-_]GitHub[-_]Token|GitHub[-_]Token)\s*:\s*[^\r\n;]+(?:;[^\r\n]*)?"
|
||||
),
|
||||
lambda match: f"{match.group(1)}: ******",
|
||||
),
|
||||
(
|
||||
re.compile(
|
||||
r"(?i)([?&](?:access_token|refresh_token|id_token|token|api_key|apikey|"
|
||||
r"access_key|client_secret|session|csrf|"
|
||||
r"x[-_]api[-_]key|x[-_]github[-_]token|github[-_]token|key|secret|password|passwd)=)"
|
||||
r"[^&\s]+"
|
||||
),
|
||||
r"\1******",
|
||||
),
|
||||
(
|
||||
re.compile(
|
||||
r"(?i)(\b(?:access_token|refresh_token|id_token|token|api_key|apikey|"
|
||||
r"access_key|client_secret|session|csrf|"
|
||||
r"x[-_]api[-_]key|x[-_]github[-_]token|github[-_]token|secret|password|passwd)"
|
||||
r"\s*=\s*)[^\s,&;]+"
|
||||
),
|
||||
r"\1******",
|
||||
),
|
||||
(
|
||||
re.compile(
|
||||
r'(?i)("(?:authorization|cookie|set-cookie|password|passwd|secret|token|api_key|apikey|'
|
||||
r'access_token|refresh_token|access_key|client_secret|session|csrf|'
|
||||
r'x[-_]api[-_]key|x[-_]github[-_]token|'
|
||||
r'github[-_]token)"\s*:\s*)"[^"]*"'
|
||||
),
|
||||
r'\1"******"',
|
||||
),
|
||||
(
|
||||
re.compile(
|
||||
r'(?i)("(?:authorization|cookie|set-cookie|password|passwd|secret|token|api_key|apikey|'
|
||||
r'access_token|refresh_token|access_key|client_secret|session|csrf|'
|
||||
r'x[-_]api[-_]key|x[-_]github[-_]token|'
|
||||
r'github[-_]token)"\s*:\s*")[^"\r\n]*$'
|
||||
),
|
||||
r'\1******',
|
||||
),
|
||||
(
|
||||
re.compile(
|
||||
r"(?i)('(?:authorization|cookie|set-cookie|password|passwd|secret|token|api_key|apikey|"
|
||||
r"access_token|refresh_token|access_key|client_secret|session|csrf|"
|
||||
r"x[-_]api[-_]key|x[-_]github[-_]token|"
|
||||
r"github[-_]token)'\s*:\s*)'[^']*'"
|
||||
),
|
||||
r"\1'******'",
|
||||
),
|
||||
(
|
||||
re.compile(
|
||||
r"(?i)('(?:authorization|cookie|set-cookie|password|passwd|secret|token|api_key|apikey|"
|
||||
r"access_token|refresh_token|access_key|client_secret|session|csrf|"
|
||||
r"x[-_]api[-_]key|x[-_]github[-_]token|"
|
||||
r"github[-_]token)'\s*:\s*)'[^'\r\n]*$"
|
||||
),
|
||||
r"\1'******",
|
||||
),
|
||||
)
|
||||
_SENSITIVE_BEARER_PATTERN = _SENSITIVE_TEXT_PATTERNS[3]
|
||||
_SENSITIVE_STANDALONE_TOKEN_PATTERNS = _SENSITIVE_TEXT_PATTERNS[4:7]
|
||||
_SENSITIVE_HEADER_PATTERN = _SENSITIVE_TEXT_PATTERNS[7]
|
||||
_SENSITIVE_PARAM_PATTERNS = _SENSITIVE_TEXT_PATTERNS[8:]
|
||||
|
||||
|
||||
def _sanitize_standalone_sensitive_tokens(text: str) -> str:
|
||||
if "sk-" not in text and "eyJ" not in text and "://" not in text:
|
||||
return text
|
||||
sanitized = text
|
||||
for pattern, replacement in _SENSITIVE_STANDALONE_TOKEN_PATTERNS:
|
||||
sanitized = pattern.sub(replacement, sanitized)
|
||||
return sanitized
|
||||
|
||||
|
||||
def _is_sensitive_key(key: Any) -> bool:
|
||||
normalized = str(key or "").strip().lower().replace("-", "_")
|
||||
return any(hint.replace("-", "_") in normalized for hint in _SENSITIVE_KEY_HINTS)
|
||||
|
||||
|
||||
def _sanitize_sensitive_text(text: str) -> str:
|
||||
sanitized = str(text or "")
|
||||
if not sanitized:
|
||||
return sanitized
|
||||
|
||||
if len(sanitized) > _SENSITIVE_TEXT_LARGE_OMIT_THRESHOLD:
|
||||
edge_chars = max(0, int(_SENSITIVE_TEXT_LARGE_EDGE_CHARS))
|
||||
head = sanitized[:edge_chars]
|
||||
tail = sanitized[-edge_chars:] if edge_chars else ""
|
||||
omitted = max(0, len(sanitized) - len(head) - len(tail))
|
||||
safe_head = _sanitize_sensitive_text(head)
|
||||
safe_tail = _sanitize_sensitive_text(tail) if tail else ""
|
||||
if safe_tail:
|
||||
return (
|
||||
f"{safe_head}... [omitted {omitted} chars from oversized log payload] "
|
||||
f"...{safe_tail}"
|
||||
)
|
||||
return f"{safe_head}... [omitted {omitted} chars from oversized log payload]"
|
||||
|
||||
sanitized = _sanitize_standalone_sensitive_tokens(sanitized)
|
||||
|
||||
if not _should_scan_sensitive_text(sanitized):
|
||||
return sanitized
|
||||
|
||||
lower_text = sanitized.lower()
|
||||
|
||||
# 1. 只有可能有 Base64 候选字符或包含 'data:' 才做前三个正则替换
|
||||
if "data:" in lower_text or "base64" in lower_text or len(sanitized) >= 1024:
|
||||
if "data:" in lower_text and "base64," in lower_text:
|
||||
sanitized = _SENSITIVE_TEXT_PATTERNS[0][0].sub(_SENSITIVE_TEXT_PATTERNS[0][1], sanitized)
|
||||
lower_text = sanitized.lower()
|
||||
if len(sanitized) >= 1024 and _has_long_base64_candidate(sanitized):
|
||||
sanitized = _redact_long_base64_runs_for_log(sanitized)
|
||||
lower_text = sanitized.lower()
|
||||
|
||||
# 2. 仅在包含 bearer 时替换 Bearer token
|
||||
if "bearer" in lower_text:
|
||||
sanitized = _SENSITIVE_BEARER_PATTERN[0].sub(_SENSITIVE_BEARER_PATTERN[1], sanitized)
|
||||
|
||||
# 3. 仅在包含相关 header 名时替换 Auth, Cookie 等头部
|
||||
if any(k in lower_text for k in ("authorization", "cookie", "set-cookie", "x-api-key", "x_api_key", "x-github-token", "x_github_token", "github-token", "github_token")):
|
||||
sanitized = _SENSITIVE_HEADER_PATTERN[0].sub(_SENSITIVE_HEADER_PATTERN[1], sanitized)
|
||||
|
||||
# 4. 仅在包含敏感关键字时执行 query/json param 相关的敏感信息提取正则
|
||||
if any(k in lower_text for k in ("access_token", "refresh_token", "id_token", "token", "api_key", "apikey", "access_key", "client_secret", "key", "secret", "password", "passwd", "session", "csrf", "x-api-key", "x_api_key", "x-github-token", "x_github_token", "github-token", "github_token")):
|
||||
for pattern, replacement in _SENSITIVE_PARAM_PATTERNS:
|
||||
sanitized = pattern.sub(replacement, sanitized)
|
||||
|
||||
return sanitized
|
||||
|
||||
|
||||
def sanitize_sensitive_data(value: Any, *, _depth: int = 0) -> Any:
|
||||
"""Return a sanitized copy suitable for logs and debug artifacts."""
|
||||
if _depth > 8:
|
||||
return "[max-depth]"
|
||||
|
||||
if isinstance(value, dict):
|
||||
sanitized_dict = {}
|
||||
for key, item in value.items():
|
||||
if _is_sensitive_key(key):
|
||||
sanitized_dict[key] = _REDACTED_TEXT
|
||||
else:
|
||||
sanitized_dict[key] = sanitize_sensitive_data(item, _depth=_depth + 1)
|
||||
return sanitized_dict
|
||||
|
||||
if isinstance(value, list):
|
||||
return [sanitize_sensitive_data(item, _depth=_depth + 1) for item in value]
|
||||
|
||||
if isinstance(value, tuple):
|
||||
return tuple(sanitize_sensitive_data(item, _depth=_depth + 1) for item in value)
|
||||
|
||||
if isinstance(value, str):
|
||||
return _sanitize_sensitive_text(value)
|
||||
|
||||
return value
|
||||
@@ -0,0 +1,130 @@
|
||||
"""
|
||||
app/core/config_parts/message_validator.py - 消息验证器模块
|
||||
"""
|
||||
import ast
|
||||
import json
|
||||
from typing import Any, Tuple, Optional, List
|
||||
from .browser_constants import BrowserConstants
|
||||
|
||||
|
||||
class MessageValidator:
|
||||
"""消息验证器"""
|
||||
|
||||
VALID_ROLES = {'user', 'assistant', 'system'}
|
||||
ROLE_ALIASES = {
|
||||
'developer': 'system',
|
||||
}
|
||||
_IMAGE_PLACEHOLDER = "[图片]"
|
||||
|
||||
@classmethod
|
||||
def _parse_multimodal_string(cls, content: str):
|
||||
"""尝试把字符串形式的多模态 content 还原成列表。"""
|
||||
text = str(content or "")
|
||||
stripped = text.strip()
|
||||
if not stripped.startswith('[') or not stripped.endswith(']'):
|
||||
return text, False
|
||||
|
||||
parsed = None
|
||||
try:
|
||||
parsed = json.loads(stripped)
|
||||
except (TypeError, ValueError, RecursionError):
|
||||
parsed = None
|
||||
|
||||
if parsed is None:
|
||||
try:
|
||||
parsed = ast.literal_eval(stripped)
|
||||
except Exception:
|
||||
parsed = None
|
||||
|
||||
if isinstance(parsed, list):
|
||||
return parsed, True
|
||||
return text, False
|
||||
|
||||
@classmethod
|
||||
def _normalize_content(cls, content: Any) -> Any:
|
||||
"""保留多模态结构,避免把图片/base64 粗暴 str() 化。"""
|
||||
if content is None:
|
||||
return ""
|
||||
if isinstance(content, dict):
|
||||
return [content]
|
||||
if isinstance(content, list):
|
||||
return content
|
||||
if isinstance(content, tuple):
|
||||
return list(content)
|
||||
if isinstance(content, str):
|
||||
parsed, ok = cls._parse_multimodal_string(content)
|
||||
return parsed if ok else content
|
||||
return str(content)
|
||||
|
||||
@classmethod
|
||||
def _effective_content_length(cls, content: Any) -> int:
|
||||
"""按网页执行真实会使用的语义估算 content 长度。"""
|
||||
normalized = cls._normalize_content(content)
|
||||
|
||||
if isinstance(normalized, str):
|
||||
text = normalized
|
||||
if text.startswith("data:image") and "base64," in text and len(text) > 1000:
|
||||
return len("[图片内容]")
|
||||
return len(text)
|
||||
|
||||
if isinstance(normalized, list):
|
||||
total = 0
|
||||
for item in normalized:
|
||||
if item is None:
|
||||
continue
|
||||
if not isinstance(item, dict):
|
||||
total += len(str(item))
|
||||
continue
|
||||
|
||||
item_type = str(item.get("type", "") or "").strip()
|
||||
if item_type == "text":
|
||||
total += len(str(item.get("text", "") or ""))
|
||||
elif item_type in {"image_url", "file", "input_file", "document", "input_audio", "audio_url", "video_url", "input_video"}:
|
||||
total += len(cls._IMAGE_PLACEHOLDER)
|
||||
else:
|
||||
total += len(str(item))
|
||||
return total
|
||||
|
||||
return len(str(normalized))
|
||||
|
||||
@classmethod
|
||||
def validate(cls, messages: Any) -> tuple:
|
||||
if messages is None:
|
||||
return False, "messages 不能为空", None
|
||||
|
||||
if not isinstance(messages, list):
|
||||
return False, "messages 应该是列表", None
|
||||
|
||||
if len(messages) == 0:
|
||||
return False, "messages 不能为空列表", None
|
||||
|
||||
message_count = len(messages)
|
||||
max_messages = int(BrowserConstants.MAX_MESSAGES_COUNT)
|
||||
if message_count > max_messages:
|
||||
return False, (
|
||||
f"消息数量超过限制(当前 {message_count} 条,最大允许 {max_messages} 条)"
|
||||
), None
|
||||
|
||||
sanitized = []
|
||||
for i, msg in enumerate(messages):
|
||||
if not isinstance(msg, dict):
|
||||
return False, f"messages[{i}] 不是字典类型", None
|
||||
|
||||
role = str(msg.get('role', 'user') or 'user').strip().lower()
|
||||
role = cls.ROLE_ALIASES.get(role, role)
|
||||
if role not in cls.VALID_ROLES:
|
||||
role = 'user'
|
||||
|
||||
content = cls._normalize_content(msg.get('content', ''))
|
||||
|
||||
content_length = cls._effective_content_length(content)
|
||||
max_length = int(BrowserConstants.MAX_MESSAGE_LENGTH)
|
||||
if content_length > max_length:
|
||||
return False, (
|
||||
f"messages[{i}].content 超过长度限制"
|
||||
f"(当前 {content_length} 字符,最大允许 {max_length} 字符)"
|
||||
), None
|
||||
|
||||
sanitized.append({'role': role, 'content': content})
|
||||
|
||||
return True, None, sanitized
|
||||
@@ -0,0 +1,274 @@
|
||||
"""
|
||||
app/core/config_parts/secure_logger.py - 安全日志器实现与日志工厂模块
|
||||
"""
|
||||
from collections import defaultdict
|
||||
import contextlib
|
||||
import contextvars
|
||||
import logging
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
from typing import Any, Dict, Optional
|
||||
|
||||
from .env_config import AppConfig
|
||||
from .browser_constants import _BrowserConstantEnabledFilter
|
||||
from .log_formatters import (
|
||||
_compact_logger_name,
|
||||
_ConsoleColorFormatter,
|
||||
_format_log_display_line,
|
||||
_web_log_handler,
|
||||
get_shared_file_log_handler,
|
||||
)
|
||||
from .cute_translator import (
|
||||
_cuteify_info_message,
|
||||
_cuteify_debug_message,
|
||||
_cuteify_warning_message,
|
||||
_cuteify_error_message,
|
||||
_add_suppressed_marker,
|
||||
)
|
||||
|
||||
# 上下文变量,存储当前请求的 request_id
|
||||
_request_context: contextvars.ContextVar[Optional[str]] = contextvars.ContextVar("request_id", default=None)
|
||||
_command_log_context: contextvars.ContextVar[Optional[Dict[str, Any]]] = contextvars.ContextVar(
|
||||
"command_log_context",
|
||||
default=None,
|
||||
)
|
||||
|
||||
_logger_setup_lock = threading.RLock()
|
||||
_logger_registry_lock = threading.Lock()
|
||||
_logger_registry: Dict[str, "SecureLogger"] = {}
|
||||
|
||||
|
||||
class SecureLogger:
|
||||
"""安全日志器,带图标和格式化(支持上下文自动注入 request_id)"""
|
||||
_debug_throttle_locks: Dict[str, threading.Lock] = defaultdict(threading.Lock)
|
||||
_debug_throttle_state: Dict[str, Dict[str, Any]] = {}
|
||||
|
||||
ICONS = {
|
||||
'DEBUG': '▫️',
|
||||
'INFO': '🔹',
|
||||
'WARNING': '⚠️',
|
||||
'ERROR': '❌',
|
||||
'SUCCESS': '✅',
|
||||
'STREAM': '🌊',
|
||||
'NETWORK': '🌐',
|
||||
}
|
||||
|
||||
# 日志级别映射
|
||||
LEVEL_MAP = {
|
||||
'DEBUG': logging.DEBUG,
|
||||
'INFO': logging.INFO,
|
||||
'WARNING': logging.WARNING,
|
||||
'ERROR': logging.ERROR,
|
||||
'CRITICAL': logging.CRITICAL,
|
||||
}
|
||||
|
||||
def __init__(self, name: str, level: Optional[int] = None):
|
||||
self._name = _compact_logger_name(name)
|
||||
|
||||
# 如果未指定级别,从环境变量获取
|
||||
if level is None:
|
||||
level = self._get_level_from_env()
|
||||
|
||||
self._level = level
|
||||
self._logger = self._setup_logger(name, level)
|
||||
|
||||
@classmethod
|
||||
def _get_level_from_env(cls) -> int:
|
||||
"""从环境变量获取日志级别"""
|
||||
level_str = AppConfig.get_log_level()
|
||||
return cls.LEVEL_MAP.get(level_str, logging.INFO)
|
||||
|
||||
def _setup_logger(self, name: str, level: int) -> logging.Logger:
|
||||
logger = logging.getLogger(name)
|
||||
|
||||
with _logger_setup_lock:
|
||||
# 防止日志向上层冒泡导致重复打印
|
||||
logger.propagate = False
|
||||
|
||||
existing_kinds = {
|
||||
getattr(handler, "_codex_secure_handler", None)
|
||||
for handler in logger.handlers
|
||||
}
|
||||
|
||||
if "console" not in existing_kinds:
|
||||
console_handler = logging.StreamHandler(sys.stdout)
|
||||
console_handler.setLevel(logging.DEBUG)
|
||||
console_handler.setFormatter(_ConsoleColorFormatter())
|
||||
console_handler.addFilter(_BrowserConstantEnabledFilter("LOG_CONSOLE_ENABLED", True))
|
||||
setattr(console_handler, "_codex_secure_handler", "console")
|
||||
logger.addHandler(console_handler)
|
||||
|
||||
file_handler = get_shared_file_log_handler()
|
||||
if file_handler is not None and file_handler not in logger.handlers:
|
||||
logger.addHandler(file_handler)
|
||||
|
||||
if _web_log_handler not in logger.handlers:
|
||||
logger.addHandler(_web_log_handler)
|
||||
|
||||
logger.setLevel(logging.DEBUG)
|
||||
return logger
|
||||
|
||||
def _format(self, level_key: str, msg: str) -> str:
|
||||
"""核心格式化逻辑(简洁版)"""
|
||||
record = logging.LogRecord(
|
||||
name=self._name,
|
||||
level=self.LEVEL_MAP.get(str(level_key or "").upper(), logging.INFO),
|
||||
pathname="",
|
||||
lineno=0,
|
||||
msg=str(msg or ""),
|
||||
args=(),
|
||||
exc_info=None,
|
||||
)
|
||||
record.codex_request_id = _request_context.get() or "SYSTEM"
|
||||
record.codex_logger_name = self._name
|
||||
formatted, _ = _format_log_display_line(record, msg)
|
||||
return formatted
|
||||
|
||||
def _make_debug_throttle_key(self, key: str) -> str:
|
||||
normalized = str(key or "").strip() or "__default__"
|
||||
return f"{self._name}:{normalized}"
|
||||
|
||||
@staticmethod
|
||||
def _coerce_bool(value: Any, default: bool = True) -> bool:
|
||||
if value is None:
|
||||
return default
|
||||
if isinstance(value, bool):
|
||||
return value
|
||||
if isinstance(value, (int, float)):
|
||||
return bool(value)
|
||||
text = str(value).strip().lower()
|
||||
if text in {"1", "true", "yes", "y", "on"}:
|
||||
return True
|
||||
if text in {"0", "false", "no", "n", "off"}:
|
||||
return False
|
||||
return default
|
||||
|
||||
def _get_effective_emit_level(self) -> Optional[int]:
|
||||
context = _command_log_context.get()
|
||||
if not isinstance(context, dict):
|
||||
return self._level
|
||||
|
||||
if not self._coerce_bool(context.get("enabled", True), True):
|
||||
return None
|
||||
|
||||
override = str(context.get("level", "GLOBAL") or "GLOBAL").strip().upper()
|
||||
if override == "GLOBAL":
|
||||
return self._level
|
||||
return self.LEVEL_MAP.get(override, self._level)
|
||||
|
||||
def _emit(self, level: int, level_key: str, msg: str, *, exc_info: bool = False):
|
||||
effective_level = self._get_effective_emit_level()
|
||||
if effective_level is None or level < effective_level:
|
||||
return
|
||||
request_id = _request_context.get() or "SYSTEM"
|
||||
original_message_text = str(msg or "")
|
||||
upper_level_key = str(level_key or "").upper()
|
||||
# P0-7:展示层翻译(cute_translator)改为惰性:由 handler 渲染时经
|
||||
# log_formatters._record_display_message 计算一次并缓存到 record 上。
|
||||
self._logger.log(
|
||||
level,
|
||||
original_message_text,
|
||||
exc_info=exc_info,
|
||||
extra={
|
||||
"codex_request_id": request_id,
|
||||
"codex_logger_name": self._name,
|
||||
"codex_message": original_message_text,
|
||||
"codex_original_message_text": original_message_text,
|
||||
"codex_display_pending": True,
|
||||
"codex_kind": upper_level_key,
|
||||
},
|
||||
)
|
||||
|
||||
def set_level(self, level: int):
|
||||
"""动态调整日志级别"""
|
||||
self._level = level
|
||||
self._logger.setLevel(logging.DEBUG)
|
||||
for handler in self._logger.handlers:
|
||||
handler.setLevel(logging.DEBUG)
|
||||
|
||||
def debug(self, msg: str):
|
||||
self._emit(logging.DEBUG, 'DEBUG', msg)
|
||||
|
||||
def debug_throttled(self, key: str, msg: str, interval_sec: float = 5.0):
|
||||
"""在高频路径里限频输出 DEBUG,并附带被抑制次数。"""
|
||||
effective_level = self._get_effective_emit_level()
|
||||
if effective_level is None or logging.DEBUG < effective_level:
|
||||
return
|
||||
|
||||
now = time.time()
|
||||
interval = max(0.0, float(interval_sec or 0.0))
|
||||
throttle_key = self._make_debug_throttle_key(key)
|
||||
suppressed = 0
|
||||
should_log = False
|
||||
|
||||
lock = self._debug_throttle_locks[throttle_key]
|
||||
with lock:
|
||||
state = self._debug_throttle_state.get(throttle_key)
|
||||
last_at = float(state.get("last_at", 0.0) or 0.0) if state else 0.0
|
||||
if state is None or (now - last_at) >= interval:
|
||||
suppressed = int(state.get("suppressed", 0) or 0) if state else 0
|
||||
self._debug_throttle_state[throttle_key] = {
|
||||
"last_at": now,
|
||||
"suppressed": 0,
|
||||
}
|
||||
should_log = True
|
||||
else:
|
||||
state["suppressed"] = int(state.get("suppressed", 0) or 0) + 1
|
||||
|
||||
if should_log:
|
||||
self.debug(_add_suppressed_marker(msg, suppressed))
|
||||
|
||||
def info(self, msg: str):
|
||||
self._emit(logging.INFO, 'INFO', msg)
|
||||
|
||||
def warning(self, msg: str):
|
||||
self._emit(logging.WARNING, 'WARNING', msg)
|
||||
|
||||
def error(self, msg: str):
|
||||
self._emit(logging.ERROR, 'ERROR', msg)
|
||||
|
||||
def exception(self, msg: str):
|
||||
self._emit(logging.ERROR, 'ERROR', msg, exc_info=True)
|
||||
|
||||
def success(self, msg: str):
|
||||
self._emit(logging.INFO, 'SUCCESS', msg)
|
||||
|
||||
def stream(self, msg: str):
|
||||
self._emit(logging.INFO, 'STREAM', msg)
|
||||
|
||||
def network(self, msg: str):
|
||||
self._emit(logging.INFO, 'NETWORK', msg)
|
||||
|
||||
@contextlib.contextmanager
|
||||
def context(self, request_id: str):
|
||||
"""上下文管理器,用于在代码块中自动设置 request_id"""
|
||||
token = _request_context.set(request_id)
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
_request_context.reset(token)
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def command_log_context(config: Optional[Dict[str, Any]] = None):
|
||||
token = _command_log_context.set(config if isinstance(config, dict) else None)
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
_command_log_context.reset(token)
|
||||
|
||||
|
||||
def get_logger(name: str) -> SecureLogger:
|
||||
"""获取 SecureLogger 实例(统一日志入口)"""
|
||||
normalized = str(name or "APP").strip() or "APP"
|
||||
with _logger_registry_lock:
|
||||
instance = _logger_registry.get(normalized)
|
||||
if instance is None:
|
||||
instance = SecureLogger(normalized)
|
||||
_logger_registry[normalized] = instance
|
||||
return instance
|
||||
|
||||
|
||||
# 创建常用 logger 实例(向后兼容)
|
||||
logger = get_logger("BROWSER")
|
||||
@@ -0,0 +1,304 @@
|
||||
"""
|
||||
app/core/config_parts/sse_formatter.py - SSE 响应格式化器模块
|
||||
"""
|
||||
import json
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
|
||||
class SSEFormatter:
|
||||
"""SSE 响应格式化器"""
|
||||
|
||||
_sequence = 0
|
||||
_sequence_lock = threading.Lock()
|
||||
|
||||
@classmethod
|
||||
def _generate_id(cls) -> str:
|
||||
timestamp = int(time.time() * 1000)
|
||||
with cls._sequence_lock:
|
||||
cls._sequence += 1
|
||||
seq = cls._sequence
|
||||
short_uuid = uuid.uuid4().hex[:6]
|
||||
return f"chatcmpl-{timestamp}-{seq}-{short_uuid}"
|
||||
|
||||
@classmethod
|
||||
def pack_chunk(
|
||||
cls,
|
||||
content: str | None = None,
|
||||
model: str = "web-browser",
|
||||
completion_id: str | None = None,
|
||||
images: list[str] | None = None,
|
||||
media: list[dict] | None = None,
|
||||
reasoning_content: str | None = None,
|
||||
) -> str:
|
||||
"""打包流式 chunk。
|
||||
|
||||
为兼容现有前端,content 仍保留 Markdown 媒体链接。
|
||||
同时补充自定义 media 字段,供需要结构化媒体数据的前端直接消费。
|
||||
"""
|
||||
chunk_id = completion_id or cls._generate_id()
|
||||
delta = {}
|
||||
if content is not None:
|
||||
delta["content"] = content
|
||||
if reasoning_content:
|
||||
delta["reasoning_content"] = reasoning_content
|
||||
if media is not None:
|
||||
delta["media"] = media
|
||||
data = {
|
||||
"id": chunk_id,
|
||||
"object": "chat.completion.chunk",
|
||||
"created": int(time.time()),
|
||||
"model": model,
|
||||
"choices": [{
|
||||
"index": 0,
|
||||
"delta": delta,
|
||||
"finish_reason": None
|
||||
}]
|
||||
}
|
||||
if media is not None:
|
||||
data["media"] = media
|
||||
return f"data: {json.dumps(data, ensure_ascii=False)}\n\n"
|
||||
|
||||
@classmethod
|
||||
def pack_finish(cls, model: str = "web-browser") -> str:
|
||||
data = {
|
||||
"id": cls._generate_id(),
|
||||
"object": "chat.completion.chunk",
|
||||
"created": int(time.time()),
|
||||
"model": model,
|
||||
"choices": [{
|
||||
"index": 0,
|
||||
"delta": {},
|
||||
"finish_reason": "stop"
|
||||
}]
|
||||
}
|
||||
return f"data: {json.dumps(data, ensure_ascii=False)}\n\ndata: [DONE]\n\n"
|
||||
|
||||
@staticmethod
|
||||
def pack_comment(comment: str = "keepalive") -> str:
|
||||
"""打包 SSE 注释帧,用于长连接保活。"""
|
||||
safe_comment = " ".join(str(comment or "keepalive").splitlines()).strip() or "keepalive"
|
||||
return f": {safe_comment}\n\n"
|
||||
|
||||
@classmethod
|
||||
def pack_keepalive(
|
||||
cls,
|
||||
model: str = "web-browser",
|
||||
completion_id: Optional[str] = None,
|
||||
) -> str:
|
||||
"""打包客户端可解析的空 OpenAI chunk 作为长连接保活。"""
|
||||
return cls.pack_chunk(
|
||||
content="",
|
||||
model=model,
|
||||
completion_id=completion_id,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def pack_error(
|
||||
message: str,
|
||||
error_type: str = "execution_error",
|
||||
code: str = "workflow_failed",
|
||||
*,
|
||||
status_code: Optional[int] = None,
|
||||
retryable: Optional[bool] = None,
|
||||
param: Optional[str] = None,
|
||||
extra: Optional[Dict[str, Any]] = None,
|
||||
) -> str:
|
||||
clean_message = str(message or "")
|
||||
clean_code = str(code or "workflow_failed")
|
||||
clean_type = str(error_type or "execution_error")
|
||||
clean_status_code = status_code
|
||||
clean_retryable = retryable
|
||||
clean_param = param
|
||||
clean_extra = dict(extra or {})
|
||||
|
||||
try:
|
||||
from app.services.error_metadata import resolve_error_metadata
|
||||
meta = resolve_error_metadata(clean_message)
|
||||
if meta:
|
||||
clean_message = meta.message
|
||||
if clean_code in {"workflow_failed", "error", "arena_page_error"} and meta.code:
|
||||
clean_code = meta.code
|
||||
if clean_type in {"execution_error", "invalid_request_error"} and meta.error_type:
|
||||
clean_type = meta.error_type
|
||||
if clean_status_code is None:
|
||||
clean_status_code = meta.status_code
|
||||
if clean_retryable is None:
|
||||
clean_retryable = meta.retryable
|
||||
if clean_param is None and meta.param is not None:
|
||||
clean_param = meta.param
|
||||
if meta.extra:
|
||||
for k, v in meta.extra.items():
|
||||
if k not in clean_extra:
|
||||
clean_extra[k] = v
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
error: Dict[str, Any] = {
|
||||
"message": clean_message,
|
||||
"type": clean_type,
|
||||
"code": clean_code,
|
||||
}
|
||||
if clean_status_code is not None:
|
||||
error["status_code"] = int(clean_status_code)
|
||||
if clean_retryable is not None:
|
||||
error["retryable"] = bool(clean_retryable)
|
||||
if clean_param is not None:
|
||||
error["param"] = clean_param
|
||||
if clean_extra:
|
||||
for k, v in clean_extra.items():
|
||||
if k not in error:
|
||||
error[k] = v
|
||||
|
||||
data = {
|
||||
"id": f"chatcmpl-error-{int(time.time() * 1000)}",
|
||||
"object": "chat.completion.chunk",
|
||||
"created": int(time.time()),
|
||||
"model": "web-browser",
|
||||
"choices": [{
|
||||
"index": 0,
|
||||
"delta": {"content": f"[错误] {clean_message}"},
|
||||
"finish_reason": None
|
||||
}],
|
||||
"error": error
|
||||
}
|
||||
return f"data: {json.dumps(data, ensure_ascii=False)}\n\n"
|
||||
|
||||
@staticmethod
|
||||
def pack_error_json(
|
||||
message: str,
|
||||
error_type: str = "execution_error",
|
||||
code: str = "workflow_failed",
|
||||
*,
|
||||
status_code: Optional[int] = None,
|
||||
retryable: Optional[bool] = None,
|
||||
param: Optional[str] = None,
|
||||
extra: Optional[Dict[str, Any]] = None,
|
||||
) -> Dict[str, Any]:
|
||||
clean_message = str(message or "")
|
||||
clean_code = str(code or "workflow_failed")
|
||||
clean_type = str(error_type or "execution_error")
|
||||
clean_status_code = status_code
|
||||
clean_retryable = retryable
|
||||
clean_param = param
|
||||
clean_extra = dict(extra or {})
|
||||
|
||||
try:
|
||||
from app.services.error_metadata import resolve_error_metadata
|
||||
meta = resolve_error_metadata(clean_message)
|
||||
if meta:
|
||||
clean_message = meta.message
|
||||
if clean_code in {"workflow_failed", "error", "arena_page_error"} and meta.code:
|
||||
clean_code = meta.code
|
||||
if clean_type in {"execution_error", "invalid_request_error"} and meta.error_type:
|
||||
clean_type = meta.error_type
|
||||
if clean_status_code is None:
|
||||
clean_status_code = meta.status_code
|
||||
if clean_retryable is None:
|
||||
clean_retryable = meta.retryable
|
||||
if clean_param is None and meta.param is not None:
|
||||
clean_param = meta.param
|
||||
if meta.extra:
|
||||
for k, v in meta.extra.items():
|
||||
if k not in clean_extra:
|
||||
clean_extra[k] = v
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
error: Dict[str, Any] = {
|
||||
"message": clean_message,
|
||||
"type": clean_type,
|
||||
"code": clean_code,
|
||||
}
|
||||
if clean_status_code is not None:
|
||||
error["status_code"] = int(clean_status_code)
|
||||
if clean_retryable is not None:
|
||||
error["retryable"] = bool(clean_retryable)
|
||||
if clean_param is not None:
|
||||
error["param"] = clean_param
|
||||
if clean_extra:
|
||||
for k, v in clean_extra.items():
|
||||
if k not in error:
|
||||
error[k] = v
|
||||
|
||||
return {"error": error}
|
||||
|
||||
@staticmethod
|
||||
def pack_non_stream(content: str, model: str = "web-browser", media: list | None = None) -> Dict:
|
||||
message = {
|
||||
"role": "assistant",
|
||||
"content": content
|
||||
}
|
||||
if media is not None:
|
||||
message["media"] = media
|
||||
|
||||
data = {
|
||||
"id": f"chatcmpl-{int(time.time() * 1000)}",
|
||||
"object": "chat.completion",
|
||||
"created": int(time.time()),
|
||||
"model": model,
|
||||
"choices": [{
|
||||
"index": 0,
|
||||
"message": message,
|
||||
"finish_reason": "stop"
|
||||
}],
|
||||
"usage": {
|
||||
"prompt_tokens": 0,
|
||||
"completion_tokens": 0,
|
||||
"total_tokens": 0
|
||||
}
|
||||
}
|
||||
if media is not None:
|
||||
data["media"] = media
|
||||
return data
|
||||
|
||||
@staticmethod
|
||||
def _build_markdown_image_block(images: list) -> str:
|
||||
refs = []
|
||||
for item in images or []:
|
||||
if isinstance(item, dict):
|
||||
ref = str(item.get("url") or item.get("data_uri") or "").strip()
|
||||
else:
|
||||
ref = str(item or "").strip()
|
||||
if ref:
|
||||
refs.append(ref)
|
||||
|
||||
if not refs:
|
||||
return ""
|
||||
|
||||
return "".join(f"\n\n" for idx, ref in enumerate(refs)) + "\n\n"
|
||||
|
||||
@classmethod
|
||||
def pack_images_chunk(cls, images: list, completion_id: str = None) -> str:
|
||||
"""
|
||||
打包携带图片的 SSE chunk。
|
||||
|
||||
为保持 OpenAI 兼容性,图片会转成 Markdown 内容,而不是放进 delta.images。
|
||||
|
||||
Args:
|
||||
images: 图片数据列表,每项符合 ImageData 格式
|
||||
completion_id: 补全 ID
|
||||
|
||||
Returns:
|
||||
SSE 格式的字符串
|
||||
|
||||
Example:
|
||||
>>> chunk = SSEFormatter.pack_images_chunk([{"kind": "url", "url": "..."}])
|
||||
"""
|
||||
markdown = cls._build_markdown_image_block(images)
|
||||
if not markdown:
|
||||
return ""
|
||||
return cls.pack_chunk(markdown, completion_id=completion_id)
|
||||
|
||||
def pack_final_chunk_with_images(self, images: list, completion_id: str = None) -> str:
|
||||
"""
|
||||
打包包含图片的最终 chunk。
|
||||
|
||||
为保持 OpenAI 兼容性,图片会转成 Markdown 内容,而不是放进 delta.images。
|
||||
"""
|
||||
markdown = self._build_markdown_image_block(images)
|
||||
if not markdown:
|
||||
return ""
|
||||
return self.pack_chunk(markdown, completion_id=completion_id)
|
||||
Reference in new issue
Block a user