Initial commit

This commit is contained in:
Alex committed 2026-11-02 22:54:14 +08:00
commit 57fc3525eb
7311 files changed
+1291398

No files matched your search

+196
View File
@@ -0,0 +1,196 @@
"""
app/core/config_parts - 配置与基础设施子模块包
"""
from .env_config import (
PROJECT_ROOT,
DEFAULT_LOG_DIR,
atomic_write_json,
_replace_file_with_retry,
classproperty,
load_dotenv,
AppConfig,
app_config,
get_tutorial_selectors_url,
)
from .browser_constants import (
BrowserConstants,
_browser_constant_bool,
_browser_constant_int,
_BrowserConstantEnabledFilter,
)
from .exceptions import (
BrowserError,
BrowserConnectionError,
ElementNotFoundError,
WorkflowError,
WorkflowCancelledError,
ConfigurationError,
)
from .sse_formatter import SSEFormatter
from .message_validator import MessageValidator
from .log_collector import LogCollector, log_collector
from .log_redaction import (
_SENSITIVE_KEY_HINTS,
_REDACTED_TEXT,
_redact_data_uri_for_log,
_redact_long_base64_for_log,
_BASE64_LOG_CHARS,
_SENSITIVE_TEXT_SCAN_HINT_RE,
_SENSITIVE_TEXT_PRECHECK_MIN_CHARS,
_SENSITIVE_TEXT_LARGE_OMIT_THRESHOLD,
_SENSITIVE_TEXT_LARGE_EDGE_CHARS,
_has_long_base64_candidate,
_redact_long_base64_runs_for_log,
_should_scan_sensitive_text,
_SENSITIVE_TEXT_PATTERNS,
_SENSITIVE_BEARER_PATTERN,
_SENSITIVE_STANDALONE_TOKEN_PATTERNS,
_SENSITIVE_HEADER_PATTERN,
_SENSITIVE_PARAM_PATTERNS,
_sanitize_standalone_sensitive_tokens,
_is_sensitive_key,
_sanitize_sensitive_text,
sanitize_sensitive_data,
)
from .log_formatters import (
_truncate_long_message,
_get_log_display_limit,
_record_request_id,
_REQUEST_SHORT_ID_PATTERN,
_request_display_tag,
_record_request_tag,
_compact_logger_name_impl,
_compact_logger_name,
_LOGGER_DISPLAY_WIDTH,
_record_logger_name,
_record_kind,
_LEVEL_BADGES,
_record_level_badge,
_replace_log_tag,
_normalize_log_display_expression,
_record_display_message,
_format_log_display_parts,
_join_log_display_parts,
_format_log_display_line,
_WebLogHandler,
_web_log_handler,
_enable_windows_ansi,
_should_use_console_color,
_ConsoleColorFormatter,
_FileLogFormatter,
_DisplayLogFormatter,
_is_windows_file_lock_error,
_SafeRotatingFileHandler,
_get_positive_int_env,
_resolve_log_dir,
get_log_file_path,
get_shared_file_log_handler,
_shared_file_log_handler,
_shared_file_log_handler_lock,
)
from .cute_translator import (
_REQUEST_FINISH_PATTERN,
_TAB_START_INDEX_PATTERN,
_TAB_START_ROUTE_PATTERN,
_CHUNKED_LONG_TEXT_PATTERN,
_CHUNKED_DONE_PATTERN,
_CHUNKED_SHORT_TEXT_PATTERN,
_CHUNKED_FIRST_BLOCK_PATTERN,
_CHUNKED_PROGRESS_PATTERN,
_VERIFY_OK_EXACT_PATTERN,
_SEND_SUCCESS_RETRY_PATTERN,
_FILE_PASTE_DONE_PATTERN,
_CLIPBOARD_OK_PATTERN,
_FILE_PASTE_UPLOAD_SIGNAL_PATTERN,
_FILE_PASTE_WEAK_SIGNAL_PATTERN,
_FILE_PASTE_INPUT_SIGNAL_PATTERN,
_FILE_PASTE_INPUT_UPLOADED_PATTERN,
_FILE_PASTE_HINT_PATTERN,
_FILE_PASTE_TEMP_FILE_PATTERN,
_STEALTH_SEND_RETRY_PATTERN,
_STEALTH_SEND_RETRY_SIMPLE_PATTERN,
_STEALTH_WARMUP_DONE_PATTERN,
_STEALTH_REVIEW_DELAY_PATTERN,
_STEALTH_CLIPBOARD_PASTE_NOCLICK_PATTERN,
_STEALTH_CLIPBOARD_PASTE_PATTERN,
_STEALTH_RANDOM_PAUSE_PATTERN,
_STEALTH_PRE_SEND_HESITATE_PATTERN,
_STEALTH_HUMAN_CLICK_PATTERN,
_SEND_ATTACHMENT_WAIT_PATTERN,
_SEND_ATTACHMENT_READY_PATTERN,
_SEND_ATTACHMENT_SETTLE_PATTERN,
_SEND_RETRY_ACTION_PATTERN,
_SEND_RETRY_WINDOW_PATTERN,
_bool_phrase,
_split_suppressed_suffix,
_add_suppressed_marker,
_restore_suppressed_hint,
_apply_cute_rules,
_cute_stage_title,
_CUTE_EXTRA_RULES,
_ERROR_KEYWORD_HINTS,
_cuteify_error_message,
_cuteify_info_message,
_cuteify_warning_message,
_cuteify_debug_message,
)
from .secure_logger import (
_request_context,
_command_log_context,
_logger_setup_lock,
_logger_registry_lock,
_logger_registry,
SecureLogger,
command_log_context,
get_logger,
logger,
)
__all__ = [
# 应用配置
'AppConfig',
'app_config',
'get_tutorial_selectors_url',
'load_dotenv',
'PROJECT_ROOT',
'DEFAULT_LOG_DIR',
'atomic_write_json',
'_replace_file_with_retry',
'classproperty',
# 浏览器常量
'BrowserConstants',
'_browser_constant_bool',
'_browser_constant_int',
'_BrowserConstantEnabledFilter',
# 日志系统与收集器
'SecureLogger',
'logger',
'get_logger',
'log_collector',
'LogCollector',
'_request_context',
'_command_log_context',
'command_log_context',
'get_log_file_path',
'get_shared_file_log_handler',
'_shared_file_log_handler',
'_shared_file_log_handler_lock',
'_logger_setup_lock',
'_logger_registry_lock',
'_logger_registry',
'sanitize_sensitive_data',
# 异常
'BrowserError',
'BrowserConnectionError',
'ElementNotFoundError',
'WorkflowError',
'WorkflowCancelledError',
'ConfigurationError',
# SSE 与消息工具
'SSEFormatter',
'MessageValidator',
]
+315
View File
@@ -0,0 +1,315 @@
"""
app/core/config_parts/browser_constants.py - 浏览器常量配置与过滤器模块
"""
import json
import logging
import os
from pathlib import Path
from typing import Any, Dict, List, Optional
from .env_config import AppConfig, PROJECT_ROOT
class BrowserConstants:
"""浏览器相关常量(从 JSON 文件加载,支持热重载)"""
# ===== 配置缓存 =====
_config: Optional[Dict] = None
_config_file = Path("config/browser_config.json")
# ===== 默认值字典 =====
_DEFAULTS = {
'DEFAULT_PORT': 9222,
'CONNECTION_TIMEOUT': 10,
'MAX_REQUEST_EXECUTE_TIME_SEC': 300.0,
'STEALTH_DELAY_MIN': 0.03,
'STEALTH_DELAY_MAX': 0.1,
'ACTION_DELAY_MIN': 0.06,
'ACTION_DELAY_MAX': 0.14,
'STEALTH_PAUSE_PROBABILITY': 0.0,
'STEALTH_PAUSE_EXTRA_MAX': 0.15,
'STEALTH_KEY_DOWN_UP_MIN': 0.015,
'STEALTH_KEY_DOWN_UP_MAX': 0.04,
'STEALTH_KEY_BETWEEN_MIN': 0.02,
'STEALTH_KEY_BETWEEN_MAX': 0.06,
'STEALTH_PASTE_SETTLE_MIN': 0.12,
'STEALTH_PASTE_SETTLE_MAX': 0.25,
'STEALTH_SKIP_PASTE_VERIFY': True,
'STEALTH_SEND_IMAGE_WAIT': 8.0,
'STEALTH_SEND_IMAGE_RETRY_INTERVAL': 1.2,
'STEALTH_MOUSE_WARMUP_ENABLED': False,
'STEALTH_CLICK_STRATEGY': 'auto',
'STEALTH_DOM_CLICK_TARGETS': ['new_chat_btn', 'input_box', 'send_btn'],
'PAGE_INTERACTION_THROTTLE_ENABLED': True,
'PAGE_INTERACTION_MAX_CONCURRENT': 3,
'PAGE_INTERACTION_MAX_WAIT': 20.0,
'PAGE_INTERACTION_MIN_INTERVAL': 0.25,
'PAGE_INTERACTION_READY_TIMEOUT': 1.5,
'PAGE_INTERACTION_STABLE_SAMPLES': 2,
'PAGE_INTERACTION_SAMPLE_INTERVAL': 0.12,
'PAGE_INTERACTION_RECT_TOLERANCE': 3,
'COORD_CLICK_READY_TIMEOUT': 0.9,
'COORD_CLICK_STABLE_SAMPLES': 2,
'COORD_CLICK_SAMPLE_INTERVAL': 0.08,
'COORD_CLICK_RECT_TOLERANCE': 3,
'COORD_CLICK_EDGE_INSET': 4,
'COORD_CLICK_RETRY_OFFSETS': [[0, 0], [4, 0], [-4, 0], [0, 4], [0, -4], [7, 3], [-7, 3]],
'WORKFLOW_WAKE_TAB_BEFORE_INTERACTION': True,
'WORKFLOW_FOCUS_EMULATION_ON_INTERACTION': True,
'DEFAULT_ELEMENT_TIMEOUT': 3,
'FALLBACK_ELEMENT_TIMEOUT': 1,
'ELEMENT_CACHE_MAX_AGE': 5.0,
'LOG_INFO_CUTE_MODE': True,
'LOG_DEBUG_CUTE_MODE': True,
'LOG_CONSOLE_ENABLED': True,
'LOG_FILE_ENABLED': True,
'LOG_WEB_COLLECTOR_ENABLED': True,
'LOG_WEB_MAX_RECORDS': 5000,
'STREAM_CHECK_INTERVAL_MIN': 0.1,
'STREAM_CHECK_INTERVAL_MAX': 1.0,
'STREAM_CHECK_INTERVAL_DEFAULT': 0.3,
'STREAM_SILENCE_THRESHOLD': 6.0,
'STREAM_MAX_TIMEOUT': 600,
'STREAM_INITIAL_WAIT': 180,
'STREAM_CONTENT_SHRINK_TOLERANCE': 3,
'STREAM_STABLE_COUNT_THRESHOLD': 5,
'STREAM_SILENCE_THRESHOLD_FALLBACK': 10.0,
'MAX_MESSAGE_LENGTH': 100000,
'MAX_MESSAGES_COUNT': 100,
'TEXT_INPUT_CHUNK_SIZE': 30000,
'STREAM_USER_MSG_WAIT': 1.5,
'STREAM_PRE_BASELINE_DELAY': 0.3,
'GLOBAL_NETWORK_INTERCEPTION_ENABLED': False,
'GLOBAL_NETWORK_INTERCEPTION_LISTEN_PATTERN': 'http',
'GLOBAL_NETWORK_INTERCEPTION_WAIT_TIMEOUT': 0.5,
'GLOBAL_NETWORK_INTERCEPTION_RETRY_DELAY': 1.0,
'GLOBAL_NETWORK_INTERCEPTION_RES_TYPES': 'XHR,Fetch,EventSource,Document',
'NETWORK_DEBUG_CAPTURE_ENABLED': False,
'NETWORK_DEBUG_CAPTURE_MAX_BODY_CHARS': 50000,
'NETWORK_DEBUG_CAPTURE_MAX_FILES_PER_REQUEST': 3,
'NETWORK_DEBUG_CAPTURE_PARSER_FILTER': '',
'NETWORK_DEBUG_CAPTURE_MAX_TOTAL_MB': 50,
'NETWORK_DEBUG_CAPTURE_RETENTION_HOURS': 24,
'CONVERSATION_TIMEOUT_THRESHOLD': 0.0,
'FORCE_NEW_CONVERSATION': False,
'REQUEST_MONITOR_ENABLED': True,
'REQUEST_MONITOR_MAX_RECORDS': 200,
'REQUEST_MONITOR_DETAIL_ENABLED': True,
'REQUEST_MONITOR_SAVE_TO_FILE': True,
'REQUEST_MONITOR_MAX_CAPTURED_RESPONSE_CHARS': 30000,
'DASHBOARD_LOG_POLL_INTERVAL_MS': 1000,
'DASHBOARD_LOG_BACKGROUND_POLL_INTERVAL_MS': 5000,
'DASHBOARD_REQUEST_HISTORY_POLL_INTERVAL_MS': 3000,
'DASHBOARD_SYSTEM_STATS_ENABLED': True,
'DASHBOARD_SYSTEM_STATS_POLL_INTERVAL_MS': 3000,
'ATTACHMENT_READY_IDLE_TIMEOUT': 8.0,
'ATTACHMENT_READY_HARD_MAX_WAIT': 90.0,
}
# ===== 类属性(会被配置文件覆盖)=====
# 连接配置
DEFAULT_PORT = 9222
CONNECTION_TIMEOUT = 10
MAX_REQUEST_EXECUTE_TIME_SEC = 300.0
# 延迟配置
STEALTH_DELAY_MIN = 0.03
STEALTH_DELAY_MAX = 0.1
ACTION_DELAY_MIN = 0.06
ACTION_DELAY_MAX = 0.14
STEALTH_PAUSE_PROBABILITY = 0.0
STEALTH_PAUSE_EXTRA_MAX = 0.15
STEALTH_KEY_DOWN_UP_MIN = 0.015
STEALTH_KEY_DOWN_UP_MAX = 0.04
STEALTH_KEY_BETWEEN_MIN = 0.02
STEALTH_KEY_BETWEEN_MAX = 0.06
STEALTH_PASTE_SETTLE_MIN = 0.12
STEALTH_PASTE_SETTLE_MAX = 0.25
STEALTH_SKIP_PASTE_VERIFY = True
STEALTH_SEND_IMAGE_WAIT = 8.0
STEALTH_SEND_IMAGE_RETRY_INTERVAL = 1.2
STEALTH_MOUSE_WARMUP_ENABLED = False
STEALTH_CLICK_STRATEGY = "auto"
STEALTH_DOM_CLICK_TARGETS = ["new_chat_btn", "input_box", "send_btn"]
PAGE_INTERACTION_THROTTLE_ENABLED = True
PAGE_INTERACTION_MAX_CONCURRENT = 3
PAGE_INTERACTION_MAX_WAIT = 20.0
PAGE_INTERACTION_MIN_INTERVAL = 0.25
PAGE_INTERACTION_READY_TIMEOUT = 1.5
PAGE_INTERACTION_STABLE_SAMPLES = 2
PAGE_INTERACTION_SAMPLE_INTERVAL = 0.12
PAGE_INTERACTION_RECT_TOLERANCE = 3
WORKFLOW_WAKE_TAB_BEFORE_INTERACTION = True
WORKFLOW_FOCUS_EMULATION_ON_INTERACTION = True
# 元素查找
DEFAULT_ELEMENT_TIMEOUT = 3
FALLBACK_ELEMENT_TIMEOUT = 1
ELEMENT_CACHE_MAX_AGE = 5.0
# 日志
LOG_INFO_CUTE_MODE = True
LOG_DEBUG_CUTE_MODE = True
LOG_CONSOLE_ENABLED = True
LOG_FILE_ENABLED = True
LOG_WEB_COLLECTOR_ENABLED = True
LOG_WEB_MAX_RECORDS = 5000
# 流式监控
STREAM_CHECK_INTERVAL_MIN = 0.1
STREAM_CHECK_INTERVAL_MAX = 1.0
STREAM_CHECK_INTERVAL_DEFAULT = 0.3
STREAM_SILENCE_THRESHOLD = 6.0
STREAM_MAX_TIMEOUT = 600
STREAM_INITIAL_WAIT = 180
# 流式监控增强配置
STREAM_CONTENT_SHRINK_TOLERANCE = 3
STREAM_STABLE_COUNT_THRESHOLD = 5
STREAM_SILENCE_THRESHOLD_FALLBACK = 10.0
# 输入验证
MAX_MESSAGE_LENGTH = 100000
MAX_MESSAGES_COUNT = 100
# 附件/图片上传就绪判定
ATTACHMENT_READY_IDLE_TIMEOUT = 8.0
ATTACHMENT_READY_HARD_MAX_WAIT = 90.0
# 文本输入
TEXT_INPUT_CHUNK_SIZE = 30000
# 两阶段 baseline 配置
STREAM_USER_MSG_WAIT = 1.5
STREAM_PRE_BASELINE_DELAY = 0.3
# 全局常驻网络监听(仅事件上报)
GLOBAL_NETWORK_INTERCEPTION_ENABLED = False
GLOBAL_NETWORK_INTERCEPTION_LISTEN_PATTERN = "http"
GLOBAL_NETWORK_INTERCEPTION_WAIT_TIMEOUT = 0.5
GLOBAL_NETWORK_INTERCEPTION_RETRY_DELAY = 1.0
# 仅捕获的资源类型(逗号分隔;"*" 表示全部)。P0-4:默认只抓 API 类请求
GLOBAL_NETWORK_INTERCEPTION_RES_TYPES = "XHR,Fetch,EventSource,Document"
# 网络解析调试捕获
NETWORK_DEBUG_CAPTURE_ENABLED = False
NETWORK_DEBUG_CAPTURE_MAX_BODY_CHARS = 50000
NETWORK_DEBUG_CAPTURE_MAX_FILES_PER_REQUEST = 3
NETWORK_DEBUG_CAPTURE_PARSER_FILTER = ""
NETWORK_DEBUG_CAPTURE_MAX_TOTAL_MB = 50
NETWORK_DEBUG_CAPTURE_RETENTION_HOURS = 24
# 对话会话控制
CONVERSATION_TIMEOUT_THRESHOLD = 0.0
FORCE_NEW_CONVERSATION = False
# 低资源运行
REQUEST_MONITOR_ENABLED = True
REQUEST_MONITOR_MAX_RECORDS = 200
REQUEST_MONITOR_DETAIL_ENABLED = True
REQUEST_MONITOR_SAVE_TO_FILE = True
REQUEST_MONITOR_MAX_CAPTURED_RESPONSE_CHARS = 30000
DASHBOARD_LOG_POLL_INTERVAL_MS = 1000
DASHBOARD_LOG_BACKGROUND_POLL_INTERVAL_MS = 5000
DASHBOARD_REQUEST_HISTORY_POLL_INTERVAL_MS = 3000
DASHBOARD_SYSTEM_STATS_ENABLED = True
DASHBOARD_SYSTEM_STATS_POLL_INTERVAL_MS = 3000
@classmethod
def _load_config(cls):
"""从文件加载配置"""
if cls._config_file.exists():
try:
with open(cls._config_file, 'r', encoding='utf-8') as f:
cls._config = json.load(f)
return
except Exception as e:
print(f"[BrowserConstants] 加载配置失败: {e}")
# 加载失败或文件不存在,使用默认值
cls._config = cls._DEFAULTS.copy()
@classmethod
def _apply_to_class_attrs(cls):
"""将配置值应用到类属性(兼容旧代码直接访问类属性的方式)"""
if cls._config is None:
cls._load_config()
for key, value in cls._config.items():
if hasattr(cls, key):
setattr(cls, key, value)
# 同步环境变量中的浏览器端口
env_port = AppConfig.get_browser_port()
if env_port:
cls.DEFAULT_PORT = env_port
@classmethod
def get(cls, key: str):
"""获取配置值(支持动态加载)"""
if cls._config is None:
cls._load_config()
return cls._config.get(key, cls._DEFAULTS.get(key))
@classmethod
def get_defaults(cls) -> Dict:
"""获取所有默认值"""
return cls._DEFAULTS.copy()
@classmethod
def reload(cls):
"""重新加载配置(热重载)"""
cls._config = None
cls._load_config()
cls._apply_to_class_attrs()
def _browser_constant_bool(key: str, default: bool = True) -> bool:
value = BrowserConstants.get(key)
if value is None:
return bool(default)
if isinstance(value, bool):
return value
if isinstance(value, (int, float)):
return bool(value)
text = str(value).strip().lower()
if text in {"1", "true", "yes", "y", "on"}:
return True
if text in {"0", "false", "no", "n", "off"}:
return False
return bool(default)
def _browser_constant_int(
key: str,
default: int,
*,
min_value: int = 0,
max_value: Optional[int] = None,
) -> int:
try:
value = int(BrowserConstants.get(key))
except Exception:
value = int(default)
value = max(int(min_value), value)
if max_value is not None:
value = min(int(max_value), value)
return value
class _BrowserConstantEnabledFilter(logging.Filter):
def __init__(self, key: str, default: bool = True):
super().__init__()
self.key = key
self.default = default
def filter(self, record: logging.LogRecord) -> bool:
return _browser_constant_bool(self.key, self.default)
# 模块加载时执行自初始化
BrowserConstants._load_config()
BrowserConstants._apply_to_class_attrs()
File diff suppressed because it is too large. Load diff
+364
View File
@@ -0,0 +1,364 @@
"""
app/core/config_parts/env_config.py - 环境变量与基础文件工具模块
"""
import os
import json
import tempfile
import time
from pathlib import Path
from typing import Any, Dict, List, Optional
PROJECT_ROOT = Path(__file__).resolve().parents[3]
DEFAULT_LOG_DIR = PROJECT_ROOT / "logs"
def atomic_write_json(path: str | Path, payload: Any) -> None:
"""Atomically write JSON to disk using a same-directory temporary file."""
target = Path(path)
target.parent.mkdir(parents=True, exist_ok=True)
fd = None
tmp_path: Optional[Path] = None
try:
fd, tmp_name = tempfile.mkstemp(
prefix=f".{target.name}.",
suffix=".tmp",
dir=str(target.parent),
)
tmp_path = Path(tmp_name)
with os.fdopen(fd, "w", encoding="utf-8", newline="\n") as f:
fd = None
json.dump(payload, f, indent=2, ensure_ascii=False)
f.write("\n")
f.flush()
os.fsync(f.fileno())
_replace_file_with_retry(tmp_path, target)
except Exception:
if fd is not None:
try:
os.close(fd)
except Exception:
pass
if tmp_path is not None:
try:
tmp_path.unlink(missing_ok=True)
except Exception:
pass
raise
def _replace_file_with_retry(source: str | Path, dest: str | Path) -> None:
"""Replace a file, retrying transient Windows sharing violations."""
source_path = Path(source)
dest_path = Path(dest)
attempts = 3 if os.name == "nt" else 1
delay = 0.02
for attempt in range(attempts):
try:
os.replace(source_path, dest_path)
return
except PermissionError:
if attempt >= attempts - 1:
raise
time.sleep(delay)
delay *= 2
class classproperty:
"""Allow config access via both `AppConfig.X` and `app_config.X`."""
def __init__(self, fget):
self.fget = fget
def __get__(self, obj, owner=None):
return self.fget(owner)
def load_dotenv(env_file: str = ".env", override: bool = False):
"""
手动加载 .env 文件(不依赖 python-dotenv)
"""
env_path = Path(env_file)
if not env_path.exists():
return
try:
with open(env_path, 'r', encoding='utf-8') as f:
for line in f:
line = line.strip()
if not line or line.startswith('#'):
continue
if '=' in line:
key, _, value = line.partition('=')
key = key.strip()
value = value.strip()
if (value.startswith('"') and value.endswith('"')) or \
(value.startswith("'") and value.endswith("'")):
value = value[1:-1]
if key:
if override or key not in os.environ:
os.environ[key] = value
except Exception as e:
print(f"[Config] 加载 .env 失败: {e}")
load_dotenv(override=os.getenv("UWAPI_DOTENV_OVERRIDE", "").lower() in ("1", "true", "yes", "on"))
class AppConfig:
"""应用配置(从环境变量读取)"""
@staticmethod
def _env_bool(name: str, default: bool = False) -> bool:
value = os.getenv(name)
if value is None or str(value).strip() == "":
return bool(default)
return str(value).strip().lower() in ("true", "1", "yes", "on")
@staticmethod
def _env_bool_secure(name: str, default: bool = False) -> bool:
"""安全开关专用布尔解析(H1):值存在但不是合法布尔字面量时 fail-closed 返回 True。
兼容逻辑:若值为 3.0.0 模板自带的占位符 ``your-secret-here``,视为未设置(回退 default),
避免因无令牌 fail-closed 开启认证导致接口与控制面板 500。
"""
value = os.getenv(name)
if value is None or str(value).strip() == "":
return bool(default)
lowered = str(value).strip().lower()
if lowered == "your-secret-here":
return bool(default)
if lowered in ("true", "1", "yes", "on"):
return True
if lowered in ("false", "0", "no", "off"):
return False
return True
@staticmethod
def _env_int(name: str, default: int) -> int:
"""安全读取整数环境变量:空值/非法值一律回落默认值,避免服务启动期崩溃。"""
raw = os.getenv(name)
if raw is None or str(raw).strip() == "":
return int(default)
try:
return int(str(raw).strip())
except (TypeError, ValueError):
return int(default)
@staticmethod
def _env_float(name: str, default: float) -> float:
"""安全读取浮点环境变量:空值/非法值一律回落默认值,避免服务启动期崩溃。"""
raw = os.getenv(name)
if raw is None or str(raw).strip() == "":
return float(default)
try:
return float(str(raw).strip())
except (TypeError, ValueError):
return float(default)
# ===== 服务配置 =====
@staticmethod
def get_host() -> str:
return os.getenv("APP_HOST", "127.0.0.1")
@staticmethod
def get_port() -> int:
return AppConfig._env_int("APP_PORT", 8199)
@staticmethod
def is_debug() -> bool:
return os.getenv("APP_DEBUG", "false").lower() in ("true", "1", "yes")
@staticmethod
def get_log_level() -> str:
return os.getenv("LOG_LEVEL", "INFO").upper()
# ===== 认证配置 =====
@staticmethod
def is_auth_enabled() -> bool:
return AppConfig._env_bool_secure("AUTH_ENABLED", False)
@staticmethod
def get_auth_token() -> str:
return os.getenv("AUTH_TOKEN", "").strip()
@staticmethod
def is_dashboard_auth_enabled() -> bool:
value = os.getenv("DASHBOARD_AUTH_ENABLED")
if value is None or str(value).strip() == "":
return AppConfig.is_auth_enabled()
return AppConfig._env_bool_secure("DASHBOARD_AUTH_ENABLED", False)
@staticmethod
def get_dashboard_auth_token() -> str:
token = os.getenv("DASHBOARD_AUTH_TOKEN", "").strip()
return token or AppConfig.get_auth_token()
# ===== CORS 配置 =====
@staticmethod
def is_cors_enabled() -> bool:
return os.getenv("CORS_ENABLED", "true").lower() in ("true", "1", "yes")
@staticmethod
def get_cors_origins() -> List[str]:
"""S1:默认不放行任何跨源来源。
控制面板与 API 同源,本来就不需要 CORS;非浏览器客户端(SDK、curl、酒馆后端)
不受 CORS 影响。需要浏览器跨源调用时请显式列出来源,``*`` 仍可显式配置但不推荐。
"""
from app.core.http_security import parse_cors_origins
return parse_cors_origins(os.getenv("CORS_ORIGINS", ""))
# ===== 浏览器配置 =====
@staticmethod
def get_browser_port() -> int:
return AppConfig._env_int("BROWSER_PORT", 9222)
@staticmethod
def is_auto_open_browser_enabled() -> bool:
return AppConfig._env_bool("AUTO_OPEN_BROWSER", True)
# ===== Dashboard 配置 =====
@staticmethod
def is_dashboard_enabled() -> bool:
return os.getenv("DASHBOARD_ENABLED", "true").lower() in ("true", "1", "yes")
@staticmethod
def get_dashboard_file() -> str:
return os.getenv("DASHBOARD_FILE", "static/index.html")
# ===== 定时服务重启守护 =====
@staticmethod
def is_scheduled_restart_enabled() -> bool:
return AppConfig._env_bool("SCHEDULED_RESTART_ENABLED", False)
@staticmethod
def get_scheduled_restart_interval_seconds() -> int:
return max(60, AppConfig._env_int("SCHEDULED_RESTART_INTERVAL_SECONDS", 10800))
@staticmethod
def get_scheduled_restart_drain_timeout_seconds() -> int:
return max(0, AppConfig._env_int("SCHEDULED_RESTART_DRAIN_TIMEOUT_SECONDS", 1800))
@staticmethod
def get_scheduled_restart_tab_state_policy() -> str:
"""Reserved for future tab-state cleanup; preserve is currently the only policy."""
return os.getenv("SCHEDULED_RESTART_TAB_STATE_POLICY", "preserve").strip().lower() or "preserve"
# ===== AI 分析配置 =====
@staticmethod
def get_helper_api_key() -> str:
return os.getenv("HELPER_API_KEY", "")
@staticmethod
def get_helper_base_url() -> str:
return os.getenv("HELPER_BASE_URL", "")
@staticmethod
def get_helper_model() -> str:
return os.getenv("HELPER_MODEL", "gpt-4")
@staticmethod
def get_helper_api_provider() -> str:
return os.getenv("HELPER_API_PROVIDER", "auto").lower()
@staticmethod
def get_max_html_chars() -> int:
return AppConfig._env_int("MAX_HTML_CHARS", 120000)
@staticmethod
def get_canvas_image_max_size() -> int:
try:
value = AppConfig._env_int("CANVAS_IMAGE_MAX_SIZE", 1024)
except Exception:
value = 1024
return max(1, value)
# ===== 错误标签页 AI 恢复配置 =====
@staticmethod
def is_tab_recovery_enabled() -> bool:
return AppConfig._env_bool("TAB_RECOVERY_ENABLED", False)
@staticmethod
def get_tab_recovery_api_url() -> str:
return os.getenv("TAB_RECOVERY_API_URL", "").strip()
@staticmethod
def get_tab_recovery_api_key() -> str:
return os.getenv("TAB_RECOVERY_API_KEY", "").strip()
@staticmethod
def get_tab_recovery_model() -> str:
return os.getenv("TAB_RECOVERY_MODEL", "gpt-4o").strip() or "gpt-4o"
@staticmethod
def get_tab_recovery_max_attempts() -> int:
try:
value = AppConfig._env_int("TAB_RECOVERY_MAX_ATTEMPTS", 1)
except Exception:
value = 1
return max(0, value)
@staticmethod
def get_tab_recovery_timeout_sec() -> float:
try:
value = AppConfig._env_float("TAB_RECOVERY_TIMEOUT_SEC", 120.0)
except Exception:
value = 120.0
return max(1.0, value)
@staticmethod
def get_tab_recovery_worker_exit_wait_sec() -> float:
try:
value = AppConfig._env_float("TAB_RECOVERY_WORKER_EXIT_WAIT_SEC", 600.0)
except Exception:
value = 600.0
return max(0.0, value)
@staticmethod
def is_tab_recovery_refresh_on_unknown() -> bool:
return AppConfig._env_bool("TAB_RECOVERY_REFRESH_ON_UNKNOWN", True)
# ===== 配置文件路径 =====
@staticmethod
def get_sites_config_file() -> str:
return os.getenv("SITES_CONFIG_FILE", "config/sites.json")
# ===== 便捷属性(支持类/实例两种访问方式)=====
@classproperty
def HOST(cls) -> str:
return cls.get_host()
@classproperty
def PORT(cls) -> int:
return cls.get_port()
@classproperty
def DEBUG(cls) -> bool:
return cls.is_debug()
@classproperty
def LOG_LEVEL(cls) -> str:
return cls.get_log_level()
@classproperty
def AUTH_TOKEN(cls) -> str:
return cls.get_auth_token()
@classproperty
def DASHBOARD_AUTH_TOKEN(cls) -> str:
return cls.get_dashboard_auth_token()
# 创建全局配置实例
app_config = AppConfig()
def get_tutorial_selectors_url() -> str:
"""返回本地控制台教程中选择器配置章节的跳转链接。"""
try:
port = AppConfig.get_port()
except Exception:
port = 8199
return f"http://127.0.0.1:{port}/static/tutorial/index.html#selectors"
+33
View File
@@ -0,0 +1,33 @@
"""
app/core/config_parts/exceptions.py - 异常定义模块
"""
class BrowserError(Exception):
"""浏览器相关错误基类"""
pass
class BrowserConnectionError(BrowserError):
"""浏览器连接错误"""
pass
class ElementNotFoundError(BrowserError):
"""元素未找到错误"""
pass
class WorkflowError(BrowserError):
"""工作流执行错误"""
pass
class WorkflowCancelledError(WorkflowError):
"""工作流被取消"""
pass
class ConfigurationError(BrowserError):
"""配置错误"""
pass
+90
View File
@@ -0,0 +1,90 @@
"""
app/core/config_parts/log_collector.py - Web 前端日志收集器模块
"""
from collections import deque
import threading
import time
from typing import Any, Dict, List, Optional, Tuple
from .browser_constants import _browser_constant_bool, _browser_constant_int
class LogCollector:
"""收集日志用于前端展示"""
def __init__(self, max_logs: int = 5000):
self.logs: deque = deque(maxlen=max_logs)
self.lock = threading.Lock()
self._next_seq = 1
self._last_clear_seq = 0
def is_enabled(self) -> bool:
return _browser_constant_bool("LOG_WEB_COLLECTOR_ENABLED", True)
def _target_max_logs(self) -> int:
return _browser_constant_int("LOG_WEB_MAX_RECORDS", 5000, min_value=0, max_value=10000)
def _sync_limits_unlocked(self) -> None:
target_max_logs = self._target_max_logs()
if self.logs.maxlen == target_max_logs:
return
self.logs = deque(list(self.logs)[-target_max_logs:] if target_max_logs else [], maxlen=target_max_logs)
def add(self, entry: Dict[str, Any]):
with self.lock:
self._sync_limits_unlocked()
if not self.is_enabled() or self.logs.maxlen == 0:
if self.logs:
self.logs.clear()
return
payload = dict(entry or {})
payload["seq"] = self._next_seq
payload.setdefault("timestamp", time.time())
payload.setdefault("level", "INFO")
payload.setdefault("kind", payload["level"])
payload.setdefault("message", "")
payload.setdefault("display_message", payload["message"])
payload.setdefault("message_text", payload["message"])
payload.setdefault("original_message_text", payload["message_text"])
payload.setdefault("message_alias", "")
payload.setdefault("logger", "")
payload.setdefault("request_id", "SYSTEM")
self.logs.append(payload)
self._next_seq += 1
def get_recent(self, since: float = 0, after_seq: int = 0) -> Tuple[List[Dict[str, Any]], int, bool]:
with self.lock:
self._sync_limits_unlocked()
if not self.is_enabled() or self.logs.maxlen == 0:
if self.logs:
self.logs.clear()
return [], self._next_seq - 1, False
cursor = max(0, int(after_seq or 0))
cleared = bool(cursor and self._last_clear_seq and cursor <= self._last_clear_seq)
if cursor > 0:
latest_seq = self._next_seq - 1
if cursor == latest_seq:
return [], latest_seq, cleared
if cursor > latest_seq:
return list(self.logs), latest_seq, True
oldest_seq = int(self.logs[0].get("seq", 0) or 0) if self.logs else latest_seq
if self.logs and cursor < oldest_seq:
return list(self.logs), latest_seq, True
logs = [
log for log in self.logs
if int(log.get("seq", 0) or 0) > cursor
]
else:
logs = [log for log in self.logs if float(log.get("timestamp", 0) or 0) > since]
return logs, self._next_seq - 1, cleared
def clear(self):
with self.lock:
self.logs.clear()
self._last_clear_seq = self._next_seq - 1
self._next_seq += 1
# 全局日志收集器实例
log_collector = LogCollector()
+652
View File
@@ -0,0 +1,652 @@
"""
app/core/config_parts/log_formatters.py - 日志格式化器与 Handler 管理模块
"""
import ctypes
import datetime
import logging
from logging.handlers import RotatingFileHandler
import os
from pathlib import Path
import re
import sys
import threading
import time
from typing import Any, List, Optional, Tuple
from .env_config import PROJECT_ROOT, DEFAULT_LOG_DIR
from .browser_constants import _BrowserConstantEnabledFilter
from .log_collector import log_collector
from .log_redaction import _sanitize_sensitive_text
_shared_file_log_handler: Optional[logging.Handler] = None
_shared_file_log_handler_lock = threading.Lock()
def _get_positive_int_env(name: str, default: int) -> int:
raw = str(os.getenv(name, "") or "").strip()
if not raw:
return default
try:
value = int(raw)
except (TypeError, ValueError, OverflowError):
return default
return value if value > 0 else default
def _truncate_long_message(text: str, max_chars: int) -> tuple[str, bool]:
raw_text = str(text or "")
if max_chars <= 0 or len(raw_text) <= max_chars:
return raw_text, False
omitted = len(raw_text) - max_chars
return (
f"{raw_text[:max_chars]}... [truncated {omitted} chars; see app.log for full sanitized text]",
True,
)
def _get_log_display_limit(env_name: str, default: int) -> int:
return _get_positive_int_env(env_name, default)
def _record_request_id(record: logging.LogRecord) -> str:
return str(getattr(record, "codex_request_id", "") or "SYSTEM")
_REQUEST_SHORT_ID_PATTERN = re.compile(r"^req-(\d+)$", re.IGNORECASE)
def _request_display_tag(request_id: Any) -> str:
raw = str(request_id or "").strip()
if not raw or raw.upper() == "SYSTEM":
return "SYSTEM"
match = _REQUEST_SHORT_ID_PATTERN.match(raw)
if match:
try:
return f"#{int(match.group(1)):03d}"
except Exception:
return f"#{match.group(1)}"
return raw if len(raw) <= 8 else f"#{raw[-6:]}"
def _record_request_tag(record: logging.LogRecord) -> str:
return _request_display_tag(_record_request_id(record))
def _compact_logger_name_impl(name: Any, max_chars: int = 16) -> str:
raw = str(name or "").strip().upper()
if not raw:
return ""
if len(raw) <= max_chars:
return raw
def cap(label: str) -> str:
label = str(label or "")
if len(label) <= max_chars:
return label
if max_chars <= 3:
return label[:max_chars]
head = max(1, (max_chars - 1) // 2)
tail = max(1, max_chars - head - 1)
return f"{label[:head]}~{label[-tail:]}"
parts = [part for part in raw.split(".") if part]
if len(parts) > 1:
prefix = ".".join(part[:1] for part in parts[:-1] if part)
tail = parts[-1]
candidate = f"{prefix}.{tail}" if prefix else tail
if len(candidate) <= max_chars:
return candidate
tail_budget = max(3, max_chars - len(prefix) - (1 if prefix else 0))
short_tail = tail[-tail_budget:]
return cap(f"{prefix}.{short_tail}" if prefix else short_tail)
if max_chars <= 3:
return raw[:max_chars]
return cap(raw)
def _compact_logger_name(name: Any, max_chars: int = 16) -> str:
res = _compact_logger_name_impl(name, max_chars)
if len(res) <= max_chars:
return res
if max_chars <= 3:
return res[:max_chars]
head = max(1, (max_chars - 1) // 2)
tail = max(1, max_chars - head - 1)
return f"{res[:head]}~{res[-tail:]}"
_LOGGER_DISPLAY_WIDTH = 12
def _record_logger_name(record: logging.LogRecord) -> str:
return _compact_logger_name(
getattr(record, "codex_logger_name", "") or getattr(record, "name", "") or "",
max_chars=_LOGGER_DISPLAY_WIDTH,
)
def _record_kind(record: logging.LogRecord) -> str:
return str(getattr(record, "codex_kind", "") or record.levelname or "INFO").upper()
# 级别徽标:控制台/展示行使用的 3 字符级别标识
_LEVEL_BADGES = {
"DEBUG": "DBG",
"INFO": "INF",
"WARNING": "WRN",
"WARN": "WRN",
"ERROR": "ERR",
"CRITICAL": "CRI",
"SUCCESS": "SUC",
"STREAM": "STM",
"NETWORK": "NET",
}
def _record_level_badge(record: logging.LogRecord) -> str:
kind = _record_kind(record)
if kind in _LEVEL_BADGES:
return _LEVEL_BADGES[kind]
level = str(record.levelname or "").upper()
return _LEVEL_BADGES.get(level, (kind[:3] or "LOG").ljust(3))
def _replace_log_tag(text: str, source_tag: str, target_tag: str) -> str:
if text.startswith(source_tag):
rest = text[len(source_tag):].lstrip()
return f"{target_tag} {rest}".rstrip()
return text
def _normalize_log_display_expression(logger_name: str, message: str) -> str:
"""Normalize legacy log expressions for display without changing raw logs."""
text = str(message or "")
if not text:
return text
input_tags = (
"[FILE_PASTE]",
"[CHUNKED_INPUT]",
"[CLIPBOARD_OK]",
"[VERIFY_OK]",
"[VERIFY_FAIL]",
"[VERIFY]",
"[INPUT_SNAPSHOT]",
"[STEALTH_VERIFY]",
)
for tag in input_tags:
normalized = _replace_log_tag(text, tag, "[INPUT]")
if normalized != text:
return normalized
for tag in ("[NetworkMonitor]", "[NETWORK_MONITOR]"):
normalized = _replace_log_tag(text, tag, "[MONIT]")
if normalized != text:
return normalized
for tag in ("[JS_EXEC]", "[CONTENT_PARSE]", "[PROBE]", "[IMAGE]", "[STEALTH_CLICK]", "[STEALTH]"):
normalized = _replace_log_tag(text, tag, "[PAGE]")
if normalized != text:
return normalized
if text.startswith("[REQUEST_TRANSPORT]"):
return _replace_log_tag(text, "[REQUEST_TRANSPORT]", "[ROUTE]")
if text.startswith("[Executor]"):
rest = text[len("[Executor]"):].lstrip()
target = "[MONIT]" if any(hint in rest for hint in ("抓流", "监听", "网络")) else "[PAGE]"
return f"{target} {rest}".rstrip()
if text.startswith("[TabPool]"):
return _replace_log_tag(text, "[TabPool]", "[POOL]")
tabpool_match = re.match(r"^TabPool\s*(?:→|->)\s*(.+)$", text, re.S)
if tabpool_match:
return f"[POOL] 标签页已被占用: tab_id={tabpool_match.group(1)}"
if text.startswith("TabPoolManager "):
return f"[POOL] {text}"
wait_done_match = re.match(r"^等待结束\s*(?:→|->)\s*(.+)$", text, re.S)
if wait_done_match:
return f"[POOL] 标签页等待结束: {wait_done_match.group(1)}"
if text.startswith("排队等待 "):
return f"[POOL] {text}"
if text.startswith("等待标签页 ") or text.startswith("等待域名路由 "):
return f"[POOL] {text}"
assign_match = re.match(r"^标签页 (.+) 分配编号 #(\d+)$", text)
if assign_match:
session_id, index_no = assign_match.groups()
return f"[POOL] 标签页分配编号: tab_id={session_id}, idx=#{index_no}"
if text.startswith("发送成功"):
return f"[SEND] {text}"
if text == "浏览器连接成功" or text == "关闭浏览器连接":
return f"[SYS] {text}"
if logger_name == "REQUEST" and text == "创建":
return "[ROUTE] 请求上下文已创建"
if logger_name == "API.CHAT" and text == "开始":
return "[ROUTE] 聊天补全请求开始处理"
return text
def _resolve_pending_display_text(record: logging.LogRecord) -> None:
"""SecureLogger 只标记 codex_display_pending;第一次渲染时才做喵化翻译并缓存。"""
if not getattr(record, "codex_display_pending", False):
return
record.codex_display_pending = False
original = str(getattr(record, "codex_original_message_text", "") or "")
try:
from .cute_translator import cuteify_display_message
record.codex_display_message_text = cuteify_display_message(
str(getattr(record, "codex_kind", "") or record.levelname or ""),
str(getattr(record, "codex_logger_name", "") or record.name or ""),
original,
)
except Exception:
record.codex_display_message_text = original
def _record_display_message(record: logging.LogRecord) -> str:
_resolve_pending_display_text(record)
message = str(getattr(record, "codex_display_message_text", "") or "")
if not message:
message = str(record.getMessage() or "")
original_message = str(getattr(record, "codex_original_message_text", "") or "")
if message == original_message or not original_message:
message = _normalize_log_display_expression(_record_logger_name(record), message)
return _sanitize_sensitive_text(message)
def _format_log_display_parts(
record: logging.LogRecord,
message: str,
*,
max_chars: int = 0,
) -> tuple[list[str], str, bool]:
"""构建展示行的各个片段。
返回 (prefix_parts, body, truncated):
- prefix_parts = [时间, 级别徽标, 请求标签, 日志器名],便于控制台按片段分别上色;
- body 已按前缀宽度做了多行缩进对齐。
"""
now = datetime.datetime.fromtimestamp(
float(getattr(record, "created", time.time()) or time.time())
).strftime("%H:%M:%S")
badge = _record_level_badge(record)
request_tag = _record_request_tag(record)
logger_name = _record_logger_name(record)
parts = [
now,
f"{badge:<3}",
f"{request_tag:<8}",
f"{logger_name:<{_LOGGER_DISPLAY_WIDTH}}",
]
prefix_width = sum(len(part) for part in parts) + 3 * len(parts)
body, truncated = _truncate_long_message(str(message or ""), max_chars)
body = body.replace("\n", "\n" + " " * prefix_width)
return parts, body, truncated
def _join_log_display_parts(parts: list[str], body: str) -> str:
return " │ ".join(parts) + " │ " + body
def _format_log_display_line(
record: logging.LogRecord,
message: str,
*,
max_chars: int = 0,
) -> tuple[str, bool]:
parts, body, truncated = _format_log_display_parts(record, message, max_chars=max_chars)
return _join_log_display_parts(parts, body), truncated
class _WebLogHandler(logging.Handler):
"""将日志发送到 Web 收集器(内部类)"""
def emit(self, record):
try:
if not log_collector.is_enabled():
return
raw_message = _sanitize_sensitive_text(str(getattr(record, "codex_message", "") or ""))
if not raw_message:
raw_message = _sanitize_sensitive_text(str(record.getMessage() or ""))
message_text = _record_display_message(record)
original_message_text = _sanitize_sensitive_text(str(
getattr(record, "codex_original_message_text", "") or raw_message
))
if not message_text:
message_text = raw_message
web_limit = _get_log_display_limit("LOG_WEB_MAX_CHARS", 2000)
message_text, message_truncated = _truncate_long_message(message_text, web_limit)
original_message_text, original_truncated = _truncate_long_message(
original_message_text,
web_limit,
)
msg, line_truncated = _format_log_display_line(
record,
message_text,
)
logger_name = _record_logger_name(record)
request_id = _record_request_id(record)
request_tag = _record_request_tag(record)
kind = _record_kind(record)
log_collector.add({
"timestamp": float(getattr(record, "created", time.time()) or time.time()),
"level": str(record.levelname or "INFO").upper(),
"kind": kind,
"message": msg,
"display_message": msg,
"message_text": message_text,
"original_message_text": original_message_text,
"message_alias": message_text if message_text != original_message_text else "",
"logger": logger_name,
"request_id": request_id,
"request_tag": request_tag,
"truncated": bool(message_truncated or original_truncated or line_truncated),
})
except Exception:
self.handleError(record)
def _enable_windows_ansi() -> bool:
"""在 Windows 控制台中尽量启用 ANSI 颜色支持。"""
if os.name != "nt":
return True
try:
kernel32 = ctypes.windll.kernel32
handle = kernel32.GetStdHandle(-11) # STD_OUTPUT_HANDLE
if handle in (0, -1):
return False
mode = ctypes.c_uint()
if kernel32.GetConsoleMode(handle, ctypes.byref(mode)) == 0:
return False
ENABLE_VIRTUAL_TERMINAL_PROCESSING = 0x0004
if mode.value & ENABLE_VIRTUAL_TERMINAL_PROCESSING:
return True
return kernel32.SetConsoleMode(
handle,
mode.value | ENABLE_VIRTUAL_TERMINAL_PROCESSING
) != 0
except Exception:
return False
def _should_use_console_color() -> bool:
"""判断当前控制台是否应启用 ANSI 颜色。"""
if os.environ.get("NO_COLOR"):
return False
if os.name == "nt":
if _enable_windows_ansi():
return True
if os.environ.get("WT_SESSION"):
return True
if os.environ.get("ANSICON"):
return True
if os.environ.get("ConEmuANSI", "").upper() == "ON":
return True
if os.environ.get("TERM_PROGRAM") == "vscode":
return True
return False
return bool(getattr(sys.stdout, "isatty", lambda: False)())
class _ConsoleColorFormatter(logging.Formatter):
"""仅用于控制台输出的彩色格式化器。"""
RESET = "\033[0m"
# 整行色调
LINE_TONES = {
"ERROR": "\033[31m",
"WARN": "\033[33m",
"KEY": "\033[94m",
"SUCCESS": "\033[92m",
"DEBUG": "\033[90m",
}
# 级别徽标色
BADGE_TONES = {
"INF": "\033[32m",
"STM": "\033[36m",
"NET": "\033[36m",
"SUC": "\033[92m",
"WRN": "\033[1;33m",
"ERR": "\033[1;31m",
"CRI": "\033[1;31m",
"DBG": "\033[90m",
}
KEY_PATTERNS = (
"[CMD] ▶ 执行:",
"[CMD] 执行:",
"[CMD] 开始执行工作流:",
"[CMD] 触发命令:",
"[CMD] 链式触发:",
"[CMD] 条件分支触发:",
"[CMD] 结果事件触发:",
)
def __init__(self):
super().__init__()
self._use_color = _should_use_console_color()
def _resolve_tone(self, record: logging.LogRecord, message: str) -> Optional[str]:
level = str(record.levelname or "").upper()
if level in ("ERROR", "CRITICAL"):
return "ERROR"
if level == "WARNING":
return "WARN"
if level == "DEBUG":
return "DEBUG"
if any(pattern in message for pattern in self.KEY_PATTERNS):
return "KEY"
if _record_kind(record) == "SUCCESS":
return "SUCCESS"
return None
def format(self, record: logging.LogRecord) -> str:
message = _record_display_message(record)
console_limit = _get_log_display_limit("LOG_CONSOLE_MAX_CHARS", 1200)
message, _ = _truncate_long_message(message, console_limit)
parts, body, _ = _format_log_display_parts(record, message)
formatted = _join_log_display_parts(parts, body)
if record.exc_info:
exc_text = self.formatException(record.exc_info)
if exc_text:
formatted = f"{formatted}\n{exc_text}"
if not self._use_color:
return formatted
tone = self._resolve_tone(record, formatted)
if tone:
color = self.LINE_TONES.get(tone)
if color:
return f"{color}{formatted}{self.RESET}"
return formatted
badge = parts[1]
badge_color = self.BADGE_TONES.get(badge.strip())
if not badge_color:
return formatted
colored_parts = list(parts)
colored_parts[1] = f"{badge_color}{badge}{self.RESET}"
colored = _join_log_display_parts(colored_parts, body)
if record.exc_info:
exc_text = self.formatException(record.exc_info)
if exc_text:
colored = f"{colored}\n{exc_text}"
return colored
class _FileLogFormatter(logging.Formatter):
"""文件日志使用结构化字段,避免控制台前缀被再次包裹。"""
def __init__(self):
super().__init__(
"%(asctime)s | %(levelname)s | %(request_tag)s | %(name)s | %(message)s",
"%Y-%m-%d %H:%M:%S",
)
def format(self, record: logging.LogRecord) -> str:
original_message = str(
getattr(record, "codex_original_message_text", "") or record.getMessage() or ""
)
message = _sanitize_sensitive_text(
_normalize_log_display_expression(_record_logger_name(record), original_message)
)
prefix = (
f"{self.formatTime(record, self.datefmt)} | "
f"{record.levelname} | "
f"{_record_request_tag(record)} | "
f"{_record_logger_name(record) or record.name} | "
)
formatted = f"{prefix}{message.replace(chr(10), chr(10) + ' ' * len(prefix))}"
if record.exc_info:
exc_text = self.formatException(record.exc_info)
if exc_text:
formatted = f"{formatted}\n{exc_text}"
return formatted
class _DisplayLogFormatter(logging.Formatter):
"""保留旧式单行展示前缀,供兼容 handler 使用。"""
def format(self, record: logging.LogRecord) -> str:
message = _record_display_message(record)
formatted, _ = _format_log_display_line(record, message)
if record.exc_info:
exc_text = self.formatException(record.exc_info)
if exc_text:
formatted = f"{formatted}\n{exc_text}"
return formatted
def _is_windows_file_lock_error(exc: BaseException) -> bool:
"""Return True for transient Windows sharing violations during file rotation."""
if os.name != "nt" or not isinstance(exc, OSError):
return False
winerror = getattr(exc, "winerror", None)
if winerror in {32, 33}:
return True
return getattr(exc, "errno", None) in {32, 33}
class _SafeRotatingFileHandler(RotatingFileHandler):
"""Windows 上轮转目标被短暂占用时,降级为继续写当前日志。"""
_ROLLOVER_RETRY_DELAY_SECONDS = 5.0
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self._rollover_retry_after = 0.0
def shouldRollover(self, record: logging.LogRecord) -> bool:
if self._rollover_retry_after and time.monotonic() < self._rollover_retry_after:
return False
return super().shouldRollover(record)
def doRollover(self) -> None:
try:
super().doRollover()
self._rollover_retry_after = 0.0
except OSError as exc:
if not _is_windows_file_lock_error(exc):
raise
self._defer_rollover()
if self.stream is None and not self.delay:
self.stream = self._open()
def handleError(self, record: logging.LogRecord) -> None:
exc = sys.exc_info()[1]
if _is_windows_file_lock_error(exc):
self._defer_rollover()
try:
if self.stream is None and not self.delay:
self.stream = self._open()
logging.FileHandler.emit(self, record)
except Exception:
pass
return
super().handleError(record)
def _defer_rollover(self) -> None:
self._rollover_retry_after = time.monotonic() + self._ROLLOVER_RETRY_DELAY_SECONDS
# 创建全局 Web 日志处理器
_web_log_handler = _WebLogHandler()
_web_log_handler.setLevel(logging.DEBUG)
_web_log_handler.setFormatter(_DisplayLogFormatter())
_web_log_handler.addFilter(_BrowserConstantEnabledFilter("LOG_WEB_COLLECTOR_ENABLED", True))
setattr(_web_log_handler, "_codex_secure_handler", "web")
def _resolve_log_dir() -> Path:
configured = str(os.getenv("LOG_DIR", "") or "").strip()
if not configured:
return DEFAULT_LOG_DIR
candidate = Path(configured)
if not candidate.is_absolute():
candidate = PROJECT_ROOT / candidate
return candidate.resolve()
def get_log_file_path() -> Path:
configured = str(os.getenv("LOG_FILE", "") or "").strip()
if configured:
candidate = Path(configured)
if not candidate.is_absolute():
candidate = PROJECT_ROOT / candidate
return candidate.resolve()
return _resolve_log_dir() / "app.log"
def get_shared_file_log_handler() -> Optional[logging.Handler]:
global _shared_file_log_handler
with _shared_file_log_handler_lock:
if _shared_file_log_handler is not None:
return _shared_file_log_handler
try:
log_file = get_log_file_path()
log_file.parent.mkdir(parents=True, exist_ok=True)
handler = _SafeRotatingFileHandler(
log_file,
maxBytes=_get_positive_int_env("LOG_MAX_BYTES", 5 * 1024 * 1024),
backupCount=_get_positive_int_env("LOG_BACKUP_COUNT", 5),
encoding="utf-8",
)
handler.setLevel(logging.DEBUG)
handler.setFormatter(_FileLogFormatter())
handler.addFilter(_BrowserConstantEnabledFilter("LOG_FILE_ENABLED", True))
setattr(handler, "_codex_secure_handler", "file")
_shared_file_log_handler = handler
except Exception as e:
try:
print(f"[Config] failed to initialize file logging: {e}", file=sys.stderr)
except Exception:
pass
_shared_file_log_handler = None
return _shared_file_log_handler
+313
View File
@@ -0,0 +1,313 @@
"""
app/core/config_parts/log_redaction.py - 日志敏感信息过滤与脱敏模块
"""
import re
from typing import Any, Match, Optional
_SENSITIVE_KEY_HINTS = (
"authorization",
"cookie",
"set-cookie",
"password",
"passwd",
"secret",
"token",
"api_key",
"apikey",
"access_key",
"client_secret",
"refresh_token",
"session",
"csrf",
)
_REDACTED_TEXT = "******"
def _redact_data_uri_for_log(match: Match) -> str:
media_type = str(match.group(1) or "media").lower()
mime_suffix = str(match.group(2) or "octet-stream").lower()
payload_len = len(str(match.group(3) or ""))
return f"data:{media_type}/{mime_suffix};base64,[omitted {payload_len} chars]"
def _redact_long_base64_for_log(match: Match) -> str:
return f"[base64 omitted: {len(match.group(0))} chars]"
_BASE64_LOG_CHARS = frozenset("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/=_-")
_SENSITIVE_TEXT_SCAN_HINT_RE = re.compile(
r"(?i)data:(?:image|audio|video)/|bearer\s+|authorization|set-cookie|cookie|"
r"x[-_]api[-_]key|x[-_]github[-_]token|github[-_]token|"
r"access_token|refresh_token|id_token|api_key|apikey|access_key|client_secret|"
r"secret|password|passwd|token|session|csrf|sk-[A-Za-z0-9_-]{8,}|"
r"eyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}|"
r"https?://[^/\s:@]+:[^/\s@]+@"
)
_SENSITIVE_TEXT_PRECHECK_MIN_CHARS = 4096
_SENSITIVE_TEXT_LARGE_OMIT_THRESHOLD = 512 * 1024
_SENSITIVE_TEXT_LARGE_EDGE_CHARS = 4096
def _has_long_base64_candidate(text: str, min_chars: int = 1024) -> bool:
run_len = 0
for char in text:
if char in _BASE64_LOG_CHARS:
run_len += 1
if run_len >= min_chars:
return True
elif char in "\r\n" and run_len:
continue
else:
run_len = 0
return False
def _redact_long_base64_runs_for_log(text: str, min_chars: int = 1024) -> str:
raw_text = str(text or "")
if not raw_text:
return raw_text
parts = []
copy_from = 0
run_start = None
run_payload_chars = 0
for index, char in enumerate(raw_text):
if char in _BASE64_LOG_CHARS:
if run_start is None:
run_start = index
run_payload_chars = 0
run_payload_chars += 1
continue
if char in "\r\n" and run_start is not None:
continue
if run_start is not None:
if run_payload_chars >= min_chars:
parts.append(raw_text[copy_from:run_start])
parts.append(f"[base64 omitted: {index - run_start} chars]")
copy_from = index
run_start = None
run_payload_chars = 0
if run_start is not None and run_payload_chars >= min_chars:
parts.append(raw_text[copy_from:run_start])
parts.append(f"[base64 omitted: {len(raw_text) - run_start} chars]")
copy_from = len(raw_text)
if not parts:
return raw_text
parts.append(raw_text[copy_from:])
return "".join(parts)
def _should_scan_sensitive_text(text: str) -> bool:
if len(text) <= _SENSITIVE_TEXT_PRECHECK_MIN_CHARS:
return True
if _SENSITIVE_TEXT_SCAN_HINT_RE.search(text):
return True
return _has_long_base64_candidate(text)
_SENSITIVE_TEXT_PATTERNS = (
(
re.compile(
r"(?i)data:(image|audio|video)/([a-zA-Z0-9.+-]{1,100});base64,"
r"([A-Za-z0-9+/=_\-\r\n]{64,})"
),
_redact_data_uri_for_log,
),
(
re.compile(r"(?<![A-Za-z0-9+/=_-])[A-Za-z0-9+/=_-]{1024,}(?![A-Za-z0-9+/=_-])"),
_redact_long_base64_for_log,
),
(
re.compile(
r"(?<![A-Za-z0-9+/=_-])"
r"(?:[A-Za-z0-9+/=_-]{64,}[\r\n]+){7,}[A-Za-z0-9+/=_-]{64,}"
r"(?![A-Za-z0-9+/=_-])"
),
_redact_long_base64_for_log,
),
(
re.compile(r"(?i)\b(Bearer)\s+([A-Za-z0-9._~+/=-]{8,})"),
r"\1 ******",
),
(
re.compile(
r"(?<![A-Za-z0-9_-])"
r"(sk-(?:proj-)?[A-Za-z0-9_-]{16,})"
r"(?![A-Za-z0-9_-])"
),
"sk-******",
),
(
re.compile(
r"(?<![A-Za-z0-9_-])"
r"(eyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{6,})"
r"(?![A-Za-z0-9_-])"
),
"[jwt omitted]",
),
(
re.compile(r"(?i)(https?://[^/\s:@]+):([^/\s@]+)(@)"),
r"\1:******\3",
),
(
re.compile(
r"(?i)\b(Authorization|Cookie|Set-Cookie|X[-_]Api[-_]Key|"
r"X[-_]GitHub[-_]Token|GitHub[-_]Token)\s*:\s*[^\r\n;]+(?:;[^\r\n]*)?"
),
lambda match: f"{match.group(1)}: ******",
),
(
re.compile(
r"(?i)([?&](?:access_token|refresh_token|id_token|token|api_key|apikey|"
r"access_key|client_secret|session|csrf|"
r"x[-_]api[-_]key|x[-_]github[-_]token|github[-_]token|key|secret|password|passwd)=)"
r"[^&\s]+"
),
r"\1******",
),
(
re.compile(
r"(?i)(\b(?:access_token|refresh_token|id_token|token|api_key|apikey|"
r"access_key|client_secret|session|csrf|"
r"x[-_]api[-_]key|x[-_]github[-_]token|github[-_]token|secret|password|passwd)"
r"\s*=\s*)[^\s,&;]+"
),
r"\1******",
),
(
re.compile(
r'(?i)("(?:authorization|cookie|set-cookie|password|passwd|secret|token|api_key|apikey|'
r'access_token|refresh_token|access_key|client_secret|session|csrf|'
r'x[-_]api[-_]key|x[-_]github[-_]token|'
r'github[-_]token)"\s*:\s*)"[^"]*"'
),
r'\1"******"',
),
(
re.compile(
r'(?i)("(?:authorization|cookie|set-cookie|password|passwd|secret|token|api_key|apikey|'
r'access_token|refresh_token|access_key|client_secret|session|csrf|'
r'x[-_]api[-_]key|x[-_]github[-_]token|'
r'github[-_]token)"\s*:\s*")[^"\r\n]*$'
),
r'\1******',
),
(
re.compile(
r"(?i)('(?:authorization|cookie|set-cookie|password|passwd|secret|token|api_key|apikey|"
r"access_token|refresh_token|access_key|client_secret|session|csrf|"
r"x[-_]api[-_]key|x[-_]github[-_]token|"
r"github[-_]token)'\s*:\s*)'[^']*'"
),
r"\1'******'",
),
(
re.compile(
r"(?i)('(?:authorization|cookie|set-cookie|password|passwd|secret|token|api_key|apikey|"
r"access_token|refresh_token|access_key|client_secret|session|csrf|"
r"x[-_]api[-_]key|x[-_]github[-_]token|"
r"github[-_]token)'\s*:\s*)'[^'\r\n]*$"
),
r"\1'******",
),
)
_SENSITIVE_BEARER_PATTERN = _SENSITIVE_TEXT_PATTERNS[3]
_SENSITIVE_STANDALONE_TOKEN_PATTERNS = _SENSITIVE_TEXT_PATTERNS[4:7]
_SENSITIVE_HEADER_PATTERN = _SENSITIVE_TEXT_PATTERNS[7]
_SENSITIVE_PARAM_PATTERNS = _SENSITIVE_TEXT_PATTERNS[8:]
def _sanitize_standalone_sensitive_tokens(text: str) -> str:
if "sk-" not in text and "eyJ" not in text and "://" not in text:
return text
sanitized = text
for pattern, replacement in _SENSITIVE_STANDALONE_TOKEN_PATTERNS:
sanitized = pattern.sub(replacement, sanitized)
return sanitized
def _is_sensitive_key(key: Any) -> bool:
normalized = str(key or "").strip().lower().replace("-", "_")
return any(hint.replace("-", "_") in normalized for hint in _SENSITIVE_KEY_HINTS)
def _sanitize_sensitive_text(text: str) -> str:
sanitized = str(text or "")
if not sanitized:
return sanitized
if len(sanitized) > _SENSITIVE_TEXT_LARGE_OMIT_THRESHOLD:
edge_chars = max(0, int(_SENSITIVE_TEXT_LARGE_EDGE_CHARS))
head = sanitized[:edge_chars]
tail = sanitized[-edge_chars:] if edge_chars else ""
omitted = max(0, len(sanitized) - len(head) - len(tail))
safe_head = _sanitize_sensitive_text(head)
safe_tail = _sanitize_sensitive_text(tail) if tail else ""
if safe_tail:
return (
f"{safe_head}... [omitted {omitted} chars from oversized log payload] "
f"...{safe_tail}"
)
return f"{safe_head}... [omitted {omitted} chars from oversized log payload]"
sanitized = _sanitize_standalone_sensitive_tokens(sanitized)
if not _should_scan_sensitive_text(sanitized):
return sanitized
lower_text = sanitized.lower()
# 1. 只有可能有 Base64 候选字符或包含 'data:' 才做前三个正则替换
if "data:" in lower_text or "base64" in lower_text or len(sanitized) >= 1024:
if "data:" in lower_text and "base64," in lower_text:
sanitized = _SENSITIVE_TEXT_PATTERNS[0][0].sub(_SENSITIVE_TEXT_PATTERNS[0][1], sanitized)
lower_text = sanitized.lower()
if len(sanitized) >= 1024 and _has_long_base64_candidate(sanitized):
sanitized = _redact_long_base64_runs_for_log(sanitized)
lower_text = sanitized.lower()
# 2. 仅在包含 bearer 时替换 Bearer token
if "bearer" in lower_text:
sanitized = _SENSITIVE_BEARER_PATTERN[0].sub(_SENSITIVE_BEARER_PATTERN[1], sanitized)
# 3. 仅在包含相关 header 名时替换 Auth, Cookie 等头部
if any(k in lower_text for k in ("authorization", "cookie", "set-cookie", "x-api-key", "x_api_key", "x-github-token", "x_github_token", "github-token", "github_token")):
sanitized = _SENSITIVE_HEADER_PATTERN[0].sub(_SENSITIVE_HEADER_PATTERN[1], sanitized)
# 4. 仅在包含敏感关键字时执行 query/json param 相关的敏感信息提取正则
if any(k in lower_text for k in ("access_token", "refresh_token", "id_token", "token", "api_key", "apikey", "access_key", "client_secret", "key", "secret", "password", "passwd", "session", "csrf", "x-api-key", "x_api_key", "x-github-token", "x_github_token", "github-token", "github_token")):
for pattern, replacement in _SENSITIVE_PARAM_PATTERNS:
sanitized = pattern.sub(replacement, sanitized)
return sanitized
def sanitize_sensitive_data(value: Any, *, _depth: int = 0) -> Any:
"""Return a sanitized copy suitable for logs and debug artifacts."""
if _depth > 8:
return "[max-depth]"
if isinstance(value, dict):
sanitized_dict = {}
for key, item in value.items():
if _is_sensitive_key(key):
sanitized_dict[key] = _REDACTED_TEXT
else:
sanitized_dict[key] = sanitize_sensitive_data(item, _depth=_depth + 1)
return sanitized_dict
if isinstance(value, list):
return [sanitize_sensitive_data(item, _depth=_depth + 1) for item in value]
if isinstance(value, tuple):
return tuple(sanitize_sensitive_data(item, _depth=_depth + 1) for item in value)
if isinstance(value, str):
return _sanitize_sensitive_text(value)
return value
+130
View File
@@ -0,0 +1,130 @@
"""
app/core/config_parts/message_validator.py - 消息验证器模块
"""
import ast
import json
from typing import Any, Tuple, Optional, List
from .browser_constants import BrowserConstants
class MessageValidator:
"""消息验证器"""
VALID_ROLES = {'user', 'assistant', 'system'}
ROLE_ALIASES = {
'developer': 'system',
}
_IMAGE_PLACEHOLDER = "[图片]"
@classmethod
def _parse_multimodal_string(cls, content: str):
"""尝试把字符串形式的多模态 content 还原成列表。"""
text = str(content or "")
stripped = text.strip()
if not stripped.startswith('[') or not stripped.endswith(']'):
return text, False
parsed = None
try:
parsed = json.loads(stripped)
except (TypeError, ValueError, RecursionError):
parsed = None
if parsed is None:
try:
parsed = ast.literal_eval(stripped)
except Exception:
parsed = None
if isinstance(parsed, list):
return parsed, True
return text, False
@classmethod
def _normalize_content(cls, content: Any) -> Any:
"""保留多模态结构,避免把图片/base64 粗暴 str() 化。"""
if content is None:
return ""
if isinstance(content, dict):
return [content]
if isinstance(content, list):
return content
if isinstance(content, tuple):
return list(content)
if isinstance(content, str):
parsed, ok = cls._parse_multimodal_string(content)
return parsed if ok else content
return str(content)
@classmethod
def _effective_content_length(cls, content: Any) -> int:
"""按网页执行真实会使用的语义估算 content 长度。"""
normalized = cls._normalize_content(content)
if isinstance(normalized, str):
text = normalized
if text.startswith("data:image") and "base64," in text and len(text) > 1000:
return len("[图片内容]")
return len(text)
if isinstance(normalized, list):
total = 0
for item in normalized:
if item is None:
continue
if not isinstance(item, dict):
total += len(str(item))
continue
item_type = str(item.get("type", "") or "").strip()
if item_type == "text":
total += len(str(item.get("text", "") or ""))
elif item_type in {"image_url", "file", "input_file", "document", "input_audio", "audio_url", "video_url", "input_video"}:
total += len(cls._IMAGE_PLACEHOLDER)
else:
total += len(str(item))
return total
return len(str(normalized))
@classmethod
def validate(cls, messages: Any) -> tuple:
if messages is None:
return False, "messages 不能为空", None
if not isinstance(messages, list):
return False, "messages 应该是列表", None
if len(messages) == 0:
return False, "messages 不能为空列表", None
message_count = len(messages)
max_messages = int(BrowserConstants.MAX_MESSAGES_COUNT)
if message_count > max_messages:
return False, (
f"消息数量超过限制(当前 {message_count} 条,最大允许 {max_messages} 条)"
), None
sanitized = []
for i, msg in enumerate(messages):
if not isinstance(msg, dict):
return False, f"messages[{i}] 不是字典类型", None
role = str(msg.get('role', 'user') or 'user').strip().lower()
role = cls.ROLE_ALIASES.get(role, role)
if role not in cls.VALID_ROLES:
role = 'user'
content = cls._normalize_content(msg.get('content', ''))
content_length = cls._effective_content_length(content)
max_length = int(BrowserConstants.MAX_MESSAGE_LENGTH)
if content_length > max_length:
return False, (
f"messages[{i}].content 超过长度限制"
f"(当前 {content_length} 字符,最大允许 {max_length} 字符)"
), None
sanitized.append({'role': role, 'content': content})
return True, None, sanitized
+274
View File
@@ -0,0 +1,274 @@
"""
app/core/config_parts/secure_logger.py - 安全日志器实现与日志工厂模块
"""
from collections import defaultdict
import contextlib
import contextvars
import logging
import sys
import threading
import time
from typing import Any, Dict, Optional
from .env_config import AppConfig
from .browser_constants import _BrowserConstantEnabledFilter
from .log_formatters import (
_compact_logger_name,
_ConsoleColorFormatter,
_format_log_display_line,
_web_log_handler,
get_shared_file_log_handler,
)
from .cute_translator import (
_cuteify_info_message,
_cuteify_debug_message,
_cuteify_warning_message,
_cuteify_error_message,
_add_suppressed_marker,
)
# 上下文变量,存储当前请求的 request_id
_request_context: contextvars.ContextVar[Optional[str]] = contextvars.ContextVar("request_id", default=None)
_command_log_context: contextvars.ContextVar[Optional[Dict[str, Any]]] = contextvars.ContextVar(
"command_log_context",
default=None,
)
_logger_setup_lock = threading.RLock()
_logger_registry_lock = threading.Lock()
_logger_registry: Dict[str, "SecureLogger"] = {}
class SecureLogger:
"""安全日志器,带图标和格式化(支持上下文自动注入 request_id)"""
_debug_throttle_locks: Dict[str, threading.Lock] = defaultdict(threading.Lock)
_debug_throttle_state: Dict[str, Dict[str, Any]] = {}
ICONS = {
'DEBUG': '▫️',
'INFO': '🔹',
'WARNING': '⚠️',
'ERROR': '❌',
'SUCCESS': '✅',
'STREAM': '🌊',
'NETWORK': '🌐',
}
# 日志级别映射
LEVEL_MAP = {
'DEBUG': logging.DEBUG,
'INFO': logging.INFO,
'WARNING': logging.WARNING,
'ERROR': logging.ERROR,
'CRITICAL': logging.CRITICAL,
}
def __init__(self, name: str, level: Optional[int] = None):
self._name = _compact_logger_name(name)
# 如果未指定级别,从环境变量获取
if level is None:
level = self._get_level_from_env()
self._level = level
self._logger = self._setup_logger(name, level)
@classmethod
def _get_level_from_env(cls) -> int:
"""从环境变量获取日志级别"""
level_str = AppConfig.get_log_level()
return cls.LEVEL_MAP.get(level_str, logging.INFO)
def _setup_logger(self, name: str, level: int) -> logging.Logger:
logger = logging.getLogger(name)
with _logger_setup_lock:
# 防止日志向上层冒泡导致重复打印
logger.propagate = False
existing_kinds = {
getattr(handler, "_codex_secure_handler", None)
for handler in logger.handlers
}
if "console" not in existing_kinds:
console_handler = logging.StreamHandler(sys.stdout)
console_handler.setLevel(logging.DEBUG)
console_handler.setFormatter(_ConsoleColorFormatter())
console_handler.addFilter(_BrowserConstantEnabledFilter("LOG_CONSOLE_ENABLED", True))
setattr(console_handler, "_codex_secure_handler", "console")
logger.addHandler(console_handler)
file_handler = get_shared_file_log_handler()
if file_handler is not None and file_handler not in logger.handlers:
logger.addHandler(file_handler)
if _web_log_handler not in logger.handlers:
logger.addHandler(_web_log_handler)
logger.setLevel(logging.DEBUG)
return logger
def _format(self, level_key: str, msg: str) -> str:
"""核心格式化逻辑(简洁版)"""
record = logging.LogRecord(
name=self._name,
level=self.LEVEL_MAP.get(str(level_key or "").upper(), logging.INFO),
pathname="",
lineno=0,
msg=str(msg or ""),
args=(),
exc_info=None,
)
record.codex_request_id = _request_context.get() or "SYSTEM"
record.codex_logger_name = self._name
formatted, _ = _format_log_display_line(record, msg)
return formatted
def _make_debug_throttle_key(self, key: str) -> str:
normalized = str(key or "").strip() or "__default__"
return f"{self._name}:{normalized}"
@staticmethod
def _coerce_bool(value: Any, default: bool = True) -> bool:
if value is None:
return default
if isinstance(value, bool):
return value
if isinstance(value, (int, float)):
return bool(value)
text = str(value).strip().lower()
if text in {"1", "true", "yes", "y", "on"}:
return True
if text in {"0", "false", "no", "n", "off"}:
return False
return default
def _get_effective_emit_level(self) -> Optional[int]:
context = _command_log_context.get()
if not isinstance(context, dict):
return self._level
if not self._coerce_bool(context.get("enabled", True), True):
return None
override = str(context.get("level", "GLOBAL") or "GLOBAL").strip().upper()
if override == "GLOBAL":
return self._level
return self.LEVEL_MAP.get(override, self._level)
def _emit(self, level: int, level_key: str, msg: str, *, exc_info: bool = False):
effective_level = self._get_effective_emit_level()
if effective_level is None or level < effective_level:
return
request_id = _request_context.get() or "SYSTEM"
original_message_text = str(msg or "")
upper_level_key = str(level_key or "").upper()
# P0-7:展示层翻译(cute_translator)改为惰性:由 handler 渲染时经
# log_formatters._record_display_message 计算一次并缓存到 record 上。
self._logger.log(
level,
original_message_text,
exc_info=exc_info,
extra={
"codex_request_id": request_id,
"codex_logger_name": self._name,
"codex_message": original_message_text,
"codex_original_message_text": original_message_text,
"codex_display_pending": True,
"codex_kind": upper_level_key,
},
)
def set_level(self, level: int):
"""动态调整日志级别"""
self._level = level
self._logger.setLevel(logging.DEBUG)
for handler in self._logger.handlers:
handler.setLevel(logging.DEBUG)
def debug(self, msg: str):
self._emit(logging.DEBUG, 'DEBUG', msg)
def debug_throttled(self, key: str, msg: str, interval_sec: float = 5.0):
"""在高频路径里限频输出 DEBUG,并附带被抑制次数。"""
effective_level = self._get_effective_emit_level()
if effective_level is None or logging.DEBUG < effective_level:
return
now = time.time()
interval = max(0.0, float(interval_sec or 0.0))
throttle_key = self._make_debug_throttle_key(key)
suppressed = 0
should_log = False
lock = self._debug_throttle_locks[throttle_key]
with lock:
state = self._debug_throttle_state.get(throttle_key)
last_at = float(state.get("last_at", 0.0) or 0.0) if state else 0.0
if state is None or (now - last_at) >= interval:
suppressed = int(state.get("suppressed", 0) or 0) if state else 0
self._debug_throttle_state[throttle_key] = {
"last_at": now,
"suppressed": 0,
}
should_log = True
else:
state["suppressed"] = int(state.get("suppressed", 0) or 0) + 1
if should_log:
self.debug(_add_suppressed_marker(msg, suppressed))
def info(self, msg: str):
self._emit(logging.INFO, 'INFO', msg)
def warning(self, msg: str):
self._emit(logging.WARNING, 'WARNING', msg)
def error(self, msg: str):
self._emit(logging.ERROR, 'ERROR', msg)
def exception(self, msg: str):
self._emit(logging.ERROR, 'ERROR', msg, exc_info=True)
def success(self, msg: str):
self._emit(logging.INFO, 'SUCCESS', msg)
def stream(self, msg: str):
self._emit(logging.INFO, 'STREAM', msg)
def network(self, msg: str):
self._emit(logging.INFO, 'NETWORK', msg)
@contextlib.contextmanager
def context(self, request_id: str):
"""上下文管理器,用于在代码块中自动设置 request_id"""
token = _request_context.set(request_id)
try:
yield
finally:
_request_context.reset(token)
@contextlib.contextmanager
def command_log_context(config: Optional[Dict[str, Any]] = None):
token = _command_log_context.set(config if isinstance(config, dict) else None)
try:
yield
finally:
_command_log_context.reset(token)
def get_logger(name: str) -> SecureLogger:
"""获取 SecureLogger 实例(统一日志入口)"""
normalized = str(name or "APP").strip() or "APP"
with _logger_registry_lock:
instance = _logger_registry.get(normalized)
if instance is None:
instance = SecureLogger(normalized)
_logger_registry[normalized] = instance
return instance
# 创建常用 logger 实例(向后兼容)
logger = get_logger("BROWSER")
+304
View File
@@ -0,0 +1,304 @@
"""
app/core/config_parts/sse_formatter.py - SSE 响应格式化器模块
"""
import json
import threading
import time
import uuid
from typing import Any, Dict, List, Optional
class SSEFormatter:
"""SSE 响应格式化器"""
_sequence = 0
_sequence_lock = threading.Lock()
@classmethod
def _generate_id(cls) -> str:
timestamp = int(time.time() * 1000)
with cls._sequence_lock:
cls._sequence += 1
seq = cls._sequence
short_uuid = uuid.uuid4().hex[:6]
return f"chatcmpl-{timestamp}-{seq}-{short_uuid}"
@classmethod
def pack_chunk(
cls,
content: str | None = None,
model: str = "web-browser",
completion_id: str | None = None,
images: list[str] | None = None,
media: list[dict] | None = None,
reasoning_content: str | None = None,
) -> str:
"""打包流式 chunk。
为兼容现有前端,content 仍保留 Markdown 媒体链接。
同时补充自定义 media 字段,供需要结构化媒体数据的前端直接消费。
"""
chunk_id = completion_id or cls._generate_id()
delta = {}
if content is not None:
delta["content"] = content
if reasoning_content:
delta["reasoning_content"] = reasoning_content
if media is not None:
delta["media"] = media
data = {
"id": chunk_id,
"object": "chat.completion.chunk",
"created": int(time.time()),
"model": model,
"choices": [{
"index": 0,
"delta": delta,
"finish_reason": None
}]
}
if media is not None:
data["media"] = media
return f"data: {json.dumps(data, ensure_ascii=False)}\n\n"
@classmethod
def pack_finish(cls, model: str = "web-browser") -> str:
data = {
"id": cls._generate_id(),
"object": "chat.completion.chunk",
"created": int(time.time()),
"model": model,
"choices": [{
"index": 0,
"delta": {},
"finish_reason": "stop"
}]
}
return f"data: {json.dumps(data, ensure_ascii=False)}\n\ndata: [DONE]\n\n"
@staticmethod
def pack_comment(comment: str = "keepalive") -> str:
"""打包 SSE 注释帧,用于长连接保活。"""
safe_comment = " ".join(str(comment or "keepalive").splitlines()).strip() or "keepalive"
return f": {safe_comment}\n\n"
@classmethod
def pack_keepalive(
cls,
model: str = "web-browser",
completion_id: Optional[str] = None,
) -> str:
"""打包客户端可解析的空 OpenAI chunk 作为长连接保活。"""
return cls.pack_chunk(
content="",
model=model,
completion_id=completion_id,
)
@staticmethod
def pack_error(
message: str,
error_type: str = "execution_error",
code: str = "workflow_failed",
*,
status_code: Optional[int] = None,
retryable: Optional[bool] = None,
param: Optional[str] = None,
extra: Optional[Dict[str, Any]] = None,
) -> str:
clean_message = str(message or "")
clean_code = str(code or "workflow_failed")
clean_type = str(error_type or "execution_error")
clean_status_code = status_code
clean_retryable = retryable
clean_param = param
clean_extra = dict(extra or {})
try:
from app.services.error_metadata import resolve_error_metadata
meta = resolve_error_metadata(clean_message)
if meta:
clean_message = meta.message
if clean_code in {"workflow_failed", "error", "arena_page_error"} and meta.code:
clean_code = meta.code
if clean_type in {"execution_error", "invalid_request_error"} and meta.error_type:
clean_type = meta.error_type
if clean_status_code is None:
clean_status_code = meta.status_code
if clean_retryable is None:
clean_retryable = meta.retryable
if clean_param is None and meta.param is not None:
clean_param = meta.param
if meta.extra:
for k, v in meta.extra.items():
if k not in clean_extra:
clean_extra[k] = v
except Exception:
pass
error: Dict[str, Any] = {
"message": clean_message,
"type": clean_type,
"code": clean_code,
}
if clean_status_code is not None:
error["status_code"] = int(clean_status_code)
if clean_retryable is not None:
error["retryable"] = bool(clean_retryable)
if clean_param is not None:
error["param"] = clean_param
if clean_extra:
for k, v in clean_extra.items():
if k not in error:
error[k] = v
data = {
"id": f"chatcmpl-error-{int(time.time() * 1000)}",
"object": "chat.completion.chunk",
"created": int(time.time()),
"model": "web-browser",
"choices": [{
"index": 0,
"delta": {"content": f"[错误] {clean_message}"},
"finish_reason": None
}],
"error": error
}
return f"data: {json.dumps(data, ensure_ascii=False)}\n\n"
@staticmethod
def pack_error_json(
message: str,
error_type: str = "execution_error",
code: str = "workflow_failed",
*,
status_code: Optional[int] = None,
retryable: Optional[bool] = None,
param: Optional[str] = None,
extra: Optional[Dict[str, Any]] = None,
) -> Dict[str, Any]:
clean_message = str(message or "")
clean_code = str(code or "workflow_failed")
clean_type = str(error_type or "execution_error")
clean_status_code = status_code
clean_retryable = retryable
clean_param = param
clean_extra = dict(extra or {})
try:
from app.services.error_metadata import resolve_error_metadata
meta = resolve_error_metadata(clean_message)
if meta:
clean_message = meta.message
if clean_code in {"workflow_failed", "error", "arena_page_error"} and meta.code:
clean_code = meta.code
if clean_type in {"execution_error", "invalid_request_error"} and meta.error_type:
clean_type = meta.error_type
if clean_status_code is None:
clean_status_code = meta.status_code
if clean_retryable is None:
clean_retryable = meta.retryable
if clean_param is None and meta.param is not None:
clean_param = meta.param
if meta.extra:
for k, v in meta.extra.items():
if k not in clean_extra:
clean_extra[k] = v
except Exception:
pass
error: Dict[str, Any] = {
"message": clean_message,
"type": clean_type,
"code": clean_code,
}
if clean_status_code is not None:
error["status_code"] = int(clean_status_code)
if clean_retryable is not None:
error["retryable"] = bool(clean_retryable)
if clean_param is not None:
error["param"] = clean_param
if clean_extra:
for k, v in clean_extra.items():
if k not in error:
error[k] = v
return {"error": error}
@staticmethod
def pack_non_stream(content: str, model: str = "web-browser", media: list | None = None) -> Dict:
message = {
"role": "assistant",
"content": content
}
if media is not None:
message["media"] = media
data = {
"id": f"chatcmpl-{int(time.time() * 1000)}",
"object": "chat.completion",
"created": int(time.time()),
"model": model,
"choices": [{
"index": 0,
"message": message,
"finish_reason": "stop"
}],
"usage": {
"prompt_tokens": 0,
"completion_tokens": 0,
"total_tokens": 0
}
}
if media is not None:
data["media"] = media
return data
@staticmethod
def _build_markdown_image_block(images: list) -> str:
refs = []
for item in images or []:
if isinstance(item, dict):
ref = str(item.get("url") or item.get("data_uri") or "").strip()
else:
ref = str(item or "").strip()
if ref:
refs.append(ref)
if not refs:
return ""
return "".join(f"\n\n![image_{idx}]({ref})" for idx, ref in enumerate(refs)) + "\n\n"
@classmethod
def pack_images_chunk(cls, images: list, completion_id: str = None) -> str:
"""
打包携带图片的 SSE chunk。
为保持 OpenAI 兼容性,图片会转成 Markdown 内容,而不是放进 delta.images。
Args:
images: 图片数据列表,每项符合 ImageData 格式
completion_id: 补全 ID
Returns:
SSE 格式的字符串
Example:
>>> chunk = SSEFormatter.pack_images_chunk([{"kind": "url", "url": "..."}])
"""
markdown = cls._build_markdown_image_block(images)
if not markdown:
return ""
return cls.pack_chunk(markdown, completion_id=completion_id)
def pack_final_chunk_with_images(self, images: list, completion_id: str = None) -> str:
"""
打包包含图片的最终 chunk。
为保持 OpenAI 兼容性,图片会转成 Markdown 内容,而不是放进 delta.images。
"""
markdown = self._build_markdown_image_block(images)
if not markdown:
return ""
return self.pack_chunk(markdown, completion_id=completion_id)