Files
universal-web-api/app/core/browser/media.py
T
2026-11-02 22:54:14 +08:00

3228 lines
140 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
# app/core/browser/media.py
import json
import time
import os
import shutil
import subprocess
import base64
import uuid
import binascii
from pathlib import Path
from datetime import datetime
from urllib.parse import urlparse, urlsplit
from typing import Optional, List, Dict, Any, Callable, TYPE_CHECKING
import requests
from app.core.config import logger, AppConfig, BrowserConstants
from app.models.schemas import (
get_enabled_modalities,
get_modality_policy,
get_modality_run_policy,
is_modality_enabled,
)
from app.core.background_image_downloader import (
background_image_downloader,
build_image_download_partition,
build_image_download_request_context,
get_image_download_partition,
normalize_remote_image_url,
)
from app.utils.remote_resource import get_public_remote_resource
from app.utils.media_safety import (
IMAGE_EXT_BY_MIME,
AUDIO_EXT_BY_MIME,
VIDEO_EXT_BY_MIME,
choose_media_extension,
is_active_content_mime,
looks_like_active_content,
)
from app.utils.site_url import extract_remote_site_domain
from app.utils.image_validation import filter_reference_images
from app.core.tab_pool import TabSession
from app.core.driver import as_element, driver_for_tab
if TYPE_CHECKING:
from .main import BrowserCore
class BrowserMediaMixin:
"""媒体文件提取、音频后处理(FFmpeg/FFprobe)、本地落盘、视觉/截图本地化混入类"""
_MEDIA_DOM_BASELINE_PROPERTY = "__universalProxyMediaBaseline"
def _capture_media_dom_baseline(self, tab, image_config: Dict[str, Any]) -> Optional[Dict[str, Any]]:
"""Mark images that existed immediately before the request is submitted."""
image_config = image_config or {}
image_enabled = bool(image_config.get("enabled")) or is_modality_enabled(
image_config.get("modalities") or {},
"image",
)
if tab is None or not image_enabled:
return None
selector = str(image_config.get("selector") or "img").strip() or "img"
token = uuid.uuid4().hex
try:
result = driver_for_tab(tab).run_js(
"""
const selector = String(arguments[0] || "img");
const token = String(arguments[1] || "");
const propertyName = String(arguments[2] || "");
const pickReference = (node) => {
try {
return String(
node.currentSrc
|| node.src
|| node.getAttribute("src")
|| ""
).trim();
} catch {
return "";
}
};
let nodes = [];
try {
// The configured selector may only match generated cards. Capture
// every image so upload previews and history cannot later become
// screenshot fallback candidates for this request.
nodes = Array.from(document.querySelectorAll("img"));
} catch (error) {
return { ok: false, error: String(error).slice(0, 160) };
}
const urls = [];
let markedCount = 0;
for (const node of nodes) {
const reference = pickReference(node);
try {
Object.defineProperty(node, propertyName, {
configurable: true,
writable: true,
value: { token, reference },
});
markedCount += 1;
} catch {
try {
node[propertyName] = { token, reference };
markedCount += 1;
} catch {}
}
if (reference && reference.length <= 8192) urls.push(reference);
}
return {
ok: true,
node_count: nodes.length,
marked_count: markedCount,
url_count: new Set(urls).size,
references: Array.from(new Set(urls)).slice(-256),
page_url: String(location.href || ""),
};
""",
selector,
token,
self._MEDIA_DOM_BASELINE_PROPERTY,
) or {}
except Exception as exc:
logger.warning(f"发送前图片基线采集失败,将使用常规媒体提取: {exc}")
return None
if not bool(result.get("ok")):
logger.warning(
"发送前图片基线采集失败,将使用常规媒体提取: "
f"{result.get('error') or 'unknown_error'}"
)
return None
baseline = {
"token": token,
"property": self._MEDIA_DOM_BASELINE_PROPERTY,
"selector": "img",
"node_count": int(result.get("node_count") or 0),
"marked_count": int(result.get("marked_count") or 0),
"url_count": int(result.get("url_count") or 0),
"references": [
str(reference or "").strip()
for reference in (result.get("references") or [])
if str(reference or "").strip()
],
"page_url": str(result.get("page_url") or ""),
}
logger.debug(
"已记录发送前图片基线: "
f"nodes={baseline['node_count']}, marked={baseline['marked_count']}, "
f"urls={baseline['url_count']}"
)
return baseline
@staticmethod
def _media_reference_key(reference: Any) -> str:
text = str(reference or "").strip()
if not text:
return ""
if text.lower().startswith(("data:", "blob:")):
return text
normalized = normalize_remote_image_url(text)
if not normalized:
return text
try:
parsed = urlparse(normalized)
except Exception:
return normalized
if parsed.scheme.lower() not in {"http", "https"}:
return normalized
host = str(parsed.hostname or "").strip().lower()
signed_query = "x-amz-signature=" in str(parsed.query or "").lower()
if host.endswith(".r2.cloudflarestorage.com") or signed_query:
return f"{parsed.scheme.lower()}://{parsed.netloc.lower()}{parsed.path}"
return normalized
@classmethod
def _filter_media_items_against_dom_baseline(
cls,
media_items: List[Dict],
media_dom_baseline: Optional[Dict[str, Any]],
) -> List[Dict]:
baseline_keys = {
cls._media_reference_key(reference)
for reference in ((media_dom_baseline or {}).get("references") or [])
}
baseline_keys.discard("")
if not baseline_keys:
return list(media_items or [])
filtered_items = []
removed_images = 0
for item in media_items or []:
if not isinstance(item, dict):
continue
media_type = str(item.get("media_type") or "image").strip().lower() or "image"
reference = item.get("url") or item.get("data_uri") or ""
if media_type == "image" and cls._media_reference_key(reference) in baseline_keys:
removed_images += 1
continue
filtered_items.append(item)
if removed_images:
logger.debug(
"发送前已存在的图片 URL 已从本轮结果剔除: "
f"count={removed_images}"
)
return filtered_items
@staticmethod
def _candidate_has_fresh_dom_image(
candidate,
image_config: Dict[str, Any],
media_dom_baseline: Optional[Dict[str, Any]],
) -> bool:
if candidate is None or not media_dom_baseline:
return False
selector = str((image_config or {}).get("selector") or "img").strip() or "img"
token = str(media_dom_baseline.get("token") or "")
property_name = str(media_dom_baseline.get("property") or "")
exclude_existing_nodes = bool(
(image_config or {}).get("request_baseline_exclude_existing_nodes")
)
if not token or not property_name:
return False
try:
return bool(
as_element(candidate).run_js(
"""
const selector = String(arguments[0] || "img");
const baselineToken = String(arguments[1] || "");
const propertyName = String(arguments[2] || "");
const excludeExistingNodes = Boolean(arguments[3]);
const nodes = [];
try {
if (this instanceof Element && this.matches(selector)) nodes.push(this);
} catch {}
try {
if (this.querySelectorAll) nodes.push(...this.querySelectorAll(selector));
} catch {}
return nodes.some((node) => {
const baseline = node[propertyName];
if (!baseline || String(baseline.token || "") !== baselineToken) return true;
if (excludeExistingNodes) return false;
let currentReference = "";
try {
currentReference = String(
node.currentSrc
|| node.src
|| node.getAttribute("src")
|| ""
).trim();
} catch {}
return currentReference !== String(baseline.reference || "");
});
""",
selector,
token,
property_name,
exclude_existing_nodes,
)
)
except Exception as exc:
logger.debug(f"本轮新增图片探测失败(按未命中处理): {exc}")
return False
@staticmethod
def _media_modalities(image_config: Dict[str, Any]) -> Dict[str, Any]:
return dict((image_config or {}).get("modalities") or {})
@staticmethod
def _media_enabled_types(image_config: Dict[str, Any]) -> List[str]:
return sorted(get_enabled_modalities((image_config or {}).get("modalities") or {}))
@staticmethod
def _media_policy(image_config: Dict[str, Any], media_type: str) -> Dict[str, Any]:
return dict(get_modality_policy((image_config or {}).get("modalities") or {}, media_type))
@staticmethod
def _media_run_policy(image_config: Dict[str, Any], media_type: str) -> str:
return get_modality_run_policy((image_config or {}).get("modalities") or {}, media_type)
@staticmethod
def _media_policy_allows_signal_wait(image_config: Dict[str, Any], media_type: str) -> bool:
return BrowserMediaMixin._media_run_policy(image_config, media_type) in {
"on_signal",
"probe_if_trigger_found",
"always_probe",
}
@staticmethod
def _media_policy_allows_audio_probe(
image_config: Dict[str, Any],
*,
signal_seen: bool = False,
) -> bool:
policy = BrowserMediaMixin._media_run_policy(image_config, "audio")
if policy == "always_probe":
return True
if policy == "probe_if_trigger_found":
return True
if policy == "on_signal" and signal_seen:
return True
return False
@staticmethod
def _media_quick_probe_timeout(image_config: Dict[str, Any], default: float = 1.0) -> float:
values = []
for media_type in BrowserMediaMixin._media_enabled_types(image_config):
policy = BrowserMediaMixin._media_policy(image_config, media_type)
try:
values.append(float(policy.get("quick_probe_timeout_seconds") or default))
except (TypeError, ValueError):
pass
if not values:
return default
return max(0.1, min(max(values), 10.0))
@staticmethod
def _media_late_wait_timeout(
image_config: Dict[str, Any],
media_type: str,
fallback: float,
) -> float:
policy = BrowserMediaMixin._media_policy(image_config, media_type)
try:
return max(0.2, min(float(policy.get("late_wait_timeout_seconds") or fallback), 300.0))
except (TypeError, ValueError):
return fallback
@staticmethod
def _media_blind_wait_timeout(
image_config: Dict[str, Any],
media_type: str,
fallback: float,
) -> float:
policy = BrowserMediaMixin._media_policy(image_config, media_type)
raw_value = policy.get("blind_wait_timeout_seconds")
if raw_value is None:
raw_value = image_config.get("blind_wait_timeout_seconds")
try:
return max(0.0, min(float(raw_value if raw_value is not None else fallback), 300.0))
except (TypeError, ValueError):
return fallback
@staticmethod
def _media_audio_capture_timeout(image_config: Dict[str, Any], fallback: float) -> float:
policy = BrowserMediaMixin._media_policy(image_config, "audio")
try:
return max(0.2, min(float(policy.get("capture_timeout_seconds") or fallback), 180.0))
except (TypeError, ValueError):
return fallback
@staticmethod
def _build_localized_image_item(
original_item: Dict[str, Any],
local_path: Path,
accessible_url: str,
*,
mime: Optional[str] = None,
byte_size: Optional[int] = None,
source: str = "local_file",
) -> Dict[str, Any]:
new_item = dict(original_item or {})
new_item["kind"] = "url"
new_item["url"] = accessible_url
new_item["data_uri"] = None
new_item["source"] = source
new_item["local_path"] = str(local_path)
if mime is not None:
new_item["mime"] = mime
if byte_size is None:
try:
byte_size = int(local_path.stat().st_size)
except Exception:
byte_size = None
if byte_size is not None:
new_item["byte_size"] = byte_size
return new_item
@staticmethod
def _read_response_bytes_with_limit(response, max_bytes: int) -> bytes:
max_bytes = max(1, int(max_bytes or 1))
content_length = str(response.headers.get("Content-Length") or "").strip()
if content_length:
try:
declared_size = int(content_length)
if declared_size > max_bytes:
raise ValueError(f"image_too_large:{declared_size}")
except ValueError as exc:
if str(exc).startswith("image_too_large:"):
raise
chunks = []
total = 0
for chunk in response.iter_content(chunk_size=1024 * 64):
if not chunk:
continue
total += len(chunk)
if total > max_bytes:
raise ValueError(f"image_too_large:{total}")
chunks.append(chunk)
return b"".join(chunks)
@staticmethod
def _estimate_base64_decoded_size(base64_data: str) -> int:
compact_base64 = "".join(str(base64_data or "").split())
if not compact_base64:
return 0
padding = compact_base64.count("=")
return max(0, (len(compact_base64) * 3) // 4 - padding)
def _localize_image_item_from_background_result(
self,
original_item: Dict[str, Any],
background_result: Optional[Dict[str, Any]],
) -> Optional[Dict[str, Any]]:
if not isinstance(background_result, dict):
return None
if str(background_result.get("status") or "") != "done":
return None
local_path_text = str(background_result.get("local_path") or "").strip()
accessible_url = str(background_result.get("accessible_url") or "").strip()
if not local_path_text or not accessible_url:
return None
local_path = Path(local_path_text)
if not local_path.exists():
return None
return self._build_localized_image_item(
original_item,
local_path,
accessible_url,
mime=str(background_result.get("mime") or "").strip() or None,
byte_size=background_result.get("byte_size"),
source=str(background_result.get("source") or "background_download"),
)
def _prefetch_remote_image_urls(
self,
tab,
image_urls: List[str],
max_size_mb: int = 10,
) -> int:
normalized_urls = []
seen = set()
for raw_url in image_urls or []:
normalized = normalize_remote_image_url(raw_url)
if not normalized or normalized in seen:
continue
seen.add(normalized)
normalized_urls.append(normalized)
if not normalized_urls or tab is None:
return 0
cookies_dict, headers = build_image_download_request_context(tab)
started = 0
for target_url in normalized_urls:
partition_key = get_image_download_partition(target_url, cookies_dict, headers)
result = background_image_downloader.start_download(
target_url,
cookies=cookies_dict,
headers=headers,
partition_key=partition_key,
max_bytes=max(1, int(max_size_mb or 10)) * 1024 * 1024,
)
if result:
started += 1
if started:
logger.debug(f"已提交后台图片预下载任务: {started} 个")
return started
def _localize_images_with_background_cache(
self,
images: List[Dict],
*,
wait_seconds: float = 0.0,
cookies: Any = None,
headers: Optional[Dict[str, str]] = None,
partition_key: Optional[str] = None,
) -> List[Dict]:
if not images:
return images
localized = list(images)
hit_count = 0
for index, item in enumerate(images):
if str(item.get("kind") or "").strip().lower() != "url":
continue
target_url = normalize_remote_image_url(item.get("url"))
if not target_url:
continue
current_partition = (
str(partition_key).strip()
if partition_key is not None
else get_image_download_partition(target_url, cookies, headers)
)
result = background_image_downloader.get_download_result(
target_url,
wait=wait_seconds > 0,
timeout=wait_seconds if wait_seconds > 0 else None,
partition_key=current_partition,
)
new_item = self._localize_image_item_from_background_result(item, result)
if new_item is None:
continue
localized[index] = new_item
hit_count += 1
if hit_count:
logger.debug(f"命中后台图片缓存: {hit_count} 张")
return localized
@staticmethod
def _append_audio_tail_silence(filepath, duration_seconds: float = 0.3):
"""为音频尾部追加一小段静音,避免播放时戛然而止。"""
from pathlib import Path
try:
target = Path(filepath)
except Exception:
return filepath
if duration_seconds <= 0 or not target.exists() or not target.is_file():
return filepath
ffmpeg_path = shutil.which("ffmpeg")
if not ffmpeg_path:
logger.debug(f"音频尾静音跳过:未找到 ffmpeg,file={target}")
return filepath
ffprobe_path = shutil.which("ffprobe")
def _probe_duration(path_obj: Path) -> float:
if not ffprobe_path:
return -1.0
cmd = [
ffprobe_path,
"-v",
"error",
"-show_entries",
"format=duration",
"-of",
"default=noprint_wrappers=1:nokey=1",
str(path_obj),
]
try:
completed = subprocess.run(
cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
text=True,
timeout=15,
check=False,
)
if completed.returncode != 0:
return -1.0
return float(str(completed.stdout or "").strip() or -1.0)
except Exception:
return -1.0
duration_text = f"{float(duration_seconds):.3f}".rstrip("0").rstrip(".")
temp_path = target.with_name(f"{target.stem}_tail{target.suffix}")
suffix = target.suffix.lower()
codec_args: list[str] = []
if suffix in {".ogg", ".oga"}:
codec_args = ["-c:a", "libvorbis", "-q:a", "5"]
elif suffix == ".mp3":
codec_args = ["-c:a", "libmp3lame", "-b:a", "128k"]
elif suffix in {".m4a", ".mp4"}:
codec_args = ["-c:a", "aac", "-b:a", "128k"]
elif suffix == ".wav":
codec_args = ["-c:a", "pcm_s16le"]
elif suffix == ".webm":
codec_args = ["-c:a", "libopus", "-b:a", "96k"]
original_duration = _probe_duration(target)
logger.debug(
f"音频尾静音开始: file={target.name}, ext={suffix or '<none>'}, "
f"append={duration_text}s, duration_before={original_duration:.3f}"
)
if original_duration <= 0:
logger.debug(
f"音频尾静音跳过:无法可靠探测原始时长,保留原文件。file={target.name}"
)
return filepath
if suffix == ".webm":
logger.debug(
f"音频尾静音跳过:暂不改写 webm,保留原文件。file={target.name}"
)
return filepath
command_variants = [[
ffmpeg_path,
"-y",
"-hide_banner",
"-loglevel",
"error",
"-i",
str(target),
"-af",
f"apad=pad_dur={duration_text}",
"-t",
f"{max(0.0, original_duration) + float(duration_seconds):.3f}" if original_duration > 0 else duration_text,
*codec_args,
str(temp_path),
]]
for cmd in command_variants:
try:
completed = subprocess.run(
cmd,
stdout=subprocess.DEVNULL,
stderr=subprocess.PIPE,
text=True,
timeout=30,
check=False,
)
except Exception as exc:
logger.debug(f"音频尾静音追加失败(ffmpeg 调用异常): {exc}")
continue
if completed.returncode == 0 and temp_path.exists() and temp_path.stat().st_size > 0:
try:
new_duration = _probe_duration(temp_path)
logger.debug(
f"音频尾静音生成成功: file={target.name}, "
f"duration_after={new_duration:.3f}, size={temp_path.stat().st_size}"
)
temp_path.replace(target)
return str(target)
except Exception as exc:
logger.debug(f"音频尾静音替换失败: {exc}")
try:
if temp_path.exists():
temp_path.unlink()
except Exception:
pass
return filepath
try:
if temp_path.exists():
temp_path.unlink()
except Exception:
pass
logger.debug(f"音频尾静音未生效,保留原文件: {target.name}")
return filepath
def _get_pending_detection_markers(
self,
image_config: Dict,
media_type: str,
marker_key: str,
) -> List[str]:
default_markers = {
"audio": {
"text_contains": [
"generated_music_content/",
"googleusercontent.com/generated_music_content/",
],
"url_contains": [
"generated_music_content/",
"googleusercontent.com/generated_music_content/",
],
"label_contains": [],
},
"video": {
"text_contains": [
"video_gen_chip/",
"googleusercontent.com/video_gen_chip/",
"正在生成视频",
"视频已准备就绪",
"come back later to check",
"i'm generating your video",
"your video is being generated",
"your video is ready",
],
"url_contains": [
"video_gen_chip/",
"googleusercontent.com/video_gen_chip/",
],
"label_contains": [],
},
}
normalized_media_type = str(media_type or "").strip().lower()
normalized_marker_key = str(marker_key or "").strip()
config_root = dict((image_config or {}).get("pending_detection") or {})
type_config = dict(config_root.get(normalized_media_type) or {})
configured = type_config.get(normalized_marker_key)
if isinstance(configured, list):
markers = [str(item or "").strip().lower() for item in configured if str(item or "").strip()]
if markers:
return markers
return [
str(item or "").strip().lower()
for item in default_markers.get(normalized_media_type, {}).get(normalized_marker_key, [])
if str(item or "").strip()
]
def _is_pending_media_text(self, text: str, image_config: Dict, media_type: str = "") -> bool:
lowered = str(text or "").strip().lower()
if not lowered:
return False
media_types = [str(media_type or "").strip().lower()] if media_type else ["audio", "video"]
for current_type in media_types:
markers = self._get_pending_detection_markers(
image_config,
current_type,
"text_contains",
)
if any(marker in lowered for marker in markers):
return True
return False
def _is_pending_media_item(self, media_item: Dict, image_config: Dict) -> bool:
if media_item.get("pending") is True:
return True
media_type = str(media_item.get("media_type") or "").strip().lower()
ref = str(media_item.get("url") or media_item.get("data_uri") or "").strip().lower()
label = str(media_item.get("label") or "").strip().lower()
if not media_type:
return False
url_markers = self._get_pending_detection_markers(
image_config,
media_type,
"url_contains",
)
label_markers = self._get_pending_detection_markers(
image_config,
media_type,
"label_contains",
)
if media_type not in {"audio", "video"}:
return False
return any(marker in ref for marker in url_markers) or any(
marker in label for marker in label_markers
)
def _filter_ready_media_items(self, media_items: List[Dict], image_config: Dict) -> List[Dict]:
return [
item for item in (media_items or [])
if not self._is_pending_media_item(item, image_config)
]
def _merge_dom_and_stream_media_items(
self,
dom_media_items: List[Dict],
stream_media_items: List[Dict],
image_config: Dict,
) -> List[Dict]:
"""Merge current-response stream media with media found in the selected DOM node."""
dom_items = [dict(item) for item in (dom_media_items or []) if isinstance(item, dict)]
stream_items = self._dedupe_media_items(
self._filter_ready_media_items(
[dict(item) for item in (stream_media_items or []) if isinstance(item, dict)],
image_config,
)
)
if not stream_items:
return self._dedupe_media_items(dom_items)
# A parsed stream image belongs to the response currently being monitored. The DOM
# selector can still resolve to an older image-only reply, so do not let that stale
# candidate replace the request-correlated stream result.
stream_images = [
item
for item in stream_items
if str(item.get("media_type") or "").strip().lower() == "image"
and str(item.get("url") or item.get("data_uri") or "").strip()
]
if stream_images:
dom_image_count = sum(
1
for item in dom_items
if str(item.get("media_type") or "").strip().lower() == "image"
)
dom_items = [
item
for item in dom_items
if str(item.get("media_type") or "").strip().lower() != "image"
]
dom_items = stream_images + dom_items
if dom_image_count:
logger.debug(
"网络流图片已替换 DOM 图片候选: "
f"stream={len(stream_images)}, dom={dom_image_count}"
)
merged = list(dom_items)
dom_types = {
str(item.get("media_type") or "").strip().lower()
for item in dom_items
if str(item.get("url") or item.get("data_uri") or "").strip()
}
appended = 0
for item in stream_items:
media_type = str(item.get("media_type") or "").strip().lower()
if not media_type or media_type in dom_types or media_type == "image":
continue
merged.append(dict(item))
appended += 1
if appended:
logger.debug(f"DOM 媒体缺失,已回退合并网络流媒体: {appended} 项")
return self._dedupe_media_items(merged)
def _should_run_media_postprocess(
self,
image_config: Dict,
*,
request_text_hint: str = "",
response_text_hint: str = "",
media_generation_state: Optional[Dict[str, Any]] = None,
stream_media_items: Optional[List[Dict[str, Any]]] = None,
dom_stream_media_items: Optional[List[Dict[str, Any]]] = None,
dom_image_detected: bool = False,
dom_final_image_urls: Optional[List[str]] = None,
) -> tuple[bool, Dict[str, Any]]:
modalities = self._media_modalities(image_config)
enabled_types = self._media_enabled_types(image_config)
stream_media_count = sum(1 for item in (stream_media_items or []) if isinstance(item, dict))
dom_stream_media_count = sum(1 for item in (dom_stream_media_items or []) if isinstance(item, dict))
dom_final_image_url_count = sum(1 for item in (dom_final_image_urls or []) if str(item or "").strip())
force_postprocess = bool((image_config or {}).get("force_postprocess"))
direct_postprocess_modalities = [
media_type
for media_type in (image_config or {}).get("direct_postprocess_modalities", [])
if media_type in enabled_types
]
media_state = dict(media_generation_state or {})
media_state_pending = bool(media_state.get("pending"))
media_state_type = str(media_state.get("media_type") or "").strip().lower()
media_state_hint = str(media_state.get("hint_text") or "").strip()
response_hint = str(response_text_hint or "").strip()
combined_hint = "\n".join(part for part in (response_hint, media_state_hint) if part).lower()
request_likely_image = self._looks_like_image_generation_request(request_text_hint)
av_pending_text_signal = (
("audio" in enabled_types or "video" in enabled_types)
and self._is_pending_media_text(combined_hint, image_config)
)
image_markers = (
"image_generation_content/",
"googleusercontent.com/image_generation_content/",
"generated image",
"generating image",
"images are being generated",
)
image_marker_hit = any(marker in combined_hint for marker in image_markers)
diagnostics: Dict[str, Any] = {
"enabled_types": enabled_types,
"run_policies": {
media_type: self._media_run_policy(image_config, media_type)
for media_type in enabled_types
},
"stream_media_count": stream_media_count,
"dom_stream_media_count": dom_stream_media_count,
"dom_image_detected": bool(dom_image_detected),
"dom_final_image_url_count": dom_final_image_url_count,
"force_postprocess": force_postprocess,
"direct_postprocess_modalities": direct_postprocess_modalities,
"media_state_pending": media_state_pending,
"media_state_type": media_state_type,
"media_state_hint_len": len(media_state_hint),
"response_hint_len": len(response_hint),
"request_hint_len": len(str(request_text_hint or "").strip()),
"request_likely_image": request_likely_image,
"av_pending_text_signal": av_pending_text_signal,
"image_marker_hit": image_marker_hit,
"decision": "",
}
if not enabled_types:
diagnostics["decision"] = "disabled"
return False, diagnostics
if stream_media_count > 0:
diagnostics["decision"] = "stream_media_items"
return True, diagnostics
if dom_stream_media_count > 0:
diagnostics["decision"] = "dom_stream_media_items"
return True, diagnostics
if dom_final_image_url_count > 0 and "image" in enabled_types:
diagnostics["decision"] = "dom_final_image_urls"
return True, diagnostics
if bool(dom_image_detected) and "image" in enabled_types:
diagnostics["decision"] = "dom_image_detected"
return True, diagnostics
if force_postprocess:
diagnostics["decision"] = "force_postprocess"
return True, diagnostics
if media_state_pending:
diagnostics["decision"] = "media_state_pending"
return True, diagnostics
if av_pending_text_signal:
diagnostics["decision"] = "audio_video_pending_text_signal"
return True, diagnostics
if "image" in enabled_types:
if request_likely_image:
diagnostics["decision"] = "request_likely_image"
return True, diagnostics
if image_marker_hit:
diagnostics["decision"] = "image_marker_hint"
return True, diagnostics
if direct_postprocess_modalities:
diagnostics["decision"] = "direct_postprocess_modalities"
return True, diagnostics
diagnostics["decision"] = "generic_dom_quick_scan"
return True, diagnostics
def _extract_media_after_stream(
self,
tab,
extractor,
image_config: Dict,
result_selector: str,
message_wrapper_selector: str = "",
completion_id: str = None,
stop_checker: Optional[Callable[[], bool]] = None,
response_text_hint: str = "",
request_text_hint: str = "",
media_generation_state: Optional[Dict[str, Any]] = None,
stream_media_items: Optional[List[Dict[str, Any]]] = None,
direct_modalities: Optional[List[str]] = None,
media_dom_baseline: Optional[Dict[str, Any]] = None,
) -> List[Dict]:
"""流式输出结束后提取多模态资源。"""
from app.core.elements import ElementFinder
from app.core.extractors.media_extractor import media_extractor
image_config = dict(image_config or {})
direct_scan_only = False
if direct_modalities:
requested_direct_modalities = {
str(item or "").strip().lower()
for item in direct_modalities
if str(item or "").strip().lower() in {"image", "audio", "video"}
}
if requested_direct_modalities:
direct_scan_only = True
scoped_modalities = {}
for media_type in ("image", "audio", "video"):
policy = self._media_policy(image_config, media_type)
if media_type not in requested_direct_modalities:
policy["enabled"] = False
policy["run_policy"] = "disabled"
scoped_modalities[media_type] = policy
image_config["modalities"] = scoped_modalities
image_config["wait_for_load"] = False
image_config["audio_capture_enabled"] = False
modalities = self._media_modalities(image_config)
only_audio_mode = (
is_modality_enabled(modalities, "audio")
and not is_modality_enabled(modalities, "image")
and not is_modality_enabled(modalities, "video")
)
debounce = 0.0 if only_audio_mode else image_config.get("debounce_seconds", 2.0)
effective_stop_checker = stop_checker or self._should_stop_checker
if debounce > 0:
elapsed = 0
step = 0.1
while elapsed < debounce:
if effective_stop_checker():
return []
time.sleep(step)
elapsed += step
finder = ElementFinder(tab)
fallback_stream_media_items = self._dedupe_media_items(
[dict(item) for item in (stream_media_items or []) if isinstance(item, dict)]
)
dom_baseline_active = bool(
media_dom_baseline
and media_dom_baseline.get("token")
and media_dom_baseline.get("property")
)
def _apply_stream_media_fallback(dom_media_items: List[Dict]) -> List[Dict]:
return self._merge_dom_and_stream_media_items(
dom_media_items,
fallback_stream_media_items,
image_config,
)
def _find_candidate_elements(timeout: float = 1.0):
primary_elements = []
fallback_elements = []
try:
if result_selector:
primary_elements = finder.find_all(result_selector, timeout=timeout) or []
except Exception as e:
logger.debug(f"主结果容器查找失败(忽略): {e}")
primary_elements = []
if primary_elements:
return primary_elements, "result_container"
try:
if message_wrapper_selector:
fallback_elements = finder.find_all(message_wrapper_selector, timeout=timeout) or []
except Exception as e:
logger.debug(f"消息包装容器查找失败(忽略): {e}")
fallback_elements = []
if fallback_elements:
logger.debug(
"主结果容器为空,回退到消息包装容器继续等待媒体渲染: "
f"primary={result_selector!r}, fallback={message_wrapper_selector!r}"
)
return fallback_elements, "message_wrapper"
return [], ""
def _audio_trigger_available(target_element=None, *, signal_seen: bool = False) -> bool:
if not is_modality_enabled(image_config.get("modalities") or {}, "audio"):
return False
if not bool(image_config.get("audio_capture_enabled", True)):
return False
if not self._media_policy_allows_audio_probe(image_config, signal_seen=signal_seen):
return False
policy = self._media_run_policy(image_config, "audio")
if policy == "always_probe":
return True
for current_target in [target_element, tab]:
if current_target is None:
continue
try:
probe_result = media_extractor.probe_audio_trigger(current_target, image_config)
except Exception:
probe_result = {}
if bool(probe_result.get("found")) or int(probe_result.get("candidate_count") or 0) > 0:
logger.debug(
"页面音频触发入口快速探测命中: "
f"policy={policy}, signal_seen={signal_seen}, "
f"candidate_count={probe_result.get('candidate_count')}, "
f"selector={probe_result.get('selector_used')!r}"
)
return True
logger.debug(
"页面音频触发入口快速探测未命中,跳过播放录音等待: "
f"policy={policy}, signal_seen={signal_seen}"
)
return False
try:
quick_probe_timeout = self._media_quick_probe_timeout(image_config, default=1.0)
elements, container_mode = _find_candidate_elements(timeout=quick_probe_timeout)
if not elements:
should_try_page_audio_capture = (
_audio_trigger_available(None, signal_seen=False)
and not effective_stop_checker()
)
if should_try_page_audio_capture:
logger.debug("结果容器为空,尝试页面级音频播放捕获回退")
captured_audio_items = self._capture_audio_via_page_playback(
tab=tab,
target_element=None,
image_config=image_config,
stop_checker=effective_stop_checker,
)
if captured_audio_items:
return _apply_stream_media_fallback(captured_audio_items)
if fallback_stream_media_items:
logger.debug("结果容器为空,直接回退到网络流媒体结果")
return list(fallback_stream_media_items)
return []
def _select_target_element(candidates):
if not candidates:
return None
strategy = str(image_config.get("final_target_strategy", "container") or "container").strip().lower()
selector = str(image_config.get("selector") or "img").strip() or "img"
def _has_media(candidate) -> bool:
try:
return bool(
as_element(candidate).run_js(
"""
const selector = String(arguments[0] || "img");
try {
if (this instanceof Element && typeof this.matches === "function" && this.matches(selector)) {
return true;
}
} catch {}
try {
return !!(this.querySelector && this.querySelector(selector));
} catch {
return false;
}
""",
selector,
)
)
except Exception:
return False
if strategy == "latest_visual_reply":
scored_candidates = []
parser_id = str(image_config.get("_parser_id", "") or "").strip().lower()
if "right" in parser_id:
column = "right"
elif "left" in parser_id:
column = "left"
else:
parser_side = str(image_config.get("_parser_target_side", "") or "").strip().lower()
column = parser_side if parser_side in {"left", "right"} else str(
image_config.get("latest_visual_column", "left") or "left"
).strip().lower()
if column not in {"left", "right"}:
column = "left"
for index, candidate in enumerate(candidates):
has_media = _has_media(candidate)
try:
rect = as_element(candidate).run_js(
"""
const rect = this.getBoundingClientRect();
return {
bottom: Number(rect && rect.bottom || 0) + Number(window.scrollY || 0),
left: Number(rect && rect.left || 0) + Number(window.scrollX || 0),
width: Number(rect && rect.width || 0),
height: Number(rect && rect.height || 0),
};
"""
) or {}
bottom = float(rect.get("bottom") or 0)
left = float(rect.get("left") or 0)
area = float(rect.get("width") or 0) * float(rect.get("height") or 0)
except Exception:
bottom = 0.0
left = 0.0
area = 0.0
horizontal_score = left if column == "right" else -left
scored_candidates.append((bottom, horizontal_score, area, index, index, left, has_media, candidate))
if len(scored_candidates) > 1:
left_edges = [item[5] for item in scored_candidates]
horizontal_span = max(left_edges) - min(left_edges)
if horizontal_span >= 80.0:
midpoint = min(left_edges) + horizontal_span / 2.0
side_candidates = [
item
for item in scored_candidates
if (
item[5] >= midpoint
if column == "right"
else item[5] <= midpoint
)
]
if side_candidates:
scored_candidates = side_candidates
if dom_baseline_active and scored_candidates:
fresh_scored = [
item
for item in scored_candidates
if self._candidate_has_fresh_dom_image(
item[7],
image_config,
media_dom_baseline,
)
]
if fresh_scored:
logger.debug(
f"[latest_visual_reply] 分栏 {column} 容器已按基线收敛到本轮新图: "
f"fresh={len(fresh_scored)}, side_total={len(scored_candidates)}"
)
scored_candidates = fresh_scored
if scored_candidates:
scored_candidates.sort(key=lambda item: item[:4], reverse=True)
best = scored_candidates[0]
logger.debug(
"[latest_visual_reply] 选中视觉最新媒体容器: "
f"index={best[4]}, column={column}, has_media={best[6]}, "
f"bottom={best[0]:.1f}, left={best[5]:.1f}, total={len(candidates)}"
)
return best[7]
if dom_baseline_active:
fresh_candidates = [
candidate
for candidate in candidates
if self._candidate_has_fresh_dom_image(
candidate,
image_config,
media_dom_baseline,
)
]
if fresh_candidates:
logger.debug(
"结果容器已按发送前基线收敛到本轮新增图片: "
f"fresh={len(fresh_candidates)}, total={len(candidates)}"
)
candidates = fresh_candidates
if strategy not in ("latest_reply", "latest_visual_reply"):
return candidates[-1]
for candidate in reversed(candidates):
if _has_media(candidate):
return candidate
return candidates[-1]
last_element = _select_target_element(elements)
if last_element is None:
return []
def _extract_media_once(target_element, quiet=False):
cfg = dict(image_config or {})
if quiet:
cfg["quiet"] = True
if media_dom_baseline:
cfg["request_baseline_token"] = str(media_dom_baseline.get("token") or "")
cfg["request_baseline_property"] = str(media_dom_baseline.get("property") or "")
if hasattr(extractor, 'extract_media'):
extracted_items = extractor.extract_media(
target_element,
config=cfg,
container_selector_fallback=result_selector
)
elif hasattr(extractor, 'extract_images'):
extracted_items = media_extractor.extract(
target_element,
config=cfg,
container_selector_fallback=result_selector
)
else:
extracted_items = media_extractor.extract(
target_element,
config=cfg,
container_selector_fallback=result_selector
)
extracted_items = list(extracted_items or [])
extracted_items = self._filter_media_items_against_dom_baseline(
extracted_items,
media_dom_baseline,
)
if media_dom_baseline and not self._candidate_has_fresh_dom_image(
target_element,
image_config,
media_dom_baseline,
):
stale_image_count = sum(
1
for item in extracted_items
if str(item.get("media_type") or "").strip().lower() == "image"
)
if stale_image_count:
logger.debug(
"发送前已存在的 DOM 图片已从本轮结果剔除: "
f"count={stale_image_count}"
)
extracted_items = [
item
for item in extracted_items
if str(item.get("media_type") or "").strip().lower() != "image"
]
return extracted_items
def _extract_fresh_media_from_page_scope(reason: str, quiet=True):
if only_audio_mode or not is_modality_enabled(modalities, "image"):
return []
if not dom_baseline_active or effective_stop_checker():
return []
fallback_selectors = image_config.get("fresh_image_fallback_selectors") or (
"main",
"body",
)
if isinstance(fallback_selectors, str):
fallback_selectors = [fallback_selectors]
roots = []
seen_root_ids = set()
for fallback_selector in fallback_selectors:
fallback_selector = str(fallback_selector or "").strip()
if not fallback_selector:
continue
try:
found_roots = finder.find_all(fallback_selector, timeout=0.5) or []
except Exception as e:
logger.debug(f"页面级图片回退根查找失败(忽略): selector={fallback_selector!r}, error={e}")
continue
for root in found_roots:
root_key = id(root)
if root_key in seen_root_ids:
continue
seen_root_ids.add(root_key)
roots.append((fallback_selector, root))
if not roots:
return []
logger.debug(
"窄范围未捕获本轮图片,启用页面级图片回退扫描: "
f"reason={reason}, roots={len(roots)}"
)
fallback_items = []
for fallback_selector, root in roots:
cfg = dict(image_config or {})
cfg["allow_container_fallback"] = True
cfg["container_selector"] = None
cfg["request_baseline_token"] = str(media_dom_baseline.get("token") or "")
cfg["request_baseline_property"] = str(media_dom_baseline.get("property") or "")
if quiet:
cfg["quiet"] = True
try:
extracted_items = media_extractor.extract(
root,
config=cfg,
container_selector_fallback="",
)
except Exception as e:
logger.debug(
"页面级图片回退扫描失败(忽略): "
f"selector={fallback_selector!r}, error={e}"
)
continue
extracted_items = list(extracted_items or [])
extracted_items = self._filter_media_items_against_dom_baseline(
extracted_items,
media_dom_baseline,
)
ready_items = self._filter_ready_media_items(extracted_items, image_config)
image_count = sum(
1
for item in ready_items
if str(item.get("media_type") or "").strip().lower() == "image"
)
if image_count:
logger.debug(
"页面级图片回退扫描捕获本轮新图: "
f"selector={fallback_selector!r}, images={image_count}"
)
fallback_items.extend(ready_items)
break
return self._dedupe_media_items(fallback_items)
request_likely_image = self._looks_like_image_generation_request(request_text_hint)
media_items = [] if only_audio_mode else _extract_media_once(last_element)
ready_media_items = _apply_stream_media_fallback(
self._filter_ready_media_items(media_items, image_config)
)
pending_media_items = [
item for item in media_items
if self._is_pending_media_item(item, image_config)
]
image_items = [item for item in ready_media_items if item.get("media_type") == "image"]
audio_items = [item for item in ready_media_items if item.get("media_type") == "audio"]
video_items = [item for item in ready_media_items if item.get("media_type") == "video"]
if (
request_likely_image
and not image_items
and not fallback_stream_media_items
and not effective_stop_checker()
):
fresh_page_media_items = _extract_fresh_media_from_page_scope("initial_empty")
if fresh_page_media_items:
ready_media_items = self._dedupe_media_items(
list(ready_media_items) + fresh_page_media_items
)
image_items = [item for item in ready_media_items if item.get("media_type") == "image"]
audio_items = [item for item in ready_media_items if item.get("media_type") == "audio"]
video_items = [item for item in ready_media_items if item.get("media_type") == "video"]
placeholder_text = ""
if not only_audio_mode:
try:
if hasattr(extractor, 'extract_text'):
placeholder_text = str(extractor.extract_text(last_element) or "")
else:
placeholder_text = str(
as_element(last_element).run_js("return this.innerText || this.textContent || ''") or ""
)
except Exception:
placeholder_text = ""
placeholder_text_lower = placeholder_text.lower()
response_text_hint_lower = str(response_text_hint or "").strip().lower()
media_state = dict(media_generation_state or {})
media_state_pending = bool(media_state.get("pending"))
media_state_type = str(media_state.get("media_type") or "").strip().lower()
media_state_hint_text = str(media_state.get("hint_text") or "").strip().lower()
combined_pending_text = "\n".join(
text for text in (placeholder_text_lower, response_text_hint_lower, media_state_hint_text)
if text
)
terminal_error_markers = image_config.get("terminal_error_text_markers") or (
"something went wrong with this response",
"please try again",
"trace id:",
)
if isinstance(terminal_error_markers, str):
terminal_error_markers = (terminal_error_markers,)
terminal_media_error = any(
str(marker or "").strip().lower() in combined_pending_text
for marker in terminal_error_markers
if str(marker or "").strip()
)
has_generated_image_hint = any(
marker in placeholder_text_lower
for marker in (
"image_generation_content/",
"googleusercontent.com/image_generation_content/",
)
)
if not has_generated_image_hint:
try:
has_generated_image_hint = bool(
as_element(last_element).run_js(
"""
return !!this.querySelector(
'.attachment-container.generated-images, '
+ '.generated-images, generated-image, single-image, '
+ '.image-button, img[src^="blob:"], img[src^="data:image"]'
);
"""
)
)
except Exception:
has_generated_image_hint = False
pending_audio_hint = self._is_pending_media_text(combined_pending_text, image_config, "audio")
pending_video_hint = self._is_pending_media_text(combined_pending_text, image_config, "video")
if pending_media_items:
pending_audio_hint = pending_audio_hint or any(
item.get("media_type") == "audio" for item in pending_media_items
)
pending_video_hint = pending_video_hint or any(
item.get("media_type") == "video" for item in pending_media_items
)
logger.debug(
"检测到占位媒体,继续等待真实结果: "
f"pending={len(pending_media_items)}, ready={len(ready_media_items)}"
)
pending_kinds = set()
if (
has_generated_image_hint
or (media_state_pending and media_state_type == "image")
) and not image_items and self._media_policy_allows_signal_wait(image_config, "image"):
pending_kinds.add("image")
if (
media_state_pending
and media_state_type == "audio"
and not (audio_items or video_items)
and self._media_policy_allows_signal_wait(image_config, "audio")
):
pending_kinds.add("audio")
if (
media_state_pending
and media_state_type == "video"
and not video_items
and self._media_policy_allows_signal_wait(image_config, "video")
):
pending_kinds.add("video")
if (
pending_audio_hint
and not (audio_items or video_items)
and self._media_policy_allows_signal_wait(image_config, "audio")
):
pending_kinds.add("audio")
if (
pending_video_hint
and not video_items
and self._media_policy_allows_signal_wait(image_config, "video")
):
pending_kinds.add("video")
if terminal_media_error and not image_items:
pending_kinds.discard("image")
logger.warning("当前回复包含终止错误且没有本轮新图,跳过图片等待与历史图回退")
if pending_kinds and not direct_scan_only and not effective_stop_checker():
base_timeout = float(image_config.get("load_timeout_seconds", 5.0) or 5.0)
late_wait_timeout = float(
image_config.get("late_render_timeout_seconds")
or max(30.0, base_timeout * 6.0)
)
for pending_kind in pending_kinds:
late_wait_timeout = max(
late_wait_timeout,
self._media_late_wait_timeout(image_config, pending_kind, late_wait_timeout),
)
state_wait_timeout = media_state.get("wait_timeout_seconds")
try:
if state_wait_timeout is not None:
late_wait_timeout = max(late_wait_timeout, float(state_wait_timeout))
except Exception:
pass
if media_state_pending and media_state_type in pending_kinds:
logger.debug(
"检测到解析器上报的待渲染媒体任务,延长媒体渲染等待窗口: "
f"{late_wait_timeout:.1f}s"
)
poll_interval = float(image_config.get("late_render_poll_seconds") or 1.0)
deadline = time.time() + late_wait_timeout
wait_satisfied = False
while time.time() < deadline and not effective_stop_checker():
time.sleep(max(0.2, poll_interval))
elements, container_mode = _find_candidate_elements(timeout=0.5)
if not elements:
continue
selected = _select_target_element(elements)
if selected is None:
continue
last_element = selected
media_items = _extract_media_once(last_element, quiet=True)
ready_media_items = _apply_stream_media_fallback(
self._filter_ready_media_items(media_items, image_config)
)
image_items = [item for item in ready_media_items if item.get("media_type") == "image"]
audio_items = [item for item in ready_media_items if item.get("media_type") == "audio"]
video_items = [item for item in ready_media_items if item.get("media_type") == "video"]
if "image" in pending_kinds and not image_items and not fallback_stream_media_items:
fresh_page_media_items = _extract_fresh_media_from_page_scope(
"pending_wait_empty",
quiet=True,
)
if fresh_page_media_items:
ready_media_items = self._dedupe_media_items(
list(ready_media_items) + fresh_page_media_items
)
image_items = [
item for item in ready_media_items if item.get("media_type") == "image"
]
audio_items = [
item for item in ready_media_items if item.get("media_type") == "audio"
]
video_items = [
item for item in ready_media_items if item.get("media_type") == "video"
]
satisfied = True
if "image" in pending_kinds and not image_items:
satisfied = False
if "audio" in pending_kinds and not (audio_items or video_items):
satisfied = False
if "video" in pending_kinds and not video_items:
satisfied = False
if satisfied:
kinds_label = ",".join(sorted(pending_kinds))
logger.debug(
f"延迟媒体渲染已捕获: kinds={kinds_label} "
f"(late_wait={late_wait_timeout:.1f}s, container={container_mode or 'unknown'})"
)
wait_satisfied = True
break
if pending_kinds and not wait_satisfied and not effective_stop_checker():
logger.warning(
"等待待渲染媒体超时,仍未拿到最终结果: "
f"kinds={','.join(sorted(pending_kinds))}, late_wait={late_wait_timeout:.1f}s"
)
should_probe_late_image_render = (
is_modality_enabled(modalities, "image")
and self._media_policy_allows_signal_wait(image_config, "image")
and not image_items
and not pending_kinds
and not only_audio_mode
and not effective_stop_checker()
and request_likely_image
and len(str(response_text_hint or "").strip()) <= 32
and not terminal_media_error
)
if should_probe_late_image_render:
late_image_wait_timeout = float(
image_config.get("late_image_render_timeout_seconds")
or max(45.0, float(image_config.get("load_timeout_seconds", 5.0) or 5.0) * 8.0)
)
late_image_wait_timeout = self._media_blind_wait_timeout(
image_config,
"image",
self._media_late_wait_timeout(
image_config,
"image",
late_image_wait_timeout,
),
)
if late_image_wait_timeout <= 0:
logger.debug("无显式占位信号,图片盲等已按配置跳过")
should_probe_late_image_render = False
if should_probe_late_image_render:
poll_interval = float(image_config.get("late_render_poll_seconds") or 1.0)
deadline = time.time() + late_image_wait_timeout
while time.time() < deadline and not effective_stop_checker():
time.sleep(max(0.2, poll_interval))
elements, container_mode = _find_candidate_elements(timeout=0.5)
if not elements:
continue
selected = _select_target_element(elements)
if selected is None:
continue
last_element = selected
media_items = _extract_media_once(last_element, quiet=True)
ready_media_items = _apply_stream_media_fallback(
self._filter_ready_media_items(media_items, image_config)
)
image_items = [item for item in ready_media_items if item.get("media_type") == "image"]
audio_items = [item for item in ready_media_items if item.get("media_type") == "audio"]
video_items = [item for item in ready_media_items if item.get("media_type") == "video"]
if not image_items and not fallback_stream_media_items:
fresh_page_media_items = _extract_fresh_media_from_page_scope(
"blind_wait_empty",
quiet=True,
)
if fresh_page_media_items:
ready_media_items = self._dedupe_media_items(
list(ready_media_items) + fresh_page_media_items
)
image_items = [
item for item in ready_media_items if item.get("media_type") == "image"
]
audio_items = [
item for item in ready_media_items if item.get("media_type") == "audio"
]
video_items = [
item for item in ready_media_items if item.get("media_type") == "video"
]
if image_items:
logger.debug(
"无显式占位信号,但已在延迟轮询中捕获到图片结果: "
f"late_wait={late_image_wait_timeout:.1f}s, container={container_mode or 'unknown'}"
)
break
should_try_audio_capture = (
_audio_trigger_available(
last_element,
signal_seen=bool("audio" in pending_kinds or pending_audio_hint or media_state_type == "audio"),
)
and not audio_items
and not effective_stop_checker()
)
if should_try_audio_capture:
captured_audio_items = self._capture_audio_via_page_playback(
tab=tab,
target_element=last_element,
image_config=image_config,
stop_checker=effective_stop_checker,
response_text_hint=response_text_hint,
)
if captured_audio_items:
ready_media_items = self._dedupe_media_items(list(ready_media_items) + captured_audio_items)
audio_items = [item for item in ready_media_items if item.get("media_type") == "audio"]
media_items = ready_media_items
try:
if image_items:
self._prefetch_remote_image_urls(
tab,
[
str(item.get("url") or "").strip()
for item in image_items
if isinstance(item, dict)
],
max_size_mb=image_config.get("max_size_mb", 10),
)
converted_images = self._try_screenshot_images_to_local(tab, last_element, image_items, image_config)
other_items = [item for item in media_items if item.get("media_type") != "image"]
media_items = converted_images + other_items
except Exception as e:
logger.warning(f"截图落盘失败(已忽略): {e}")
try:
media_items = self._persist_data_uri_media_to_local(
media_items,
max_size_mb=image_config.get("max_size_mb", 10),
)
except Exception as e:
logger.warning(f"data uri 落盘失败(已忽略): {e}")
uploaded_images = image_config.get("uploaded_image_paths") or []
if uploaded_images and media_items:
media_items = filter_reference_images(
media_items,
uploaded_images,
tab=tab,
logger_context="media_after_stream",
)
# 汇总并打印多模态媒体提取完成的日志
final_images = [item for item in media_items if item.get("media_type") == "image"]
final_videos = [item for item in media_items if item.get("media_type") == "video"]
final_audios = [item for item in media_items if item.get("media_type") == "audio"]
logger.debug(
f"提取完成: {len(final_images)} 张图片, {len(final_videos)} 个视频, {len(final_audios)} 个音频"
)
return media_items
except Exception as e:
logger.warning(f"多模态提取异常: {e}")
return []
def _capture_audio_via_page_playback(
self,
tab,
target_element,
image_config: Dict,
stop_checker: Optional[Callable[[], bool]] = None,
response_text_hint: str = "",
) -> List[Dict]:
"""回退方案:触发页面播放按钮并从隐藏音频/ WebAudio 输出中捕获音频。"""
from app.core.extractors.media_extractor import media_extractor
if not tab:
return []
effective_stop_checker = stop_checker or self._should_stop_checker
if effective_stop_checker():
return []
prepared = media_extractor.prepare_page_audio_capture(tab, image_config)
if not prepared:
return []
network_capture = dict(image_config.get("audio_network_capture") or {})
network_capture_enabled = bool(network_capture.get("enabled", False))
if network_capture_enabled and not media_extractor.install_audio_network_probe(tab, image_config):
network_capture_enabled = False
activation_result = media_extractor.activate_audio_trigger_surface(target_element or tab)
if activation_result:
logger.debug(
"页面音频操作区激活结果: "
f"ok={bool(activation_result.get('ok'))}, "
f"activated_count={activation_result.get('activated_count')}"
)
trigger_target = target_element or tab
trigger_result: Dict[str, Any] = {}
trigger_targets = [trigger_target]
if target_element is not None and target_element is not tab:
trigger_targets.append(tab)
for attempt in range(3):
if effective_stop_checker():
return []
if attempt > 0:
time.sleep(0.35)
media_extractor.activate_audio_trigger_surface(target_element or tab)
for current_target in trigger_targets:
trigger_result = media_extractor.trigger_audio_playback(current_target, image_config)
if bool(trigger_result.get("clicked")):
trigger_result["attempt"] = attempt + 1
trigger_result["target_scope"] = "tab" if current_target is tab else "message"
break
if bool(trigger_result.get("clicked")):
break
if not bool(trigger_result.get("clicked")):
logger.debug(
"页面音频捕获未触发: "
f"selector={trigger_result.get('selector_used')!r}, "
f"labels={trigger_result.get('labels_used')!r}, "
f"candidate_count={trigger_result.get('candidate_count')}, "
f"debug_matches={trigger_result.get('debug_matches')!r}, "
f"nearby_candidates={trigger_result.get('nearby_candidates')!r}, "
f"visible_button_samples={trigger_result.get('visible_button_samples')!r}"
)
return []
logger.debug(
"页面音频捕获已触发: "
f"attempt={trigger_result.get('attempt')}, "
f"scope={trigger_result.get('target_scope')}, "
f"text={trigger_result.get('text')!r}, "
f"score={trigger_result.get('score')}"
)
effective_response_text_hint = str(response_text_hint or "").strip()
if target_element is not None and len(effective_response_text_hint) <= 8:
try:
dom_response_text = str(
as_element(target_element).run_js(
"""
const text = (this.innerText || this.textContent || "").trim();
return text;
"""
) or ""
).strip()
except Exception:
dom_response_text = ""
if len(dom_response_text) > len(effective_response_text_hint):
logger.debug(
"页面音频捕获文本提示已回退 to 当前回复 DOM 文本: "
f"old_len={len(effective_response_text_hint)}, new_len={len(dom_response_text)}"
)
effective_response_text_hint = dom_response_text
if network_capture_enabled:
network_audio_items = media_extractor.capture_network_audio(
tab=tab,
config=image_config,
stop_checker=effective_stop_checker,
response_text_hint=effective_response_text_hint,
)
if network_audio_items:
logger.debug(f"网络音频捕获成功: {len(network_audio_items)} 项")
return network_audio_items
max_wait = float(
image_config.get("audio_capture_max_wait_seconds")
or max(8.0, float(image_config.get("load_timeout_seconds", 5.0) or 5.0) * 1.8)
)
max_wait = self._media_audio_capture_timeout(image_config, max_wait)
hint_text = str(effective_response_text_hint or "").strip()
if hint_text:
try:
chars_per_second = max(
1.0,
float(image_config.get("audio_capture_estimated_chars_per_second") or 4.8),
)
min_wait = max(
1.0,
float(image_config.get("audio_capture_min_wait_seconds") or 2.0),
)
padding_seconds = max(
0.0,
float(image_config.get("audio_capture_wait_padding_seconds") or 1.2),
)
hard_cap = max(
min_wait,
float(image_config.get("audio_capture_hard_max_wait_seconds") or max_wait),
)
fallback_wait = max_wait
estimated_wait = min_wait + (len(hint_text) / chars_per_second) + padding_seconds
max_wait = min(max(min_wait, estimated_wait), hard_cap)
logger.debug(
"页面音频捕获动态等待窗口: "
f"text_len={len(hint_text)}, max_wait={max_wait:.1f}s, "
f"fallback_wait={fallback_wait:.1f}s, "
f"chars_per_second={chars_per_second:.1f}"
)
except (TypeError, ValueError):
pass
poll_interval = max(0.1, float(image_config.get("audio_capture_poll_seconds") or 0.25))
silence_seconds = max(0.4, float(image_config.get("audio_capture_silence_seconds") or 1.2))
activity_silence_seconds = max(
0.2,
float(image_config.get("audio_capture_activity_silence_seconds") or 0.65),
)
terminal_settle_seconds = max(
0.0,
float(image_config.get("audio_capture_terminal_settle_seconds") or 0.35),
)
deadline = time.time() + max_wait
has_seen_data = False
has_seen_activity = False
terminal_deadline = 0.0
while time.time() < deadline and not effective_stop_checker():
time.sleep(poll_interval)
status = media_extractor.get_page_audio_capture_status(tab)
if not isinstance(status, dict):
continue
if bool(status.get("has_data")):
has_seen_data = True
active_recordings = int(status.get("active_recordings") or 0)
last_data_at_ms = int(status.get("last_data_at") or 0)
last_active_at_ms = int(status.get("last_active_at") or 0)
playing_media_elements = int(status.get("playing_media_elements") or 0)
terminal_playback_elements = int(status.get("terminal_playback_elements") or 0)
if last_active_at_ms:
has_seen_activity = True
if has_seen_data and active_recordings <= 0:
break
if has_seen_data and playing_media_elements <= 0 and terminal_playback_elements > 0:
if terminal_deadline <= 0:
terminal_deadline = time.time() + terminal_settle_seconds
elif time.time() >= terminal_deadline:
break
else:
terminal_deadline = 0.0
if has_seen_activity and last_active_at_ms:
activity_silence_elapsed = (time.time() * 1000.0 - last_active_at_ms) / 1000.0
if activity_silence_elapsed >= activity_silence_seconds:
break
if has_seen_data and last_data_at_ms:
silence_elapsed = (time.time() * 1000.0 - last_data_at_ms) / 1000.0
if silence_elapsed >= silence_seconds:
break
if effective_stop_checker():
return []
captured_items = media_extractor.export_page_audio_capture(tab, image_config)
if captured_items:
status = media_extractor.get_page_audio_capture_status(tab)
if isinstance(status, dict):
logger.debug(
"页面播放音频捕获成功: "
f"{len(captured_items)} 项, "
f"version={status.get('version')}, "
f"tracked_media={status.get('tracked_media_elements')}, "
f"tracked_web_audio={status.get('tracked_web_audio')}, "
f"active={status.get('active_recordings')}, "
f"chunks={status.get('total_chunks')}, "
f"last_data_at={status.get('last_data_at')}, "
f"last_active_at={status.get('last_active_at')}, "
f"peak_rms={status.get('peak_rms')}"
)
if not captured_items:
status = media_extractor.get_page_audio_capture_status(tab)
if isinstance(status, dict):
logger.debug(
"页面音频捕获未导出到任何音频数据: "
f"version={status.get('version')}, "
f"tracked_media={status.get('tracked_media_elements')}, "
f"tracked_web_audio={status.get('tracked_web_audio')}, "
f"active={status.get('active_recordings')}, "
f"chunks={status.get('total_chunks')}, "
f"has_data={status.get('has_data')}, "
f"errors={status.get('recent_errors')}, "
f"events={status.get('recent_events')}"
)
else:
logger.debug("页面音频捕获未导出到任何音频数据")
if network_capture_enabled:
try:
media_extractor.capture_network_audio(
tab=tab,
config=image_config,
stop_checker=lambda: True,
response_text_hint=effective_response_text_hint,
)
except Exception:
pass
try:
browser_tts_items = media_extractor.capture_browser_tts_fallback(
tab=tab,
config=image_config,
stop_checker=effective_stop_checker,
response_text_hint=effective_response_text_hint,
)
except Exception as exc:
logger.debug(f"浏览器 TTS 兜底失败(已忽略): {exc}")
browser_tts_items = []
if browser_tts_items:
return browser_tts_items
return captured_items
def _resolve_media_ref(self, media_item: Dict) -> str:
ref = str(media_item.get("url") or media_item.get("data_uri") or "").strip()
if not ref:
return ""
if ref.startswith("/") and not ref.startswith("//"):
public_base = os.getenv("PUBLIC_BASE_URL", "").strip()
if public_base:
return public_base.rstrip("/") + ref
return f"http://{AppConfig.get_host()}:{AppConfig.get_port()}{ref}"
return ref
def _build_media_markdown_block(self, media_items: List[Dict]) -> str:
image_blocks = []
audio_lines = []
video_lines = []
for item in media_items or []:
ref = self._resolve_media_ref(item)
if not ref:
continue
media_type = str(item.get("media_type") or "image").lower()
if media_type == "image":
image_blocks.append(f"\n\n![image_{len(image_blocks)}]({ref})")
continue
label = item.get("label") or item.get("mime") or ""
label_suffix = f" - {label}" if label else ""
if media_type == "audio":
audio_lines.append(f"[audio_{len(audio_lines)}]({ref}){label_suffix}")
elif media_type == "video":
video_lines.append(f"[video_{len(video_lines)}]({ref}){label_suffix}")
blocks = []
if image_blocks:
blocks.append("".join(image_blocks))
if audio_lines:
blocks.append("\n\n" + "\n".join(audio_lines))
if video_lines:
blocks.append("\n\n" + "\n".join(video_lines))
if not blocks:
return ""
return "".join(blocks) + "\n\n"
def _prepare_media_items_for_response(self, media_items: List[Dict]) -> List[Dict]:
result = []
for item in media_items or []:
entry = dict(item)
ref = self._resolve_media_ref(item)
if entry.get("kind") == "url":
entry["url"] = ref or entry.get("url")
if ref and entry.get("kind") != "url":
entry["kind"] = "url"
entry["url"] = ref
entry.pop("data_uri", None)
entry.pop("local_path", None)
result.append(entry)
return result
def _persist_remote_media_urls_to_local(
self,
media_items: List[Dict],
tab=None,
max_size_mb: int = 10,
image_config: Optional[Dict[str, Any]] = None,
) -> List[Dict]:
"""Download remote audio/video URLs to local files so downstream clients get stable local URLs."""
if not media_items or tab is None:
return media_items
save_dir = Path("download_images")
save_dir.mkdir(exist_ok=True)
max_bytes = max(1, int(max_size_mb)) * 1024 * 1024
# S8/S9:扩展名统一由 app.utils.media_safety 白名单决定(不再使用本地 ext_map / URL 后缀)
cookies_dict, headers = build_image_download_request_context(tab, accept="*/*")
partition_key = build_image_download_partition(cookies_dict, headers)
result = []
is_image_gen = bool(
(image_config or {}).get("arena_image_generation")
or (image_config or {}).get("_arena_image_generation_active")
or (image_config or {}).get("require_localized_images")
)
bg_timeout = float(
(image_config or {}).get("background_download_wait_seconds")
or (image_config or {}).get("download_wait_seconds")
or (3.5 if is_image_gen else 1.0)
)
for item in media_items:
filepath = None
if item.get("kind") != "url":
result.append(item)
continue
media_type = str(item.get("media_type") or "").lower()
if media_type not in {"audio", "video"}:
if media_type == "image":
target_url = normalize_remote_image_url(item.get("url"))
target_partition = (
get_image_download_partition(target_url, cookies_dict, headers)
if target_url
else partition_key
)
background_result = background_image_downloader.get_download_result(
target_url,
wait=True,
timeout=bg_timeout,
partition_key=target_partition,
) if target_url else None
localized_item = self._localize_image_item_from_background_result(item, background_result)
result.append(localized_item if localized_item is not None else item)
continue
result.append(item)
continue
url = str(item.get("url") or "").strip()
if not (url.startswith("http://") or url.startswith("https://")):
result.append(item)
continue
try:
credential_origin_url = headers.get("Referer")
target_host = str(urlparse(url).hostname or "").lower().rstrip(".")
referer_host = str(
urlparse(str(headers.get("Referer") or "")).hostname or ""
).lower().rstrip(".")
if (
target_host == "contribution.usercontent.google.com"
and referer_host == "gemini.google.com"
):
credential_origin_url = url
response = get_public_remote_resource(
url,
cookies=cookies_dict,
headers=headers,
credential_origin_url=credential_origin_url,
timeout=(8, 30),
stream=True,
)
except Exception as exc:
logger.warning(f"{media_type} 下载失败,保留远程链接: {exc}")
result.append(item)
continue
try:
if response.status_code != 200:
logger.warning(f"{media_type} 下载失败,HTTP {response.status_code}")
result.append(item)
continue
content_type = str(response.headers.get("Content-Type") or "").split(";", 1)[0].strip().lower()
if media_type == "audio" and "audio" not in content_type:
logger.warning(f"音频下载返回非音频类型,保留远程链接: {content_type or 'unknown'}")
result.append(item)
continue
if media_type == "video" and "video" not in content_type:
logger.warning(f"视频下载返回非视频类型,保留远程链接: {content_type or 'unknown'}")
result.append(item)
continue
# S9:扩展名只来自白名单;远端 URL 的 .html/.svg 后缀永远不会被采用
ext = choose_media_extension(media_type, content_type, url)
if not ext:
logger.warning(f"{media_type} 响应类型不安全,保留远程链接: {content_type or 'unknown'}")
result.append(item)
continue
filename = f"{int(time.time())}_{uuid.uuid4().hex[:8]}{ext}"
filepath = save_dir / filename
written = 0
sniffed = False
with filepath.open("wb") as handle:
for chunk in response.iter_content(chunk_size=1024 * 64):
if not chunk:
continue
if not sniffed:
sniffed = True
# 谎报 Content-Type(如 video/custom)但实际是 HTML/SVG 的响应在写盘前拦截
if looks_like_active_content(chunk[:1024]):
raise ValueError("unsafe_media_content")
written += len(chunk)
if written > max_bytes:
raise ValueError(f"media_too_large:{written}")
handle.write(chunk)
new_item = dict(item)
new_item["url"] = f"/media/{filename}"
new_item["mime"] = content_type or item.get("mime")
new_item["byte_size"] = written
new_item["source"] = "local_file"
new_item["local_path"] = str(filepath)
result.append(new_item)
logger.debug(f"✅ {media_type} 已保存到本地: {filename} ({written} bytes)")
except Exception as exc:
logger.warning(f"{media_type} 落盘失败,保留远程链接: {exc}")
try:
if filepath is not None and filepath.exists():
filepath.unlink()
except Exception:
pass
result.append(item)
finally:
response.close()
return result
@staticmethod
def _remote_image_urls_match(left: str, right: str) -> bool:
left_url = str(left or "").strip()
right_url = str(right or "").strip()
if not left_url or not right_url:
return False
if left_url == right_url:
return True
try:
left_parsed = urlparse(left_url)
right_parsed = urlparse(right_url)
except Exception:
return False
return bool(
left_parsed.scheme.lower() == right_parsed.scheme.lower()
and left_parsed.netloc.lower() == right_parsed.netloc.lower()
and left_parsed.path
and right_parsed.path
and left_parsed.path == right_parsed.path
)
def _wait_for_image_element_ready(
self,
img_ele,
target_url: str,
*,
timeout_seconds: float,
poll_seconds: float = 0.2,
) -> tuple[bool, Dict[str, Any]]:
"""Wait until the matched DOM image has decoded dimensions before screenshotting it."""
try:
timeout = max(0.0, float(timeout_seconds or 0.0))
except (TypeError, ValueError):
timeout = 0.0
try:
poll = max(0.05, float(poll_seconds or 0.2))
except (TypeError, ValueError):
poll = 0.2
deadline = time.monotonic() + timeout
last_state: Dict[str, Any] = {}
while True:
try:
raw_state = as_element(img_ele).run_js(
"""
return (() => {
const src = String(
this.currentSrc
|| this.getAttribute('src')
|| this.src
|| ''
).trim();
return {
src,
complete: Boolean(this.complete),
natural_width: Number(this.naturalWidth || 0),
natural_height: Number(this.naturalHeight || 0),
};
})();
"""
)
last_state = dict(raw_state) if isinstance(raw_state, dict) else {}
except Exception as exc:
last_state = {"error": str(exc)[:120]}
source_matches = self._remote_image_urls_match(
target_url,
str(last_state.get("src") or ""),
)
try:
natural_width = max(0, int(float(last_state.get("natural_width") or 0)))
except (TypeError, ValueError):
natural_width = 0
try:
natural_height = max(0, int(float(last_state.get("natural_height") or 0)))
except (TypeError, ValueError):
natural_height = 0
last_state["natural_width"] = natural_width
last_state["natural_height"] = natural_height
last_state["source_matches"] = source_matches
if (
source_matches
and bool(last_state.get("complete"))
and natural_width > 0
and natural_height > 0
):
return True, last_state
remaining = deadline - time.monotonic()
if remaining <= 0:
return False, last_state
time.sleep(min(poll, remaining))
@staticmethod
def _expand_image_element_for_screenshot(img_ele) -> bool:
"""Temporarily render an image at its decoded dimensions for screenshot fallback."""
try:
return bool(as_element(img_ele).run_js(
"""
return (() => {
const image = this;
const width = Number(image.naturalWidth || 0);
const height = Number(image.naturalHeight || 0);
if (!(width > 0 && height > 0)) return false;
const marker = '__universalProxyScreenshotStyle';
if (!Object.prototype.hasOwnProperty.call(image, marker)) {
Object.defineProperty(image, marker, {
configurable: true,
value: {
hadStyle: image.hasAttribute('style'),
style: image.getAttribute('style') || '',
},
});
}
image.style.setProperty('display', 'block', 'important');
image.style.setProperty('box-sizing', 'content-box', 'important');
image.style.setProperty('width', `${width}px`, 'important');
image.style.setProperty('height', `${height}px`, 'important');
image.style.setProperty('min-width', '0', 'important');
image.style.setProperty('min-height', '0', 'important');
image.style.setProperty('max-width', 'none', 'important');
image.style.setProperty('max-height', 'none', 'important');
image.style.setProperty('object-fit', 'fill', 'important');
image.style.setProperty('position', 'fixed', 'important');
image.style.setProperty('left', '0', 'important');
image.style.setProperty('top', '0', 'important');
image.style.setProperty('right', 'auto', 'important');
image.style.setProperty('bottom', 'auto', 'important');
image.style.setProperty('z-index', '2147483647', 'important');
image.style.setProperty('transform', 'none', 'important');
image.style.setProperty('clip-path', 'none', 'important');
return true;
})();
"""
))
except Exception as exc:
logger.debug(f"展开图片以供完整截图失败(忽略): {exc}")
return False
@staticmethod
def _restore_image_element_after_screenshot(img_ele) -> None:
try:
as_element(img_ele).run_js(
"""
(() => {
const image = this;
const marker = '__universalProxyScreenshotStyle';
const snapshot = image[marker];
if (!snapshot) return;
if (snapshot.hadStyle) {
image.setAttribute('style', snapshot.style);
} else {
image.removeAttribute('style');
}
try {
delete image[marker];
} catch {
image[marker] = undefined;
}
})();
"""
)
except Exception as exc:
logger.debug(f"恢复图片截图样式失败(忽略): {exc}")
def _try_screenshot_images_to_local(self, tab, last_element, images: List[Dict], image_config: Dict = None) -> List[Dict]:
"""
优先下载图片(更精准),下载失败才截图。
基于实测 API:img_ele.attr('src'), page.cookies(), get_screenshot(path)
"""
if not images:
return images
image_config = image_config or {}
baseline_keys = {
self._media_reference_key(reference)
for reference in (image_config.get("request_baseline_references") or [])
}
baseline_keys.discard("")
# Only images absent from the pre-submit baseline may be localized. This
# protects against an uploaded input or history item being mislabeled as
# stream output before any download or screenshot fallback can use it.
candidates = []
excluded_baseline_images = 0
for item in images:
candidate = dict(item or {})
reference = candidate.get("url") or candidate.get("data_uri") or ""
if (
str(candidate.get("media_type") or "image").strip().lower() == "image"
and self._media_reference_key(reference) in baseline_keys
):
excluded_baseline_images += 1
continue
candidates.append(candidate)
if excluded_baseline_images:
logger.warning(
"截图本地化拒绝发送前已存在的图片资源: "
f"count={excluded_baseline_images}"
)
remote_indexes = []
for idx, item in enumerate(candidates):
if str(item.get("kind") or "").strip().lower() != "url":
continue
url = str(item.get("url") or "").strip()
if url.startswith("http://") or url.startswith("https://"):
remote_indexes.append(idx)
if not remote_indexes:
return candidates
out_dir = Path("download_images")
out_dir.mkdir(exist_ok=True)
selector = image_config.get("selector", "img")
# S8:不再接受 image/svg+xml(与后台下载器一致),扩展名来自统一白名单
ext_map = dict(IMAGE_EXT_BY_MIME)
scoped_selector = str(selector or "img").strip() or "img"
# Arena 图片卡片在流结束后才会补上动画 class;此时严格的预设 selector
# 会短暂匹配不到已解码的 img。因此严格 selector 仅用于优先定位,通用 img
# 作为按 URL 精确匹配的回退候选,不能按位置直接使用。
candidate_selectors = [scoped_selector]
if scoped_selector != "img":
candidate_selectors.append("img")
reply_ele_entries = []
page_ele_entries = []
candidates_initially_scanned = False
baseline_token = str(image_config.get("request_baseline_token") or "")
baseline_property = str(image_config.get("request_baseline_property") or "")
exclude_existing_nodes = bool(image_config.get("request_baseline_exclude_existing_nodes"))
effective_stop_checker = getattr(self, "_should_stop_checker", lambda: False)
def _append_image_candidates(root, scope_name: str, target_list: List[Dict]) -> None:
if root is None:
return
seen_ids = {id(e["element"]) for e in target_list}
for candidate_selector in candidate_selectors:
try:
candidates = as_element(root).find_all(f"css:{candidate_selector}", timeout=0.5) or []
except Exception as exc:
logger.debug(
f"读取{scope_name}图片候选失败(忽略): "
f"selector={candidate_selector!r}, error={exc}"
)
continue
logger.debug(
f"图片定位:{scope_name}使用 '{candidate_selector}',"
f"找到 {len(candidates)} 个"
)
for ele in candidates:
if id(ele) in seen_ids:
continue
seen_ids.add(id(ele))
try:
baseline_probe = """
const propertyName = String(arguments[0] || '');
const baselineToken = String(arguments[1] || '');
const src = String(
this.currentSrc
|| this.getAttribute('src')
|| this.src
|| ''
).trim();
const baseline = propertyName ? this[propertyName] : null;
return {
src,
is_preexisting: Boolean(
propertyName
&& baselineToken
&& baseline
&& String(baseline.token || '') === baselineToken
),
baseline_reference: String(baseline && baseline.reference || ''),
};
"""
try:
entry_state = as_element(ele).run_js(
baseline_probe,
baseline_property,
baseline_token,
)
except TypeError:
# Some lightweight test and legacy element adapters only
# support a script without arguments. They cannot expose a
# baseline marker, so retain the strict URL match and treat
# the candidate as unmarked rather than failing extraction.
entry_state = {
"src": as_element(ele).run_js(
"""
return String(
this.currentSrc
|| this.getAttribute('src')
|| this.src
|| ''
).trim();
"""
),
}
if not isinstance(entry_state, dict):
entry_state = {}
img_src = str(
entry_state.get("src")
or ele.attr('src')
or ele.link
or ""
).strip()
except Exception as exc:
logger.debug(f"读取图片元素地址失败(忽略): {exc}")
img_src = ""
entry_state = {}
target_list.append({
"element": ele,
"src": img_src,
"is_preexisting": bool(entry_state.get("is_preexisting")),
"baseline_reference": str(entry_state.get("baseline_reference") or ""),
"used": False,
})
def _ensure_initial_candidates_scanned() -> None:
nonlocal candidates_initially_scanned
if not candidates_initially_scanned:
candidates_initially_scanned = True
_append_image_candidates(last_element, "当前回复", reply_ele_entries)
# Battle 模式下 last_element 可能指向另一侧回复。整页候选只用于后续 URL
# 精确匹配,不能作为位置兜底,否则会截到旧回复或另一侧的图片。
_append_image_candidates(tab, "整页", page_ele_entries)
cookies_dict, headers = build_image_download_request_context(tab)
is_image_gen = bool(
image_config.get("arena_image_generation")
or image_config.get("_arena_image_generation_active")
or image_config.get("require_localized_images")
)
prefetch_wait_seconds = float(
image_config.get("background_download_wait_seconds")
or image_config.get("download_wait_seconds")
or (3.5 if is_image_gen else 1.0)
)
try:
max_image_bytes = max(1, int(image_config.get("max_size_mb") or 10)) * 1024 * 1024
except (TypeError, ValueError):
max_image_bytes = 10 * 1024 * 1024
try:
screenshot_ready_wait_seconds = max(
0.0,
float(
image_config.get("screenshot_ready_wait_seconds")
if image_config.get("screenshot_ready_wait_seconds") is not None
else image_config.get("load_timeout_seconds", 5.0)
),
)
except (TypeError, ValueError):
screenshot_ready_wait_seconds = 5.0
try:
screenshot_ready_poll_seconds = max(
0.05,
float(image_config.get("screenshot_ready_poll_seconds") or 0.2),
)
except (TypeError, ValueError):
screenshot_ready_poll_seconds = 0.2
new_images = self._localize_images_with_background_cache(
candidates,
wait_seconds=max(0.0, prefetch_wait_seconds),
cookies=cookies_dict,
headers=headers,
)
def _claim_image_element(target_url: str):
for entry_list in (reply_ele_entries, page_ele_entries):
for entry in reversed(entry_list):
if entry["used"]:
continue
if not self._remote_image_urls_match(target_url, entry["src"]):
continue
if entry["is_preexisting"] and (
exclude_existing_nodes
or self._remote_image_urls_match(target_url, entry["baseline_reference"])
):
continue
entry["used"] = True
return entry["element"]
return None
def _wait_for_matching_image_element(target_url: str):
"""Wait for late-rendered images before giving up screenshot fallback."""
_ensure_initial_candidates_scanned()
matched = _claim_image_element(target_url)
if matched is not None:
return matched
deadline = time.monotonic() + screenshot_ready_wait_seconds
while time.monotonic() < deadline:
if effective_stop_checker():
break
_append_image_candidates(last_element, "当前回复(等待渲染)", reply_ele_entries)
matched = _claim_image_element(target_url)
if matched is not None:
return matched
_append_image_candidates(tab, "整页(等待渲染)", page_ele_entries)
matched = _claim_image_element(target_url)
if matched is not None:
return matched
time.sleep(min(screenshot_ready_poll_seconds, max(0.0, deadline - time.monotonic())))
return None
localized_count = 0
for target_index in reversed(remote_indexes):
if effective_stop_checker():
break
target_image = new_images[target_index]
target_url = normalize_remote_image_url(target_image.get("url"))
if not target_url:
continue
target_partition_key = get_image_download_partition(target_url, cookies_dict, headers)
background_result = background_image_downloader.get_download_result(
target_url,
wait=False,
partition_key=target_partition_key,
)
background_status = str((background_result or {}).get("status") or "").strip().lower()
if background_status in {"queued", "downloading"}:
if is_image_gen:
# 生图模式下,若后台下载仍在进行,给予同步阻塞等待(最多 prefetch_wait_seconds),确保图片落盘
background_result = background_image_downloader.get_download_result(
target_url,
wait=True,
timeout=prefetch_wait_seconds,
partition_key=target_partition_key,
)
background_status = str((background_result or {}).get("status") or "").strip().lower()
if background_status in {"queued", "downloading"}:
# The prefetch worker is deliberately independent from response
# rendering. Waiting here made the browser sit on a spinner for up
# to 60 seconds, then started a duplicate foreground download.
logger.debug(
f"图片后台下载仍在进行,保留远程地址并继续渲染: index={target_index}, "
f"status={background_status}"
)
continue
localized_item = self._localize_image_item_from_background_result(
target_image,
background_result,
)
if localized_item is not None:
new_images[target_index] = localized_item
localized_count += 1
continue
saved = False
saved_mime = None
base_name = f"{int(time.time())}_{uuid.uuid4().hex[:8]}"
ext = ".png"
filename = f"{base_name}{ext}"
out_path = out_dir / filename
response = None
download_started_at = time.monotonic()
try:
parsed_target_url = urlsplit(target_url)
target_label = f"{parsed_target_url.scheme}://{parsed_target_url.netloc}{parsed_target_url.path[:80]}"
logger.debug(f"尝试下载图片[{target_index}]: {target_label}")
response = get_public_remote_resource(
target_url,
cookies=cookies_dict,
headers=headers,
credential_origin_url=headers.get("Referer"),
timeout=(8, 15),
stream=True,
)
response_received_at = time.monotonic()
content_length_header = str(response.headers.get("Content-Length") or "").strip()
logger.debug(
f"图片下载响应[{target_index}]: status={response.status_code}, "
f"content_type={str(response.headers.get('Content-Type') or '').split(';', 1)[0].strip().lower() or 'unknown'}, "
f"content_length={content_length_header or 'unknown'}, "
f"response_wait={response_received_at - download_started_at:.3f}s"
)
if response.status_code == 200:
content_type = str(response.headers.get('Content-Type') or '').split(";", 1)[0].strip().lower()
if 'image' not in content_type or is_active_content_mime(content_type):
logger.debug(f"下载内容无效: type: {content_type or 'unknown'}")
else:
read_started_at = time.monotonic()
content = self._read_response_bytes_with_limit(response, max_image_bytes)
read_finished_at = time.monotonic()
if len(content) > 1000 and looks_like_active_content(content[:1024]):
# S8:Content-Type 声称图片但内容是 SVG/HTML 标记 → 不落盘,走截图回退
logger.debug(f"下载内容疑似活动文档,拒绝落盘: type: {content_type}")
elif len(content) > 1000:
ext = ext_map.get(content_type, ext)
filename = f"{base_name}{ext}"
out_path = out_dir / filename
out_path.write_bytes(content)
saved = True
saved_mime = content_type or None
background_image_downloader.register_downloaded_file(
target_url,
local_path=out_path,
accessible_url=f"/download_images/{filename}",
mime=content_type or None,
byte_size=len(content),
source="inline_download",
partition_key=target_partition_key,
)
logger.debug(
f"✅ 下载成功: {filename} ({len(content)} bytes), "
f"read={read_finished_at - read_started_at:.3f}s, "
f"write_and_register={time.monotonic() - read_finished_at:.3f}s, "
f"total={time.monotonic() - download_started_at:.3f}s"
)
else:
logger.debug(f"下载内容无效: {len(content)} bytes, type: {content_type}")
else:
logger.debug(f"下载失败: HTTP {response.status_code}")
except Exception as e:
logger.debug(
f"下载异常,将尝试截图: elapsed={time.monotonic() - download_started_at:.3f}s, "
f"error={str(e)[:100]}"
)
finally:
if response is not None:
try:
response.close()
except Exception:
pass
img_ele = _wait_for_matching_image_element(target_url) if not saved else None
if not saved and img_ele is not None:
image_ready, ready_state = self._wait_for_image_element_ready(
img_ele,
target_url,
timeout_seconds=screenshot_ready_wait_seconds,
poll_seconds=screenshot_ready_poll_seconds,
)
if not image_ready:
logger.warning(
f"图片[{target_index}] DOM 仍处于加载状态,跳过截图回退并保留远程链接:"
f"complete={bool(ready_state.get('complete'))}, "
f"natural={int(ready_state.get('natural_width') or 0)}x"
f"{int(ready_state.get('natural_height') or 0)}, "
f"source_matches={bool(ready_state.get('source_matches'))}"
)
else:
if out_path.suffix.lower() != ".png":
ext = ".png"
filename = f"{base_name}{ext}"
out_path = out_dir / filename
logger.debug(f"图片[{target_index}] 已确认加载完成,回退到截图方式")
expanded_for_screenshot = False
try:
expanded_for_screenshot = self._expand_image_element_for_screenshot(img_ele)
if expanded_for_screenshot:
logger.debug(f"图片[{target_index}] 已临时展开为自然尺寸后截图")
try:
img_ele.get_screenshot(str(out_path))
finally:
if expanded_for_screenshot:
self._restore_image_element_after_screenshot(img_ele)
still_ready, _ = self._wait_for_image_element_ready(
img_ele,
target_url,
timeout_seconds=0.0,
)
if not still_ready:
if out_path.exists():
out_path.unlink()
logger.warning(
f"图片[{target_index}] 截图期间 DOM 图片状态发生变化,已丢弃截图"
)
elif out_path.exists() and out_path.stat().st_size > 0:
saved = True
saved_mime = "image/png"
background_image_downloader.register_downloaded_file(
target_url,
local_path=out_path,
accessible_url=f"/download_images/{out_path.name}",
mime=saved_mime,
byte_size=int(out_path.stat().st_size),
source="screenshot_fallback",
partition_key=target_partition_key,
)
except Exception as e:
if out_path.exists():
try:
out_path.unlink()
except Exception:
pass
logger.warning(f"截图失败: {e}")
elif not saved:
logger.warning(
f"图片[{target_index}] 未找到与目标 URL 匹配的 DOM 节点,跳过截图回退"
)
if not saved:
logger.warning(f"图片[{target_index}] 保存失败:下载和截图均失败")
continue
local_url = f"/download_images/{out_path.name}"
new_images[target_index] = self._build_localized_image_item(
target_image,
out_path,
local_url,
mime=saved_mime,
byte_size=int(out_path.stat().st_size),
source="local_file",
)
localized_count += 1
if localized_count > 0:
logger.debug(f"✅ 图片本地化完成: {localized_count}/{len(remote_indexes)} 张")
return new_images
def _persist_data_uri_media_to_local(
self,
media_items: List[Dict],
*,
max_size_mb: int = 10,
) -> List[Dict]:
"""Persist extracted data-uri media so downstream Markdown can reuse the existing URL flow."""
if not media_items:
return media_items
save_dir = Path("download_images")
save_dir.mkdir(exist_ok=True)
try:
max_bytes = max(1, int(max_size_mb or 10)) * 1024 * 1024
except (TypeError, ValueError):
max_bytes = 10 * 1024 * 1024
ext_map = {**IMAGE_EXT_BY_MIME, **AUDIO_EXT_BY_MIME, **VIDEO_EXT_BY_MIME}
result = []
for item in media_items:
if item.get("kind") != "data_uri":
result.append(item)
continue
data_uri = str(item.get("data_uri") or "").strip()
if not data_uri.startswith("data:"):
result.append(item)
continue
try:
header, b64_data = data_uri.split(",", 1)
mime = header.split(";", 1)[0].split(":", 1)[1].lower()
if is_active_content_mime(mime):
# S8:data:image/svg+xml / text/html 等不落盘到同源目录
raise ValueError(f"unsafe_data_uri_mime:{mime}")
ext = ext_map.get(mime, ".png")
compact_b64 = "".join(str(b64_data or "").split())
padding = compact_b64.count("=")
estimated_size = max(0, (len(compact_b64) * 3) // 4 - padding)
if estimated_size > max_bytes:
raise ValueError(f"data_uri_too_large:{estimated_size}")
media_bytes = base64.b64decode(compact_b64)
if len(media_bytes) > max_bytes:
raise ValueError(f"data_uri_too_large:{len(media_bytes)}")
if looks_like_active_content(media_bytes[:1024]):
raise ValueError("unsafe_data_uri_content")
except (ValueError, IndexError, binascii.Error) as e:
logger.warning(f"data uri 解析失败,保留原媒体数据: {e}")
result.append(item)
continue
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
filename = f"{timestamp}_{uuid.uuid4().hex[:8]}{ext}"
filepath = save_dir / filename
try:
filepath.write_bytes(media_bytes)
if str(item.get("media_type") or "").strip().lower() == "audio":
logger.debug(f"data-uri 音频已落盘,准备追加尾静音: {filepath.name}")
self._append_audio_tail_silence(filepath, duration_seconds=0.3)
except Exception as e:
logger.warning(f"data uri 保存失败,保留原媒体数据: {e}")
result.append(item)
continue
new_item = dict(item)
new_item["kind"] = "url"
media_type = str(item.get("media_type") or "").strip().lower()
if media_type in {"audio", "video"}:
new_item["url"] = f"/media/{filename}"
else:
new_item["url"] = f"/download_images/{filename}"
new_item["data_uri"] = None
new_item["mime"] = mime
new_item["byte_size"] = len(media_bytes)
new_item["source"] = "local_file"
new_item["local_path"] = str(filepath)
result.append(new_item)
return result
def _dedupe_media_items(self, media_items: List[Dict]) -> List[Dict]:
result = []
seen = set()
for item in media_items or []:
media_type = str(item.get("media_type") or "")
ref = str(item.get("url") or item.get("data_uri") or "")
key = (media_type, ref)
if key in seen:
continue
seen.add(key)
result.append(item)
return result
def _retry_pending_media_from_response_text(
self,
session: TabSession,
full_content: str,
preset_name: Optional[str] = None,
stop_checker: Optional[Callable[[], bool]] = None,
) -> List[Dict]:
hint_text = str(full_content or "").strip()
if not hint_text:
return []
try:
domain = str(getattr(session, "current_domain", "") or "").strip()
if not domain:
tab = getattr(session, "tab", None)
current_url = str(getattr(tab, "url", "") or "").strip()
domain = extract_remote_site_domain(current_url)
if not domain:
return []
config_engine = self._get_config_engine()
effective_preset_name = preset_name if preset_name is not None else session.preset_name
tab = getattr(session, "tab", None)
if tab is None:
return []
site_config = config_engine.get_site_config(
domain,
html_content=lambda: getattr(tab, "html", ""),
preset_name=effective_preset_name,
)
if not site_config:
return []
image_config = site_config.get("image_extraction", {}) or {}
modalities = image_config.get("modalities") or {}
image_extraction_enabled = bool(image_config.get("enabled", False)) or any(
is_modality_enabled(modalities, key) for key in ("image", "audio", "video")
)
if not image_extraction_enabled:
return []
pending_hit = any(
self._is_pending_media_text(hint_text, image_config, media_type)
for media_type in ("audio", "video")
)
if not pending_hit:
return []
selectors = site_config.get("selectors", {}) or {}
result_selector = str(selectors.get("result_container", "") or "").strip()
if not result_selector:
return []
extractor = config_engine.get_site_extractor(domain, preset_name=effective_preset_name)
logger.debug(f"[{session.id}] 非流式响应命中占位媒体文本,触发二次提取等待")
return self._extract_media_after_stream(
tab=tab,
extractor=extractor,
image_config=image_config,
result_selector=result_selector,
message_wrapper_selector=str(selectors.get("message_wrapper", "") or "").strip(),
stop_checker=stop_checker,
response_text_hint=hint_text,
)
except Exception as e:
logger.warning(f"[{session.id}] 非流式二次媒体提取失败(已忽略): {e}")
return []
def _download_url_images(self, images: List[Dict], tab=None) -> List[Dict]:
"""
在浏览器内通过 Canvas 压缩图片,保存到本地并返回可访问 URL
流程:
1. 浏览器 Canvas 压缩 → base64
2. 后端解码 → 保存到 download_images/
3. 返回 /download_images/xxx.jpg URL
"""
import base64
import uuid
from pathlib import Path
from datetime import datetime
result = []
# 确保目录存在
save_dir = Path("download_images")
save_dir.mkdir(exist_ok=True)
canvas_image_max_size = AppConfig.get_canvas_image_max_size()
max_canvas_image_bytes = max(1, canvas_image_max_size * canvas_image_max_size * 4)
for img in images:
if img.get('kind') != 'url':
result.append(img)
continue
url = img.get('url')
if not url:
result.append(img)
continue
if not tab:
result.append(img)
continue
try:
# 🔑 在浏览器中用 Canvas 加载并压缩图片
js_code = """
(async function(imageUrl, configuredMaxSize) {
return new Promise((resolve) => {
const img = new Image();
img.crossOrigin = 'anonymous';
img.onload = function() {
try {
// 限制最大尺寸
const MAX_SIZE = Math.max(1, Math.floor(Number(configuredMaxSize) || 1024));
let width = img.naturalWidth;
let height = img.naturalHeight;
if (width > MAX_SIZE || height > MAX_SIZE) {
if (width > height) {
height = Math.round(height * MAX_SIZE / width);
width = MAX_SIZE;
} else {
width = Math.round(width * MAX_SIZE / height);
height = MAX_SIZE;
}
}
const canvas = document.createElement('canvas');
canvas.width = width;
canvas.height = height;
const ctx = canvas.getContext('2d');
ctx.drawImage(img, 0, 0, width, height);
// 转为 JPEG
const dataUri = canvas.toDataURL('image/jpeg', 0.85);
resolve({
success: true,
dataUri: dataUri,
width: width,
height: height
});
} catch (e) {
resolve({ success: false, error: 'Canvas: ' + e.message });
}
};
img.onerror = function() {
resolve({ success: false, error: 'Load failed' });
};
setTimeout(() => resolve({ success: false, error: 'Timeout' }), 15000);
img.src = imageUrl;
});
})(arguments[0], arguments[1]);
"""
# ===== PROBE: 验证 run_js 是否等待 Promise,并检查图片/Fetch 可用性 =====
probe_js = """
(function(u){
try {
// 1) 最小同步返回测试
const sync_ok = { ok: true, type: typeof u, head: String(u).slice(0, 40) };
// 2) Promise 返回测试(不返回大对象)
const promise_test = Promise.resolve({ promise_ok: true });
// 3) 图片加载测试(不画 canvas,不导 dataUri,避免大返回)
const img_test = new Promise((resolve) => {
const img = new Image();
let done = false;
img.onload = () => {
if (done) return;
done = true;
resolve({ img_onload: true, w: img.naturalWidth, h: img.naturalHeight });
};
img.onerror = () => {
if (done) return;
done = true;
resolve({ img_onerror: true });
};
setTimeout(() => {
if (done) return;
done = true;
resolve({ img_timeout: true });
}, 6000);
img.src = u;
});
// 4) fetch 测试(只返回 status,不读 body)
const fetch_test = (async () => {
try {
const r = await fetch(u, { method: 'GET' });
return { fetch_ok: true, status: r.status, redirected: r.redirected };
} catch (e) {
return { fetch_error: String(e).slice(0, 120) };
}
})();
// 关键:返回一个对象,包含同步字段 + Promise 字段
// 如果 run_js 不等待 Promise,你只能拿到一个“未解析”的东西或 None
return Promise.all([promise_test, img_test, fetch_test]).then(all => {
return {
sync: sync_ok,
promise: all[0],
img: all[1],
fetch: all[2]
};
});
} catch(e) {
return { probe_exception: String(e).slice(0, 160) };
}
})(arguments[0]);
"""
probe_result = driver_for_tab(tab).run_js(probe_js, url)
logger.info(f"[PROBE_JS] probe_result_type={type(probe_result).__name__}, value={str(probe_result)[:500]}")
download_result = driver_for_tab(tab).run_js(js_code, url, canvas_image_max_size)
logger.info(f"[PROBE_JS] canvas_result_type={type(download_result).__name__}, value={str(download_result)[:300]}")
if download_result and download_result.get('success'):
data_uri = download_result['dataUri']
# 解析 base64
# 格式: data:image/jpeg;base64,/9j/4AAQSkZJRg...
if ',' in data_uri:
header, b64_data = data_uri.split(',', 1)
mime = 'image/jpeg'
if 'png' in header:
mime = 'image/png'
ext = '.png'
else:
ext = '.jpg'
# 解码并保存
compact_b64 = "".join(str(b64_data or "").split())
estimated_size = self._estimate_base64_decoded_size(compact_b64)
if estimated_size > max_canvas_image_bytes:
raise ValueError(f"canvas_image_too_large:{estimated_size}")
image_bytes = base64.b64decode(compact_b64)
if len(image_bytes) > max_canvas_image_bytes:
raise ValueError(f"canvas_image_too_large:{len(image_bytes)}")
# 生成唯一文件名
timestamp = datetime.now().strftime('%Y%m%d_%H%M%S')
unique_id = uuid.uuid4().hex[:8]
filename = f"{timestamp}_{unique_id}{ext}"
filepath = save_dir / filename
# 写入文件
with open(filepath, 'wb') as f:
f.write(image_bytes)
# 构建可访问的 URL
accessible_url = f"/download_images/{filename}"
new_img = img.copy()
new_img['kind'] = 'url'
new_img['url'] = accessible_url
new_img['data_uri'] = None
new_img['mime'] = mime
new_img['width'] = download_result['width']
new_img['height'] = download_result['height']
new_img['byte_size'] = len(image_bytes)
new_img['source'] = 'local_file'
new_img['local_path'] = str(filepath)
result.append(new_img)
logger.info(f"✅ 图片已保存: {filename} ({len(image_bytes)} bytes)")
continue
error_msg = download_result.get('error', 'Unknown') if download_result else 'No result'
logger.warning(f"⚠️ 图片处理失败: {error_msg}")
except Exception as e:
logger.warning(f"⚠️ 图片保存异常: {str(e)[:100]}")
# 失败时保留原 URL
result.append(img)
return result