3228 lines
140 KiB
Python
3228 lines
140 KiB
Python
# app/core/browser/media.py
|
||
|
||
import json
|
||
import time
|
||
import os
|
||
import shutil
|
||
import subprocess
|
||
import base64
|
||
import uuid
|
||
import binascii
|
||
from pathlib import Path
|
||
from datetime import datetime
|
||
from urllib.parse import urlparse, urlsplit
|
||
from typing import Optional, List, Dict, Any, Callable, TYPE_CHECKING
|
||
import requests
|
||
|
||
from app.core.config import logger, AppConfig, BrowserConstants
|
||
from app.models.schemas import (
|
||
get_enabled_modalities,
|
||
get_modality_policy,
|
||
get_modality_run_policy,
|
||
is_modality_enabled,
|
||
)
|
||
from app.core.background_image_downloader import (
|
||
background_image_downloader,
|
||
build_image_download_partition,
|
||
build_image_download_request_context,
|
||
get_image_download_partition,
|
||
normalize_remote_image_url,
|
||
)
|
||
from app.utils.remote_resource import get_public_remote_resource
|
||
from app.utils.media_safety import (
|
||
IMAGE_EXT_BY_MIME,
|
||
AUDIO_EXT_BY_MIME,
|
||
VIDEO_EXT_BY_MIME,
|
||
choose_media_extension,
|
||
is_active_content_mime,
|
||
looks_like_active_content,
|
||
)
|
||
from app.utils.site_url import extract_remote_site_domain
|
||
from app.utils.image_validation import filter_reference_images
|
||
from app.core.tab_pool import TabSession
|
||
from app.core.driver import as_element, driver_for_tab
|
||
|
||
if TYPE_CHECKING:
|
||
from .main import BrowserCore
|
||
|
||
|
||
class BrowserMediaMixin:
|
||
"""媒体文件提取、音频后处理(FFmpeg/FFprobe)、本地落盘、视觉/截图本地化混入类"""
|
||
|
||
_MEDIA_DOM_BASELINE_PROPERTY = "__universalProxyMediaBaseline"
|
||
|
||
def _capture_media_dom_baseline(self, tab, image_config: Dict[str, Any]) -> Optional[Dict[str, Any]]:
|
||
"""Mark images that existed immediately before the request is submitted."""
|
||
image_config = image_config or {}
|
||
image_enabled = bool(image_config.get("enabled")) or is_modality_enabled(
|
||
image_config.get("modalities") or {},
|
||
"image",
|
||
)
|
||
if tab is None or not image_enabled:
|
||
return None
|
||
|
||
selector = str(image_config.get("selector") or "img").strip() or "img"
|
||
token = uuid.uuid4().hex
|
||
try:
|
||
result = driver_for_tab(tab).run_js(
|
||
"""
|
||
const selector = String(arguments[0] || "img");
|
||
const token = String(arguments[1] || "");
|
||
const propertyName = String(arguments[2] || "");
|
||
const pickReference = (node) => {
|
||
try {
|
||
return String(
|
||
node.currentSrc
|
||
|| node.src
|
||
|| node.getAttribute("src")
|
||
|| ""
|
||
).trim();
|
||
} catch {
|
||
return "";
|
||
}
|
||
};
|
||
|
||
let nodes = [];
|
||
try {
|
||
// The configured selector may only match generated cards. Capture
|
||
// every image so upload previews and history cannot later become
|
||
// screenshot fallback candidates for this request.
|
||
nodes = Array.from(document.querySelectorAll("img"));
|
||
} catch (error) {
|
||
return { ok: false, error: String(error).slice(0, 160) };
|
||
}
|
||
|
||
const urls = [];
|
||
let markedCount = 0;
|
||
for (const node of nodes) {
|
||
const reference = pickReference(node);
|
||
try {
|
||
Object.defineProperty(node, propertyName, {
|
||
configurable: true,
|
||
writable: true,
|
||
value: { token, reference },
|
||
});
|
||
markedCount += 1;
|
||
} catch {
|
||
try {
|
||
node[propertyName] = { token, reference };
|
||
markedCount += 1;
|
||
} catch {}
|
||
}
|
||
if (reference && reference.length <= 8192) urls.push(reference);
|
||
}
|
||
|
||
return {
|
||
ok: true,
|
||
node_count: nodes.length,
|
||
marked_count: markedCount,
|
||
url_count: new Set(urls).size,
|
||
references: Array.from(new Set(urls)).slice(-256),
|
||
page_url: String(location.href || ""),
|
||
};
|
||
""",
|
||
selector,
|
||
token,
|
||
self._MEDIA_DOM_BASELINE_PROPERTY,
|
||
) or {}
|
||
except Exception as exc:
|
||
logger.warning(f"发送前图片基线采集失败,将使用常规媒体提取: {exc}")
|
||
return None
|
||
|
||
if not bool(result.get("ok")):
|
||
logger.warning(
|
||
"发送前图片基线采集失败,将使用常规媒体提取: "
|
||
f"{result.get('error') or 'unknown_error'}"
|
||
)
|
||
return None
|
||
|
||
baseline = {
|
||
"token": token,
|
||
"property": self._MEDIA_DOM_BASELINE_PROPERTY,
|
||
"selector": "img",
|
||
"node_count": int(result.get("node_count") or 0),
|
||
"marked_count": int(result.get("marked_count") or 0),
|
||
"url_count": int(result.get("url_count") or 0),
|
||
"references": [
|
||
str(reference or "").strip()
|
||
for reference in (result.get("references") or [])
|
||
if str(reference or "").strip()
|
||
],
|
||
"page_url": str(result.get("page_url") or ""),
|
||
}
|
||
logger.debug(
|
||
"已记录发送前图片基线: "
|
||
f"nodes={baseline['node_count']}, marked={baseline['marked_count']}, "
|
||
f"urls={baseline['url_count']}"
|
||
)
|
||
return baseline
|
||
|
||
@staticmethod
|
||
def _media_reference_key(reference: Any) -> str:
|
||
text = str(reference or "").strip()
|
||
if not text:
|
||
return ""
|
||
if text.lower().startswith(("data:", "blob:")):
|
||
return text
|
||
|
||
normalized = normalize_remote_image_url(text)
|
||
if not normalized:
|
||
return text
|
||
try:
|
||
parsed = urlparse(normalized)
|
||
except Exception:
|
||
return normalized
|
||
if parsed.scheme.lower() not in {"http", "https"}:
|
||
return normalized
|
||
host = str(parsed.hostname or "").strip().lower()
|
||
signed_query = "x-amz-signature=" in str(parsed.query or "").lower()
|
||
if host.endswith(".r2.cloudflarestorage.com") or signed_query:
|
||
return f"{parsed.scheme.lower()}://{parsed.netloc.lower()}{parsed.path}"
|
||
return normalized
|
||
|
||
@classmethod
|
||
def _filter_media_items_against_dom_baseline(
|
||
cls,
|
||
media_items: List[Dict],
|
||
media_dom_baseline: Optional[Dict[str, Any]],
|
||
) -> List[Dict]:
|
||
baseline_keys = {
|
||
cls._media_reference_key(reference)
|
||
for reference in ((media_dom_baseline or {}).get("references") or [])
|
||
}
|
||
baseline_keys.discard("")
|
||
if not baseline_keys:
|
||
return list(media_items or [])
|
||
|
||
filtered_items = []
|
||
removed_images = 0
|
||
for item in media_items or []:
|
||
if not isinstance(item, dict):
|
||
continue
|
||
media_type = str(item.get("media_type") or "image").strip().lower() or "image"
|
||
reference = item.get("url") or item.get("data_uri") or ""
|
||
if media_type == "image" and cls._media_reference_key(reference) in baseline_keys:
|
||
removed_images += 1
|
||
continue
|
||
filtered_items.append(item)
|
||
|
||
if removed_images:
|
||
logger.debug(
|
||
"发送前已存在的图片 URL 已从本轮结果剔除: "
|
||
f"count={removed_images}"
|
||
)
|
||
return filtered_items
|
||
|
||
@staticmethod
|
||
def _candidate_has_fresh_dom_image(
|
||
candidate,
|
||
image_config: Dict[str, Any],
|
||
media_dom_baseline: Optional[Dict[str, Any]],
|
||
) -> bool:
|
||
if candidate is None or not media_dom_baseline:
|
||
return False
|
||
|
||
selector = str((image_config or {}).get("selector") or "img").strip() or "img"
|
||
token = str(media_dom_baseline.get("token") or "")
|
||
property_name = str(media_dom_baseline.get("property") or "")
|
||
exclude_existing_nodes = bool(
|
||
(image_config or {}).get("request_baseline_exclude_existing_nodes")
|
||
)
|
||
if not token or not property_name:
|
||
return False
|
||
|
||
try:
|
||
return bool(
|
||
as_element(candidate).run_js(
|
||
"""
|
||
const selector = String(arguments[0] || "img");
|
||
const baselineToken = String(arguments[1] || "");
|
||
const propertyName = String(arguments[2] || "");
|
||
const excludeExistingNodes = Boolean(arguments[3]);
|
||
const nodes = [];
|
||
try {
|
||
if (this instanceof Element && this.matches(selector)) nodes.push(this);
|
||
} catch {}
|
||
try {
|
||
if (this.querySelectorAll) nodes.push(...this.querySelectorAll(selector));
|
||
} catch {}
|
||
|
||
return nodes.some((node) => {
|
||
const baseline = node[propertyName];
|
||
if (!baseline || String(baseline.token || "") !== baselineToken) return true;
|
||
if (excludeExistingNodes) return false;
|
||
let currentReference = "";
|
||
try {
|
||
currentReference = String(
|
||
node.currentSrc
|
||
|| node.src
|
||
|| node.getAttribute("src")
|
||
|| ""
|
||
).trim();
|
||
} catch {}
|
||
return currentReference !== String(baseline.reference || "");
|
||
});
|
||
""",
|
||
selector,
|
||
token,
|
||
property_name,
|
||
exclude_existing_nodes,
|
||
)
|
||
)
|
||
except Exception as exc:
|
||
logger.debug(f"本轮新增图片探测失败(按未命中处理): {exc}")
|
||
return False
|
||
|
||
@staticmethod
|
||
def _media_modalities(image_config: Dict[str, Any]) -> Dict[str, Any]:
|
||
return dict((image_config or {}).get("modalities") or {})
|
||
|
||
@staticmethod
|
||
def _media_enabled_types(image_config: Dict[str, Any]) -> List[str]:
|
||
return sorted(get_enabled_modalities((image_config or {}).get("modalities") or {}))
|
||
|
||
@staticmethod
|
||
def _media_policy(image_config: Dict[str, Any], media_type: str) -> Dict[str, Any]:
|
||
return dict(get_modality_policy((image_config or {}).get("modalities") or {}, media_type))
|
||
|
||
@staticmethod
|
||
def _media_run_policy(image_config: Dict[str, Any], media_type: str) -> str:
|
||
return get_modality_run_policy((image_config or {}).get("modalities") or {}, media_type)
|
||
|
||
@staticmethod
|
||
def _media_policy_allows_signal_wait(image_config: Dict[str, Any], media_type: str) -> bool:
|
||
return BrowserMediaMixin._media_run_policy(image_config, media_type) in {
|
||
"on_signal",
|
||
"probe_if_trigger_found",
|
||
"always_probe",
|
||
}
|
||
|
||
@staticmethod
|
||
def _media_policy_allows_audio_probe(
|
||
image_config: Dict[str, Any],
|
||
*,
|
||
signal_seen: bool = False,
|
||
) -> bool:
|
||
policy = BrowserMediaMixin._media_run_policy(image_config, "audio")
|
||
if policy == "always_probe":
|
||
return True
|
||
if policy == "probe_if_trigger_found":
|
||
return True
|
||
if policy == "on_signal" and signal_seen:
|
||
return True
|
||
return False
|
||
|
||
@staticmethod
|
||
def _media_quick_probe_timeout(image_config: Dict[str, Any], default: float = 1.0) -> float:
|
||
values = []
|
||
for media_type in BrowserMediaMixin._media_enabled_types(image_config):
|
||
policy = BrowserMediaMixin._media_policy(image_config, media_type)
|
||
try:
|
||
values.append(float(policy.get("quick_probe_timeout_seconds") or default))
|
||
except (TypeError, ValueError):
|
||
pass
|
||
if not values:
|
||
return default
|
||
return max(0.1, min(max(values), 10.0))
|
||
|
||
@staticmethod
|
||
def _media_late_wait_timeout(
|
||
image_config: Dict[str, Any],
|
||
media_type: str,
|
||
fallback: float,
|
||
) -> float:
|
||
policy = BrowserMediaMixin._media_policy(image_config, media_type)
|
||
try:
|
||
return max(0.2, min(float(policy.get("late_wait_timeout_seconds") or fallback), 300.0))
|
||
except (TypeError, ValueError):
|
||
return fallback
|
||
|
||
@staticmethod
|
||
def _media_blind_wait_timeout(
|
||
image_config: Dict[str, Any],
|
||
media_type: str,
|
||
fallback: float,
|
||
) -> float:
|
||
policy = BrowserMediaMixin._media_policy(image_config, media_type)
|
||
raw_value = policy.get("blind_wait_timeout_seconds")
|
||
if raw_value is None:
|
||
raw_value = image_config.get("blind_wait_timeout_seconds")
|
||
try:
|
||
return max(0.0, min(float(raw_value if raw_value is not None else fallback), 300.0))
|
||
except (TypeError, ValueError):
|
||
return fallback
|
||
|
||
@staticmethod
|
||
def _media_audio_capture_timeout(image_config: Dict[str, Any], fallback: float) -> float:
|
||
policy = BrowserMediaMixin._media_policy(image_config, "audio")
|
||
try:
|
||
return max(0.2, min(float(policy.get("capture_timeout_seconds") or fallback), 180.0))
|
||
except (TypeError, ValueError):
|
||
return fallback
|
||
|
||
@staticmethod
|
||
def _build_localized_image_item(
|
||
original_item: Dict[str, Any],
|
||
local_path: Path,
|
||
accessible_url: str,
|
||
*,
|
||
mime: Optional[str] = None,
|
||
byte_size: Optional[int] = None,
|
||
source: str = "local_file",
|
||
) -> Dict[str, Any]:
|
||
new_item = dict(original_item or {})
|
||
new_item["kind"] = "url"
|
||
new_item["url"] = accessible_url
|
||
new_item["data_uri"] = None
|
||
new_item["source"] = source
|
||
new_item["local_path"] = str(local_path)
|
||
if mime is not None:
|
||
new_item["mime"] = mime
|
||
if byte_size is None:
|
||
try:
|
||
byte_size = int(local_path.stat().st_size)
|
||
except Exception:
|
||
byte_size = None
|
||
if byte_size is not None:
|
||
new_item["byte_size"] = byte_size
|
||
return new_item
|
||
|
||
@staticmethod
|
||
def _read_response_bytes_with_limit(response, max_bytes: int) -> bytes:
|
||
max_bytes = max(1, int(max_bytes or 1))
|
||
content_length = str(response.headers.get("Content-Length") or "").strip()
|
||
if content_length:
|
||
try:
|
||
declared_size = int(content_length)
|
||
if declared_size > max_bytes:
|
||
raise ValueError(f"image_too_large:{declared_size}")
|
||
except ValueError as exc:
|
||
if str(exc).startswith("image_too_large:"):
|
||
raise
|
||
|
||
chunks = []
|
||
total = 0
|
||
for chunk in response.iter_content(chunk_size=1024 * 64):
|
||
if not chunk:
|
||
continue
|
||
total += len(chunk)
|
||
if total > max_bytes:
|
||
raise ValueError(f"image_too_large:{total}")
|
||
chunks.append(chunk)
|
||
return b"".join(chunks)
|
||
|
||
@staticmethod
|
||
def _estimate_base64_decoded_size(base64_data: str) -> int:
|
||
compact_base64 = "".join(str(base64_data or "").split())
|
||
if not compact_base64:
|
||
return 0
|
||
padding = compact_base64.count("=")
|
||
return max(0, (len(compact_base64) * 3) // 4 - padding)
|
||
|
||
def _localize_image_item_from_background_result(
|
||
self,
|
||
original_item: Dict[str, Any],
|
||
background_result: Optional[Dict[str, Any]],
|
||
) -> Optional[Dict[str, Any]]:
|
||
if not isinstance(background_result, dict):
|
||
return None
|
||
if str(background_result.get("status") or "") != "done":
|
||
return None
|
||
|
||
local_path_text = str(background_result.get("local_path") or "").strip()
|
||
accessible_url = str(background_result.get("accessible_url") or "").strip()
|
||
if not local_path_text or not accessible_url:
|
||
return None
|
||
|
||
local_path = Path(local_path_text)
|
||
if not local_path.exists():
|
||
return None
|
||
|
||
return self._build_localized_image_item(
|
||
original_item,
|
||
local_path,
|
||
accessible_url,
|
||
mime=str(background_result.get("mime") or "").strip() or None,
|
||
byte_size=background_result.get("byte_size"),
|
||
source=str(background_result.get("source") or "background_download"),
|
||
)
|
||
|
||
def _prefetch_remote_image_urls(
|
||
self,
|
||
tab,
|
||
image_urls: List[str],
|
||
max_size_mb: int = 10,
|
||
) -> int:
|
||
normalized_urls = []
|
||
seen = set()
|
||
for raw_url in image_urls or []:
|
||
normalized = normalize_remote_image_url(raw_url)
|
||
if not normalized or normalized in seen:
|
||
continue
|
||
seen.add(normalized)
|
||
normalized_urls.append(normalized)
|
||
|
||
if not normalized_urls or tab is None:
|
||
return 0
|
||
|
||
cookies_dict, headers = build_image_download_request_context(tab)
|
||
started = 0
|
||
for target_url in normalized_urls:
|
||
partition_key = get_image_download_partition(target_url, cookies_dict, headers)
|
||
result = background_image_downloader.start_download(
|
||
target_url,
|
||
cookies=cookies_dict,
|
||
headers=headers,
|
||
partition_key=partition_key,
|
||
max_bytes=max(1, int(max_size_mb or 10)) * 1024 * 1024,
|
||
)
|
||
if result:
|
||
started += 1
|
||
if started:
|
||
logger.debug(f"已提交后台图片预下载任务: {started} 个")
|
||
return started
|
||
|
||
def _localize_images_with_background_cache(
|
||
self,
|
||
images: List[Dict],
|
||
*,
|
||
wait_seconds: float = 0.0,
|
||
cookies: Any = None,
|
||
headers: Optional[Dict[str, str]] = None,
|
||
partition_key: Optional[str] = None,
|
||
) -> List[Dict]:
|
||
if not images:
|
||
return images
|
||
|
||
localized = list(images)
|
||
hit_count = 0
|
||
for index, item in enumerate(images):
|
||
if str(item.get("kind") or "").strip().lower() != "url":
|
||
continue
|
||
target_url = normalize_remote_image_url(item.get("url"))
|
||
if not target_url:
|
||
continue
|
||
|
||
current_partition = (
|
||
str(partition_key).strip()
|
||
if partition_key is not None
|
||
else get_image_download_partition(target_url, cookies, headers)
|
||
)
|
||
result = background_image_downloader.get_download_result(
|
||
target_url,
|
||
wait=wait_seconds > 0,
|
||
timeout=wait_seconds if wait_seconds > 0 else None,
|
||
partition_key=current_partition,
|
||
)
|
||
new_item = self._localize_image_item_from_background_result(item, result)
|
||
if new_item is None:
|
||
continue
|
||
localized[index] = new_item
|
||
hit_count += 1
|
||
|
||
if hit_count:
|
||
logger.debug(f"命中后台图片缓存: {hit_count} 张")
|
||
return localized
|
||
|
||
@staticmethod
|
||
def _append_audio_tail_silence(filepath, duration_seconds: float = 0.3):
|
||
"""为音频尾部追加一小段静音,避免播放时戛然而止。"""
|
||
from pathlib import Path
|
||
|
||
try:
|
||
target = Path(filepath)
|
||
except Exception:
|
||
return filepath
|
||
|
||
if duration_seconds <= 0 or not target.exists() or not target.is_file():
|
||
return filepath
|
||
|
||
ffmpeg_path = shutil.which("ffmpeg")
|
||
if not ffmpeg_path:
|
||
logger.debug(f"音频尾静音跳过:未找到 ffmpeg,file={target}")
|
||
return filepath
|
||
|
||
ffprobe_path = shutil.which("ffprobe")
|
||
|
||
def _probe_duration(path_obj: Path) -> float:
|
||
if not ffprobe_path:
|
||
return -1.0
|
||
cmd = [
|
||
ffprobe_path,
|
||
"-v",
|
||
"error",
|
||
"-show_entries",
|
||
"format=duration",
|
||
"-of",
|
||
"default=noprint_wrappers=1:nokey=1",
|
||
str(path_obj),
|
||
]
|
||
try:
|
||
completed = subprocess.run(
|
||
cmd,
|
||
stdout=subprocess.PIPE,
|
||
stderr=subprocess.PIPE,
|
||
text=True,
|
||
timeout=15,
|
||
check=False,
|
||
)
|
||
if completed.returncode != 0:
|
||
return -1.0
|
||
return float(str(completed.stdout or "").strip() or -1.0)
|
||
except Exception:
|
||
return -1.0
|
||
|
||
duration_text = f"{float(duration_seconds):.3f}".rstrip("0").rstrip(".")
|
||
temp_path = target.with_name(f"{target.stem}_tail{target.suffix}")
|
||
suffix = target.suffix.lower()
|
||
codec_args: list[str] = []
|
||
if suffix in {".ogg", ".oga"}:
|
||
codec_args = ["-c:a", "libvorbis", "-q:a", "5"]
|
||
elif suffix == ".mp3":
|
||
codec_args = ["-c:a", "libmp3lame", "-b:a", "128k"]
|
||
elif suffix in {".m4a", ".mp4"}:
|
||
codec_args = ["-c:a", "aac", "-b:a", "128k"]
|
||
elif suffix == ".wav":
|
||
codec_args = ["-c:a", "pcm_s16le"]
|
||
elif suffix == ".webm":
|
||
codec_args = ["-c:a", "libopus", "-b:a", "96k"]
|
||
|
||
original_duration = _probe_duration(target)
|
||
logger.debug(
|
||
f"音频尾静音开始: file={target.name}, ext={suffix or '<none>'}, "
|
||
f"append={duration_text}s, duration_before={original_duration:.3f}"
|
||
)
|
||
|
||
if original_duration <= 0:
|
||
logger.debug(
|
||
f"音频尾静音跳过:无法可靠探测原始时长,保留原文件。file={target.name}"
|
||
)
|
||
return filepath
|
||
|
||
if suffix == ".webm":
|
||
logger.debug(
|
||
f"音频尾静音跳过:暂不改写 webm,保留原文件。file={target.name}"
|
||
)
|
||
return filepath
|
||
|
||
command_variants = [[
|
||
ffmpeg_path,
|
||
"-y",
|
||
"-hide_banner",
|
||
"-loglevel",
|
||
"error",
|
||
"-i",
|
||
str(target),
|
||
"-af",
|
||
f"apad=pad_dur={duration_text}",
|
||
"-t",
|
||
f"{max(0.0, original_duration) + float(duration_seconds):.3f}" if original_duration > 0 else duration_text,
|
||
*codec_args,
|
||
str(temp_path),
|
||
]]
|
||
|
||
for cmd in command_variants:
|
||
try:
|
||
completed = subprocess.run(
|
||
cmd,
|
||
stdout=subprocess.DEVNULL,
|
||
stderr=subprocess.PIPE,
|
||
text=True,
|
||
timeout=30,
|
||
check=False,
|
||
)
|
||
except Exception as exc:
|
||
logger.debug(f"音频尾静音追加失败(ffmpeg 调用异常): {exc}")
|
||
continue
|
||
|
||
if completed.returncode == 0 and temp_path.exists() and temp_path.stat().st_size > 0:
|
||
try:
|
||
new_duration = _probe_duration(temp_path)
|
||
logger.debug(
|
||
f"音频尾静音生成成功: file={target.name}, "
|
||
f"duration_after={new_duration:.3f}, size={temp_path.stat().st_size}"
|
||
)
|
||
temp_path.replace(target)
|
||
return str(target)
|
||
except Exception as exc:
|
||
logger.debug(f"音频尾静音替换失败: {exc}")
|
||
try:
|
||
if temp_path.exists():
|
||
temp_path.unlink()
|
||
except Exception:
|
||
pass
|
||
return filepath
|
||
|
||
try:
|
||
if temp_path.exists():
|
||
temp_path.unlink()
|
||
except Exception:
|
||
pass
|
||
logger.debug(f"音频尾静音未生效,保留原文件: {target.name}")
|
||
return filepath
|
||
|
||
def _get_pending_detection_markers(
|
||
self,
|
||
image_config: Dict,
|
||
media_type: str,
|
||
marker_key: str,
|
||
) -> List[str]:
|
||
default_markers = {
|
||
"audio": {
|
||
"text_contains": [
|
||
"generated_music_content/",
|
||
"googleusercontent.com/generated_music_content/",
|
||
],
|
||
"url_contains": [
|
||
"generated_music_content/",
|
||
"googleusercontent.com/generated_music_content/",
|
||
],
|
||
"label_contains": [],
|
||
},
|
||
"video": {
|
||
"text_contains": [
|
||
"video_gen_chip/",
|
||
"googleusercontent.com/video_gen_chip/",
|
||
"正在生成视频",
|
||
"视频已准备就绪",
|
||
"come back later to check",
|
||
"i'm generating your video",
|
||
"your video is being generated",
|
||
"your video is ready",
|
||
],
|
||
"url_contains": [
|
||
"video_gen_chip/",
|
||
"googleusercontent.com/video_gen_chip/",
|
||
],
|
||
"label_contains": [],
|
||
},
|
||
}
|
||
|
||
normalized_media_type = str(media_type or "").strip().lower()
|
||
normalized_marker_key = str(marker_key or "").strip()
|
||
config_root = dict((image_config or {}).get("pending_detection") or {})
|
||
type_config = dict(config_root.get(normalized_media_type) or {})
|
||
configured = type_config.get(normalized_marker_key)
|
||
|
||
if isinstance(configured, list):
|
||
markers = [str(item or "").strip().lower() for item in configured if str(item or "").strip()]
|
||
if markers:
|
||
return markers
|
||
|
||
return [
|
||
str(item or "").strip().lower()
|
||
for item in default_markers.get(normalized_media_type, {}).get(normalized_marker_key, [])
|
||
if str(item or "").strip()
|
||
]
|
||
|
||
def _is_pending_media_text(self, text: str, image_config: Dict, media_type: str = "") -> bool:
|
||
lowered = str(text or "").strip().lower()
|
||
if not lowered:
|
||
return False
|
||
|
||
media_types = [str(media_type or "").strip().lower()] if media_type else ["audio", "video"]
|
||
for current_type in media_types:
|
||
markers = self._get_pending_detection_markers(
|
||
image_config,
|
||
current_type,
|
||
"text_contains",
|
||
)
|
||
if any(marker in lowered for marker in markers):
|
||
return True
|
||
return False
|
||
|
||
def _is_pending_media_item(self, media_item: Dict, image_config: Dict) -> bool:
|
||
if media_item.get("pending") is True:
|
||
return True
|
||
|
||
media_type = str(media_item.get("media_type") or "").strip().lower()
|
||
ref = str(media_item.get("url") or media_item.get("data_uri") or "").strip().lower()
|
||
label = str(media_item.get("label") or "").strip().lower()
|
||
|
||
if not media_type:
|
||
return False
|
||
|
||
url_markers = self._get_pending_detection_markers(
|
||
image_config,
|
||
media_type,
|
||
"url_contains",
|
||
)
|
||
label_markers = self._get_pending_detection_markers(
|
||
image_config,
|
||
media_type,
|
||
"label_contains",
|
||
)
|
||
|
||
if media_type not in {"audio", "video"}:
|
||
return False
|
||
|
||
return any(marker in ref for marker in url_markers) or any(
|
||
marker in label for marker in label_markers
|
||
)
|
||
|
||
def _filter_ready_media_items(self, media_items: List[Dict], image_config: Dict) -> List[Dict]:
|
||
return [
|
||
item for item in (media_items or [])
|
||
if not self._is_pending_media_item(item, image_config)
|
||
]
|
||
|
||
def _merge_dom_and_stream_media_items(
|
||
self,
|
||
dom_media_items: List[Dict],
|
||
stream_media_items: List[Dict],
|
||
image_config: Dict,
|
||
) -> List[Dict]:
|
||
"""Merge current-response stream media with media found in the selected DOM node."""
|
||
dom_items = [dict(item) for item in (dom_media_items or []) if isinstance(item, dict)]
|
||
stream_items = self._dedupe_media_items(
|
||
self._filter_ready_media_items(
|
||
[dict(item) for item in (stream_media_items or []) if isinstance(item, dict)],
|
||
image_config,
|
||
)
|
||
)
|
||
if not stream_items:
|
||
return self._dedupe_media_items(dom_items)
|
||
|
||
# A parsed stream image belongs to the response currently being monitored. The DOM
|
||
# selector can still resolve to an older image-only reply, so do not let that stale
|
||
# candidate replace the request-correlated stream result.
|
||
stream_images = [
|
||
item
|
||
for item in stream_items
|
||
if str(item.get("media_type") or "").strip().lower() == "image"
|
||
and str(item.get("url") or item.get("data_uri") or "").strip()
|
||
]
|
||
if stream_images:
|
||
dom_image_count = sum(
|
||
1
|
||
for item in dom_items
|
||
if str(item.get("media_type") or "").strip().lower() == "image"
|
||
)
|
||
dom_items = [
|
||
item
|
||
for item in dom_items
|
||
if str(item.get("media_type") or "").strip().lower() != "image"
|
||
]
|
||
dom_items = stream_images + dom_items
|
||
if dom_image_count:
|
||
logger.debug(
|
||
"网络流图片已替换 DOM 图片候选: "
|
||
f"stream={len(stream_images)}, dom={dom_image_count}"
|
||
)
|
||
|
||
merged = list(dom_items)
|
||
dom_types = {
|
||
str(item.get("media_type") or "").strip().lower()
|
||
for item in dom_items
|
||
if str(item.get("url") or item.get("data_uri") or "").strip()
|
||
}
|
||
|
||
appended = 0
|
||
for item in stream_items:
|
||
media_type = str(item.get("media_type") or "").strip().lower()
|
||
if not media_type or media_type in dom_types or media_type == "image":
|
||
continue
|
||
merged.append(dict(item))
|
||
appended += 1
|
||
|
||
if appended:
|
||
logger.debug(f"DOM 媒体缺失,已回退合并网络流媒体: {appended} 项")
|
||
|
||
return self._dedupe_media_items(merged)
|
||
|
||
def _should_run_media_postprocess(
|
||
self,
|
||
image_config: Dict,
|
||
*,
|
||
request_text_hint: str = "",
|
||
response_text_hint: str = "",
|
||
media_generation_state: Optional[Dict[str, Any]] = None,
|
||
stream_media_items: Optional[List[Dict[str, Any]]] = None,
|
||
dom_stream_media_items: Optional[List[Dict[str, Any]]] = None,
|
||
dom_image_detected: bool = False,
|
||
dom_final_image_urls: Optional[List[str]] = None,
|
||
) -> tuple[bool, Dict[str, Any]]:
|
||
modalities = self._media_modalities(image_config)
|
||
enabled_types = self._media_enabled_types(image_config)
|
||
stream_media_count = sum(1 for item in (stream_media_items or []) if isinstance(item, dict))
|
||
dom_stream_media_count = sum(1 for item in (dom_stream_media_items or []) if isinstance(item, dict))
|
||
dom_final_image_url_count = sum(1 for item in (dom_final_image_urls or []) if str(item or "").strip())
|
||
force_postprocess = bool((image_config or {}).get("force_postprocess"))
|
||
direct_postprocess_modalities = [
|
||
media_type
|
||
for media_type in (image_config or {}).get("direct_postprocess_modalities", [])
|
||
if media_type in enabled_types
|
||
]
|
||
media_state = dict(media_generation_state or {})
|
||
media_state_pending = bool(media_state.get("pending"))
|
||
media_state_type = str(media_state.get("media_type") or "").strip().lower()
|
||
media_state_hint = str(media_state.get("hint_text") or "").strip()
|
||
response_hint = str(response_text_hint or "").strip()
|
||
combined_hint = "\n".join(part for part in (response_hint, media_state_hint) if part).lower()
|
||
request_likely_image = self._looks_like_image_generation_request(request_text_hint)
|
||
av_pending_text_signal = (
|
||
("audio" in enabled_types or "video" in enabled_types)
|
||
and self._is_pending_media_text(combined_hint, image_config)
|
||
)
|
||
image_markers = (
|
||
"image_generation_content/",
|
||
"googleusercontent.com/image_generation_content/",
|
||
"generated image",
|
||
"generating image",
|
||
"images are being generated",
|
||
)
|
||
image_marker_hit = any(marker in combined_hint for marker in image_markers)
|
||
diagnostics: Dict[str, Any] = {
|
||
"enabled_types": enabled_types,
|
||
"run_policies": {
|
||
media_type: self._media_run_policy(image_config, media_type)
|
||
for media_type in enabled_types
|
||
},
|
||
"stream_media_count": stream_media_count,
|
||
"dom_stream_media_count": dom_stream_media_count,
|
||
"dom_image_detected": bool(dom_image_detected),
|
||
"dom_final_image_url_count": dom_final_image_url_count,
|
||
"force_postprocess": force_postprocess,
|
||
"direct_postprocess_modalities": direct_postprocess_modalities,
|
||
"media_state_pending": media_state_pending,
|
||
"media_state_type": media_state_type,
|
||
"media_state_hint_len": len(media_state_hint),
|
||
"response_hint_len": len(response_hint),
|
||
"request_hint_len": len(str(request_text_hint or "").strip()),
|
||
"request_likely_image": request_likely_image,
|
||
"av_pending_text_signal": av_pending_text_signal,
|
||
"image_marker_hit": image_marker_hit,
|
||
"decision": "",
|
||
}
|
||
|
||
if not enabled_types:
|
||
diagnostics["decision"] = "disabled"
|
||
return False, diagnostics
|
||
|
||
if stream_media_count > 0:
|
||
diagnostics["decision"] = "stream_media_items"
|
||
return True, diagnostics
|
||
if dom_stream_media_count > 0:
|
||
diagnostics["decision"] = "dom_stream_media_items"
|
||
return True, diagnostics
|
||
if dom_final_image_url_count > 0 and "image" in enabled_types:
|
||
diagnostics["decision"] = "dom_final_image_urls"
|
||
return True, diagnostics
|
||
if bool(dom_image_detected) and "image" in enabled_types:
|
||
diagnostics["decision"] = "dom_image_detected"
|
||
return True, diagnostics
|
||
|
||
if force_postprocess:
|
||
diagnostics["decision"] = "force_postprocess"
|
||
return True, diagnostics
|
||
|
||
if media_state_pending:
|
||
diagnostics["decision"] = "media_state_pending"
|
||
return True, diagnostics
|
||
|
||
if av_pending_text_signal:
|
||
diagnostics["decision"] = "audio_video_pending_text_signal"
|
||
return True, diagnostics
|
||
|
||
if "image" in enabled_types:
|
||
if request_likely_image:
|
||
diagnostics["decision"] = "request_likely_image"
|
||
return True, diagnostics
|
||
|
||
if image_marker_hit:
|
||
diagnostics["decision"] = "image_marker_hint"
|
||
return True, diagnostics
|
||
|
||
if direct_postprocess_modalities:
|
||
diagnostics["decision"] = "direct_postprocess_modalities"
|
||
return True, diagnostics
|
||
|
||
diagnostics["decision"] = "generic_dom_quick_scan"
|
||
return True, diagnostics
|
||
|
||
def _extract_media_after_stream(
|
||
self,
|
||
tab,
|
||
extractor,
|
||
image_config: Dict,
|
||
result_selector: str,
|
||
message_wrapper_selector: str = "",
|
||
completion_id: str = None,
|
||
stop_checker: Optional[Callable[[], bool]] = None,
|
||
response_text_hint: str = "",
|
||
request_text_hint: str = "",
|
||
media_generation_state: Optional[Dict[str, Any]] = None,
|
||
stream_media_items: Optional[List[Dict[str, Any]]] = None,
|
||
direct_modalities: Optional[List[str]] = None,
|
||
media_dom_baseline: Optional[Dict[str, Any]] = None,
|
||
) -> List[Dict]:
|
||
"""流式输出结束后提取多模态资源。"""
|
||
from app.core.elements import ElementFinder
|
||
from app.core.extractors.media_extractor import media_extractor
|
||
|
||
image_config = dict(image_config or {})
|
||
direct_scan_only = False
|
||
if direct_modalities:
|
||
requested_direct_modalities = {
|
||
str(item or "").strip().lower()
|
||
for item in direct_modalities
|
||
if str(item or "").strip().lower() in {"image", "audio", "video"}
|
||
}
|
||
if requested_direct_modalities:
|
||
direct_scan_only = True
|
||
scoped_modalities = {}
|
||
for media_type in ("image", "audio", "video"):
|
||
policy = self._media_policy(image_config, media_type)
|
||
if media_type not in requested_direct_modalities:
|
||
policy["enabled"] = False
|
||
policy["run_policy"] = "disabled"
|
||
scoped_modalities[media_type] = policy
|
||
image_config["modalities"] = scoped_modalities
|
||
image_config["wait_for_load"] = False
|
||
image_config["audio_capture_enabled"] = False
|
||
modalities = self._media_modalities(image_config)
|
||
only_audio_mode = (
|
||
is_modality_enabled(modalities, "audio")
|
||
and not is_modality_enabled(modalities, "image")
|
||
and not is_modality_enabled(modalities, "video")
|
||
)
|
||
debounce = 0.0 if only_audio_mode else image_config.get("debounce_seconds", 2.0)
|
||
effective_stop_checker = stop_checker or self._should_stop_checker
|
||
if debounce > 0:
|
||
elapsed = 0
|
||
step = 0.1
|
||
while elapsed < debounce:
|
||
if effective_stop_checker():
|
||
return []
|
||
time.sleep(step)
|
||
elapsed += step
|
||
|
||
finder = ElementFinder(tab)
|
||
fallback_stream_media_items = self._dedupe_media_items(
|
||
[dict(item) for item in (stream_media_items or []) if isinstance(item, dict)]
|
||
)
|
||
dom_baseline_active = bool(
|
||
media_dom_baseline
|
||
and media_dom_baseline.get("token")
|
||
and media_dom_baseline.get("property")
|
||
)
|
||
|
||
def _apply_stream_media_fallback(dom_media_items: List[Dict]) -> List[Dict]:
|
||
return self._merge_dom_and_stream_media_items(
|
||
dom_media_items,
|
||
fallback_stream_media_items,
|
||
image_config,
|
||
)
|
||
|
||
def _find_candidate_elements(timeout: float = 1.0):
|
||
primary_elements = []
|
||
fallback_elements = []
|
||
|
||
try:
|
||
if result_selector:
|
||
primary_elements = finder.find_all(result_selector, timeout=timeout) or []
|
||
except Exception as e:
|
||
logger.debug(f"主结果容器查找失败(忽略): {e}")
|
||
primary_elements = []
|
||
|
||
if primary_elements:
|
||
return primary_elements, "result_container"
|
||
|
||
try:
|
||
if message_wrapper_selector:
|
||
fallback_elements = finder.find_all(message_wrapper_selector, timeout=timeout) or []
|
||
except Exception as e:
|
||
logger.debug(f"消息包装容器查找失败(忽略): {e}")
|
||
fallback_elements = []
|
||
|
||
if fallback_elements:
|
||
logger.debug(
|
||
"主结果容器为空,回退到消息包装容器继续等待媒体渲染: "
|
||
f"primary={result_selector!r}, fallback={message_wrapper_selector!r}"
|
||
)
|
||
return fallback_elements, "message_wrapper"
|
||
|
||
return [], ""
|
||
|
||
def _audio_trigger_available(target_element=None, *, signal_seen: bool = False) -> bool:
|
||
if not is_modality_enabled(image_config.get("modalities") or {}, "audio"):
|
||
return False
|
||
if not bool(image_config.get("audio_capture_enabled", True)):
|
||
return False
|
||
if not self._media_policy_allows_audio_probe(image_config, signal_seen=signal_seen):
|
||
return False
|
||
policy = self._media_run_policy(image_config, "audio")
|
||
if policy == "always_probe":
|
||
return True
|
||
for current_target in [target_element, tab]:
|
||
if current_target is None:
|
||
continue
|
||
try:
|
||
probe_result = media_extractor.probe_audio_trigger(current_target, image_config)
|
||
except Exception:
|
||
probe_result = {}
|
||
if bool(probe_result.get("found")) or int(probe_result.get("candidate_count") or 0) > 0:
|
||
logger.debug(
|
||
"页面音频触发入口快速探测命中: "
|
||
f"policy={policy}, signal_seen={signal_seen}, "
|
||
f"candidate_count={probe_result.get('candidate_count')}, "
|
||
f"selector={probe_result.get('selector_used')!r}"
|
||
)
|
||
return True
|
||
logger.debug(
|
||
"页面音频触发入口快速探测未命中,跳过播放录音等待: "
|
||
f"policy={policy}, signal_seen={signal_seen}"
|
||
)
|
||
return False
|
||
|
||
try:
|
||
quick_probe_timeout = self._media_quick_probe_timeout(image_config, default=1.0)
|
||
elements, container_mode = _find_candidate_elements(timeout=quick_probe_timeout)
|
||
if not elements:
|
||
should_try_page_audio_capture = (
|
||
_audio_trigger_available(None, signal_seen=False)
|
||
and not effective_stop_checker()
|
||
)
|
||
if should_try_page_audio_capture:
|
||
logger.debug("结果容器为空,尝试页面级音频播放捕获回退")
|
||
captured_audio_items = self._capture_audio_via_page_playback(
|
||
tab=tab,
|
||
target_element=None,
|
||
image_config=image_config,
|
||
stop_checker=effective_stop_checker,
|
||
)
|
||
if captured_audio_items:
|
||
return _apply_stream_media_fallback(captured_audio_items)
|
||
|
||
if fallback_stream_media_items:
|
||
logger.debug("结果容器为空,直接回退到网络流媒体结果")
|
||
return list(fallback_stream_media_items)
|
||
return []
|
||
|
||
def _select_target_element(candidates):
|
||
if not candidates:
|
||
return None
|
||
|
||
strategy = str(image_config.get("final_target_strategy", "container") or "container").strip().lower()
|
||
|
||
selector = str(image_config.get("selector") or "img").strip() or "img"
|
||
|
||
def _has_media(candidate) -> bool:
|
||
try:
|
||
return bool(
|
||
as_element(candidate).run_js(
|
||
"""
|
||
const selector = String(arguments[0] || "img");
|
||
try {
|
||
if (this instanceof Element && typeof this.matches === "function" && this.matches(selector)) {
|
||
return true;
|
||
}
|
||
} catch {}
|
||
try {
|
||
return !!(this.querySelector && this.querySelector(selector));
|
||
} catch {
|
||
return false;
|
||
}
|
||
""",
|
||
selector,
|
||
)
|
||
)
|
||
except Exception:
|
||
return False
|
||
|
||
if strategy == "latest_visual_reply":
|
||
scored_candidates = []
|
||
parser_id = str(image_config.get("_parser_id", "") or "").strip().lower()
|
||
if "right" in parser_id:
|
||
column = "right"
|
||
elif "left" in parser_id:
|
||
column = "left"
|
||
else:
|
||
parser_side = str(image_config.get("_parser_target_side", "") or "").strip().lower()
|
||
column = parser_side if parser_side in {"left", "right"} else str(
|
||
image_config.get("latest_visual_column", "left") or "left"
|
||
).strip().lower()
|
||
if column not in {"left", "right"}:
|
||
column = "left"
|
||
for index, candidate in enumerate(candidates):
|
||
has_media = _has_media(candidate)
|
||
try:
|
||
rect = as_element(candidate).run_js(
|
||
"""
|
||
const rect = this.getBoundingClientRect();
|
||
return {
|
||
bottom: Number(rect && rect.bottom || 0) + Number(window.scrollY || 0),
|
||
left: Number(rect && rect.left || 0) + Number(window.scrollX || 0),
|
||
width: Number(rect && rect.width || 0),
|
||
height: Number(rect && rect.height || 0),
|
||
};
|
||
"""
|
||
) or {}
|
||
bottom = float(rect.get("bottom") or 0)
|
||
left = float(rect.get("left") or 0)
|
||
area = float(rect.get("width") or 0) * float(rect.get("height") or 0)
|
||
except Exception:
|
||
bottom = 0.0
|
||
left = 0.0
|
||
area = 0.0
|
||
horizontal_score = left if column == "right" else -left
|
||
scored_candidates.append((bottom, horizontal_score, area, index, index, left, has_media, candidate))
|
||
|
||
if len(scored_candidates) > 1:
|
||
left_edges = [item[5] for item in scored_candidates]
|
||
horizontal_span = max(left_edges) - min(left_edges)
|
||
if horizontal_span >= 80.0:
|
||
midpoint = min(left_edges) + horizontal_span / 2.0
|
||
side_candidates = [
|
||
item
|
||
for item in scored_candidates
|
||
if (
|
||
item[5] >= midpoint
|
||
if column == "right"
|
||
else item[5] <= midpoint
|
||
)
|
||
]
|
||
if side_candidates:
|
||
scored_candidates = side_candidates
|
||
|
||
if dom_baseline_active and scored_candidates:
|
||
fresh_scored = [
|
||
item
|
||
for item in scored_candidates
|
||
if self._candidate_has_fresh_dom_image(
|
||
item[7],
|
||
image_config,
|
||
media_dom_baseline,
|
||
)
|
||
]
|
||
if fresh_scored:
|
||
logger.debug(
|
||
f"[latest_visual_reply] 分栏 {column} 容器已按基线收敛到本轮新图: "
|
||
f"fresh={len(fresh_scored)}, side_total={len(scored_candidates)}"
|
||
)
|
||
scored_candidates = fresh_scored
|
||
|
||
if scored_candidates:
|
||
scored_candidates.sort(key=lambda item: item[:4], reverse=True)
|
||
best = scored_candidates[0]
|
||
logger.debug(
|
||
"[latest_visual_reply] 选中视觉最新媒体容器: "
|
||
f"index={best[4]}, column={column}, has_media={best[6]}, "
|
||
f"bottom={best[0]:.1f}, left={best[5]:.1f}, total={len(candidates)}"
|
||
)
|
||
return best[7]
|
||
|
||
if dom_baseline_active:
|
||
fresh_candidates = [
|
||
candidate
|
||
for candidate in candidates
|
||
if self._candidate_has_fresh_dom_image(
|
||
candidate,
|
||
image_config,
|
||
media_dom_baseline,
|
||
)
|
||
]
|
||
if fresh_candidates:
|
||
logger.debug(
|
||
"结果容器已按发送前基线收敛到本轮新增图片: "
|
||
f"fresh={len(fresh_candidates)}, total={len(candidates)}"
|
||
)
|
||
candidates = fresh_candidates
|
||
|
||
if strategy not in ("latest_reply", "latest_visual_reply"):
|
||
return candidates[-1]
|
||
|
||
for candidate in reversed(candidates):
|
||
if _has_media(candidate):
|
||
return candidate
|
||
|
||
return candidates[-1]
|
||
|
||
last_element = _select_target_element(elements)
|
||
if last_element is None:
|
||
return []
|
||
|
||
def _extract_media_once(target_element, quiet=False):
|
||
cfg = dict(image_config or {})
|
||
if quiet:
|
||
cfg["quiet"] = True
|
||
if media_dom_baseline:
|
||
cfg["request_baseline_token"] = str(media_dom_baseline.get("token") or "")
|
||
cfg["request_baseline_property"] = str(media_dom_baseline.get("property") or "")
|
||
if hasattr(extractor, 'extract_media'):
|
||
extracted_items = extractor.extract_media(
|
||
target_element,
|
||
config=cfg,
|
||
container_selector_fallback=result_selector
|
||
)
|
||
elif hasattr(extractor, 'extract_images'):
|
||
extracted_items = media_extractor.extract(
|
||
target_element,
|
||
config=cfg,
|
||
container_selector_fallback=result_selector
|
||
)
|
||
else:
|
||
extracted_items = media_extractor.extract(
|
||
target_element,
|
||
config=cfg,
|
||
container_selector_fallback=result_selector
|
||
)
|
||
|
||
extracted_items = list(extracted_items or [])
|
||
extracted_items = self._filter_media_items_against_dom_baseline(
|
||
extracted_items,
|
||
media_dom_baseline,
|
||
)
|
||
if media_dom_baseline and not self._candidate_has_fresh_dom_image(
|
||
target_element,
|
||
image_config,
|
||
media_dom_baseline,
|
||
):
|
||
stale_image_count = sum(
|
||
1
|
||
for item in extracted_items
|
||
if str(item.get("media_type") or "").strip().lower() == "image"
|
||
)
|
||
if stale_image_count:
|
||
logger.debug(
|
||
"发送前已存在的 DOM 图片已从本轮结果剔除: "
|
||
f"count={stale_image_count}"
|
||
)
|
||
extracted_items = [
|
||
item
|
||
for item in extracted_items
|
||
if str(item.get("media_type") or "").strip().lower() != "image"
|
||
]
|
||
return extracted_items
|
||
|
||
def _extract_fresh_media_from_page_scope(reason: str, quiet=True):
|
||
if only_audio_mode or not is_modality_enabled(modalities, "image"):
|
||
return []
|
||
if not dom_baseline_active or effective_stop_checker():
|
||
return []
|
||
|
||
fallback_selectors = image_config.get("fresh_image_fallback_selectors") or (
|
||
"main",
|
||
"body",
|
||
)
|
||
if isinstance(fallback_selectors, str):
|
||
fallback_selectors = [fallback_selectors]
|
||
|
||
roots = []
|
||
seen_root_ids = set()
|
||
for fallback_selector in fallback_selectors:
|
||
fallback_selector = str(fallback_selector or "").strip()
|
||
if not fallback_selector:
|
||
continue
|
||
try:
|
||
found_roots = finder.find_all(fallback_selector, timeout=0.5) or []
|
||
except Exception as e:
|
||
logger.debug(f"页面级图片回退根查找失败(忽略): selector={fallback_selector!r}, error={e}")
|
||
continue
|
||
for root in found_roots:
|
||
root_key = id(root)
|
||
if root_key in seen_root_ids:
|
||
continue
|
||
seen_root_ids.add(root_key)
|
||
roots.append((fallback_selector, root))
|
||
|
||
if not roots:
|
||
return []
|
||
|
||
logger.debug(
|
||
"窄范围未捕获本轮图片,启用页面级图片回退扫描: "
|
||
f"reason={reason}, roots={len(roots)}"
|
||
)
|
||
fallback_items = []
|
||
for fallback_selector, root in roots:
|
||
cfg = dict(image_config or {})
|
||
cfg["allow_container_fallback"] = True
|
||
cfg["container_selector"] = None
|
||
cfg["request_baseline_token"] = str(media_dom_baseline.get("token") or "")
|
||
cfg["request_baseline_property"] = str(media_dom_baseline.get("property") or "")
|
||
if quiet:
|
||
cfg["quiet"] = True
|
||
try:
|
||
extracted_items = media_extractor.extract(
|
||
root,
|
||
config=cfg,
|
||
container_selector_fallback="",
|
||
)
|
||
except Exception as e:
|
||
logger.debug(
|
||
"页面级图片回退扫描失败(忽略): "
|
||
f"selector={fallback_selector!r}, error={e}"
|
||
)
|
||
continue
|
||
extracted_items = list(extracted_items or [])
|
||
extracted_items = self._filter_media_items_against_dom_baseline(
|
||
extracted_items,
|
||
media_dom_baseline,
|
||
)
|
||
ready_items = self._filter_ready_media_items(extracted_items, image_config)
|
||
image_count = sum(
|
||
1
|
||
for item in ready_items
|
||
if str(item.get("media_type") or "").strip().lower() == "image"
|
||
)
|
||
if image_count:
|
||
logger.debug(
|
||
"页面级图片回退扫描捕获本轮新图: "
|
||
f"selector={fallback_selector!r}, images={image_count}"
|
||
)
|
||
fallback_items.extend(ready_items)
|
||
break
|
||
|
||
return self._dedupe_media_items(fallback_items)
|
||
|
||
request_likely_image = self._looks_like_image_generation_request(request_text_hint)
|
||
media_items = [] if only_audio_mode else _extract_media_once(last_element)
|
||
ready_media_items = _apply_stream_media_fallback(
|
||
self._filter_ready_media_items(media_items, image_config)
|
||
)
|
||
pending_media_items = [
|
||
item for item in media_items
|
||
if self._is_pending_media_item(item, image_config)
|
||
]
|
||
|
||
image_items = [item for item in ready_media_items if item.get("media_type") == "image"]
|
||
audio_items = [item for item in ready_media_items if item.get("media_type") == "audio"]
|
||
video_items = [item for item in ready_media_items if item.get("media_type") == "video"]
|
||
|
||
if (
|
||
request_likely_image
|
||
and not image_items
|
||
and not fallback_stream_media_items
|
||
and not effective_stop_checker()
|
||
):
|
||
fresh_page_media_items = _extract_fresh_media_from_page_scope("initial_empty")
|
||
if fresh_page_media_items:
|
||
ready_media_items = self._dedupe_media_items(
|
||
list(ready_media_items) + fresh_page_media_items
|
||
)
|
||
image_items = [item for item in ready_media_items if item.get("media_type") == "image"]
|
||
audio_items = [item for item in ready_media_items if item.get("media_type") == "audio"]
|
||
video_items = [item for item in ready_media_items if item.get("media_type") == "video"]
|
||
|
||
placeholder_text = ""
|
||
if not only_audio_mode:
|
||
try:
|
||
if hasattr(extractor, 'extract_text'):
|
||
placeholder_text = str(extractor.extract_text(last_element) or "")
|
||
else:
|
||
placeholder_text = str(
|
||
as_element(last_element).run_js("return this.innerText || this.textContent || ''") or ""
|
||
)
|
||
except Exception:
|
||
placeholder_text = ""
|
||
|
||
placeholder_text_lower = placeholder_text.lower()
|
||
response_text_hint_lower = str(response_text_hint or "").strip().lower()
|
||
media_state = dict(media_generation_state or {})
|
||
media_state_pending = bool(media_state.get("pending"))
|
||
media_state_type = str(media_state.get("media_type") or "").strip().lower()
|
||
media_state_hint_text = str(media_state.get("hint_text") or "").strip().lower()
|
||
combined_pending_text = "\n".join(
|
||
text for text in (placeholder_text_lower, response_text_hint_lower, media_state_hint_text)
|
||
if text
|
||
)
|
||
terminal_error_markers = image_config.get("terminal_error_text_markers") or (
|
||
"something went wrong with this response",
|
||
"please try again",
|
||
"trace id:",
|
||
)
|
||
if isinstance(terminal_error_markers, str):
|
||
terminal_error_markers = (terminal_error_markers,)
|
||
terminal_media_error = any(
|
||
str(marker or "").strip().lower() in combined_pending_text
|
||
for marker in terminal_error_markers
|
||
if str(marker or "").strip()
|
||
)
|
||
|
||
has_generated_image_hint = any(
|
||
marker in placeholder_text_lower
|
||
for marker in (
|
||
"image_generation_content/",
|
||
"googleusercontent.com/image_generation_content/",
|
||
)
|
||
)
|
||
|
||
if not has_generated_image_hint:
|
||
try:
|
||
has_generated_image_hint = bool(
|
||
as_element(last_element).run_js(
|
||
"""
|
||
return !!this.querySelector(
|
||
'.attachment-container.generated-images, '
|
||
+ '.generated-images, generated-image, single-image, '
|
||
+ '.image-button, img[src^="blob:"], img[src^="data:image"]'
|
||
);
|
||
"""
|
||
)
|
||
)
|
||
except Exception:
|
||
has_generated_image_hint = False
|
||
|
||
pending_audio_hint = self._is_pending_media_text(combined_pending_text, image_config, "audio")
|
||
pending_video_hint = self._is_pending_media_text(combined_pending_text, image_config, "video")
|
||
|
||
if pending_media_items:
|
||
pending_audio_hint = pending_audio_hint or any(
|
||
item.get("media_type") == "audio" for item in pending_media_items
|
||
)
|
||
pending_video_hint = pending_video_hint or any(
|
||
item.get("media_type") == "video" for item in pending_media_items
|
||
)
|
||
logger.debug(
|
||
"检测到占位媒体,继续等待真实结果: "
|
||
f"pending={len(pending_media_items)}, ready={len(ready_media_items)}"
|
||
)
|
||
|
||
pending_kinds = set()
|
||
if (
|
||
has_generated_image_hint
|
||
or (media_state_pending and media_state_type == "image")
|
||
) and not image_items and self._media_policy_allows_signal_wait(image_config, "image"):
|
||
pending_kinds.add("image")
|
||
if (
|
||
media_state_pending
|
||
and media_state_type == "audio"
|
||
and not (audio_items or video_items)
|
||
and self._media_policy_allows_signal_wait(image_config, "audio")
|
||
):
|
||
pending_kinds.add("audio")
|
||
if (
|
||
media_state_pending
|
||
and media_state_type == "video"
|
||
and not video_items
|
||
and self._media_policy_allows_signal_wait(image_config, "video")
|
||
):
|
||
pending_kinds.add("video")
|
||
if (
|
||
pending_audio_hint
|
||
and not (audio_items or video_items)
|
||
and self._media_policy_allows_signal_wait(image_config, "audio")
|
||
):
|
||
pending_kinds.add("audio")
|
||
if (
|
||
pending_video_hint
|
||
and not video_items
|
||
and self._media_policy_allows_signal_wait(image_config, "video")
|
||
):
|
||
pending_kinds.add("video")
|
||
|
||
if terminal_media_error and not image_items:
|
||
pending_kinds.discard("image")
|
||
logger.warning("当前回复包含终止错误且没有本轮新图,跳过图片等待与历史图回退")
|
||
|
||
if pending_kinds and not direct_scan_only and not effective_stop_checker():
|
||
base_timeout = float(image_config.get("load_timeout_seconds", 5.0) or 5.0)
|
||
late_wait_timeout = float(
|
||
image_config.get("late_render_timeout_seconds")
|
||
or max(30.0, base_timeout * 6.0)
|
||
)
|
||
for pending_kind in pending_kinds:
|
||
late_wait_timeout = max(
|
||
late_wait_timeout,
|
||
self._media_late_wait_timeout(image_config, pending_kind, late_wait_timeout),
|
||
)
|
||
state_wait_timeout = media_state.get("wait_timeout_seconds")
|
||
try:
|
||
if state_wait_timeout is not None:
|
||
late_wait_timeout = max(late_wait_timeout, float(state_wait_timeout))
|
||
except Exception:
|
||
pass
|
||
if media_state_pending and media_state_type in pending_kinds:
|
||
logger.debug(
|
||
"检测到解析器上报的待渲染媒体任务,延长媒体渲染等待窗口: "
|
||
f"{late_wait_timeout:.1f}s"
|
||
)
|
||
poll_interval = float(image_config.get("late_render_poll_seconds") or 1.0)
|
||
deadline = time.time() + late_wait_timeout
|
||
wait_satisfied = False
|
||
|
||
while time.time() < deadline and not effective_stop_checker():
|
||
time.sleep(max(0.2, poll_interval))
|
||
elements, container_mode = _find_candidate_elements(timeout=0.5)
|
||
if not elements:
|
||
continue
|
||
selected = _select_target_element(elements)
|
||
if selected is None:
|
||
continue
|
||
last_element = selected
|
||
media_items = _extract_media_once(last_element, quiet=True)
|
||
ready_media_items = _apply_stream_media_fallback(
|
||
self._filter_ready_media_items(media_items, image_config)
|
||
)
|
||
image_items = [item for item in ready_media_items if item.get("media_type") == "image"]
|
||
audio_items = [item for item in ready_media_items if item.get("media_type") == "audio"]
|
||
video_items = [item for item in ready_media_items if item.get("media_type") == "video"]
|
||
|
||
if "image" in pending_kinds and not image_items and not fallback_stream_media_items:
|
||
fresh_page_media_items = _extract_fresh_media_from_page_scope(
|
||
"pending_wait_empty",
|
||
quiet=True,
|
||
)
|
||
if fresh_page_media_items:
|
||
ready_media_items = self._dedupe_media_items(
|
||
list(ready_media_items) + fresh_page_media_items
|
||
)
|
||
image_items = [
|
||
item for item in ready_media_items if item.get("media_type") == "image"
|
||
]
|
||
audio_items = [
|
||
item for item in ready_media_items if item.get("media_type") == "audio"
|
||
]
|
||
video_items = [
|
||
item for item in ready_media_items if item.get("media_type") == "video"
|
||
]
|
||
|
||
satisfied = True
|
||
if "image" in pending_kinds and not image_items:
|
||
satisfied = False
|
||
if "audio" in pending_kinds and not (audio_items or video_items):
|
||
satisfied = False
|
||
if "video" in pending_kinds and not video_items:
|
||
satisfied = False
|
||
|
||
if satisfied:
|
||
kinds_label = ",".join(sorted(pending_kinds))
|
||
logger.debug(
|
||
f"延迟媒体渲染已捕获: kinds={kinds_label} "
|
||
f"(late_wait={late_wait_timeout:.1f}s, container={container_mode or 'unknown'})"
|
||
)
|
||
wait_satisfied = True
|
||
break
|
||
|
||
if pending_kinds and not wait_satisfied and not effective_stop_checker():
|
||
logger.warning(
|
||
"等待待渲染媒体超时,仍未拿到最终结果: "
|
||
f"kinds={','.join(sorted(pending_kinds))}, late_wait={late_wait_timeout:.1f}s"
|
||
)
|
||
|
||
should_probe_late_image_render = (
|
||
is_modality_enabled(modalities, "image")
|
||
and self._media_policy_allows_signal_wait(image_config, "image")
|
||
and not image_items
|
||
and not pending_kinds
|
||
and not only_audio_mode
|
||
and not effective_stop_checker()
|
||
and request_likely_image
|
||
and len(str(response_text_hint or "").strip()) <= 32
|
||
and not terminal_media_error
|
||
)
|
||
if should_probe_late_image_render:
|
||
late_image_wait_timeout = float(
|
||
image_config.get("late_image_render_timeout_seconds")
|
||
or max(45.0, float(image_config.get("load_timeout_seconds", 5.0) or 5.0) * 8.0)
|
||
)
|
||
late_image_wait_timeout = self._media_blind_wait_timeout(
|
||
image_config,
|
||
"image",
|
||
self._media_late_wait_timeout(
|
||
image_config,
|
||
"image",
|
||
late_image_wait_timeout,
|
||
),
|
||
)
|
||
if late_image_wait_timeout <= 0:
|
||
logger.debug("无显式占位信号,图片盲等已按配置跳过")
|
||
should_probe_late_image_render = False
|
||
|
||
if should_probe_late_image_render:
|
||
poll_interval = float(image_config.get("late_render_poll_seconds") or 1.0)
|
||
deadline = time.time() + late_image_wait_timeout
|
||
|
||
while time.time() < deadline and not effective_stop_checker():
|
||
time.sleep(max(0.2, poll_interval))
|
||
elements, container_mode = _find_candidate_elements(timeout=0.5)
|
||
if not elements:
|
||
continue
|
||
|
||
selected = _select_target_element(elements)
|
||
if selected is None:
|
||
continue
|
||
last_element = selected
|
||
media_items = _extract_media_once(last_element, quiet=True)
|
||
ready_media_items = _apply_stream_media_fallback(
|
||
self._filter_ready_media_items(media_items, image_config)
|
||
)
|
||
image_items = [item for item in ready_media_items if item.get("media_type") == "image"]
|
||
audio_items = [item for item in ready_media_items if item.get("media_type") == "audio"]
|
||
video_items = [item for item in ready_media_items if item.get("media_type") == "video"]
|
||
|
||
if not image_items and not fallback_stream_media_items:
|
||
fresh_page_media_items = _extract_fresh_media_from_page_scope(
|
||
"blind_wait_empty",
|
||
quiet=True,
|
||
)
|
||
if fresh_page_media_items:
|
||
ready_media_items = self._dedupe_media_items(
|
||
list(ready_media_items) + fresh_page_media_items
|
||
)
|
||
image_items = [
|
||
item for item in ready_media_items if item.get("media_type") == "image"
|
||
]
|
||
audio_items = [
|
||
item for item in ready_media_items if item.get("media_type") == "audio"
|
||
]
|
||
video_items = [
|
||
item for item in ready_media_items if item.get("media_type") == "video"
|
||
]
|
||
|
||
if image_items:
|
||
logger.debug(
|
||
"无显式占位信号,但已在延迟轮询中捕获到图片结果: "
|
||
f"late_wait={late_image_wait_timeout:.1f}s, container={container_mode or 'unknown'}"
|
||
)
|
||
break
|
||
|
||
should_try_audio_capture = (
|
||
_audio_trigger_available(
|
||
last_element,
|
||
signal_seen=bool("audio" in pending_kinds or pending_audio_hint or media_state_type == "audio"),
|
||
)
|
||
and not audio_items
|
||
and not effective_stop_checker()
|
||
)
|
||
if should_try_audio_capture:
|
||
captured_audio_items = self._capture_audio_via_page_playback(
|
||
tab=tab,
|
||
target_element=last_element,
|
||
image_config=image_config,
|
||
stop_checker=effective_stop_checker,
|
||
response_text_hint=response_text_hint,
|
||
)
|
||
if captured_audio_items:
|
||
ready_media_items = self._dedupe_media_items(list(ready_media_items) + captured_audio_items)
|
||
audio_items = [item for item in ready_media_items if item.get("media_type") == "audio"]
|
||
|
||
media_items = ready_media_items
|
||
|
||
try:
|
||
if image_items:
|
||
self._prefetch_remote_image_urls(
|
||
tab,
|
||
[
|
||
str(item.get("url") or "").strip()
|
||
for item in image_items
|
||
if isinstance(item, dict)
|
||
],
|
||
max_size_mb=image_config.get("max_size_mb", 10),
|
||
)
|
||
converted_images = self._try_screenshot_images_to_local(tab, last_element, image_items, image_config)
|
||
other_items = [item for item in media_items if item.get("media_type") != "image"]
|
||
media_items = converted_images + other_items
|
||
except Exception as e:
|
||
logger.warning(f"截图落盘失败(已忽略): {e}")
|
||
|
||
try:
|
||
media_items = self._persist_data_uri_media_to_local(
|
||
media_items,
|
||
max_size_mb=image_config.get("max_size_mb", 10),
|
||
)
|
||
except Exception as e:
|
||
logger.warning(f"data uri 落盘失败(已忽略): {e}")
|
||
|
||
uploaded_images = image_config.get("uploaded_image_paths") or []
|
||
if uploaded_images and media_items:
|
||
media_items = filter_reference_images(
|
||
media_items,
|
||
uploaded_images,
|
||
tab=tab,
|
||
logger_context="media_after_stream",
|
||
)
|
||
|
||
# 汇总并打印多模态媒体提取完成的日志
|
||
final_images = [item for item in media_items if item.get("media_type") == "image"]
|
||
final_videos = [item for item in media_items if item.get("media_type") == "video"]
|
||
final_audios = [item for item in media_items if item.get("media_type") == "audio"]
|
||
logger.debug(
|
||
f"提取完成: {len(final_images)} 张图片, {len(final_videos)} 个视频, {len(final_audios)} 个音频"
|
||
)
|
||
|
||
return media_items
|
||
|
||
except Exception as e:
|
||
logger.warning(f"多模态提取异常: {e}")
|
||
return []
|
||
|
||
def _capture_audio_via_page_playback(
|
||
self,
|
||
tab,
|
||
target_element,
|
||
image_config: Dict,
|
||
stop_checker: Optional[Callable[[], bool]] = None,
|
||
response_text_hint: str = "",
|
||
) -> List[Dict]:
|
||
"""回退方案:触发页面播放按钮并从隐藏音频/ WebAudio 输出中捕获音频。"""
|
||
from app.core.extractors.media_extractor import media_extractor
|
||
|
||
if not tab:
|
||
return []
|
||
|
||
effective_stop_checker = stop_checker or self._should_stop_checker
|
||
if effective_stop_checker():
|
||
return []
|
||
|
||
prepared = media_extractor.prepare_page_audio_capture(tab, image_config)
|
||
if not prepared:
|
||
return []
|
||
|
||
network_capture = dict(image_config.get("audio_network_capture") or {})
|
||
network_capture_enabled = bool(network_capture.get("enabled", False))
|
||
if network_capture_enabled and not media_extractor.install_audio_network_probe(tab, image_config):
|
||
network_capture_enabled = False
|
||
|
||
activation_result = media_extractor.activate_audio_trigger_surface(target_element or tab)
|
||
if activation_result:
|
||
logger.debug(
|
||
"页面音频操作区激活结果: "
|
||
f"ok={bool(activation_result.get('ok'))}, "
|
||
f"activated_count={activation_result.get('activated_count')}"
|
||
)
|
||
|
||
trigger_target = target_element or tab
|
||
trigger_result: Dict[str, Any] = {}
|
||
trigger_targets = [trigger_target]
|
||
if target_element is not None and target_element is not tab:
|
||
trigger_targets.append(tab)
|
||
|
||
for attempt in range(3):
|
||
if effective_stop_checker():
|
||
return []
|
||
if attempt > 0:
|
||
time.sleep(0.35)
|
||
media_extractor.activate_audio_trigger_surface(target_element or tab)
|
||
|
||
for current_target in trigger_targets:
|
||
trigger_result = media_extractor.trigger_audio_playback(current_target, image_config)
|
||
if bool(trigger_result.get("clicked")):
|
||
trigger_result["attempt"] = attempt + 1
|
||
trigger_result["target_scope"] = "tab" if current_target is tab else "message"
|
||
break
|
||
if bool(trigger_result.get("clicked")):
|
||
break
|
||
|
||
if not bool(trigger_result.get("clicked")):
|
||
logger.debug(
|
||
"页面音频捕获未触发: "
|
||
f"selector={trigger_result.get('selector_used')!r}, "
|
||
f"labels={trigger_result.get('labels_used')!r}, "
|
||
f"candidate_count={trigger_result.get('candidate_count')}, "
|
||
f"debug_matches={trigger_result.get('debug_matches')!r}, "
|
||
f"nearby_candidates={trigger_result.get('nearby_candidates')!r}, "
|
||
f"visible_button_samples={trigger_result.get('visible_button_samples')!r}"
|
||
)
|
||
return []
|
||
|
||
logger.debug(
|
||
"页面音频捕获已触发: "
|
||
f"attempt={trigger_result.get('attempt')}, "
|
||
f"scope={trigger_result.get('target_scope')}, "
|
||
f"text={trigger_result.get('text')!r}, "
|
||
f"score={trigger_result.get('score')}"
|
||
)
|
||
|
||
effective_response_text_hint = str(response_text_hint or "").strip()
|
||
if target_element is not None and len(effective_response_text_hint) <= 8:
|
||
try:
|
||
dom_response_text = str(
|
||
as_element(target_element).run_js(
|
||
"""
|
||
const text = (this.innerText || this.textContent || "").trim();
|
||
return text;
|
||
"""
|
||
) or ""
|
||
).strip()
|
||
except Exception:
|
||
dom_response_text = ""
|
||
if len(dom_response_text) > len(effective_response_text_hint):
|
||
logger.debug(
|
||
"页面音频捕获文本提示已回退 to 当前回复 DOM 文本: "
|
||
f"old_len={len(effective_response_text_hint)}, new_len={len(dom_response_text)}"
|
||
)
|
||
effective_response_text_hint = dom_response_text
|
||
|
||
if network_capture_enabled:
|
||
network_audio_items = media_extractor.capture_network_audio(
|
||
tab=tab,
|
||
config=image_config,
|
||
stop_checker=effective_stop_checker,
|
||
response_text_hint=effective_response_text_hint,
|
||
)
|
||
if network_audio_items:
|
||
logger.debug(f"网络音频捕获成功: {len(network_audio_items)} 项")
|
||
return network_audio_items
|
||
|
||
max_wait = float(
|
||
image_config.get("audio_capture_max_wait_seconds")
|
||
or max(8.0, float(image_config.get("load_timeout_seconds", 5.0) or 5.0) * 1.8)
|
||
)
|
||
max_wait = self._media_audio_capture_timeout(image_config, max_wait)
|
||
hint_text = str(effective_response_text_hint or "").strip()
|
||
if hint_text:
|
||
try:
|
||
chars_per_second = max(
|
||
1.0,
|
||
float(image_config.get("audio_capture_estimated_chars_per_second") or 4.8),
|
||
)
|
||
min_wait = max(
|
||
1.0,
|
||
float(image_config.get("audio_capture_min_wait_seconds") or 2.0),
|
||
)
|
||
padding_seconds = max(
|
||
0.0,
|
||
float(image_config.get("audio_capture_wait_padding_seconds") or 1.2),
|
||
)
|
||
hard_cap = max(
|
||
min_wait,
|
||
float(image_config.get("audio_capture_hard_max_wait_seconds") or max_wait),
|
||
)
|
||
fallback_wait = max_wait
|
||
estimated_wait = min_wait + (len(hint_text) / chars_per_second) + padding_seconds
|
||
max_wait = min(max(min_wait, estimated_wait), hard_cap)
|
||
logger.debug(
|
||
"页面音频捕获动态等待窗口: "
|
||
f"text_len={len(hint_text)}, max_wait={max_wait:.1f}s, "
|
||
f"fallback_wait={fallback_wait:.1f}s, "
|
||
f"chars_per_second={chars_per_second:.1f}"
|
||
)
|
||
except (TypeError, ValueError):
|
||
pass
|
||
poll_interval = max(0.1, float(image_config.get("audio_capture_poll_seconds") or 0.25))
|
||
silence_seconds = max(0.4, float(image_config.get("audio_capture_silence_seconds") or 1.2))
|
||
activity_silence_seconds = max(
|
||
0.2,
|
||
float(image_config.get("audio_capture_activity_silence_seconds") or 0.65),
|
||
)
|
||
terminal_settle_seconds = max(
|
||
0.0,
|
||
float(image_config.get("audio_capture_terminal_settle_seconds") or 0.35),
|
||
)
|
||
deadline = time.time() + max_wait
|
||
has_seen_data = False
|
||
has_seen_activity = False
|
||
terminal_deadline = 0.0
|
||
|
||
while time.time() < deadline and not effective_stop_checker():
|
||
time.sleep(poll_interval)
|
||
status = media_extractor.get_page_audio_capture_status(tab)
|
||
if not isinstance(status, dict):
|
||
continue
|
||
|
||
if bool(status.get("has_data")):
|
||
has_seen_data = True
|
||
|
||
active_recordings = int(status.get("active_recordings") or 0)
|
||
last_data_at_ms = int(status.get("last_data_at") or 0)
|
||
last_active_at_ms = int(status.get("last_active_at") or 0)
|
||
playing_media_elements = int(status.get("playing_media_elements") or 0)
|
||
terminal_playback_elements = int(status.get("terminal_playback_elements") or 0)
|
||
if last_active_at_ms:
|
||
has_seen_activity = True
|
||
if has_seen_data and active_recordings <= 0:
|
||
break
|
||
if has_seen_data and playing_media_elements <= 0 and terminal_playback_elements > 0:
|
||
if terminal_deadline <= 0:
|
||
terminal_deadline = time.time() + terminal_settle_seconds
|
||
elif time.time() >= terminal_deadline:
|
||
break
|
||
else:
|
||
terminal_deadline = 0.0
|
||
|
||
if has_seen_activity and last_active_at_ms:
|
||
activity_silence_elapsed = (time.time() * 1000.0 - last_active_at_ms) / 1000.0
|
||
if activity_silence_elapsed >= activity_silence_seconds:
|
||
break
|
||
|
||
if has_seen_data and last_data_at_ms:
|
||
silence_elapsed = (time.time() * 1000.0 - last_data_at_ms) / 1000.0
|
||
if silence_elapsed >= silence_seconds:
|
||
break
|
||
|
||
if effective_stop_checker():
|
||
return []
|
||
|
||
captured_items = media_extractor.export_page_audio_capture(tab, image_config)
|
||
if captured_items:
|
||
status = media_extractor.get_page_audio_capture_status(tab)
|
||
if isinstance(status, dict):
|
||
logger.debug(
|
||
"页面播放音频捕获成功: "
|
||
f"{len(captured_items)} 项, "
|
||
f"version={status.get('version')}, "
|
||
f"tracked_media={status.get('tracked_media_elements')}, "
|
||
f"tracked_web_audio={status.get('tracked_web_audio')}, "
|
||
f"active={status.get('active_recordings')}, "
|
||
f"chunks={status.get('total_chunks')}, "
|
||
f"last_data_at={status.get('last_data_at')}, "
|
||
f"last_active_at={status.get('last_active_at')}, "
|
||
f"peak_rms={status.get('peak_rms')}"
|
||
)
|
||
if not captured_items:
|
||
status = media_extractor.get_page_audio_capture_status(tab)
|
||
if isinstance(status, dict):
|
||
logger.debug(
|
||
"页面音频捕获未导出到任何音频数据: "
|
||
f"version={status.get('version')}, "
|
||
f"tracked_media={status.get('tracked_media_elements')}, "
|
||
f"tracked_web_audio={status.get('tracked_web_audio')}, "
|
||
f"active={status.get('active_recordings')}, "
|
||
f"chunks={status.get('total_chunks')}, "
|
||
f"has_data={status.get('has_data')}, "
|
||
f"errors={status.get('recent_errors')}, "
|
||
f"events={status.get('recent_events')}"
|
||
)
|
||
else:
|
||
logger.debug("页面音频捕获未导出到任何音频数据")
|
||
if network_capture_enabled:
|
||
try:
|
||
media_extractor.capture_network_audio(
|
||
tab=tab,
|
||
config=image_config,
|
||
stop_checker=lambda: True,
|
||
response_text_hint=effective_response_text_hint,
|
||
)
|
||
except Exception:
|
||
pass
|
||
try:
|
||
browser_tts_items = media_extractor.capture_browser_tts_fallback(
|
||
tab=tab,
|
||
config=image_config,
|
||
stop_checker=effective_stop_checker,
|
||
response_text_hint=effective_response_text_hint,
|
||
)
|
||
except Exception as exc:
|
||
logger.debug(f"浏览器 TTS 兜底失败(已忽略): {exc}")
|
||
browser_tts_items = []
|
||
if browser_tts_items:
|
||
return browser_tts_items
|
||
return captured_items
|
||
|
||
def _resolve_media_ref(self, media_item: Dict) -> str:
|
||
ref = str(media_item.get("url") or media_item.get("data_uri") or "").strip()
|
||
if not ref:
|
||
return ""
|
||
|
||
if ref.startswith("/") and not ref.startswith("//"):
|
||
public_base = os.getenv("PUBLIC_BASE_URL", "").strip()
|
||
if public_base:
|
||
return public_base.rstrip("/") + ref
|
||
return f"http://{AppConfig.get_host()}:{AppConfig.get_port()}{ref}"
|
||
|
||
return ref
|
||
|
||
def _build_media_markdown_block(self, media_items: List[Dict]) -> str:
|
||
image_blocks = []
|
||
audio_lines = []
|
||
video_lines = []
|
||
|
||
for item in media_items or []:
|
||
ref = self._resolve_media_ref(item)
|
||
if not ref:
|
||
continue
|
||
|
||
media_type = str(item.get("media_type") or "image").lower()
|
||
if media_type == "image":
|
||
image_blocks.append(f"\n\n")
|
||
continue
|
||
|
||
label = item.get("label") or item.get("mime") or ""
|
||
label_suffix = f" - {label}" if label else ""
|
||
if media_type == "audio":
|
||
audio_lines.append(f"[audio_{len(audio_lines)}]({ref}){label_suffix}")
|
||
elif media_type == "video":
|
||
video_lines.append(f"[video_{len(video_lines)}]({ref}){label_suffix}")
|
||
|
||
blocks = []
|
||
if image_blocks:
|
||
blocks.append("".join(image_blocks))
|
||
if audio_lines:
|
||
blocks.append("\n\n" + "\n".join(audio_lines))
|
||
if video_lines:
|
||
blocks.append("\n\n" + "\n".join(video_lines))
|
||
|
||
if not blocks:
|
||
return ""
|
||
|
||
return "".join(blocks) + "\n\n"
|
||
|
||
def _prepare_media_items_for_response(self, media_items: List[Dict]) -> List[Dict]:
|
||
result = []
|
||
for item in media_items or []:
|
||
entry = dict(item)
|
||
ref = self._resolve_media_ref(item)
|
||
if entry.get("kind") == "url":
|
||
entry["url"] = ref or entry.get("url")
|
||
if ref and entry.get("kind") != "url":
|
||
entry["kind"] = "url"
|
||
entry["url"] = ref
|
||
entry.pop("data_uri", None)
|
||
entry.pop("local_path", None)
|
||
result.append(entry)
|
||
return result
|
||
|
||
def _persist_remote_media_urls_to_local(
|
||
self,
|
||
media_items: List[Dict],
|
||
tab=None,
|
||
max_size_mb: int = 10,
|
||
image_config: Optional[Dict[str, Any]] = None,
|
||
) -> List[Dict]:
|
||
"""Download remote audio/video URLs to local files so downstream clients get stable local URLs."""
|
||
if not media_items or tab is None:
|
||
return media_items
|
||
|
||
save_dir = Path("download_images")
|
||
save_dir.mkdir(exist_ok=True)
|
||
max_bytes = max(1, int(max_size_mb)) * 1024 * 1024
|
||
|
||
# S8/S9:扩展名统一由 app.utils.media_safety 白名单决定(不再使用本地 ext_map / URL 后缀)
|
||
|
||
cookies_dict, headers = build_image_download_request_context(tab, accept="*/*")
|
||
partition_key = build_image_download_partition(cookies_dict, headers)
|
||
|
||
result = []
|
||
is_image_gen = bool(
|
||
(image_config or {}).get("arena_image_generation")
|
||
or (image_config or {}).get("_arena_image_generation_active")
|
||
or (image_config or {}).get("require_localized_images")
|
||
)
|
||
bg_timeout = float(
|
||
(image_config or {}).get("background_download_wait_seconds")
|
||
or (image_config or {}).get("download_wait_seconds")
|
||
or (3.5 if is_image_gen else 1.0)
|
||
)
|
||
for item in media_items:
|
||
filepath = None
|
||
if item.get("kind") != "url":
|
||
result.append(item)
|
||
continue
|
||
|
||
media_type = str(item.get("media_type") or "").lower()
|
||
if media_type not in {"audio", "video"}:
|
||
if media_type == "image":
|
||
target_url = normalize_remote_image_url(item.get("url"))
|
||
target_partition = (
|
||
get_image_download_partition(target_url, cookies_dict, headers)
|
||
if target_url
|
||
else partition_key
|
||
)
|
||
background_result = background_image_downloader.get_download_result(
|
||
target_url,
|
||
wait=True,
|
||
timeout=bg_timeout,
|
||
partition_key=target_partition,
|
||
) if target_url else None
|
||
localized_item = self._localize_image_item_from_background_result(item, background_result)
|
||
result.append(localized_item if localized_item is not None else item)
|
||
continue
|
||
result.append(item)
|
||
continue
|
||
|
||
url = str(item.get("url") or "").strip()
|
||
if not (url.startswith("http://") or url.startswith("https://")):
|
||
result.append(item)
|
||
continue
|
||
|
||
try:
|
||
credential_origin_url = headers.get("Referer")
|
||
target_host = str(urlparse(url).hostname or "").lower().rstrip(".")
|
||
referer_host = str(
|
||
urlparse(str(headers.get("Referer") or "")).hostname or ""
|
||
).lower().rstrip(".")
|
||
if (
|
||
target_host == "contribution.usercontent.google.com"
|
||
and referer_host == "gemini.google.com"
|
||
):
|
||
credential_origin_url = url
|
||
response = get_public_remote_resource(
|
||
url,
|
||
cookies=cookies_dict,
|
||
headers=headers,
|
||
credential_origin_url=credential_origin_url,
|
||
timeout=(8, 30),
|
||
stream=True,
|
||
)
|
||
except Exception as exc:
|
||
logger.warning(f"{media_type} 下载失败,保留远程链接: {exc}")
|
||
result.append(item)
|
||
continue
|
||
|
||
try:
|
||
if response.status_code != 200:
|
||
logger.warning(f"{media_type} 下载失败,HTTP {response.status_code}")
|
||
result.append(item)
|
||
continue
|
||
|
||
content_type = str(response.headers.get("Content-Type") or "").split(";", 1)[0].strip().lower()
|
||
if media_type == "audio" and "audio" not in content_type:
|
||
logger.warning(f"音频下载返回非音频类型,保留远程链接: {content_type or 'unknown'}")
|
||
result.append(item)
|
||
continue
|
||
if media_type == "video" and "video" not in content_type:
|
||
logger.warning(f"视频下载返回非视频类型,保留远程链接: {content_type or 'unknown'}")
|
||
result.append(item)
|
||
continue
|
||
|
||
# S9:扩展名只来自白名单;远端 URL 的 .html/.svg 后缀永远不会被采用
|
||
ext = choose_media_extension(media_type, content_type, url)
|
||
if not ext:
|
||
logger.warning(f"{media_type} 响应类型不安全,保留远程链接: {content_type or 'unknown'}")
|
||
result.append(item)
|
||
continue
|
||
|
||
filename = f"{int(time.time())}_{uuid.uuid4().hex[:8]}{ext}"
|
||
filepath = save_dir / filename
|
||
|
||
written = 0
|
||
sniffed = False
|
||
with filepath.open("wb") as handle:
|
||
for chunk in response.iter_content(chunk_size=1024 * 64):
|
||
if not chunk:
|
||
continue
|
||
if not sniffed:
|
||
sniffed = True
|
||
# 谎报 Content-Type(如 video/custom)但实际是 HTML/SVG 的响应在写盘前拦截
|
||
if looks_like_active_content(chunk[:1024]):
|
||
raise ValueError("unsafe_media_content")
|
||
written += len(chunk)
|
||
if written > max_bytes:
|
||
raise ValueError(f"media_too_large:{written}")
|
||
handle.write(chunk)
|
||
|
||
new_item = dict(item)
|
||
new_item["url"] = f"/media/{filename}"
|
||
new_item["mime"] = content_type or item.get("mime")
|
||
new_item["byte_size"] = written
|
||
new_item["source"] = "local_file"
|
||
new_item["local_path"] = str(filepath)
|
||
result.append(new_item)
|
||
logger.debug(f"✅ {media_type} 已保存到本地: {filename} ({written} bytes)")
|
||
except Exception as exc:
|
||
logger.warning(f"{media_type} 落盘失败,保留远程链接: {exc}")
|
||
try:
|
||
if filepath is not None and filepath.exists():
|
||
filepath.unlink()
|
||
except Exception:
|
||
pass
|
||
result.append(item)
|
||
finally:
|
||
response.close()
|
||
|
||
return result
|
||
|
||
@staticmethod
|
||
def _remote_image_urls_match(left: str, right: str) -> bool:
|
||
left_url = str(left or "").strip()
|
||
right_url = str(right or "").strip()
|
||
if not left_url or not right_url:
|
||
return False
|
||
if left_url == right_url:
|
||
return True
|
||
|
||
try:
|
||
left_parsed = urlparse(left_url)
|
||
right_parsed = urlparse(right_url)
|
||
except Exception:
|
||
return False
|
||
return bool(
|
||
left_parsed.scheme.lower() == right_parsed.scheme.lower()
|
||
and left_parsed.netloc.lower() == right_parsed.netloc.lower()
|
||
and left_parsed.path
|
||
and right_parsed.path
|
||
and left_parsed.path == right_parsed.path
|
||
)
|
||
|
||
def _wait_for_image_element_ready(
|
||
self,
|
||
img_ele,
|
||
target_url: str,
|
||
*,
|
||
timeout_seconds: float,
|
||
poll_seconds: float = 0.2,
|
||
) -> tuple[bool, Dict[str, Any]]:
|
||
"""Wait until the matched DOM image has decoded dimensions before screenshotting it."""
|
||
try:
|
||
timeout = max(0.0, float(timeout_seconds or 0.0))
|
||
except (TypeError, ValueError):
|
||
timeout = 0.0
|
||
try:
|
||
poll = max(0.05, float(poll_seconds or 0.2))
|
||
except (TypeError, ValueError):
|
||
poll = 0.2
|
||
|
||
deadline = time.monotonic() + timeout
|
||
last_state: Dict[str, Any] = {}
|
||
while True:
|
||
try:
|
||
raw_state = as_element(img_ele).run_js(
|
||
"""
|
||
return (() => {
|
||
const src = String(
|
||
this.currentSrc
|
||
|| this.getAttribute('src')
|
||
|| this.src
|
||
|| ''
|
||
).trim();
|
||
return {
|
||
src,
|
||
complete: Boolean(this.complete),
|
||
natural_width: Number(this.naturalWidth || 0),
|
||
natural_height: Number(this.naturalHeight || 0),
|
||
};
|
||
})();
|
||
"""
|
||
)
|
||
last_state = dict(raw_state) if isinstance(raw_state, dict) else {}
|
||
except Exception as exc:
|
||
last_state = {"error": str(exc)[:120]}
|
||
|
||
source_matches = self._remote_image_urls_match(
|
||
target_url,
|
||
str(last_state.get("src") or ""),
|
||
)
|
||
try:
|
||
natural_width = max(0, int(float(last_state.get("natural_width") or 0)))
|
||
except (TypeError, ValueError):
|
||
natural_width = 0
|
||
try:
|
||
natural_height = max(0, int(float(last_state.get("natural_height") or 0)))
|
||
except (TypeError, ValueError):
|
||
natural_height = 0
|
||
last_state["natural_width"] = natural_width
|
||
last_state["natural_height"] = natural_height
|
||
last_state["source_matches"] = source_matches
|
||
if (
|
||
source_matches
|
||
and bool(last_state.get("complete"))
|
||
and natural_width > 0
|
||
and natural_height > 0
|
||
):
|
||
return True, last_state
|
||
|
||
remaining = deadline - time.monotonic()
|
||
if remaining <= 0:
|
||
return False, last_state
|
||
time.sleep(min(poll, remaining))
|
||
|
||
@staticmethod
|
||
def _expand_image_element_for_screenshot(img_ele) -> bool:
|
||
"""Temporarily render an image at its decoded dimensions for screenshot fallback."""
|
||
try:
|
||
return bool(as_element(img_ele).run_js(
|
||
"""
|
||
return (() => {
|
||
const image = this;
|
||
const width = Number(image.naturalWidth || 0);
|
||
const height = Number(image.naturalHeight || 0);
|
||
if (!(width > 0 && height > 0)) return false;
|
||
|
||
const marker = '__universalProxyScreenshotStyle';
|
||
if (!Object.prototype.hasOwnProperty.call(image, marker)) {
|
||
Object.defineProperty(image, marker, {
|
||
configurable: true,
|
||
value: {
|
||
hadStyle: image.hasAttribute('style'),
|
||
style: image.getAttribute('style') || '',
|
||
},
|
||
});
|
||
}
|
||
|
||
image.style.setProperty('display', 'block', 'important');
|
||
image.style.setProperty('box-sizing', 'content-box', 'important');
|
||
image.style.setProperty('width', `${width}px`, 'important');
|
||
image.style.setProperty('height', `${height}px`, 'important');
|
||
image.style.setProperty('min-width', '0', 'important');
|
||
image.style.setProperty('min-height', '0', 'important');
|
||
image.style.setProperty('max-width', 'none', 'important');
|
||
image.style.setProperty('max-height', 'none', 'important');
|
||
image.style.setProperty('object-fit', 'fill', 'important');
|
||
image.style.setProperty('position', 'fixed', 'important');
|
||
image.style.setProperty('left', '0', 'important');
|
||
image.style.setProperty('top', '0', 'important');
|
||
image.style.setProperty('right', 'auto', 'important');
|
||
image.style.setProperty('bottom', 'auto', 'important');
|
||
image.style.setProperty('z-index', '2147483647', 'important');
|
||
image.style.setProperty('transform', 'none', 'important');
|
||
image.style.setProperty('clip-path', 'none', 'important');
|
||
return true;
|
||
})();
|
||
"""
|
||
))
|
||
except Exception as exc:
|
||
logger.debug(f"展开图片以供完整截图失败(忽略): {exc}")
|
||
return False
|
||
|
||
@staticmethod
|
||
def _restore_image_element_after_screenshot(img_ele) -> None:
|
||
try:
|
||
as_element(img_ele).run_js(
|
||
"""
|
||
(() => {
|
||
const image = this;
|
||
const marker = '__universalProxyScreenshotStyle';
|
||
const snapshot = image[marker];
|
||
if (!snapshot) return;
|
||
if (snapshot.hadStyle) {
|
||
image.setAttribute('style', snapshot.style);
|
||
} else {
|
||
image.removeAttribute('style');
|
||
}
|
||
try {
|
||
delete image[marker];
|
||
} catch {
|
||
image[marker] = undefined;
|
||
}
|
||
})();
|
||
"""
|
||
)
|
||
except Exception as exc:
|
||
logger.debug(f"恢复图片截图样式失败(忽略): {exc}")
|
||
|
||
def _try_screenshot_images_to_local(self, tab, last_element, images: List[Dict], image_config: Dict = None) -> List[Dict]:
|
||
"""
|
||
优先下载图片(更精准),下载失败才截图。
|
||
基于实测 API:img_ele.attr('src'), page.cookies(), get_screenshot(path)
|
||
"""
|
||
if not images:
|
||
return images
|
||
|
||
image_config = image_config or {}
|
||
baseline_keys = {
|
||
self._media_reference_key(reference)
|
||
for reference in (image_config.get("request_baseline_references") or [])
|
||
}
|
||
baseline_keys.discard("")
|
||
|
||
# Only images absent from the pre-submit baseline may be localized. This
|
||
# protects against an uploaded input or history item being mislabeled as
|
||
# stream output before any download or screenshot fallback can use it.
|
||
candidates = []
|
||
excluded_baseline_images = 0
|
||
for item in images:
|
||
candidate = dict(item or {})
|
||
reference = candidate.get("url") or candidate.get("data_uri") or ""
|
||
if (
|
||
str(candidate.get("media_type") or "image").strip().lower() == "image"
|
||
and self._media_reference_key(reference) in baseline_keys
|
||
):
|
||
excluded_baseline_images += 1
|
||
continue
|
||
candidates.append(candidate)
|
||
if excluded_baseline_images:
|
||
logger.warning(
|
||
"截图本地化拒绝发送前已存在的图片资源: "
|
||
f"count={excluded_baseline_images}"
|
||
)
|
||
|
||
remote_indexes = []
|
||
for idx, item in enumerate(candidates):
|
||
if str(item.get("kind") or "").strip().lower() != "url":
|
||
continue
|
||
url = str(item.get("url") or "").strip()
|
||
if url.startswith("http://") or url.startswith("https://"):
|
||
remote_indexes.append(idx)
|
||
|
||
if not remote_indexes:
|
||
return candidates
|
||
|
||
out_dir = Path("download_images")
|
||
out_dir.mkdir(exist_ok=True)
|
||
|
||
selector = image_config.get("selector", "img")
|
||
# S8:不再接受 image/svg+xml(与后台下载器一致),扩展名来自统一白名单
|
||
ext_map = dict(IMAGE_EXT_BY_MIME)
|
||
|
||
scoped_selector = str(selector or "img").strip() or "img"
|
||
# Arena 图片卡片在流结束后才会补上动画 class;此时严格的预设 selector
|
||
# 会短暂匹配不到已解码的 img。因此严格 selector 仅用于优先定位,通用 img
|
||
# 作为按 URL 精确匹配的回退候选,不能按位置直接使用。
|
||
candidate_selectors = [scoped_selector]
|
||
if scoped_selector != "img":
|
||
candidate_selectors.append("img")
|
||
reply_ele_entries = []
|
||
page_ele_entries = []
|
||
candidates_initially_scanned = False
|
||
baseline_token = str(image_config.get("request_baseline_token") or "")
|
||
baseline_property = str(image_config.get("request_baseline_property") or "")
|
||
exclude_existing_nodes = bool(image_config.get("request_baseline_exclude_existing_nodes"))
|
||
effective_stop_checker = getattr(self, "_should_stop_checker", lambda: False)
|
||
|
||
def _append_image_candidates(root, scope_name: str, target_list: List[Dict]) -> None:
|
||
if root is None:
|
||
return
|
||
seen_ids = {id(e["element"]) for e in target_list}
|
||
for candidate_selector in candidate_selectors:
|
||
try:
|
||
candidates = as_element(root).find_all(f"css:{candidate_selector}", timeout=0.5) or []
|
||
except Exception as exc:
|
||
logger.debug(
|
||
f"读取{scope_name}图片候选失败(忽略): "
|
||
f"selector={candidate_selector!r}, error={exc}"
|
||
)
|
||
continue
|
||
|
||
logger.debug(
|
||
f"图片定位:{scope_name}使用 '{candidate_selector}',"
|
||
f"找到 {len(candidates)} 个"
|
||
)
|
||
for ele in candidates:
|
||
if id(ele) in seen_ids:
|
||
continue
|
||
seen_ids.add(id(ele))
|
||
try:
|
||
baseline_probe = """
|
||
const propertyName = String(arguments[0] || '');
|
||
const baselineToken = String(arguments[1] || '');
|
||
const src = String(
|
||
this.currentSrc
|
||
|| this.getAttribute('src')
|
||
|| this.src
|
||
|| ''
|
||
).trim();
|
||
const baseline = propertyName ? this[propertyName] : null;
|
||
return {
|
||
src,
|
||
is_preexisting: Boolean(
|
||
propertyName
|
||
&& baselineToken
|
||
&& baseline
|
||
&& String(baseline.token || '') === baselineToken
|
||
),
|
||
baseline_reference: String(baseline && baseline.reference || ''),
|
||
};
|
||
"""
|
||
try:
|
||
entry_state = as_element(ele).run_js(
|
||
baseline_probe,
|
||
baseline_property,
|
||
baseline_token,
|
||
)
|
||
except TypeError:
|
||
# Some lightweight test and legacy element adapters only
|
||
# support a script without arguments. They cannot expose a
|
||
# baseline marker, so retain the strict URL match and treat
|
||
# the candidate as unmarked rather than failing extraction.
|
||
entry_state = {
|
||
"src": as_element(ele).run_js(
|
||
"""
|
||
return String(
|
||
this.currentSrc
|
||
|| this.getAttribute('src')
|
||
|| this.src
|
||
|| ''
|
||
).trim();
|
||
"""
|
||
),
|
||
}
|
||
if not isinstance(entry_state, dict):
|
||
entry_state = {}
|
||
img_src = str(
|
||
entry_state.get("src")
|
||
or ele.attr('src')
|
||
or ele.link
|
||
or ""
|
||
).strip()
|
||
except Exception as exc:
|
||
logger.debug(f"读取图片元素地址失败(忽略): {exc}")
|
||
img_src = ""
|
||
entry_state = {}
|
||
target_list.append({
|
||
"element": ele,
|
||
"src": img_src,
|
||
"is_preexisting": bool(entry_state.get("is_preexisting")),
|
||
"baseline_reference": str(entry_state.get("baseline_reference") or ""),
|
||
"used": False,
|
||
})
|
||
|
||
def _ensure_initial_candidates_scanned() -> None:
|
||
nonlocal candidates_initially_scanned
|
||
if not candidates_initially_scanned:
|
||
candidates_initially_scanned = True
|
||
_append_image_candidates(last_element, "当前回复", reply_ele_entries)
|
||
# Battle 模式下 last_element 可能指向另一侧回复。整页候选只用于后续 URL
|
||
# 精确匹配,不能作为位置兜底,否则会截到旧回复或另一侧的图片。
|
||
_append_image_candidates(tab, "整页", page_ele_entries)
|
||
|
||
cookies_dict, headers = build_image_download_request_context(tab)
|
||
is_image_gen = bool(
|
||
image_config.get("arena_image_generation")
|
||
or image_config.get("_arena_image_generation_active")
|
||
or image_config.get("require_localized_images")
|
||
)
|
||
prefetch_wait_seconds = float(
|
||
image_config.get("background_download_wait_seconds")
|
||
or image_config.get("download_wait_seconds")
|
||
or (3.5 if is_image_gen else 1.0)
|
||
)
|
||
try:
|
||
max_image_bytes = max(1, int(image_config.get("max_size_mb") or 10)) * 1024 * 1024
|
||
except (TypeError, ValueError):
|
||
max_image_bytes = 10 * 1024 * 1024
|
||
try:
|
||
screenshot_ready_wait_seconds = max(
|
||
0.0,
|
||
float(
|
||
image_config.get("screenshot_ready_wait_seconds")
|
||
if image_config.get("screenshot_ready_wait_seconds") is not None
|
||
else image_config.get("load_timeout_seconds", 5.0)
|
||
),
|
||
)
|
||
except (TypeError, ValueError):
|
||
screenshot_ready_wait_seconds = 5.0
|
||
try:
|
||
screenshot_ready_poll_seconds = max(
|
||
0.05,
|
||
float(image_config.get("screenshot_ready_poll_seconds") or 0.2),
|
||
)
|
||
except (TypeError, ValueError):
|
||
screenshot_ready_poll_seconds = 0.2
|
||
|
||
new_images = self._localize_images_with_background_cache(
|
||
candidates,
|
||
wait_seconds=max(0.0, prefetch_wait_seconds),
|
||
cookies=cookies_dict,
|
||
headers=headers,
|
||
)
|
||
|
||
def _claim_image_element(target_url: str):
|
||
for entry_list in (reply_ele_entries, page_ele_entries):
|
||
for entry in reversed(entry_list):
|
||
if entry["used"]:
|
||
continue
|
||
if not self._remote_image_urls_match(target_url, entry["src"]):
|
||
continue
|
||
if entry["is_preexisting"] and (
|
||
exclude_existing_nodes
|
||
or self._remote_image_urls_match(target_url, entry["baseline_reference"])
|
||
):
|
||
continue
|
||
entry["used"] = True
|
||
return entry["element"]
|
||
|
||
return None
|
||
|
||
def _wait_for_matching_image_element(target_url: str):
|
||
"""Wait for late-rendered images before giving up screenshot fallback."""
|
||
_ensure_initial_candidates_scanned()
|
||
matched = _claim_image_element(target_url)
|
||
if matched is not None:
|
||
return matched
|
||
|
||
deadline = time.monotonic() + screenshot_ready_wait_seconds
|
||
while time.monotonic() < deadline:
|
||
if effective_stop_checker():
|
||
break
|
||
_append_image_candidates(last_element, "当前回复(等待渲染)", reply_ele_entries)
|
||
matched = _claim_image_element(target_url)
|
||
if matched is not None:
|
||
return matched
|
||
_append_image_candidates(tab, "整页(等待渲染)", page_ele_entries)
|
||
matched = _claim_image_element(target_url)
|
||
if matched is not None:
|
||
return matched
|
||
time.sleep(min(screenshot_ready_poll_seconds, max(0.0, deadline - time.monotonic())))
|
||
return None
|
||
localized_count = 0
|
||
|
||
for target_index in reversed(remote_indexes):
|
||
if effective_stop_checker():
|
||
break
|
||
target_image = new_images[target_index]
|
||
target_url = normalize_remote_image_url(target_image.get("url"))
|
||
if not target_url:
|
||
continue
|
||
target_partition_key = get_image_download_partition(target_url, cookies_dict, headers)
|
||
|
||
background_result = background_image_downloader.get_download_result(
|
||
target_url,
|
||
wait=False,
|
||
partition_key=target_partition_key,
|
||
)
|
||
background_status = str((background_result or {}).get("status") or "").strip().lower()
|
||
if background_status in {"queued", "downloading"}:
|
||
if is_image_gen:
|
||
# 生图模式下,若后台下载仍在进行,给予同步阻塞等待(最多 prefetch_wait_seconds),确保图片落盘
|
||
background_result = background_image_downloader.get_download_result(
|
||
target_url,
|
||
wait=True,
|
||
timeout=prefetch_wait_seconds,
|
||
partition_key=target_partition_key,
|
||
)
|
||
background_status = str((background_result or {}).get("status") or "").strip().lower()
|
||
|
||
if background_status in {"queued", "downloading"}:
|
||
# The prefetch worker is deliberately independent from response
|
||
# rendering. Waiting here made the browser sit on a spinner for up
|
||
# to 60 seconds, then started a duplicate foreground download.
|
||
logger.debug(
|
||
f"图片后台下载仍在进行,保留远程地址并继续渲染: index={target_index}, "
|
||
f"status={background_status}"
|
||
)
|
||
continue
|
||
localized_item = self._localize_image_item_from_background_result(
|
||
target_image,
|
||
background_result,
|
||
)
|
||
if localized_item is not None:
|
||
new_images[target_index] = localized_item
|
||
localized_count += 1
|
||
continue
|
||
|
||
saved = False
|
||
saved_mime = None
|
||
|
||
base_name = f"{int(time.time())}_{uuid.uuid4().hex[:8]}"
|
||
ext = ".png"
|
||
filename = f"{base_name}{ext}"
|
||
out_path = out_dir / filename
|
||
|
||
response = None
|
||
download_started_at = time.monotonic()
|
||
try:
|
||
parsed_target_url = urlsplit(target_url)
|
||
target_label = f"{parsed_target_url.scheme}://{parsed_target_url.netloc}{parsed_target_url.path[:80]}"
|
||
logger.debug(f"尝试下载图片[{target_index}]: {target_label}")
|
||
response = get_public_remote_resource(
|
||
target_url,
|
||
cookies=cookies_dict,
|
||
headers=headers,
|
||
credential_origin_url=headers.get("Referer"),
|
||
timeout=(8, 15),
|
||
stream=True,
|
||
)
|
||
response_received_at = time.monotonic()
|
||
content_length_header = str(response.headers.get("Content-Length") or "").strip()
|
||
logger.debug(
|
||
f"图片下载响应[{target_index}]: status={response.status_code}, "
|
||
f"content_type={str(response.headers.get('Content-Type') or '').split(';', 1)[0].strip().lower() or 'unknown'}, "
|
||
f"content_length={content_length_header or 'unknown'}, "
|
||
f"response_wait={response_received_at - download_started_at:.3f}s"
|
||
)
|
||
|
||
if response.status_code == 200:
|
||
content_type = str(response.headers.get('Content-Type') or '').split(";", 1)[0].strip().lower()
|
||
if 'image' not in content_type or is_active_content_mime(content_type):
|
||
logger.debug(f"下载内容无效: type: {content_type or 'unknown'}")
|
||
else:
|
||
read_started_at = time.monotonic()
|
||
content = self._read_response_bytes_with_limit(response, max_image_bytes)
|
||
read_finished_at = time.monotonic()
|
||
|
||
if len(content) > 1000 and looks_like_active_content(content[:1024]):
|
||
# S8:Content-Type 声称图片但内容是 SVG/HTML 标记 → 不落盘,走截图回退
|
||
logger.debug(f"下载内容疑似活动文档,拒绝落盘: type: {content_type}")
|
||
elif len(content) > 1000:
|
||
ext = ext_map.get(content_type, ext)
|
||
filename = f"{base_name}{ext}"
|
||
out_path = out_dir / filename
|
||
out_path.write_bytes(content)
|
||
saved = True
|
||
saved_mime = content_type or None
|
||
background_image_downloader.register_downloaded_file(
|
||
target_url,
|
||
local_path=out_path,
|
||
accessible_url=f"/download_images/{filename}",
|
||
mime=content_type or None,
|
||
byte_size=len(content),
|
||
source="inline_download",
|
||
partition_key=target_partition_key,
|
||
)
|
||
logger.debug(
|
||
f"✅ 下载成功: {filename} ({len(content)} bytes), "
|
||
f"read={read_finished_at - read_started_at:.3f}s, "
|
||
f"write_and_register={time.monotonic() - read_finished_at:.3f}s, "
|
||
f"total={time.monotonic() - download_started_at:.3f}s"
|
||
)
|
||
else:
|
||
logger.debug(f"下载内容无效: {len(content)} bytes, type: {content_type}")
|
||
else:
|
||
logger.debug(f"下载失败: HTTP {response.status_code}")
|
||
except Exception as e:
|
||
logger.debug(
|
||
f"下载异常,将尝试截图: elapsed={time.monotonic() - download_started_at:.3f}s, "
|
||
f"error={str(e)[:100]}"
|
||
)
|
||
finally:
|
||
if response is not None:
|
||
try:
|
||
response.close()
|
||
except Exception:
|
||
pass
|
||
|
||
img_ele = _wait_for_matching_image_element(target_url) if not saved else None
|
||
if not saved and img_ele is not None:
|
||
image_ready, ready_state = self._wait_for_image_element_ready(
|
||
img_ele,
|
||
target_url,
|
||
timeout_seconds=screenshot_ready_wait_seconds,
|
||
poll_seconds=screenshot_ready_poll_seconds,
|
||
)
|
||
if not image_ready:
|
||
logger.warning(
|
||
f"图片[{target_index}] DOM 仍处于加载状态,跳过截图回退并保留远程链接:"
|
||
f"complete={bool(ready_state.get('complete'))}, "
|
||
f"natural={int(ready_state.get('natural_width') or 0)}x"
|
||
f"{int(ready_state.get('natural_height') or 0)}, "
|
||
f"source_matches={bool(ready_state.get('source_matches'))}"
|
||
)
|
||
else:
|
||
if out_path.suffix.lower() != ".png":
|
||
ext = ".png"
|
||
filename = f"{base_name}{ext}"
|
||
out_path = out_dir / filename
|
||
logger.debug(f"图片[{target_index}] 已确认加载完成,回退到截图方式")
|
||
expanded_for_screenshot = False
|
||
try:
|
||
expanded_for_screenshot = self._expand_image_element_for_screenshot(img_ele)
|
||
if expanded_for_screenshot:
|
||
logger.debug(f"图片[{target_index}] 已临时展开为自然尺寸后截图")
|
||
try:
|
||
img_ele.get_screenshot(str(out_path))
|
||
finally:
|
||
if expanded_for_screenshot:
|
||
self._restore_image_element_after_screenshot(img_ele)
|
||
|
||
still_ready, _ = self._wait_for_image_element_ready(
|
||
img_ele,
|
||
target_url,
|
||
timeout_seconds=0.0,
|
||
)
|
||
if not still_ready:
|
||
if out_path.exists():
|
||
out_path.unlink()
|
||
logger.warning(
|
||
f"图片[{target_index}] 截图期间 DOM 图片状态发生变化,已丢弃截图"
|
||
)
|
||
elif out_path.exists() and out_path.stat().st_size > 0:
|
||
saved = True
|
||
saved_mime = "image/png"
|
||
background_image_downloader.register_downloaded_file(
|
||
target_url,
|
||
local_path=out_path,
|
||
accessible_url=f"/download_images/{out_path.name}",
|
||
mime=saved_mime,
|
||
byte_size=int(out_path.stat().st_size),
|
||
source="screenshot_fallback",
|
||
partition_key=target_partition_key,
|
||
)
|
||
except Exception as e:
|
||
if out_path.exists():
|
||
try:
|
||
out_path.unlink()
|
||
except Exception:
|
||
pass
|
||
logger.warning(f"截图失败: {e}")
|
||
elif not saved:
|
||
logger.warning(
|
||
f"图片[{target_index}] 未找到与目标 URL 匹配的 DOM 节点,跳过截图回退"
|
||
)
|
||
|
||
if not saved:
|
||
logger.warning(f"图片[{target_index}] 保存失败:下载和截图均失败")
|
||
continue
|
||
|
||
local_url = f"/download_images/{out_path.name}"
|
||
new_images[target_index] = self._build_localized_image_item(
|
||
target_image,
|
||
out_path,
|
||
local_url,
|
||
mime=saved_mime,
|
||
byte_size=int(out_path.stat().st_size),
|
||
source="local_file",
|
||
)
|
||
localized_count += 1
|
||
|
||
if localized_count > 0:
|
||
logger.debug(f"✅ 图片本地化完成: {localized_count}/{len(remote_indexes)} 张")
|
||
|
||
return new_images
|
||
|
||
def _persist_data_uri_media_to_local(
|
||
self,
|
||
media_items: List[Dict],
|
||
*,
|
||
max_size_mb: int = 10,
|
||
) -> List[Dict]:
|
||
"""Persist extracted data-uri media so downstream Markdown can reuse the existing URL flow."""
|
||
if not media_items:
|
||
return media_items
|
||
|
||
save_dir = Path("download_images")
|
||
save_dir.mkdir(exist_ok=True)
|
||
try:
|
||
max_bytes = max(1, int(max_size_mb or 10)) * 1024 * 1024
|
||
except (TypeError, ValueError):
|
||
max_bytes = 10 * 1024 * 1024
|
||
|
||
ext_map = {**IMAGE_EXT_BY_MIME, **AUDIO_EXT_BY_MIME, **VIDEO_EXT_BY_MIME}
|
||
|
||
result = []
|
||
for item in media_items:
|
||
if item.get("kind") != "data_uri":
|
||
result.append(item)
|
||
continue
|
||
|
||
data_uri = str(item.get("data_uri") or "").strip()
|
||
if not data_uri.startswith("data:"):
|
||
result.append(item)
|
||
continue
|
||
|
||
try:
|
||
header, b64_data = data_uri.split(",", 1)
|
||
mime = header.split(";", 1)[0].split(":", 1)[1].lower()
|
||
if is_active_content_mime(mime):
|
||
# S8:data:image/svg+xml / text/html 等不落盘到同源目录
|
||
raise ValueError(f"unsafe_data_uri_mime:{mime}")
|
||
ext = ext_map.get(mime, ".png")
|
||
compact_b64 = "".join(str(b64_data or "").split())
|
||
padding = compact_b64.count("=")
|
||
estimated_size = max(0, (len(compact_b64) * 3) // 4 - padding)
|
||
if estimated_size > max_bytes:
|
||
raise ValueError(f"data_uri_too_large:{estimated_size}")
|
||
media_bytes = base64.b64decode(compact_b64)
|
||
if len(media_bytes) > max_bytes:
|
||
raise ValueError(f"data_uri_too_large:{len(media_bytes)}")
|
||
if looks_like_active_content(media_bytes[:1024]):
|
||
raise ValueError("unsafe_data_uri_content")
|
||
except (ValueError, IndexError, binascii.Error) as e:
|
||
logger.warning(f"data uri 解析失败,保留原媒体数据: {e}")
|
||
result.append(item)
|
||
continue
|
||
|
||
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
|
||
filename = f"{timestamp}_{uuid.uuid4().hex[:8]}{ext}"
|
||
filepath = save_dir / filename
|
||
|
||
try:
|
||
filepath.write_bytes(media_bytes)
|
||
if str(item.get("media_type") or "").strip().lower() == "audio":
|
||
logger.debug(f"data-uri 音频已落盘,准备追加尾静音: {filepath.name}")
|
||
self._append_audio_tail_silence(filepath, duration_seconds=0.3)
|
||
except Exception as e:
|
||
logger.warning(f"data uri 保存失败,保留原媒体数据: {e}")
|
||
result.append(item)
|
||
continue
|
||
|
||
new_item = dict(item)
|
||
new_item["kind"] = "url"
|
||
media_type = str(item.get("media_type") or "").strip().lower()
|
||
if media_type in {"audio", "video"}:
|
||
new_item["url"] = f"/media/{filename}"
|
||
else:
|
||
new_item["url"] = f"/download_images/{filename}"
|
||
new_item["data_uri"] = None
|
||
new_item["mime"] = mime
|
||
new_item["byte_size"] = len(media_bytes)
|
||
new_item["source"] = "local_file"
|
||
new_item["local_path"] = str(filepath)
|
||
result.append(new_item)
|
||
|
||
return result
|
||
|
||
def _dedupe_media_items(self, media_items: List[Dict]) -> List[Dict]:
|
||
result = []
|
||
seen = set()
|
||
for item in media_items or []:
|
||
media_type = str(item.get("media_type") or "")
|
||
ref = str(item.get("url") or item.get("data_uri") or "")
|
||
key = (media_type, ref)
|
||
if key in seen:
|
||
continue
|
||
seen.add(key)
|
||
result.append(item)
|
||
return result
|
||
|
||
def _retry_pending_media_from_response_text(
|
||
self,
|
||
session: TabSession,
|
||
full_content: str,
|
||
preset_name: Optional[str] = None,
|
||
stop_checker: Optional[Callable[[], bool]] = None,
|
||
) -> List[Dict]:
|
||
hint_text = str(full_content or "").strip()
|
||
if not hint_text:
|
||
return []
|
||
|
||
try:
|
||
domain = str(getattr(session, "current_domain", "") or "").strip()
|
||
if not domain:
|
||
tab = getattr(session, "tab", None)
|
||
current_url = str(getattr(tab, "url", "") or "").strip()
|
||
domain = extract_remote_site_domain(current_url)
|
||
if not domain:
|
||
return []
|
||
|
||
config_engine = self._get_config_engine()
|
||
effective_preset_name = preset_name if preset_name is not None else session.preset_name
|
||
tab = getattr(session, "tab", None)
|
||
if tab is None:
|
||
return []
|
||
|
||
site_config = config_engine.get_site_config(
|
||
domain,
|
||
html_content=lambda: getattr(tab, "html", ""),
|
||
preset_name=effective_preset_name,
|
||
)
|
||
if not site_config:
|
||
return []
|
||
|
||
image_config = site_config.get("image_extraction", {}) or {}
|
||
modalities = image_config.get("modalities") or {}
|
||
image_extraction_enabled = bool(image_config.get("enabled", False)) or any(
|
||
is_modality_enabled(modalities, key) for key in ("image", "audio", "video")
|
||
)
|
||
if not image_extraction_enabled:
|
||
return []
|
||
|
||
pending_hit = any(
|
||
self._is_pending_media_text(hint_text, image_config, media_type)
|
||
for media_type in ("audio", "video")
|
||
)
|
||
if not pending_hit:
|
||
return []
|
||
|
||
selectors = site_config.get("selectors", {}) or {}
|
||
result_selector = str(selectors.get("result_container", "") or "").strip()
|
||
if not result_selector:
|
||
return []
|
||
|
||
extractor = config_engine.get_site_extractor(domain, preset_name=effective_preset_name)
|
||
|
||
logger.debug(f"[{session.id}] 非流式响应命中占位媒体文本,触发二次提取等待")
|
||
return self._extract_media_after_stream(
|
||
tab=tab,
|
||
extractor=extractor,
|
||
image_config=image_config,
|
||
result_selector=result_selector,
|
||
message_wrapper_selector=str(selectors.get("message_wrapper", "") or "").strip(),
|
||
stop_checker=stop_checker,
|
||
response_text_hint=hint_text,
|
||
)
|
||
except Exception as e:
|
||
logger.warning(f"[{session.id}] 非流式二次媒体提取失败(已忽略): {e}")
|
||
return []
|
||
|
||
def _download_url_images(self, images: List[Dict], tab=None) -> List[Dict]:
|
||
"""
|
||
在浏览器内通过 Canvas 压缩图片,保存到本地并返回可访问 URL
|
||
|
||
流程:
|
||
1. 浏览器 Canvas 压缩 → base64
|
||
2. 后端解码 → 保存到 download_images/
|
||
3. 返回 /download_images/xxx.jpg URL
|
||
"""
|
||
import base64
|
||
import uuid
|
||
from pathlib import Path
|
||
from datetime import datetime
|
||
|
||
result = []
|
||
|
||
# 确保目录存在
|
||
save_dir = Path("download_images")
|
||
save_dir.mkdir(exist_ok=True)
|
||
canvas_image_max_size = AppConfig.get_canvas_image_max_size()
|
||
max_canvas_image_bytes = max(1, canvas_image_max_size * canvas_image_max_size * 4)
|
||
|
||
for img in images:
|
||
if img.get('kind') != 'url':
|
||
result.append(img)
|
||
continue
|
||
|
||
url = img.get('url')
|
||
if not url:
|
||
result.append(img)
|
||
continue
|
||
|
||
if not tab:
|
||
result.append(img)
|
||
continue
|
||
|
||
try:
|
||
# 🔑 在浏览器中用 Canvas 加载并压缩图片
|
||
js_code = """
|
||
(async function(imageUrl, configuredMaxSize) {
|
||
return new Promise((resolve) => {
|
||
const img = new Image();
|
||
img.crossOrigin = 'anonymous';
|
||
|
||
img.onload = function() {
|
||
try {
|
||
// 限制最大尺寸
|
||
const MAX_SIZE = Math.max(1, Math.floor(Number(configuredMaxSize) || 1024));
|
||
let width = img.naturalWidth;
|
||
let height = img.naturalHeight;
|
||
|
||
if (width > MAX_SIZE || height > MAX_SIZE) {
|
||
if (width > height) {
|
||
height = Math.round(height * MAX_SIZE / width);
|
||
width = MAX_SIZE;
|
||
} else {
|
||
width = Math.round(width * MAX_SIZE / height);
|
||
height = MAX_SIZE;
|
||
}
|
||
}
|
||
|
||
const canvas = document.createElement('canvas');
|
||
canvas.width = width;
|
||
canvas.height = height;
|
||
|
||
const ctx = canvas.getContext('2d');
|
||
ctx.drawImage(img, 0, 0, width, height);
|
||
|
||
// 转为 JPEG
|
||
const dataUri = canvas.toDataURL('image/jpeg', 0.85);
|
||
|
||
resolve({
|
||
success: true,
|
||
dataUri: dataUri,
|
||
width: width,
|
||
height: height
|
||
});
|
||
} catch (e) {
|
||
resolve({ success: false, error: 'Canvas: ' + e.message });
|
||
}
|
||
};
|
||
|
||
img.onerror = function() {
|
||
resolve({ success: false, error: 'Load failed' });
|
||
};
|
||
|
||
setTimeout(() => resolve({ success: false, error: 'Timeout' }), 15000);
|
||
img.src = imageUrl;
|
||
});
|
||
})(arguments[0], arguments[1]);
|
||
"""
|
||
|
||
# ===== PROBE: 验证 run_js 是否等待 Promise,并检查图片/Fetch 可用性 =====
|
||
probe_js = """
|
||
(function(u){
|
||
try {
|
||
// 1) 最小同步返回测试
|
||
const sync_ok = { ok: true, type: typeof u, head: String(u).slice(0, 40) };
|
||
|
||
// 2) Promise 返回测试(不返回大对象)
|
||
const promise_test = Promise.resolve({ promise_ok: true });
|
||
|
||
// 3) 图片加载测试(不画 canvas,不导 dataUri,避免大返回)
|
||
const img_test = new Promise((resolve) => {
|
||
const img = new Image();
|
||
let done = false;
|
||
|
||
img.onload = () => {
|
||
if (done) return;
|
||
done = true;
|
||
resolve({ img_onload: true, w: img.naturalWidth, h: img.naturalHeight });
|
||
};
|
||
img.onerror = () => {
|
||
if (done) return;
|
||
done = true;
|
||
resolve({ img_onerror: true });
|
||
};
|
||
|
||
setTimeout(() => {
|
||
if (done) return;
|
||
done = true;
|
||
resolve({ img_timeout: true });
|
||
}, 6000);
|
||
|
||
img.src = u;
|
||
});
|
||
|
||
// 4) fetch 测试(只返回 status,不读 body)
|
||
const fetch_test = (async () => {
|
||
try {
|
||
const r = await fetch(u, { method: 'GET' });
|
||
return { fetch_ok: true, status: r.status, redirected: r.redirected };
|
||
} catch (e) {
|
||
return { fetch_error: String(e).slice(0, 120) };
|
||
}
|
||
})();
|
||
|
||
// 关键:返回一个对象,包含同步字段 + Promise 字段
|
||
// 如果 run_js 不等待 Promise,你只能拿到一个“未解析”的东西或 None
|
||
return Promise.all([promise_test, img_test, fetch_test]).then(all => {
|
||
return {
|
||
sync: sync_ok,
|
||
promise: all[0],
|
||
img: all[1],
|
||
fetch: all[2]
|
||
};
|
||
});
|
||
} catch(e) {
|
||
return { probe_exception: String(e).slice(0, 160) };
|
||
}
|
||
})(arguments[0]);
|
||
"""
|
||
|
||
probe_result = driver_for_tab(tab).run_js(probe_js, url)
|
||
logger.info(f"[PROBE_JS] probe_result_type={type(probe_result).__name__}, value={str(probe_result)[:500]}")
|
||
|
||
download_result = driver_for_tab(tab).run_js(js_code, url, canvas_image_max_size)
|
||
|
||
logger.info(f"[PROBE_JS] canvas_result_type={type(download_result).__name__}, value={str(download_result)[:300]}")
|
||
if download_result and download_result.get('success'):
|
||
data_uri = download_result['dataUri']
|
||
|
||
# 解析 base64
|
||
# 格式: data:image/jpeg;base64,/9j/4AAQSkZJRg...
|
||
if ',' in data_uri:
|
||
header, b64_data = data_uri.split(',', 1)
|
||
mime = 'image/jpeg'
|
||
if 'png' in header:
|
||
mime = 'image/png'
|
||
ext = '.png'
|
||
else:
|
||
ext = '.jpg'
|
||
|
||
# 解码并保存
|
||
compact_b64 = "".join(str(b64_data or "").split())
|
||
estimated_size = self._estimate_base64_decoded_size(compact_b64)
|
||
if estimated_size > max_canvas_image_bytes:
|
||
raise ValueError(f"canvas_image_too_large:{estimated_size}")
|
||
image_bytes = base64.b64decode(compact_b64)
|
||
if len(image_bytes) > max_canvas_image_bytes:
|
||
raise ValueError(f"canvas_image_too_large:{len(image_bytes)}")
|
||
|
||
# 生成唯一文件名
|
||
timestamp = datetime.now().strftime('%Y%m%d_%H%M%S')
|
||
unique_id = uuid.uuid4().hex[:8]
|
||
filename = f"{timestamp}_{unique_id}{ext}"
|
||
filepath = save_dir / filename
|
||
|
||
# 写入文件
|
||
with open(filepath, 'wb') as f:
|
||
f.write(image_bytes)
|
||
|
||
# 构建可访问的 URL
|
||
accessible_url = f"/download_images/{filename}"
|
||
|
||
new_img = img.copy()
|
||
new_img['kind'] = 'url'
|
||
new_img['url'] = accessible_url
|
||
new_img['data_uri'] = None
|
||
new_img['mime'] = mime
|
||
new_img['width'] = download_result['width']
|
||
new_img['height'] = download_result['height']
|
||
new_img['byte_size'] = len(image_bytes)
|
||
new_img['source'] = 'local_file'
|
||
new_img['local_path'] = str(filepath)
|
||
|
||
result.append(new_img)
|
||
logger.info(f"✅ 图片已保存: {filename} ({len(image_bytes)} bytes)")
|
||
continue
|
||
|
||
error_msg = download_result.get('error', 'Unknown') if download_result else 'No result'
|
||
logger.warning(f"⚠️ 图片处理失败: {error_msg}")
|
||
|
||
except Exception as e:
|
||
logger.warning(f"⚠️ 图片保存异常: {str(e)[:100]}")
|
||
|
||
# 失败时保留原 URL
|
||
result.append(img)
|
||
|
||
return result
|