""" app/core/background_image_downloader.py 线程安全的后台图片下载器: - 复用主线程提取的 cookies / headers 发起异步下载 - 直接落盘到 download_images/ - 维护 原始 URL -> 本地文件 元数据缓存 """ import hashlib import mimetypes import threading import time import uuid from collections import OrderedDict from concurrent.futures import ThreadPoolExecutor from pathlib import Path from typing import Any, Dict, Optional, Tuple from urllib.parse import urlparse, urlsplit, urlunsplit import requests from app.core.config import logger from app.utils.remote_resource import ( get_public_remote_resource, normalize_remote_http_url, remote_url_origin, ) DEFAULT_IMAGE_ACCEPT = "image/avif,image/webp,image/apng,image/*,*/*;q=0.8" REMOTE_IMAGE_URL_TRAILING_WRAPPERS = ")]}" REMOTE_IMAGE_URL_TRAILING_SENTENCE_PUNCTUATION = ".,;:!?" REMOTE_IMAGE_URL_PATH_EXTENSIONS = (".png", ".jpg", ".jpeg", ".webp", ".gif", ".bmp", ".svg", ".avif") _IMAGE_CONTENT_TYPE_EXT_MAP = { "image/jpeg": ".jpg", "image/jpg": ".jpg", "image/png": ".png", "image/webp": ".webp", "image/gif": ".gif", "image/bmp": ".bmp", "image/avif": ".avif", } _SAFE_IMAGE_EXTENSIONS = frozenset(_IMAGE_CONTENT_TYPE_EXT_MAP.values()) _PENDING_DOWNLOAD_STATUSES = {"queued", "downloading"} def _strip_remote_image_url_trailing_punctuation(url: str) -> str: text = str(url or "").strip() if not text: return "" suffix_chars = REMOTE_IMAGE_URL_TRAILING_WRAPPERS + REMOTE_IMAGE_URL_TRAILING_SENTENCE_PUNCTUATION suffix_start = len(text) while suffix_start > 0 and text[suffix_start - 1] in suffix_chars: suffix_start -= 1 if suffix_start == len(text): return text candidate = text[:suffix_start].rstrip() suffix = text[suffix_start:] if not candidate: return text if all(ch in REMOTE_IMAGE_URL_TRAILING_WRAPPERS for ch in suffix): return candidate try: parsed = urlsplit(candidate) except Exception: return text if parsed.query or parsed.fragment: return text if parsed.path.lower().endswith(REMOTE_IMAGE_URL_PATH_EXTENSIONS): return candidate return text def normalize_remote_image_url(url: str) -> str: normalized = _strip_remote_image_url_trailing_punctuation(url) return normalize_remote_http_url(normalized) def extract_tab_cookies(tab): cookie_jar = requests.cookies.RequestsCookieJar() if tab is None: return cookie_jar source_url = str(getattr(tab, "url", "") or "") source_host = str(urlsplit(source_url).hostname or "").strip().lower().rstrip(".") try: cookies_list = tab.cookies() except Exception as exc: logger.debug(f"后台图片下载读取 cookies 失败(忽略): {exc}") return cookie_jar if not cookies_list: return cookie_jar for cookie in cookies_list: if isinstance(cookie, dict) and "name" in cookie and "value" in cookie: domain = str(cookie.get("domain") or source_host).strip().lower() path = str(cookie.get("path") or "/").strip() or "/" if not domain: continue cookie_jar.set( str(cookie["name"]), str(cookie["value"]), domain=domain, path=path, secure=bool(cookie.get("secure", False)), ) return cookie_jar def build_image_request_headers(tab, accept: str = DEFAULT_IMAGE_ACCEPT) -> Dict[str, str]: return { "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36", "Referer": str(getattr(tab, "url", "") or "") if tab is not None else "", "Accept": str(accept or DEFAULT_IMAGE_ACCEPT), } def build_image_download_request_context( tab, accept: str = DEFAULT_IMAGE_ACCEPT, ) -> Tuple[Any, Dict[str, str]]: return extract_tab_cookies(tab), build_image_request_headers(tab, accept=accept) def build_image_download_partition(cookies: Any, headers: Optional[Dict[str, str]] = None) -> str: source_origin = remote_url_origin((headers or {}).get("Referer")) or "no-origin" cookie_parts = [] try: for cookie in cookies or []: cookie_parts.append( "\x1f".join(( str(getattr(cookie, "domain", "") or ""), str(getattr(cookie, "path", "") or "/"), str(getattr(cookie, "name", "") or ""), str(getattr(cookie, "value", "") or ""), )) ) except Exception: cookie_parts = [repr(cookies)] if source_origin == "no-origin" and not cookie_parts: return "public" material = source_origin + "\x1e" + "\x1e".join(sorted(cookie_parts)) return hashlib.sha256(material.encode("utf-8", errors="replace")).hexdigest()[:24] def get_image_download_partition( url: Any, cookies: Any, headers: Optional[Dict[str, str]] = None, ) -> str: """Use one cache entry for signed public object-store URLs across browser views.""" normalized = normalize_remote_image_url(url) try: parsed = urlsplit(normalized) host = str(parsed.hostname or "").lower().rstrip(".") query = str(parsed.query or "").lower() except Exception: host = "" query = "" if host.endswith(".r2.cloudflarestorage.com") and "x-amz-signature=" in query: return "signed-public" return build_image_download_partition(cookies, headers) class BackgroundImageDownloader: """线程安全的后台图片下载器。""" def __init__( self, save_dir: str | Path = "download_images", *, max_workers: int = 4, min_bytes: int = 1000, max_bytes: int = 10 * 1024 * 1024, max_entries: int = 1000, max_pending: int = 100, max_age_seconds: float = 24 * 60 * 60, max_total_bytes: int = 1024 * 1024 * 1024, ): self._save_dir = Path(save_dir) self._executor = ThreadPoolExecutor( max_workers=max(1, int(max_workers or 1)), thread_name_prefix="bg-image", ) self._lock = threading.RLock() self._entries: OrderedDict[str, Dict[str, Any]] = OrderedDict() self._min_bytes = max(1, int(min_bytes or 1)) self._max_bytes = max(self._min_bytes, int(max_bytes or (10 * 1024 * 1024))) self._max_entries = max(1, int(max_entries or 1)) self._max_pending = max(1, int(max_pending or 1)) self._max_age_seconds = max(60.0, float(max_age_seconds or (24 * 60 * 60))) self._max_total_bytes = max(self._max_bytes, int(max_total_bytes or (1024 * 1024 * 1024))) self._pending_count = 0 self._shutdown = False self._last_disk_cleanup_at = 0.0 def start_download( self, url: str, *, cookies: Any = None, headers: Optional[Dict[str, str]] = None, partition_key: str = "", max_bytes: Optional[int] = None, ) -> Dict[str, Any]: normalized = normalize_remote_image_url(url) if not normalized: return {} partition = str( partition_key or get_image_download_partition(normalized, cookies, headers) ).strip() or "public" cache_key = self._cache_key(normalized, partition) effective_max_bytes = self._max_bytes if max_bytes is not None: try: effective_max_bytes = max(self._min_bytes, int(max_bytes)) except (TypeError, ValueError): effective_max_bytes = self._max_bytes with self._lock: if self._shutdown: return {} self._cleanup_disk_if_due_locked() entry = self._entries.get(cache_key) if entry is not None: self._entries.move_to_end(cache_key) if entry and str(entry.get("status") or "") == "done": local_path = Path(str(entry.get("local_path") or "")) if local_path.exists(): return self._snapshot_entry(entry) entry = None if entry and str(entry.get("status") or "") in {"queued", "downloading"}: return self._snapshot_entry(entry) if self._pending_count >= self._max_pending: logger.warning( f"后台图片下载队列已达上限,跳过预取: pending={self._pending_count}, limit={self._max_pending}" ) return {} entry = { "url": normalized, "partition_key": partition, "local_path": None, "accessible_url": None, "mime": None, "byte_size": None, "error": None, "max_bytes": effective_max_bytes, "started_at": time.time(), "updated_at": time.time(), "_event": threading.Event(), } self._set_entry_status_locked(entry, "queued") self._entries[cache_key] = entry self._entries.move_to_end(cache_key) self._prune_entries_locked() try: entry["_future"] = self._executor.submit( self._download_worker, cache_key, normalized, cookies, dict(headers or {}), effective_max_bytes, ) except RuntimeError: self._set_entry_status_locked(entry, "failed") entry["error"] = "downloader_shutdown" event = entry.get("_event") if isinstance(event, threading.Event): event.set() return {} return self._snapshot_entry(entry) def get_download_result( self, url: str, *, wait: bool = False, timeout: Optional[float] = None, partition_key: str = "", ) -> Optional[Dict[str, Any]]: normalized = normalize_remote_image_url(url) if not normalized: return None cache_key = self._cache_key(normalized, str(partition_key or "public")) with self._lock: entry = self._entries.get(cache_key) if entry is None: return None self._entries.move_to_end(cache_key) event = entry.get("_event") status = str(entry.get("status") or "") event_pending = isinstance(event, threading.Event) and not event.is_set() if wait and isinstance(event, threading.Event) and ( status in {"queued", "downloading"} or event_pending ): try: event.wait(None if timeout is None else max(0.0, float(timeout))) except Exception: pass with self._lock: latest = self._entries.get(cache_key) if latest is None: return None self._entries.move_to_end(cache_key) return self._snapshot_entry(latest) def register_downloaded_file( self, url: str, *, local_path: str | Path, accessible_url: Optional[str] = None, mime: Optional[str] = None, byte_size: Optional[int] = None, source: str = "local_file", partition_key: str = "", ) -> Optional[Dict[str, Any]]: normalized = normalize_remote_image_url(url) if not normalized: return None path_obj = Path(local_path) if not path_obj.exists(): return None if accessible_url is None: accessible_url = f"/download_images/{path_obj.name}" if mime is None: mime = self._guess_mime_type(path_obj) if byte_size is None: try: byte_size = int(path_obj.stat().st_size) except Exception: byte_size = None partition = str(partition_key or "public").strip() or "public" cache_key = self._cache_key(normalized, partition) with self._lock: entry = self._entries.get(cache_key) or { "url": normalized, "partition_key": partition, "_event": threading.Event(), "started_at": time.time(), } entry.update({ "local_path": str(path_obj), "accessible_url": str(accessible_url or ""), "mime": mime, "byte_size": byte_size, "error": None, "source": str(source or "local_file"), "updated_at": time.time(), }) self._set_entry_status_locked(entry, "done") event = entry.get("_event") if not isinstance(event, threading.Event): event = threading.Event() entry["_event"] = event event.set() self._entries[cache_key] = entry self._entries.move_to_end(cache_key) self._prune_entries_locked() return self._snapshot_entry(entry) def _download_worker( self, cache_key: str, url: str, cookies: Any, headers: Dict[str, str], max_bytes: int, ) -> None: if self._is_shutdown(): return response = None temp_path: Optional[Path] = None final_path: Optional[Path] = None started_at = time.monotonic() download_id = hashlib.sha256(cache_key.encode("utf-8", errors="replace")).hexdigest()[:12] self._update_entry(cache_key, url, status="downloading", error=None) def _close_response() -> None: if response is not None: try: response.close() except Exception: pass try: self._save_dir.mkdir(parents=True, exist_ok=True) logger.debug(f"[IMG_DOWNLOAD] start id={download_id}") response = get_public_remote_resource( url, cookies=cookies, headers=headers or None, credential_origin_url=headers.get("Referer"), timeout=(8, 20), stream=True, ) response_at = time.monotonic() if response.status_code != 200: raise ValueError(f"http_{response.status_code}") content_type = str(response.headers.get("Content-Type") or "").split(";", 1)[0].strip().lower() if content_type and not content_type.startswith("image/"): raise ValueError(f"invalid_content_type:{content_type}") path_ext = Path(urlparse(url).path or "").suffix.lower() if content_type == "image/svg+xml" or (not content_type and path_ext == ".svg"): raise ValueError("unsafe_image_type:image/svg+xml") content_length = str(response.headers.get("Content-Length") or "").strip() if content_length: try: expected_size = int(content_length) except ValueError: expected_size = 0 if expected_size > max_bytes: raise ValueError(f"image_too_large:{expected_size}") logger.debug( f"[IMG_DOWNLOAD] headers id={download_id}, status={response.status_code}, " f"content_length={content_length or 'unknown'}, " f"response_wait={response_at - started_at:.3f}s" ) ext = self._pick_extension(content_type, url) filename = f"{int(time.time())}_{uuid.uuid4().hex[:8]}{ext}" temp_path = self._save_dir / f"{filename}.part" final_path = self._save_dir / filename written = 0 first_chunk_at: Optional[float] = None with temp_path.open("wb") as handle: for chunk in response.iter_content(chunk_size=1024 * 64): if self._is_shutdown(): raise RuntimeError("downloader_shutdown") if not chunk: continue if first_chunk_at is None: first_chunk_at = time.monotonic() written += len(chunk) if written > max_bytes: raise ValueError(f"image_too_large:{written}") handle.write(chunk) read_finished_at = time.monotonic() if self._is_shutdown(): raise RuntimeError("downloader_shutdown") if written < self._min_bytes: raise ValueError(f"image_too_small:{written}") try: with temp_path.open("rb") as handle: detected_ext = self._detect_safe_image_extension(handle.read(32)) except Exception: detected_ext = None if not detected_ext: raise ValueError("invalid_image_payload") if detected_ext != ext: final_path = self._save_dir / f"{Path(filename).stem}{detected_ext}" filename = final_path.name temp_path.replace(final_path) accessible_url = f"/download_images/{filename}" _close_response() self._update_entry( cache_key, url, status="done", local_path=str(final_path), accessible_url=accessible_url, mime=content_type or self._guess_mime_type(final_path), byte_size=written, error=None, source="background_download", ) if not self._is_shutdown(): first_byte_wait = ( f"{first_chunk_at - response_at:.3f}s" if first_chunk_at is not None else "none" ) logger.debug( f"后台图片下载完成: {filename} ({written} bytes), " f"id={download_id}, first_byte_wait={first_byte_wait}, " f"read={read_finished_at - response_at:.3f}s, " f"total={read_finished_at - started_at:.3f}s" ) except Exception as exc: try: if temp_path is not None and temp_path.exists(): temp_path.unlink() except Exception: pass _close_response() self._update_entry(cache_key, url, status="failed", error=str(exc)) if not self._is_shutdown(): logger.debug( f"后台图片下载失败(忽略): id={download_id}, " f"elapsed={time.monotonic() - started_at:.3f}s, error={str(exc)[:160]}" ) finally: _close_response() def _update_entry(self, cache_key: str, url: str, **changes: Any) -> None: normalized = normalize_remote_image_url(url) if not normalized: return with self._lock: entry = self._entries.get(cache_key) if self._shutdown and entry is None: return if self._shutdown and str(changes.get("status") or "") not in {"failed"}: return if entry is None: entry = { "url": normalized, "_event": threading.Event(), "started_at": time.time(), } self._entries[cache_key] = entry next_status = changes.get("status") if "status" in changes else None if "status" in changes: changes = {key: value for key, value in changes.items() if key != "status"} entry.update(changes) if next_status is not None: self._set_entry_status_locked(entry, next_status) entry["updated_at"] = time.time() self._entries.move_to_end(cache_key) event = entry.get("_event") if not isinstance(event, threading.Event): event = threading.Event() entry["_event"] = event status = str(entry.get("status") or "") if status in {"done", "failed"}: event.set() self._prune_entries_locked() def _set_entry_status_locked(self, entry: Dict[str, Any], status: Any) -> None: old_status = str((entry or {}).get("status") or "") next_status = str(status or "") if old_status != next_status: old_pending = old_status in _PENDING_DOWNLOAD_STATUSES next_pending = next_status in _PENDING_DOWNLOAD_STATUSES if old_pending and not next_pending: self._pending_count = max(0, self._pending_count - 1) elif next_pending and not old_pending: self._pending_count += 1 entry["status"] = next_status def _prune_entries_locked(self) -> None: now = time.time() expired_keys = [ key for key, entry in self._entries.items() if str((entry or {}).get("status") or "") in {"done", "failed"} and now - float((entry or {}).get("updated_at") or (entry or {}).get("started_at") or now) > self._max_age_seconds ] for key in expired_keys: self._remove_entry_locked(key) overflow = len(self._entries) - self._max_entries if overflow > 0: keys_to_remove = [] for key, entry in self._entries.items(): if len(keys_to_remove) >= overflow: break status = str((entry or {}).get("status") or "") if status in {"queued", "downloading"}: continue keys_to_remove.append(key) for key in keys_to_remove: self._remove_entry_locked(key) total_bytes = sum( max(0, int((entry or {}).get("byte_size") or 0)) for entry in self._entries.values() if str((entry or {}).get("status") or "") == "done" ) if total_bytes > self._max_total_bytes: for key, entry in list(self._entries.items()): if total_bytes <= self._max_total_bytes: break if str((entry or {}).get("status") or "") != "done": continue total_bytes -= max(0, int((entry or {}).get("byte_size") or 0)) self._remove_entry_locked(key) def _remove_entry_locked(self, key: str) -> None: entry = self._entries.pop(key, None) if not isinstance(entry, dict): return local_path = str(entry.get("local_path") or "").strip() if not local_path: return if any( str((other or {}).get("local_path") or "").strip() == local_path for other in self._entries.values() ): return self._unlink_managed_file(local_path) def _unlink_managed_file(self, local_path: str | Path) -> None: try: root = self._save_dir.resolve() target = Path(local_path).resolve() target.relative_to(root) if target.is_file(): target.unlink(missing_ok=True) except (OSError, ValueError): return def _cleanup_disk_if_due_locked(self) -> None: now = time.time() if now - self._last_disk_cleanup_at < 300: return self._last_disk_cleanup_at = now cutoff = now - self._max_age_seconds try: for path in self._save_dir.iterdir(): if not path.is_file(): continue try: if path.stat().st_mtime < cutoff: path.unlink(missing_ok=True) except OSError: continue except OSError: return def shutdown(self) -> None: with self._lock: if self._shutdown: return self._shutdown = True for entry in self._entries.values(): status = str((entry or {}).get("status") or "") if status not in {"queued", "downloading"}: continue self._set_entry_status_locked(entry, "failed") entry["error"] = "downloader_shutdown" entry["updated_at"] = time.time() event = entry.get("_event") if isinstance(event, threading.Event): event.set() self._executor.shutdown(wait=False, cancel_futures=True) def _is_shutdown(self) -> bool: with self._lock: return bool(self._shutdown) def _snapshot_entry(self, entry: Dict[str, Any]) -> Dict[str, Any]: if not isinstance(entry, dict): return {} return { key: value for key, value in entry.items() if not str(key).startswith("_") } @staticmethod def _cache_key(url: str, partition_key: str) -> str: return f"{str(partition_key or 'public').strip() or 'public'}\x1f{url}" @staticmethod def _guess_mime_type(path_obj: Path) -> Optional[str]: try: guessed, _ = mimetypes.guess_type(path_obj.name) return guessed except Exception: return None @staticmethod def _pick_extension(content_type: str, url: str) -> str: ext = _IMAGE_CONTENT_TYPE_EXT_MAP.get(str(content_type or "").lower()) if ext: return ext path_ext = Path(urlparse(url).path or "").suffix.lower() if path_ext in _SAFE_IMAGE_EXTENSIONS: return path_ext return ".png" @staticmethod def _detect_safe_image_extension(header: bytes) -> Optional[str]: data = bytes(header or b"") if data.startswith(b"\x89PNG\r\n\x1a\n"): return ".png" if data.startswith(b"\xff\xd8\xff"): return ".jpg" if data.startswith((b"GIF87a", b"GIF89a")): return ".gif" if data.startswith(b"BM"): return ".bmp" if len(data) >= 12 and data.startswith(b"RIFF") and data[8:12] == b"WEBP": return ".webp" if len(data) >= 16 and data[4:8] == b"ftyp" and data[8:12] in {b"avif", b"avis"}: return ".avif" return None background_image_downloader = BackgroundImageDownloader() __all__ = [ "BackgroundImageDownloader", "DEFAULT_IMAGE_ACCEPT", "background_image_downloader", "build_image_download_partition", "get_image_download_partition", "build_image_download_request_context", "build_image_request_headers", "extract_tab_cookies", "normalize_remote_image_url", ]