Files
universal-web-api/app/core/background_image_downloader.py
T
2026-11-02 22:54:14 +08:00

748 lines
27 KiB
Python

"""
app/core/background_image_downloader.py
线程安全的后台图片下载器:
- 复用主线程提取的 cookies / headers 发起异步下载
- 直接落盘到 download_images/
- 维护 原始 URL -> 本地文件 元数据缓存
"""
import hashlib
import mimetypes
import threading
import time
import uuid
from collections import OrderedDict
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path
from typing import Any, Dict, Optional, Tuple
from urllib.parse import urlparse, urlsplit, urlunsplit
import requests
from app.core.config import logger
from app.utils.remote_resource import (
get_public_remote_resource,
normalize_remote_http_url,
remote_url_origin,
)
DEFAULT_IMAGE_ACCEPT = "image/avif,image/webp,image/apng,image/*,*/*;q=0.8"
REMOTE_IMAGE_URL_TRAILING_WRAPPERS = ")]}"
REMOTE_IMAGE_URL_TRAILING_SENTENCE_PUNCTUATION = ".,;:!?"
REMOTE_IMAGE_URL_PATH_EXTENSIONS = (".png", ".jpg", ".jpeg", ".webp", ".gif", ".bmp", ".svg", ".avif")
_IMAGE_CONTENT_TYPE_EXT_MAP = {
"image/jpeg": ".jpg",
"image/jpg": ".jpg",
"image/png": ".png",
"image/webp": ".webp",
"image/gif": ".gif",
"image/bmp": ".bmp",
"image/avif": ".avif",
}
_SAFE_IMAGE_EXTENSIONS = frozenset(_IMAGE_CONTENT_TYPE_EXT_MAP.values())
_PENDING_DOWNLOAD_STATUSES = {"queued", "downloading"}
def _strip_remote_image_url_trailing_punctuation(url: str) -> str:
text = str(url or "").strip()
if not text:
return ""
suffix_chars = REMOTE_IMAGE_URL_TRAILING_WRAPPERS + REMOTE_IMAGE_URL_TRAILING_SENTENCE_PUNCTUATION
suffix_start = len(text)
while suffix_start > 0 and text[suffix_start - 1] in suffix_chars:
suffix_start -= 1
if suffix_start == len(text):
return text
candidate = text[:suffix_start].rstrip()
suffix = text[suffix_start:]
if not candidate:
return text
if all(ch in REMOTE_IMAGE_URL_TRAILING_WRAPPERS for ch in suffix):
return candidate
try:
parsed = urlsplit(candidate)
except Exception:
return text
if parsed.query or parsed.fragment:
return text
if parsed.path.lower().endswith(REMOTE_IMAGE_URL_PATH_EXTENSIONS):
return candidate
return text
def normalize_remote_image_url(url: str) -> str:
normalized = _strip_remote_image_url_trailing_punctuation(url)
return normalize_remote_http_url(normalized)
def extract_tab_cookies(tab):
cookie_jar = requests.cookies.RequestsCookieJar()
if tab is None:
return cookie_jar
source_url = str(getattr(tab, "url", "") or "")
source_host = str(urlsplit(source_url).hostname or "").strip().lower().rstrip(".")
try:
cookies_list = tab.cookies()
except Exception as exc:
logger.debug(f"后台图片下载读取 cookies 失败(忽略): {exc}")
return cookie_jar
if not cookies_list:
return cookie_jar
for cookie in cookies_list:
if isinstance(cookie, dict) and "name" in cookie and "value" in cookie:
domain = str(cookie.get("domain") or source_host).strip().lower()
path = str(cookie.get("path") or "/").strip() or "/"
if not domain:
continue
cookie_jar.set(
str(cookie["name"]),
str(cookie["value"]),
domain=domain,
path=path,
secure=bool(cookie.get("secure", False)),
)
return cookie_jar
def build_image_request_headers(tab, accept: str = DEFAULT_IMAGE_ACCEPT) -> Dict[str, str]:
return {
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
"Referer": str(getattr(tab, "url", "") or "") if tab is not None else "",
"Accept": str(accept or DEFAULT_IMAGE_ACCEPT),
}
def build_image_download_request_context(
tab,
accept: str = DEFAULT_IMAGE_ACCEPT,
) -> Tuple[Any, Dict[str, str]]:
return extract_tab_cookies(tab), build_image_request_headers(tab, accept=accept)
def build_image_download_partition(cookies: Any, headers: Optional[Dict[str, str]] = None) -> str:
source_origin = remote_url_origin((headers or {}).get("Referer")) or "no-origin"
cookie_parts = []
try:
for cookie in cookies or []:
cookie_parts.append(
"\x1f".join((
str(getattr(cookie, "domain", "") or ""),
str(getattr(cookie, "path", "") or "/"),
str(getattr(cookie, "name", "") or ""),
str(getattr(cookie, "value", "") or ""),
))
)
except Exception:
cookie_parts = [repr(cookies)]
if source_origin == "no-origin" and not cookie_parts:
return "public"
material = source_origin + "\x1e" + "\x1e".join(sorted(cookie_parts))
return hashlib.sha256(material.encode("utf-8", errors="replace")).hexdigest()[:24]
def get_image_download_partition(
url: Any,
cookies: Any,
headers: Optional[Dict[str, str]] = None,
) -> str:
"""Use one cache entry for signed public object-store URLs across browser views."""
normalized = normalize_remote_image_url(url)
try:
parsed = urlsplit(normalized)
host = str(parsed.hostname or "").lower().rstrip(".")
query = str(parsed.query or "").lower()
except Exception:
host = ""
query = ""
if host.endswith(".r2.cloudflarestorage.com") and "x-amz-signature=" in query:
return "signed-public"
return build_image_download_partition(cookies, headers)
class BackgroundImageDownloader:
"""线程安全的后台图片下载器。"""
def __init__(
self,
save_dir: str | Path = "download_images",
*,
max_workers: int = 4,
min_bytes: int = 1000,
max_bytes: int = 10 * 1024 * 1024,
max_entries: int = 1000,
max_pending: int = 100,
max_age_seconds: float = 24 * 60 * 60,
max_total_bytes: int = 1024 * 1024 * 1024,
):
self._save_dir = Path(save_dir)
self._executor = ThreadPoolExecutor(
max_workers=max(1, int(max_workers or 1)),
thread_name_prefix="bg-image",
)
self._lock = threading.RLock()
self._entries: OrderedDict[str, Dict[str, Any]] = OrderedDict()
self._min_bytes = max(1, int(min_bytes or 1))
self._max_bytes = max(self._min_bytes, int(max_bytes or (10 * 1024 * 1024)))
self._max_entries = max(1, int(max_entries or 1))
self._max_pending = max(1, int(max_pending or 1))
self._max_age_seconds = max(60.0, float(max_age_seconds or (24 * 60 * 60)))
self._max_total_bytes = max(self._max_bytes, int(max_total_bytes or (1024 * 1024 * 1024)))
self._pending_count = 0
self._shutdown = False
self._last_disk_cleanup_at = 0.0
def start_download(
self,
url: str,
*,
cookies: Any = None,
headers: Optional[Dict[str, str]] = None,
partition_key: str = "",
max_bytes: Optional[int] = None,
) -> Dict[str, Any]:
normalized = normalize_remote_image_url(url)
if not normalized:
return {}
partition = str(
partition_key or get_image_download_partition(normalized, cookies, headers)
).strip() or "public"
cache_key = self._cache_key(normalized, partition)
effective_max_bytes = self._max_bytes
if max_bytes is not None:
try:
effective_max_bytes = max(self._min_bytes, int(max_bytes))
except (TypeError, ValueError):
effective_max_bytes = self._max_bytes
with self._lock:
if self._shutdown:
return {}
self._cleanup_disk_if_due_locked()
entry = self._entries.get(cache_key)
if entry is not None:
self._entries.move_to_end(cache_key)
if entry and str(entry.get("status") or "") == "done":
local_path = Path(str(entry.get("local_path") or ""))
if local_path.exists():
return self._snapshot_entry(entry)
entry = None
if entry and str(entry.get("status") or "") in {"queued", "downloading"}:
return self._snapshot_entry(entry)
if self._pending_count >= self._max_pending:
logger.warning(
f"后台图片下载队列已达上限,跳过预取: pending={self._pending_count}, limit={self._max_pending}"
)
return {}
entry = {
"url": normalized,
"partition_key": partition,
"local_path": None,
"accessible_url": None,
"mime": None,
"byte_size": None,
"error": None,
"max_bytes": effective_max_bytes,
"started_at": time.time(),
"updated_at": time.time(),
"_event": threading.Event(),
}
self._set_entry_status_locked(entry, "queued")
self._entries[cache_key] = entry
self._entries.move_to_end(cache_key)
self._prune_entries_locked()
try:
entry["_future"] = self._executor.submit(
self._download_worker,
cache_key,
normalized,
cookies,
dict(headers or {}),
effective_max_bytes,
)
except RuntimeError:
self._set_entry_status_locked(entry, "failed")
entry["error"] = "downloader_shutdown"
event = entry.get("_event")
if isinstance(event, threading.Event):
event.set()
return {}
return self._snapshot_entry(entry)
def get_download_result(
self,
url: str,
*,
wait: bool = False,
timeout: Optional[float] = None,
partition_key: str = "",
) -> Optional[Dict[str, Any]]:
normalized = normalize_remote_image_url(url)
if not normalized:
return None
cache_key = self._cache_key(normalized, str(partition_key or "public"))
with self._lock:
entry = self._entries.get(cache_key)
if entry is None:
return None
self._entries.move_to_end(cache_key)
event = entry.get("_event")
status = str(entry.get("status") or "")
event_pending = isinstance(event, threading.Event) and not event.is_set()
if wait and isinstance(event, threading.Event) and (
status in {"queued", "downloading"} or event_pending
):
try:
event.wait(None if timeout is None else max(0.0, float(timeout)))
except Exception:
pass
with self._lock:
latest = self._entries.get(cache_key)
if latest is None:
return None
self._entries.move_to_end(cache_key)
return self._snapshot_entry(latest)
def register_downloaded_file(
self,
url: str,
*,
local_path: str | Path,
accessible_url: Optional[str] = None,
mime: Optional[str] = None,
byte_size: Optional[int] = None,
source: str = "local_file",
partition_key: str = "",
) -> Optional[Dict[str, Any]]:
normalized = normalize_remote_image_url(url)
if not normalized:
return None
path_obj = Path(local_path)
if not path_obj.exists():
return None
if accessible_url is None:
accessible_url = f"/download_images/{path_obj.name}"
if mime is None:
mime = self._guess_mime_type(path_obj)
if byte_size is None:
try:
byte_size = int(path_obj.stat().st_size)
except Exception:
byte_size = None
partition = str(partition_key or "public").strip() or "public"
cache_key = self._cache_key(normalized, partition)
with self._lock:
entry = self._entries.get(cache_key) or {
"url": normalized,
"partition_key": partition,
"_event": threading.Event(),
"started_at": time.time(),
}
entry.update({
"local_path": str(path_obj),
"accessible_url": str(accessible_url or ""),
"mime": mime,
"byte_size": byte_size,
"error": None,
"source": str(source or "local_file"),
"updated_at": time.time(),
})
self._set_entry_status_locked(entry, "done")
event = entry.get("_event")
if not isinstance(event, threading.Event):
event = threading.Event()
entry["_event"] = event
event.set()
self._entries[cache_key] = entry
self._entries.move_to_end(cache_key)
self._prune_entries_locked()
return self._snapshot_entry(entry)
def _download_worker(
self,
cache_key: str,
url: str,
cookies: Any,
headers: Dict[str, str],
max_bytes: int,
) -> None:
if self._is_shutdown():
return
response = None
temp_path: Optional[Path] = None
final_path: Optional[Path] = None
started_at = time.monotonic()
download_id = hashlib.sha256(cache_key.encode("utf-8", errors="replace")).hexdigest()[:12]
self._update_entry(cache_key, url, status="downloading", error=None)
def _close_response() -> None:
if response is not None:
try:
response.close()
except Exception:
pass
try:
self._save_dir.mkdir(parents=True, exist_ok=True)
logger.debug(f"[IMG_DOWNLOAD] start id={download_id}")
response = get_public_remote_resource(
url,
cookies=cookies,
headers=headers or None,
credential_origin_url=headers.get("Referer"),
timeout=(8, 20),
stream=True,
)
response_at = time.monotonic()
if response.status_code != 200:
raise ValueError(f"http_{response.status_code}")
content_type = str(response.headers.get("Content-Type") or "").split(";", 1)[0].strip().lower()
if content_type and not content_type.startswith("image/"):
raise ValueError(f"invalid_content_type:{content_type}")
path_ext = Path(urlparse(url).path or "").suffix.lower()
if content_type == "image/svg+xml" or (not content_type and path_ext == ".svg"):
raise ValueError("unsafe_image_type:image/svg+xml")
content_length = str(response.headers.get("Content-Length") or "").strip()
if content_length:
try:
expected_size = int(content_length)
except ValueError:
expected_size = 0
if expected_size > max_bytes:
raise ValueError(f"image_too_large:{expected_size}")
logger.debug(
f"[IMG_DOWNLOAD] headers id={download_id}, status={response.status_code}, "
f"content_length={content_length or 'unknown'}, "
f"response_wait={response_at - started_at:.3f}s"
)
ext = self._pick_extension(content_type, url)
filename = f"{int(time.time())}_{uuid.uuid4().hex[:8]}{ext}"
temp_path = self._save_dir / f"{filename}.part"
final_path = self._save_dir / filename
written = 0
first_chunk_at: Optional[float] = None
with temp_path.open("wb") as handle:
for chunk in response.iter_content(chunk_size=1024 * 64):
if self._is_shutdown():
raise RuntimeError("downloader_shutdown")
if not chunk:
continue
if first_chunk_at is None:
first_chunk_at = time.monotonic()
written += len(chunk)
if written > max_bytes:
raise ValueError(f"image_too_large:{written}")
handle.write(chunk)
read_finished_at = time.monotonic()
if self._is_shutdown():
raise RuntimeError("downloader_shutdown")
if written < self._min_bytes:
raise ValueError(f"image_too_small:{written}")
try:
with temp_path.open("rb") as handle:
detected_ext = self._detect_safe_image_extension(handle.read(32))
except Exception:
detected_ext = None
if not detected_ext:
raise ValueError("invalid_image_payload")
if detected_ext != ext:
final_path = self._save_dir / f"{Path(filename).stem}{detected_ext}"
filename = final_path.name
temp_path.replace(final_path)
accessible_url = f"/download_images/{filename}"
_close_response()
self._update_entry(
cache_key,
url,
status="done",
local_path=str(final_path),
accessible_url=accessible_url,
mime=content_type or self._guess_mime_type(final_path),
byte_size=written,
error=None,
source="background_download",
)
if not self._is_shutdown():
first_byte_wait = (
f"{first_chunk_at - response_at:.3f}s"
if first_chunk_at is not None
else "none"
)
logger.debug(
f"后台图片下载完成: {filename} ({written} bytes), "
f"id={download_id}, first_byte_wait={first_byte_wait}, "
f"read={read_finished_at - response_at:.3f}s, "
f"total={read_finished_at - started_at:.3f}s"
)
except Exception as exc:
try:
if temp_path is not None and temp_path.exists():
temp_path.unlink()
except Exception:
pass
_close_response()
self._update_entry(cache_key, url, status="failed", error=str(exc))
if not self._is_shutdown():
logger.debug(
f"后台图片下载失败(忽略): id={download_id}, "
f"elapsed={time.monotonic() - started_at:.3f}s, error={str(exc)[:160]}"
)
finally:
_close_response()
def _update_entry(self, cache_key: str, url: str, **changes: Any) -> None:
normalized = normalize_remote_image_url(url)
if not normalized:
return
with self._lock:
entry = self._entries.get(cache_key)
if self._shutdown and entry is None:
return
if self._shutdown and str(changes.get("status") or "") not in {"failed"}:
return
if entry is None:
entry = {
"url": normalized,
"_event": threading.Event(),
"started_at": time.time(),
}
self._entries[cache_key] = entry
next_status = changes.get("status") if "status" in changes else None
if "status" in changes:
changes = {key: value for key, value in changes.items() if key != "status"}
entry.update(changes)
if next_status is not None:
self._set_entry_status_locked(entry, next_status)
entry["updated_at"] = time.time()
self._entries.move_to_end(cache_key)
event = entry.get("_event")
if not isinstance(event, threading.Event):
event = threading.Event()
entry["_event"] = event
status = str(entry.get("status") or "")
if status in {"done", "failed"}:
event.set()
self._prune_entries_locked()
def _set_entry_status_locked(self, entry: Dict[str, Any], status: Any) -> None:
old_status = str((entry or {}).get("status") or "")
next_status = str(status or "")
if old_status != next_status:
old_pending = old_status in _PENDING_DOWNLOAD_STATUSES
next_pending = next_status in _PENDING_DOWNLOAD_STATUSES
if old_pending and not next_pending:
self._pending_count = max(0, self._pending_count - 1)
elif next_pending and not old_pending:
self._pending_count += 1
entry["status"] = next_status
def _prune_entries_locked(self) -> None:
now = time.time()
expired_keys = [
key
for key, entry in self._entries.items()
if str((entry or {}).get("status") or "") in {"done", "failed"}
and now - float((entry or {}).get("updated_at") or (entry or {}).get("started_at") or now)
> self._max_age_seconds
]
for key in expired_keys:
self._remove_entry_locked(key)
overflow = len(self._entries) - self._max_entries
if overflow > 0:
keys_to_remove = []
for key, entry in self._entries.items():
if len(keys_to_remove) >= overflow:
break
status = str((entry or {}).get("status") or "")
if status in {"queued", "downloading"}:
continue
keys_to_remove.append(key)
for key in keys_to_remove:
self._remove_entry_locked(key)
total_bytes = sum(
max(0, int((entry or {}).get("byte_size") or 0))
for entry in self._entries.values()
if str((entry or {}).get("status") or "") == "done"
)
if total_bytes > self._max_total_bytes:
for key, entry in list(self._entries.items()):
if total_bytes <= self._max_total_bytes:
break
if str((entry or {}).get("status") or "") != "done":
continue
total_bytes -= max(0, int((entry or {}).get("byte_size") or 0))
self._remove_entry_locked(key)
def _remove_entry_locked(self, key: str) -> None:
entry = self._entries.pop(key, None)
if not isinstance(entry, dict):
return
local_path = str(entry.get("local_path") or "").strip()
if not local_path:
return
if any(
str((other or {}).get("local_path") or "").strip() == local_path
for other in self._entries.values()
):
return
self._unlink_managed_file(local_path)
def _unlink_managed_file(self, local_path: str | Path) -> None:
try:
root = self._save_dir.resolve()
target = Path(local_path).resolve()
target.relative_to(root)
if target.is_file():
target.unlink(missing_ok=True)
except (OSError, ValueError):
return
def _cleanup_disk_if_due_locked(self) -> None:
now = time.time()
if now - self._last_disk_cleanup_at < 300:
return
self._last_disk_cleanup_at = now
cutoff = now - self._max_age_seconds
try:
for path in self._save_dir.iterdir():
if not path.is_file():
continue
try:
if path.stat().st_mtime < cutoff:
path.unlink(missing_ok=True)
except OSError:
continue
except OSError:
return
def shutdown(self) -> None:
with self._lock:
if self._shutdown:
return
self._shutdown = True
for entry in self._entries.values():
status = str((entry or {}).get("status") or "")
if status not in {"queued", "downloading"}:
continue
self._set_entry_status_locked(entry, "failed")
entry["error"] = "downloader_shutdown"
entry["updated_at"] = time.time()
event = entry.get("_event")
if isinstance(event, threading.Event):
event.set()
self._executor.shutdown(wait=False, cancel_futures=True)
def _is_shutdown(self) -> bool:
with self._lock:
return bool(self._shutdown)
def _snapshot_entry(self, entry: Dict[str, Any]) -> Dict[str, Any]:
if not isinstance(entry, dict):
return {}
return {
key: value
for key, value in entry.items()
if not str(key).startswith("_")
}
@staticmethod
def _cache_key(url: str, partition_key: str) -> str:
return f"{str(partition_key or 'public').strip() or 'public'}\x1f{url}"
@staticmethod
def _guess_mime_type(path_obj: Path) -> Optional[str]:
try:
guessed, _ = mimetypes.guess_type(path_obj.name)
return guessed
except Exception:
return None
@staticmethod
def _pick_extension(content_type: str, url: str) -> str:
ext = _IMAGE_CONTENT_TYPE_EXT_MAP.get(str(content_type or "").lower())
if ext:
return ext
path_ext = Path(urlparse(url).path or "").suffix.lower()
if path_ext in _SAFE_IMAGE_EXTENSIONS:
return path_ext
return ".png"
@staticmethod
def _detect_safe_image_extension(header: bytes) -> Optional[str]:
data = bytes(header or b"")
if data.startswith(b"\x89PNG\r\n\x1a\n"):
return ".png"
if data.startswith(b"\xff\xd8\xff"):
return ".jpg"
if data.startswith((b"GIF87a", b"GIF89a")):
return ".gif"
if data.startswith(b"BM"):
return ".bmp"
if len(data) >= 12 and data.startswith(b"RIFF") and data[8:12] == b"WEBP":
return ".webp"
if len(data) >= 16 and data[4:8] == b"ftyp" and data[8:12] in {b"avif", b"avis"}:
return ".avif"
return None
background_image_downloader = BackgroundImageDownloader()
__all__ = [
"BackgroundImageDownloader",
"DEFAULT_IMAGE_ACCEPT",
"background_image_downloader",
"build_image_download_partition",
"get_image_download_partition",
"build_image_download_request_context",
"build_image_request_headers",
"extract_tab_cookies",
"normalize_remote_image_url",
]