"""Security helpers for fetching user-controlled remote resources.""" from __future__ import annotations import contextlib import ipaddress import socket import threading from typing import Any, Dict, Iterable, Iterator, Optional, Tuple from urllib.parse import urljoin, urlsplit, urlunsplit import requests ALLOWED_REMOTE_SCHEMES = frozenset({"http", "https"}) MAX_REMOTE_REDIRECTS = 4 _PROXY_FAKE_IP_NETWORKS = ( ipaddress.ip_network("198.18.0.0/15"), # IPv4 Fake-IP (RFC 2544 benchmark, Clash/Mihomo/Sing-box) ipaddress.ip_network("2001:2::/48"), # IPv6 Fake-IP (RFC 5180 benchmark, Clash/Mihomo/Sing-box standard) ipaddress.ip_network("fc00::/7"), # IPv6 ULA (RFC 4193, legacy/custom Fake-IP e.g. fdfe::/64) ) _PROXY_FAKE_IP_HTTPS_SUFFIXES = ( # Google & Gemini CDN / Media "googleusercontent.com", "usercontent.google.com", "gstatic.com", "googleapis.com", "google.com", # Cloudflare R2 & CDN "r2.cloudflarestorage.com", "cloudflare.com", "cloudflareinsights.com", # GitHub CDN "githubusercontent.com", "raw.githubusercontent.com", "github.com", # OpenAI CDN "oaistatic.com", "oaiusercontent.com", "openai.com", # Anthropic & Claude CDN "anthropic.com", "claude.ai", # Microsoft & Bing CDN "bing.com", "msn.com", "microsoft.com", "azureedge.net", "windows.net", # Other mainstream AI & media hosting CDNs "deepseek.com", "grok.com", "x.ai", "together.ai", "together.xyz", "replicate.delivery", ) class UnsafeRemoteResourceError(ValueError): """Raised when a remote URL can reach a non-public network address.""" def normalize_remote_http_url(value: Any) -> str: raw = str(value or "").strip() if not raw: return "" try: parsed = urlsplit(raw) except Exception: return "" scheme = str(parsed.scheme or "").lower() hostname = str(parsed.hostname or "").strip().lower().rstrip(".") if scheme not in ALLOWED_REMOTE_SCHEMES or not hostname: return "" if parsed.username is not None or parsed.password is not None: return "" try: port = parsed.port except ValueError: return "" host_for_netloc = f"[{hostname}]" if ":" in hostname else hostname netloc = f"{host_for_netloc}:{port}" if port is not None else host_for_netloc return urlunsplit((scheme, netloc, parsed.path or "/", parsed.query, "")) def remote_url_origin(value: Any) -> str: normalized = normalize_remote_http_url(value) if not normalized: return "" parsed = urlsplit(normalized) default_port = 80 if parsed.scheme == "http" else 443 port = parsed.port or default_port host = str(parsed.hostname or "").lower().rstrip(".") host_for_netloc = f"[{host}]" if ":" in host else host return f"{parsed.scheme}://{host_for_netloc}:{port}" def is_same_remote_origin(left: Any, right: Any) -> bool: left_origin = remote_url_origin(left) return bool(left_origin and left_origin == remote_url_origin(right)) def _is_public_ip(address: str) -> bool: try: ip = ipaddress.ip_address(str(address or "").split("%", 1)[0]) except ValueError: return False return bool(ip.is_global) def _resolve_addresses(hostname: str) -> tuple[str, ...]: try: results = socket.getaddrinfo(hostname, None, type=socket.SOCK_STREAM) except OSError as exc: raise UnsafeRemoteResourceError(f"remote_dns_failed:{hostname}") from exc addresses = tuple(sorted({str(item[4][0]) for item in results if item and item[4]})) if not addresses: raise UnsafeRemoteResourceError(f"remote_dns_empty:{hostname}") return addresses def _is_proxy_fake_ip(address: str) -> bool: try: ip = ipaddress.ip_address(str(address or "").split("%", 1)[0]) except ValueError: return False return any(ip in network for network in _PROXY_FAKE_IP_NETWORKS) def _allows_proxy_fake_ip(hostname: str, scheme: str, addresses: Iterable[str]) -> bool: host = str(hostname or "").strip().lower().rstrip(".") if scheme != "https" or not any( host == suffix or host.endswith(f".{suffix}") for suffix in _PROXY_FAKE_IP_HTTPS_SUFFIXES ): return False resolved = tuple(addresses) return bool(resolved) and all( _is_public_ip(address) or _is_proxy_fake_ip(address) for address in resolved ) def resolve_public_addresses(hostname: str) -> tuple[str, ...]: host = str(hostname or "").strip().lower().rstrip(".") if not host: raise UnsafeRemoteResourceError("remote_host_missing") try: literal = ipaddress.ip_address(host.split("%", 1)[0]) except ValueError: literal = None if literal is not None: if not literal.is_global: raise UnsafeRemoteResourceError(f"remote_address_not_public:{literal}") return (str(literal),) addresses = _resolve_addresses(host) for address in addresses: if not _is_public_ip(address): raise UnsafeRemoteResourceError(f"remote_address_not_public:{address}") return addresses def validate_public_remote_url(value: Any) -> str: normalized = normalize_remote_http_url(value) if not normalized: raise UnsafeRemoteResourceError("remote_url_invalid") parsed = urlsplit(normalized) resolve_public_addresses(str(parsed.hostname or "")) return normalized def _validate_fetch_target(value: Any) -> Tuple[str, Tuple[str, ...]]: """Validate a fetch target and return (normalized_url, checked_addresses).""" normalized = normalize_remote_http_url(value) if not normalized: raise UnsafeRemoteResourceError("remote_url_invalid") parsed = urlsplit(normalized) host = str(parsed.hostname or "") try: return normalized, resolve_public_addresses(host) except UnsafeRemoteResourceError: addresses = _resolve_addresses(host) if _allows_proxy_fake_ip(host, parsed.scheme, addresses): return normalized, tuple(addresses) blocked = next( (address for address in addresses if not _is_public_ip(address)), addresses[0], ) raise UnsafeRemoteResourceError(f"remote_address_not_public:{blocked}") def _validate_fetch_remote_url(value: Any) -> str: """Validate a fetch target, accounting for HTTPS CDN names behind TUN fake DNS.""" return _validate_fetch_target(value)[0] # --------------------------------------------------------------------------- # S11:DNS 固定(pinning) # # 校验阶段解析出的地址必须就是实际连接的地址,否则攻击者控制的 DNS 可以在「校验」与 # 「连接」之间把答案换成内网地址(DNS rebinding)。做法:在 urllib3 建立 TCP 连接的 # 唯一入口 `urllib3.util.connection.create_connection` 外面包一层,只在 # get_public_remote_resource 调用期间、且只对本线程登记过的主机名生效——连接直接打到 # 已校验的 IP,而 HTTPConnection.host 保持原主机名,所以 TLS SNI / 证书校验 / Host 头都不变。 # 未登记的主机名(包括走 HTTP(S)_PROXY 时的代理主机)原样放行:代理场景由可信出网代理解析目标。 # --------------------------------------------------------------------------- _pin_state = threading.local() _pin_install_lock = threading.Lock() _pin_installed = False def _current_pins() -> Dict[str, Tuple[str, ...]]: pins = getattr(_pin_state, "pins", None) if pins is None: pins = {} _pin_state.pins = pins return pins def _install_pinned_create_connection() -> None: global _pin_installed if _pin_installed: return with _pin_install_lock: if _pin_installed: return try: from urllib3.util import connection as urllib3_connection except Exception: # pragma: no cover - urllib3 is a hard dependency of requests return original = urllib3_connection.create_connection def pinned_create_connection(address, *args, **kwargs): host, port = address[0], address[1] pinned = _current_pins().get(str(host or "").strip().lower().rstrip(".")) if not pinned: return original(address, *args, **kwargs) last_error: Optional[BaseException] = None for ip in pinned: try: return original((ip, port), *args, **kwargs) except OSError as exc: last_error = exc raise last_error or OSError(f"remote_pinned_connect_failed:{host}") pinned_create_connection.__uwapi_pinned__ = True # type: ignore[attr-defined] urllib3_connection.create_connection = pinned_create_connection _pin_installed = True @contextlib.contextmanager def _pinned_dns(url: str, addresses: Iterable[str]) -> Iterator[None]: host = str(urlsplit(url).hostname or "").strip().lower().rstrip(".") resolved = tuple(str(item) for item in addresses if str(item or "").strip()) if not host or not resolved: yield return _install_pinned_create_connection() pins = _current_pins() previous = pins.get(host) pins[host] = resolved try: yield finally: if previous is None: pins.pop(host, None) else: pins[host] = previous def _headers_for_target( headers: Optional[Dict[str, str]], *, target_url: str, credential_origin: str, ) -> Dict[str, str]: result = dict(headers or {}) if not credential_origin or remote_url_origin(target_url) != credential_origin: sensitive = {"authorization", "cookie", "referer"} result = {k: v for k, v in result.items() if str(k).strip().lower() not in sensitive} return result def get_public_remote_resource( url: Any, *, headers: Optional[Dict[str, str]] = None, cookies: Any = None, credential_origin_url: Any = None, timeout: Any = (8, 20), stream: bool = True, max_redirects: int = MAX_REMOTE_REDIRECTS, ): """GET a public URL, validating every redirect and scoping credentials.""" current, addresses = _validate_fetch_target(url) credential_origin = remote_url_origin(credential_origin_url) redirects_left = max(0, int(max_redirects)) while True: same_origin = bool(credential_origin and remote_url_origin(current) == credential_origin) # S11:连接固定到刚校验过的地址(连接在 requests.get 返回前已建立,stream 读取复用同一 socket) with _pinned_dns(current, addresses): response = requests.get( current, headers=_headers_for_target( headers, target_url=current, credential_origin=credential_origin, ), cookies=cookies if same_origin else None, timeout=timeout, allow_redirects=False, stream=stream, ) if response.status_code not in {301, 302, 303, 307, 308}: return response location = str(response.headers.get("Location") or "").strip() response.close() if not location: raise UnsafeRemoteResourceError("remote_redirect_missing_location") if redirects_left <= 0: raise UnsafeRemoteResourceError("remote_redirect_limit") redirects_left -= 1 current, addresses = _validate_fetch_target(urljoin(current, location)) __all__ = [ "UnsafeRemoteResourceError", "get_public_remote_resource", "is_same_remote_origin", "normalize_remote_http_url", "remote_url_origin", "resolve_public_addresses", "validate_public_remote_url", ]