Initial commit

This commit is contained in:
Alex committed 2026-11-02 22:54:14 +08:00
commit 57fc3525eb
7311 files changed
+1291398

No files matched your search

+343
View File
@@ -0,0 +1,343 @@
"""Security helpers for fetching user-controlled remote resources."""
from __future__ import annotations
import contextlib
import ipaddress
import socket
import threading
from typing import Any, Dict, Iterable, Iterator, Optional, Tuple
from urllib.parse import urljoin, urlsplit, urlunsplit
import requests
ALLOWED_REMOTE_SCHEMES = frozenset({"http", "https"})
MAX_REMOTE_REDIRECTS = 4
_PROXY_FAKE_IP_NETWORKS = (
ipaddress.ip_network("198.18.0.0/15"), # IPv4 Fake-IP (RFC 2544 benchmark, Clash/Mihomo/Sing-box)
ipaddress.ip_network("2001:2::/48"), # IPv6 Fake-IP (RFC 5180 benchmark, Clash/Mihomo/Sing-box standard)
ipaddress.ip_network("fc00::/7"), # IPv6 ULA (RFC 4193, legacy/custom Fake-IP e.g. fdfe::/64)
)
_PROXY_FAKE_IP_HTTPS_SUFFIXES = (
# Google & Gemini CDN / Media
"googleusercontent.com",
"usercontent.google.com",
"gstatic.com",
"googleapis.com",
"google.com",
# Cloudflare R2 & CDN
"r2.cloudflarestorage.com",
"cloudflare.com",
"cloudflareinsights.com",
# GitHub CDN
"githubusercontent.com",
"raw.githubusercontent.com",
"github.com",
# OpenAI CDN
"oaistatic.com",
"oaiusercontent.com",
"openai.com",
# Anthropic & Claude CDN
"anthropic.com",
"claude.ai",
# Microsoft & Bing CDN
"bing.com",
"msn.com",
"microsoft.com",
"azureedge.net",
"windows.net",
# Other mainstream AI & media hosting CDNs
"deepseek.com",
"grok.com",
"x.ai",
"together.ai",
"together.xyz",
"replicate.delivery",
)
class UnsafeRemoteResourceError(ValueError):
"""Raised when a remote URL can reach a non-public network address."""
def normalize_remote_http_url(value: Any) -> str:
raw = str(value or "").strip()
if not raw:
return ""
try:
parsed = urlsplit(raw)
except Exception:
return ""
scheme = str(parsed.scheme or "").lower()
hostname = str(parsed.hostname or "").strip().lower().rstrip(".")
if scheme not in ALLOWED_REMOTE_SCHEMES or not hostname:
return ""
if parsed.username is not None or parsed.password is not None:
return ""
try:
port = parsed.port
except ValueError:
return ""
host_for_netloc = f"[{hostname}]" if ":" in hostname else hostname
netloc = f"{host_for_netloc}:{port}" if port is not None else host_for_netloc
return urlunsplit((scheme, netloc, parsed.path or "/", parsed.query, ""))
def remote_url_origin(value: Any) -> str:
normalized = normalize_remote_http_url(value)
if not normalized:
return ""
parsed = urlsplit(normalized)
default_port = 80 if parsed.scheme == "http" else 443
port = parsed.port or default_port
host = str(parsed.hostname or "").lower().rstrip(".")
host_for_netloc = f"[{host}]" if ":" in host else host
return f"{parsed.scheme}://{host_for_netloc}:{port}"
def is_same_remote_origin(left: Any, right: Any) -> bool:
left_origin = remote_url_origin(left)
return bool(left_origin and left_origin == remote_url_origin(right))
def _is_public_ip(address: str) -> bool:
try:
ip = ipaddress.ip_address(str(address or "").split("%", 1)[0])
except ValueError:
return False
return bool(ip.is_global)
def _resolve_addresses(hostname: str) -> tuple[str, ...]:
try:
results = socket.getaddrinfo(hostname, None, type=socket.SOCK_STREAM)
except OSError as exc:
raise UnsafeRemoteResourceError(f"remote_dns_failed:{hostname}") from exc
addresses = tuple(sorted({str(item[4][0]) for item in results if item and item[4]}))
if not addresses:
raise UnsafeRemoteResourceError(f"remote_dns_empty:{hostname}")
return addresses
def _is_proxy_fake_ip(address: str) -> bool:
try:
ip = ipaddress.ip_address(str(address or "").split("%", 1)[0])
except ValueError:
return False
return any(ip in network for network in _PROXY_FAKE_IP_NETWORKS)
def _allows_proxy_fake_ip(hostname: str, scheme: str, addresses: Iterable[str]) -> bool:
host = str(hostname or "").strip().lower().rstrip(".")
if scheme != "https" or not any(
host == suffix or host.endswith(f".{suffix}")
for suffix in _PROXY_FAKE_IP_HTTPS_SUFFIXES
):
return False
resolved = tuple(addresses)
return bool(resolved) and all(
_is_public_ip(address) or _is_proxy_fake_ip(address)
for address in resolved
)
def resolve_public_addresses(hostname: str) -> tuple[str, ...]:
host = str(hostname or "").strip().lower().rstrip(".")
if not host:
raise UnsafeRemoteResourceError("remote_host_missing")
try:
literal = ipaddress.ip_address(host.split("%", 1)[0])
except ValueError:
literal = None
if literal is not None:
if not literal.is_global:
raise UnsafeRemoteResourceError(f"remote_address_not_public:{literal}")
return (str(literal),)
addresses = _resolve_addresses(host)
for address in addresses:
if not _is_public_ip(address):
raise UnsafeRemoteResourceError(f"remote_address_not_public:{address}")
return addresses
def validate_public_remote_url(value: Any) -> str:
normalized = normalize_remote_http_url(value)
if not normalized:
raise UnsafeRemoteResourceError("remote_url_invalid")
parsed = urlsplit(normalized)
resolve_public_addresses(str(parsed.hostname or ""))
return normalized
def _validate_fetch_target(value: Any) -> Tuple[str, Tuple[str, ...]]:
"""Validate a fetch target and return (normalized_url, checked_addresses)."""
normalized = normalize_remote_http_url(value)
if not normalized:
raise UnsafeRemoteResourceError("remote_url_invalid")
parsed = urlsplit(normalized)
host = str(parsed.hostname or "")
try:
return normalized, resolve_public_addresses(host)
except UnsafeRemoteResourceError:
addresses = _resolve_addresses(host)
if _allows_proxy_fake_ip(host, parsed.scheme, addresses):
return normalized, tuple(addresses)
blocked = next(
(address for address in addresses if not _is_public_ip(address)),
addresses[0],
)
raise UnsafeRemoteResourceError(f"remote_address_not_public:{blocked}")
def _validate_fetch_remote_url(value: Any) -> str:
"""Validate a fetch target, accounting for HTTPS CDN names behind TUN fake DNS."""
return _validate_fetch_target(value)[0]
# ---------------------------------------------------------------------------
# S11:DNS 固定(pinning)
#
# 校验阶段解析出的地址必须就是实际连接的地址,否则攻击者控制的 DNS 可以在「校验」与
# 「连接」之间把答案换成内网地址(DNS rebinding)。做法:在 urllib3 建立 TCP 连接的
# 唯一入口 `urllib3.util.connection.create_connection` 外面包一层,只在
# get_public_remote_resource 调用期间、且只对本线程登记过的主机名生效——连接直接打到
# 已校验的 IP,而 HTTPConnection.host 保持原主机名,所以 TLS SNI / 证书校验 / Host 头都不变。
# 未登记的主机名(包括走 HTTP(S)_PROXY 时的代理主机)原样放行:代理场景由可信出网代理解析目标。
# ---------------------------------------------------------------------------
_pin_state = threading.local()
_pin_install_lock = threading.Lock()
_pin_installed = False
def _current_pins() -> Dict[str, Tuple[str, ...]]:
pins = getattr(_pin_state, "pins", None)
if pins is None:
pins = {}
_pin_state.pins = pins
return pins
def _install_pinned_create_connection() -> None:
global _pin_installed
if _pin_installed:
return
with _pin_install_lock:
if _pin_installed:
return
try:
from urllib3.util import connection as urllib3_connection
except Exception: # pragma: no cover - urllib3 is a hard dependency of requests
return
original = urllib3_connection.create_connection
def pinned_create_connection(address, *args, **kwargs):
host, port = address[0], address[1]
pinned = _current_pins().get(str(host or "").strip().lower().rstrip("."))
if not pinned:
return original(address, *args, **kwargs)
last_error: Optional[BaseException] = None
for ip in pinned:
try:
return original((ip, port), *args, **kwargs)
except OSError as exc:
last_error = exc
raise last_error or OSError(f"remote_pinned_connect_failed:{host}")
pinned_create_connection.__uwapi_pinned__ = True # type: ignore[attr-defined]
urllib3_connection.create_connection = pinned_create_connection
_pin_installed = True
@contextlib.contextmanager
def _pinned_dns(url: str, addresses: Iterable[str]) -> Iterator[None]:
host = str(urlsplit(url).hostname or "").strip().lower().rstrip(".")
resolved = tuple(str(item) for item in addresses if str(item or "").strip())
if not host or not resolved:
yield
return
_install_pinned_create_connection()
pins = _current_pins()
previous = pins.get(host)
pins[host] = resolved
try:
yield
finally:
if previous is None:
pins.pop(host, None)
else:
pins[host] = previous
def _headers_for_target(
headers: Optional[Dict[str, str]],
*,
target_url: str,
credential_origin: str,
) -> Dict[str, str]:
result = dict(headers or {})
if not credential_origin or remote_url_origin(target_url) != credential_origin:
sensitive = {"authorization", "cookie", "referer"}
result = {k: v for k, v in result.items() if str(k).strip().lower() not in sensitive}
return result
def get_public_remote_resource(
url: Any,
*,
headers: Optional[Dict[str, str]] = None,
cookies: Any = None,
credential_origin_url: Any = None,
timeout: Any = (8, 20),
stream: bool = True,
max_redirects: int = MAX_REMOTE_REDIRECTS,
):
"""GET a public URL, validating every redirect and scoping credentials."""
current, addresses = _validate_fetch_target(url)
credential_origin = remote_url_origin(credential_origin_url)
redirects_left = max(0, int(max_redirects))
while True:
same_origin = bool(credential_origin and remote_url_origin(current) == credential_origin)
# S11:连接固定到刚校验过的地址(连接在 requests.get 返回前已建立,stream 读取复用同一 socket)
with _pinned_dns(current, addresses):
response = requests.get(
current,
headers=_headers_for_target(
headers,
target_url=current,
credential_origin=credential_origin,
),
cookies=cookies if same_origin else None,
timeout=timeout,
allow_redirects=False,
stream=stream,
)
if response.status_code not in {301, 302, 303, 307, 308}:
return response
location = str(response.headers.get("Location") or "").strip()
response.close()
if not location:
raise UnsafeRemoteResourceError("remote_redirect_missing_location")
if redirects_left <= 0:
raise UnsafeRemoteResourceError("remote_redirect_limit")
redirects_left -= 1
current, addresses = _validate_fetch_target(urljoin(current, location))
__all__ = [
"UnsafeRemoteResourceError",
"get_public_remote_resource",
"is_same_remote_origin",
"normalize_remote_http_url",
"remote_url_origin",
"resolve_public_addresses",
"validate_public_remote_url",
]