Initial commit

This commit is contained in:
Alex committed 2026-11-02 22:54:14 +08:00
commit 57fc3525eb
7311 files changed
+1291398

No files matched your search

File diff suppressed because it is too large. Load diff
@@ -0,0 +1,14 @@
"""Export tldextract's public interface."""
from . import _version
from .tldextract import ExtractResult, TLDExtract, extract, update
__version__: str = _version.version
__all__ = [
"__version__",
"extract",
"ExtractResult",
"TLDExtract",
"update",
]
@@ -0,0 +1,6 @@
"""tldextract __main__."""
from .cli import main
if __name__ == "__main__":
main()
@@ -0,0 +1,24 @@
# file generated by vcs-versioning
# don't change, don't track in version control
from __future__ import annotations
__all__ = [
"__version__",
"__version_tuple__",
"version",
"version_tuple",
"__commit_id__",
"commit_id",
]
version: str
__version__: str
__version_tuple__: tuple[int | str, ...]
version_tuple: tuple[int | str, ...]
commit_id: str | None
__commit_id__: str | None
__version__ = version = '5.4.0'
__version_tuple__ = version_tuple = (5, 4, 0)
__commit_id__ = commit_id = 'j6937495ce'
+238
View File
@@ -0,0 +1,238 @@
"""Helpers."""
import errno
import hashlib
import json
import logging
import os
import sys
from collections.abc import Callable, Hashable, Iterable
from pathlib import Path
from typing import (
TypeVar,
cast,
)
import requests
from filelock import FileLock
LOG = logging.getLogger(__name__)
_DID_LOG_UNABLE_TO_CACHE = False
T = TypeVar("T")
def get_pkg_unique_identifier() -> str:
"""Generate an identifier unique to the python version, tldextract version, and python instance.
This will prevent interference between virtualenvs and issues that might arise when installing
a new version of tldextract
"""
try:
from tldextract._version import version
except ImportError:
version = "dev"
tldextract_version = "tldextract-" + version
python_env_name = os.path.basename(sys.prefix)
# just to handle the edge case of two identically named python environments
python_binary_path_short_hash = hashlib.md5(
sys.prefix.encode("utf-8"), usedforsecurity=False
).hexdigest()[:6]
python_version = ".".join([str(v) for v in sys.version_info[:-1]])
identifier_parts = [
python_version,
python_env_name,
python_binary_path_short_hash,
tldextract_version,
]
pkg_identifier = "__".join(identifier_parts)
return pkg_identifier
def get_cache_dir() -> str:
"""Get a cache dir that we have permission to write to.
Try to follow the XDG standard, but if that doesn't work fallback to the package directory
http://specifications.freedesktop.org/basedir-spec/basedir-spec-latest.html
"""
cache_dir = os.environ.get("TLDEXTRACT_CACHE", None)
if cache_dir is not None:
return cache_dir
xdg_cache_home = os.getenv("XDG_CACHE_HOME", None)
if xdg_cache_home is None:
user_home = os.getenv("HOME", None)
if user_home:
xdg_cache_home = str(Path(user_home, ".cache"))
if xdg_cache_home is not None:
return str(
Path(xdg_cache_home, "python-tldextract", get_pkg_unique_identifier())
)
# fallback to trying to use package directory itself
return str(Path(os.path.dirname(__file__), ".suffix_cache"))
class DiskCache:
"""Disk _cache that only works for jsonable values."""
def __init__(self, cache_dir: str | None, lock_timeout: int = 20):
"""Construct a disk cache in the given directory."""
self.enabled = bool(cache_dir)
self.cache_dir = os.path.expanduser(str(cache_dir) or "")
self.lock_timeout = lock_timeout
# using a unique extension provides some safety that an incorrectly set cache_dir
# combined with a call to `.clear()` wont wipe someones hard drive
self.file_ext = ".tldextract.json"
def get(self, namespace: str, key: str | dict[str, Hashable]) -> object:
"""Retrieve a value from the disk cache."""
if not self.enabled:
raise KeyError("Cache is disabled")
cache_filepath = self._key_to_cachefile_path(namespace, key)
if not os.path.isfile(cache_filepath):
raise KeyError("namespace: " + namespace + " key: " + repr(key))
try:
with open(cache_filepath) as cache_file:
return json.load(cache_file)
except (OSError, ValueError) as exc:
raise KeyError("namespace: " + namespace + " key: " + repr(key)) from exc
def set( # noqa: A003
self, namespace: str, key: str | dict[str, Hashable], value: object
) -> None:
"""Set a value in the disk cache."""
if not self.enabled:
return
cache_filepath = self._key_to_cachefile_path(namespace, key)
try:
_make_dir(cache_filepath)
with open(cache_filepath, "w") as cache_file:
json.dump(value, cache_file)
except OSError as ioe:
global _DID_LOG_UNABLE_TO_CACHE
if not _DID_LOG_UNABLE_TO_CACHE:
LOG.warning(
"unable to cache %s.%s in %s. This could refresh the "
"Public Suffix List over HTTP every app startup. "
"Construct your `TLDExtract` with a writable `cache_dir` or "
"set `cache_dir=None` to silence this warning. %s",
namespace,
key,
cache_filepath,
ioe,
)
_DID_LOG_UNABLE_TO_CACHE = True
def clear(self) -> None:
"""Clear the disk cache."""
for root, _, files in os.walk(self.cache_dir):
for filename in files:
if filename.endswith(self.file_ext) or filename.endswith(
self.file_ext + ".lock"
):
try:
os.unlink(str(Path(root, filename)))
except FileNotFoundError:
pass
except OSError as exc:
# errno.ENOENT == "No such file or directory"
# https://docs.python.org/2/library/errno.html#errno.ENOENT
if exc.errno != errno.ENOENT:
raise
def _key_to_cachefile_path(
self, namespace: str, key: str | dict[str, Hashable]
) -> str:
namespace_path = str(Path(self.cache_dir, namespace))
hashed_key = _make_cache_key(key)
cache_path = str(Path(namespace_path, hashed_key + self.file_ext))
return cache_path
def run_and_cache(
self,
func: Callable[..., T],
namespace: str,
kwargs: dict[str, Hashable],
hashed_argnames: Iterable[str],
) -> T:
"""Get a url but cache the response."""
if not self.enabled:
return func(**kwargs)
key_args = {k: v for k, v in kwargs.items() if k in hashed_argnames}
cache_filepath = self._key_to_cachefile_path(namespace, key_args)
lock_path = cache_filepath + ".lock"
try:
_make_dir(cache_filepath)
except OSError as ioe:
global _DID_LOG_UNABLE_TO_CACHE
if not _DID_LOG_UNABLE_TO_CACHE:
LOG.warning(
"unable to cache %s.%s in %s. This could refresh the "
"Public Suffix List over HTTP every app startup. "
"Construct your `TLDExtract` with a writable `cache_dir` or "
"set `cache_dir=None` to silence this warning. %s",
namespace,
key_args,
cache_filepath,
ioe,
)
_DID_LOG_UNABLE_TO_CACHE = True
return func(**kwargs)
with FileLock(lock_path, timeout=self.lock_timeout):
try:
result = cast(T, self.get(namespace=namespace, key=key_args))
except KeyError:
result = func(**kwargs)
self.set(namespace=namespace, key=key_args, value=result)
return result
def cached_fetch_url(
self, session: requests.Session, url: str, timeout: float | int | None
) -> str:
"""Get a url but cache the response."""
return self.run_and_cache(
func=_fetch_url,
namespace="urls",
kwargs={"session": session, "url": url, "timeout": timeout},
hashed_argnames=["url"],
)
def _fetch_url(session: requests.Session, url: str, timeout: int | None) -> str:
response = session.get(url, timeout=timeout)
response.raise_for_status()
text = response.text
if not isinstance(text, str):
text = str(text, "utf-8")
return text
def _make_cache_key(inputs: str | dict[str, Hashable]) -> str:
key = repr(inputs)
return hashlib.md5(key.encode("utf8"), usedforsecurity=False).hexdigest()
def _make_dir(filename: str) -> None:
"""Make a directory if it doesn't already exist."""
if not os.path.exists(os.path.dirname(filename)):
try:
os.makedirs(os.path.dirname(filename))
except OSError as exc: # Guard against race condition
if exc.errno != errno.EEXIST:
raise
+119
View File
@@ -0,0 +1,119 @@
"""tldextract CLI."""
import argparse
import dataclasses
import json
import logging
import os.path
import pathlib
import sys
from ._version import version as __version__
from .tldextract import TLDExtract
def main() -> None:
"""Tldextract CLI main command."""
logging.basicConfig()
parser = argparse.ArgumentParser(
prog="tldextract", description="Parse hostname from a url or fqdn"
)
parser.add_argument(
"--version", action="version", version="%(prog)s " + __version__
)
parser.add_argument(
"-j",
"--json",
default=False,
action="store_true",
help="output in json format",
)
parser.add_argument(
"input", metavar="fqdn|url", type=str, nargs="*", help="fqdn or url"
)
parser.add_argument(
"-u",
"--update",
default=False,
action="store_true",
help="force fetch the latest TLD definitions",
)
parser.add_argument(
"--suffix_list_url",
action="append",
required=False,
help="use an alternate URL or local file for TLD definitions",
)
parser.add_argument(
"-c", "--cache_dir", help="use an alternate TLD definition caching folder"
)
parser.add_argument(
"-p",
"--include_psl_private_domains",
"--private_domains",
default=False,
action="store_true",
help="Include private domains",
)
parser.add_argument(
"--no_fallback_to_snapshot",
default=True,
action="store_false",
dest="fallback_to_snapshot",
help="Don't fall back to the package's snapshot of the suffix list",
)
args = parser.parse_args()
obj_kwargs = {
"include_psl_private_domains": args.include_psl_private_domains,
"fallback_to_snapshot": args.fallback_to_snapshot,
}
if args.cache_dir:
obj_kwargs["cache_dir"] = args.cache_dir
if args.suffix_list_url is not None:
suffix_list_urls = []
for source in args.suffix_list_url:
if os.path.isfile(source):
as_path_uri = pathlib.Path(os.path.abspath(source)).as_uri()
suffix_list_urls.append(as_path_uri)
else:
suffix_list_urls.append(source)
obj_kwargs["suffix_list_urls"] = suffix_list_urls
tld_extract = TLDExtract(**obj_kwargs)
if args.update:
tld_extract.update(True)
elif not args.input:
parser.print_usage()
sys.exit(1)
for i in args.input:
ext = tld_extract(i)
if args.json:
properties = (
"fqdn",
"ipv4",
"ipv6",
"registered_domain",
"reverse_domain_name",
"top_domain_under_public_suffix",
"top_domain_under_registry_suffix",
)
print(
json.dumps(
{
**dataclasses.asdict(ext),
**{prop: getattr(ext, prop) for prop in properties},
}
)
)
else:
print(f"{ext.subdomain} {ext.domain} {ext.suffix}")
Whitespace-only changes.
@@ -0,0 +1,69 @@
"""tldextract helpers for testing and fetching remote resources."""
import re
from ipaddress import AddressValueError, IPv6Address
from urllib.parse import scheme_chars
IP_RE = re.compile(
r"^(?:(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.)"
r"{3}(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])$",
re.ASCII,
)
scheme_chars_set = set(scheme_chars)
def lenient_netloc(url: str) -> str:
"""Extract the host portion of a URL-like string.
Parse more leniently than urllib.parse.{urlparse,urlsplit}, preserving
casing and brackets around IPv6 addresses.
"""
authority = (
_schemeless_url(url).partition("/")[0].partition("?")[0].partition("#")[0]
)
return _host_from_authority(authority)
def _host_from_authority(authority: str) -> str:
"""Extract a case-preserving host from a parsed URL's authority."""
after_userinfo = authority.rpartition("@")[-1]
if after_userinfo and after_userinfo[0] == "[":
maybe_ipv6 = after_userinfo.partition("]")
if maybe_ipv6[1] == "]":
return f"{maybe_ipv6[0]}]"
hostname = after_userinfo.partition(":")[0].strip()
without_root_label = hostname.rstrip(".\u3002\uff0e\uff61")
return without_root_label
def _schemeless_url(url: str) -> str:
double_slashes_start = url.find("//")
if double_slashes_start == 0:
return url[2:]
if (
double_slashes_start < 2
or url[double_slashes_start - 1] != ":"
or set(url[: double_slashes_start - 1]) - scheme_chars_set
):
return url
return url[double_slashes_start + 2 :]
def looks_like_ip(maybe_ip: str) -> bool:
"""Check whether the given str looks like an IPv4 address."""
if not maybe_ip[0].isdecimal():
return False
return IP_RE.fullmatch(maybe_ip) is not None
def looks_like_ipv6(maybe_ip: str) -> bool:
"""Check whether the given str looks like an IPv6 address."""
try:
IPv6Address(maybe_ip)
except AddressValueError:
return False
return True
@@ -0,0 +1,118 @@
"""tldextract helpers for testing and fetching remote resources."""
import logging
import pkgutil
import re
from collections.abc import Sequence
from typing import cast
import requests
from requests_file import FileAdapter
from .cache import DiskCache
LOG = logging.getLogger("tldextract")
PUBLIC_SUFFIX_RE = re.compile(r"^(?P<suffix>[.*!]*\w[\S]*)", re.UNICODE | re.MULTILINE)
PUBLIC_PRIVATE_SUFFIX_SEPARATOR = "// ===BEGIN PRIVATE DOMAINS==="
class SuffixListNotFound(LookupError): # noqa: N818
"""A recoverable error while looking up a suffix list.
Recoverable because you can specify backups, or use this library's bundled
snapshot.
"""
def find_first_response(
cache: DiskCache,
urls: Sequence[str],
cache_fetch_timeout: float | int | None = None,
session: requests.Session | None = None,
) -> str:
"""Decode the first successfully fetched URL, from UTF-8 encoding to Python unicode."""
session_created = False
if session is None:
session = requests.Session()
session.mount("file://", FileAdapter())
session_created = True
try:
for url in urls:
try:
return cache.cached_fetch_url(
session=session, url=url, timeout=cache_fetch_timeout
)
except requests.exceptions.RequestException:
LOG.warning(
"Exception reading Public Suffix List url %s", url, exc_info=True
)
finally:
# Ensure the session is always closed if it's constructed in the method
if session_created:
session.close()
raise SuffixListNotFound(
"No remote Public Suffix List found. Consider using a mirror, or avoid this"
" fetch by constructing your TLDExtract with `suffix_list_urls=()`."
)
def extract_tlds_from_suffix_list(suffix_list_text: str) -> tuple[list[str], list[str]]:
"""Parse the raw suffix list text for its different designations of suffixes."""
public_text, _, private_text = suffix_list_text.partition(
PUBLIC_PRIVATE_SUFFIX_SEPARATOR
)
public_tlds = [m.group("suffix") for m in PUBLIC_SUFFIX_RE.finditer(public_text)]
private_tlds = [m.group("suffix") for m in PUBLIC_SUFFIX_RE.finditer(private_text)]
return public_tlds, private_tlds
def get_suffix_lists(
cache: DiskCache,
urls: Sequence[str],
cache_fetch_timeout: float | int | None,
fallback_to_snapshot: bool,
session: requests.Session | None = None,
) -> tuple[list[str], list[str]]:
"""Fetch, parse, and cache the suffix lists."""
return cache.run_and_cache(
func=_get_suffix_lists,
namespace="publicsuffix.org-tlds",
kwargs={
"cache": cache,
"urls": urls,
"cache_fetch_timeout": cache_fetch_timeout,
"fallback_to_snapshot": fallback_to_snapshot,
"session": session,
},
hashed_argnames=["urls", "fallback_to_snapshot"],
)
def _get_suffix_lists(
cache: DiskCache,
urls: Sequence[str],
cache_fetch_timeout: float | int | None,
fallback_to_snapshot: bool,
session: requests.Session | None = None,
) -> tuple[list[str], list[str]]:
"""Fetch, parse, and cache the suffix lists."""
try:
text = find_first_response(
cache, urls, cache_fetch_timeout=cache_fetch_timeout, session=session
)
except SuffixListNotFound as exc:
if fallback_to_snapshot:
maybe_pkg_data = pkgutil.get_data("tldextract", ".tld_set_snapshot")
# package maintainers guarantee file is included
pkg_data = cast(bytes, maybe_pkg_data)
text = pkg_data.decode("utf-8")
else:
raise exc
public_tlds, private_tlds = extract_tlds_from_suffix_list(text)
return public_tlds, private_tlds
@@ -0,0 +1,801 @@
"""`tldextract` accurately separates a URL's subdomain, domain, and public suffix.
It does this via the Public Suffix List (PSL).
>>> import tldextract
>>> tldextract.extract("http://forums.news.cnn.com/")
ExtractResult(subdomain='forums.news', domain='cnn', suffix='com', is_private=False)
>>> tldextract.extract("http://forums.bbc.co.uk/") # United Kingdom
ExtractResult(subdomain='forums', domain='bbc', suffix='co.uk', is_private=False)
>>> tldextract.extract("http://www.worldbank.org.kg/") # Kyrgyzstan
ExtractResult(subdomain='www', domain='worldbank', suffix='org.kg', is_private=False)
Note subdomain and suffix are _optional_. Not all URL-like inputs have a
subdomain or a valid suffix.
>>> tldextract.extract("google.com")
ExtractResult(subdomain='', domain='google', suffix='com', is_private=False)
>>> tldextract.extract("google.notavalidsuffix")
ExtractResult(subdomain='google', domain='notavalidsuffix', suffix='', is_private=False)
>>> tldextract.extract("http://127.0.0.1:8080/deployed/")
ExtractResult(subdomain='', domain='127.0.0.1', suffix='', is_private=False)
To rejoin the original hostname, if it was indeed a valid, registered hostname:
>>> ext = tldextract.extract("http://forums.bbc.co.uk")
>>> ext.top_domain_under_public_suffix
'bbc.co.uk'
>>> ext.fqdn
'forums.bbc.co.uk'
"""
from __future__ import annotations
import os
import pathlib
import urllib.parse
import warnings
from collections.abc import Collection, Sequence
from dataclasses import dataclass, field
from functools import wraps
import idna
import requests
from .cache import DiskCache, get_cache_dir
from .remote import (
_host_from_authority,
lenient_netloc,
looks_like_ip,
looks_like_ipv6,
)
from .suffix_list import get_suffix_lists
def _get_default_cache_fetch_timeout() -> str | None:
"""Read the default PSL fetch timeout from the environment."""
return os.environ.get(
"TLDEXTRACT_DEFAULT_FETCH_TIMEOUT",
os.environ.get("TLDEXTRACT_CACHE_TIMEOUT"),
)
CACHE_TIMEOUT = _get_default_cache_fetch_timeout()
PUBLIC_SUFFIX_LIST_URLS = (
"https://publicsuffix.org/list/public_suffix_list.dat",
"https://raw.githubusercontent.com/publicsuffix/list/master/public_suffix_list.dat",
)
@dataclass(order=True)
class ExtractResult:
"""A URL's extracted subdomain, domain, and suffix.
These first 3 fields are what most users of this library will care about.
They are the split, non-overlapping hostname components of the input URL.
They can be used to rebuild the original URL's hostname.
The extracted strings generally preserve the input's casing and Unicode or
ASCII-compatible encoding (ACE, commonly called Punycode). They are not
normalized for comparison or validated as real hostnames.
Beyond the first 3 fields, the class contains metadata fields, like a flag
that indicates if the input URL's suffix is from a private domain.
"""
subdomain: str
"""All subdomains beneath the domain of the input URL, if it contained any such subdomains, or else the empty string."""
domain: str
"""The topmost domain of the input URL, if it contained a domain name, or else everything hostname-like in the input.
If the input URL didn't contain a real domain name, the `suffix` field will
be empty, and this field will catch values like an IP address, or
private network hostnames like "localhost".
"""
suffix: str
"""The public suffix of the input URL, if it contained one, or else the empty string.
If `include_psl_private_domains` was set to `False`, this field is the same
as `registry_suffix`, i.e. a domain under which people can register
subdomains through a registrar. If `include_psl_private_domains` was set to
`True`, this field may be a PSL private domain, like "blogspot.com".
"""
is_private: bool
"""Whether the input URL belongs in the Public Suffix List's private domains.
If `include_psl_private_domains` was set to `False`, this field is always
`False`.
"""
registry_suffix: str = field(repr=False)
"""The registry suffix of the input URL, if it contained one, or else the empty string.
This field is a domain under which people can register subdomains through a
registar.
This field is unaffected by the `include_psl_private_domains` setting. If
`include_psl_private_domains` was set to `False`, this field is always the
same as `suffix`.
.. versionadded:: 5.3.0
"""
@property
def fqdn(self) -> str:
"""The Fully Qualified Domain Name (FQDN), if there is a proper `domain` and `suffix`, or else the empty string.
>>> extract("http://forums.bbc.co.uk/path/to/file").fqdn
'forums.bbc.co.uk'
>>> extract("http://localhost:8080").fqdn
''
"""
if self.suffix and (self.domain or self.is_private):
return ".".join(i for i in (self.subdomain, self.domain, self.suffix) if i)
return ""
@property
def ipv4(self) -> str:
"""The IPv4 address, if that is what the input domain/URL was, or else the empty string.
>>> extract("http://127.0.0.1/path/to/file").ipv4
'127.0.0.1'
>>> extract("http://127.0.0.1.1/path/to/file").ipv4
''
>>> extract("http://256.1.1.1").ipv4
''
"""
if (
self.domain
and not (self.suffix or self.subdomain)
and looks_like_ip(self.domain)
):
return self.domain
return ""
@property
def ipv6(self) -> str:
"""The IPv6 address, if that is what the input domain/URL was, or else the empty string.
>>> extract(
... "http://[aBcD:ef01:2345:6789:aBcD:ef01:127.0.0.1]/path/to/file"
... ).ipv6
'aBcD:ef01:2345:6789:aBcD:ef01:127.0.0.1'
>>> extract(
... "http://[aBcD:ef01:2345:6789:aBcD:ef01:127.0.0.1.1]/path/to/file"
... ).ipv6
''
>>> extract("http://[aBcD:ef01:2345:6789:aBcD:ef01:256.0.0.1]").ipv6
''
"""
min_num_ipv6_chars = 4
if (
len(self.domain) >= min_num_ipv6_chars
and self.domain[0] == "["
and self.domain[-1] == "]"
and not (self.suffix or self.subdomain)
):
debracketed = self.domain[1:-1]
if looks_like_ipv6(debracketed):
return debracketed
return ""
@property
def registered_domain(self) -> str:
"""The `domain` and `suffix` fields joined with a dot, if they're both set, or else the empty string.
>>> extract("http://forums.bbc.co.uk").registered_domain
'bbc.co.uk'
>>> extract("http://localhost:8080").registered_domain
''
.. deprecated:: 5.3.0
Use `top_domain_under_public_suffix` instead, which has the same
behavior but a more accurate name.
.. versionremoved:: 6.0.0
This property will be removed in the next major version.
This is an alias for the `top_domain_under_public_suffix` property.
`registered_domain` is so called because is roughly the domain the
owner paid to register with a registrar or, in the case of a private
domain, "registered" with the domain owner. If the input was not
something one could register, this property returns the empty string.
To distinguish the case of private domains, consider Blogspot, which is
in the PSL's private domains. If `include_psl_private_domains` was set
to `False`, the `registered_domain` property of a Blogspot URL
represents the domain the owner of Blogspot registered with a
registrar, i.e. Google registered "blogspot.com". If
`include_psl_private_domains=True`, the `registered_domain` property
represents the "blogspot.com" _subdomain_ the owner of a blog
"registered" with Blogspot.
>>> extract(
... "http://waiterrant.blogspot.com", include_psl_private_domains=False
... ).registered_domain
'blogspot.com'
>>> extract(
... "http://waiterrant.blogspot.com", include_psl_private_domains=True
... ).registered_domain
'waiterrant.blogspot.com'
To always get the same joined string, regardless of the
`include_psl_private_domains` setting, consider the
`top_domain_under_registry_suffix` property.
"""
warnings.warn(
"The 'registered_domain' property is deprecated and will be removed in the next major version. "
"Use 'top_domain_under_public_suffix' instead, which has the same behavior but a more accurate name.",
DeprecationWarning,
stacklevel=2,
)
return self.top_domain_under_public_suffix
@property
def reverse_domain_name(self) -> str:
"""The domain name in Reverse Domain Name Notation.
Joins extracted components of the input URL in reverse domain name
notation. The suffix is used as the leftmost component, followed by the
domain, then followed by the subdomain with its parts reversed.
Reverse Domain Name Notation is typically used to organize namespaces
for packages and plugins. Technically, a full reversal would reverse
the parts of the suffix, e.g. "co.uk" would become "uk.co", but this is
not done in practice when Reverse Domain Name Notation is called for.
So this property leaves the `suffix` part in its original order.
>>> extract("login.example.com").reverse_domain_name
'com.example.login'
>>> extract("login.example.co.uk").reverse_domain_name
'co.uk.example.login'
>>> extract("localhost").reverse_domain_name
'localhost'
"""
stack = [i for i in (self.suffix, self.domain) if i]
if self.subdomain:
stack.extend(reversed(self.subdomain.split(".")))
return ".".join(stack)
@property
def top_domain_under_registry_suffix(self) -> str:
"""The rightmost domain label and `registry_suffix` joined with a dot, if such a domain is available and `registry_suffix` is set, or else the empty string.
The rightmost domain label might be in the `domain` field, or, if the
input URL's suffix is a PSL private domain, in the public suffix
`suffix` field.
If the input was not in the PSL's private domains, this property is
equivalent to `top_domain_under_public_suffix`.
>>> extract(
... "http://waiterrant.blogspot.com", include_psl_private_domains=False
... ).top_domain_under_registry_suffix
'blogspot.com'
>>> extract(
... "http://waiterrant.blogspot.com", include_psl_private_domains=True
... ).top_domain_under_registry_suffix
'blogspot.com'
>>> extract("http://localhost:8080").top_domain_under_registry_suffix
''
.. versionadded:: 5.3.0
"""
top_domain_under_public_suffix = self.top_domain_under_public_suffix
if not top_domain_under_public_suffix or not self.is_private:
return top_domain_under_public_suffix
num_labels = self.registry_suffix.count(".") + 2
return ".".join(top_domain_under_public_suffix.split(".")[-num_labels:])
@property
def top_domain_under_public_suffix(self) -> str:
"""The `domain` and `suffix` fields joined with a dot, if they're both set, or else the empty string.
>>> extract("http://forums.bbc.co.uk").top_domain_under_public_suffix
'bbc.co.uk'
>>> extract("http://localhost:8080").top_domain_under_public_suffix
''
.. versionadded:: 5.3.0
"""
if self.suffix and self.domain:
return f"{self.domain}.{self.suffix}"
return ""
class _UseEnvironmentUrls:
"""Mark omitted URLs so the constructor can read the environment."""
_USE_ENVIRONMENT_URLS = _UseEnvironmentUrls()
class TLDExtract:
"""A callable for extracting, subdomain, domain, and suffix components from a URL."""
# TODO: too-many-arguments
def __init__(
self,
cache_dir: str | None = get_cache_dir(),
suffix_list_urls: Sequence[str] | _UseEnvironmentUrls = _USE_ENVIRONMENT_URLS,
fallback_to_snapshot: bool = True,
include_psl_private_domains: bool = False,
extra_suffixes: Sequence[str] = (),
cache_fetch_timeout: str | float | None = CACHE_TIMEOUT,
) -> None:
"""Construct a callable for extracting subdomain, domain, and suffix components from a URL.
Upon calling it, it first checks for a JSON in `cache_dir`. By default,
the `cache_dir` follows the XDG Base Directory standard, e.g.
`$HOME/.cache/python-tldextract`, falling back to the tldextract package
directory only when no suitable user cache location can be determined.
You can disable the caching functionality of this module by setting
`cache_dir` to `None`.
If the cached version does not exist, such as on the first run, HTTP
request the URLs in `suffix_list_urls` in order, and use the first
successful response for public suffix definitions. Subsequent, untried
URLs are ignored. The default URLs are the latest version of the
Mozilla Public Suffix List and its mirror, but any similar document URL
could be specified. Local files can be specified by using the `file://`
protocol (see `urllib2` documentation). To disable HTTP requests, set
this to an empty sequence. The default can also be set with the
newline-delimited environment variable `TLDEXTRACT_PUBLIC_SUFFIX_LIST_URLS`.
An empty value disables HTTP requests. Entries naming existing local
files are converted to `file://` URLs, as with the `--suffix_list_url`
CLI option. New instances read the environment when constructed; an
explicit `suffix_list_urls` argument takes precedence.
If there is no cached version loaded and no data is found from the `suffix_list_urls`,
the module will fall back to the included TLD set snapshot. If you do not want
this behavior, you may set `fallback_to_snapshot` to `False`, and an exception will be
raised instead.
The Public Suffix List includes a list of "private domains" as TLDs,
such as blogspot.com. These do not fit `tldextract`'s definition of a
suffix, so these domains are excluded by default. If you'd like them
included instead, set `include_psl_private_domains` to `True`.
You can specify additional suffixes in the `extra_suffixes` argument.
These will be merged into whatever public suffix definitions are
already in use by `tldextract`, above.
`cache_fetch_timeout` is passed unmodified to the underlying request object
per the requests documentation here:
http://docs.python-requests.org/en/master/user/advanced/#timeouts
`cache_fetch_timeout` can also be set to a single value with the
environment variable `TLDEXTRACT_DEFAULT_FETCH_TIMEOUT`, like so:
TLDEXTRACT_DEFAULT_FETCH_TIMEOUT="1.2"
When set this way, the same timeout value will be used for both connect
and read timeouts. The older `TLDEXTRACT_CACHE_TIMEOUT` is also supported.
"""
if isinstance(suffix_list_urls, _UseEnvironmentUrls):
suffix_list_urls = _suffix_list_urls_from_env()
suffix_list_urls = suffix_list_urls or ()
self.suffix_list_urls = tuple(
url.strip() for url in suffix_list_urls if url.strip()
)
self.fallback_to_snapshot = fallback_to_snapshot
if not (self.suffix_list_urls or cache_dir or self.fallback_to_snapshot):
raise ValueError(
"The arguments you have provided disable all ways for tldextract "
"to obtain data. Please provide a suffix list data, a cache_dir, "
"or set `fallback_to_snapshot` to `True`."
)
self.include_psl_private_domains = include_psl_private_domains
self.extra_suffixes = extra_suffixes
self._extractor: _PublicSuffixListTLDExtractor | None = None
self.cache_fetch_timeout = (
float(cache_fetch_timeout)
if isinstance(cache_fetch_timeout, str)
else cache_fetch_timeout
)
self._cache = DiskCache(cache_dir)
def __call__(
self,
url: str,
include_psl_private_domains: bool | None = None,
session: requests.Session | None = None,
) -> ExtractResult:
"""Alias for `extract_str`."""
return self.extract_str(url, include_psl_private_domains, session=session)
def extract_str(
self,
url: str,
include_psl_private_domains: bool | None = None,
session: requests.Session | None = None,
) -> ExtractResult:
"""Take a string URL and splits it into its subdomain, domain, and suffix components.
Args:
url: The URL string to extract components from
include_psl_private_domains: Whether to treat PSL private domains as suffixes.
If None, uses the instance default.
session: Optional requests.Session for HTTP configuration (e.g., proxies)
Returns:
ExtractResult: Named tuple containing subdomain, domain, suffix, and metadata
Examples:
Basic extraction:
>>> extractor = TLDExtract()
>>> extractor.extract_str("http://forums.news.cnn.com/")
ExtractResult(subdomain='forums.news', domain='cnn', suffix='com', is_private=False)
>>> extractor.extract_str("http://forums.bbc.co.uk/")
ExtractResult(subdomain='forums', domain='bbc', suffix='co.uk', is_private=False)
Using a custom session:
>>> import requests
>>> session = requests.Session()
>>> # customize your session here
>>> with session:
... extractor.extract_str(
... "http://forums.news.cnn.com/", session=session
... )
ExtractResult(subdomain='forums.news', domain='cnn', suffix='com', is_private=False)
"""
return self._extract_netloc(
lenient_netloc(url), include_psl_private_domains, session=session
)
def extract_urllib(
self,
url: urllib.parse.ParseResult | urllib.parse.SplitResult,
include_psl_private_domains: bool | None = None,
session: requests.Session | None = None,
) -> ExtractResult:
"""Extract components from a pre-parsed URL object.
Args:
url: ParseResult or SplitResult from urllib.parse methods
include_psl_private_domains: Whether to treat PSL private domains as suffixes.
If None, uses the instance default.
session: Optional requests.Session for HTTP configuration
Returns:
ExtractResult: Named tuple containing subdomain, domain, suffix, and metadata
Note:
This method is faster than `extract_str` since the URL is already parsed.
Examples:
>>> extractor = TLDExtract()
>>> extractor.extract_urllib(
... urllib.parse.urlsplit("http://forums.news.cnn.com/")
... )
ExtractResult(subdomain='forums.news', domain='cnn', suffix='com', is_private=False)
>>> extractor.extract_urllib(
... urllib.parse.urlsplit("http://forums.bbc.co.uk/")
... )
ExtractResult(subdomain='forums', domain='bbc', suffix='co.uk', is_private=False)
"""
return self._extract_netloc(
_host_from_authority(url.netloc),
include_psl_private_domains,
session=session,
)
def _extract_netloc(
self,
netloc: str,
include_psl_private_domains: bool | None,
session: requests.Session | None = None,
) -> ExtractResult:
netloc_with_ascii_dots = (
netloc.replace("\u3002", "\u002e")
.replace("\uff0e", "\u002e")
.replace("\uff61", "\u002e")
)
min_num_ipv6_chars = 4
if (
len(netloc_with_ascii_dots) >= min_num_ipv6_chars
and netloc_with_ascii_dots[0] == "["
and netloc_with_ascii_dots[-1] == "]"
and looks_like_ipv6(netloc_with_ascii_dots[1:-1])
):
return ExtractResult(
"", netloc_with_ascii_dots, "", is_private=False, registry_suffix=""
)
labels = netloc_with_ascii_dots.split(".")
maybe_indexes = self._get_tld_extractor(session).suffix_index(
labels, include_psl_private_domains=include_psl_private_domains
)
num_ipv4_labels = 4
if (
not maybe_indexes
and len(labels) == num_ipv4_labels
and looks_like_ip(netloc_with_ascii_dots)
):
return ExtractResult(
"", netloc_with_ascii_dots, "", is_private=False, registry_suffix=""
)
elif not maybe_indexes:
return ExtractResult(
subdomain=".".join(labels[:-1]),
domain=labels[-1],
suffix="",
is_private=False,
registry_suffix="",
)
(
(public_suffix_index, public_suffix_node),
(registry_suffix_index, registry_suffix_node),
) = maybe_indexes
subdomain = (
".".join(labels[: public_suffix_index - 1])
if public_suffix_index >= 2
else ""
)
domain = labels[public_suffix_index - 1] if public_suffix_index > 0 else ""
public_suffix = ".".join(labels[public_suffix_index:])
registry_suffix = (
".".join(labels[registry_suffix_index:])
if public_suffix_node.is_private
else public_suffix
)
return ExtractResult(
subdomain=subdomain,
domain=domain,
suffix=public_suffix,
is_private=public_suffix_node.is_private,
registry_suffix=registry_suffix,
)
def update(
self, fetch_now: bool = False, session: requests.Session | None = None
) -> None:
"""Clear cache and force fresh suffix list fetch on next extraction.
Args:
fetch_now: If True, immediately fetch updated suffix lists
session: Optional requests.Session for HTTP configuration
"""
self._extractor = None
self._cache.clear()
if fetch_now:
self._get_tld_extractor(session=session)
@property
def tlds(self, session: requests.Session | None = None) -> list[str]:
"""The list of TLDs used by default.
This will vary based on `include_psl_private_domains` and `extra_suffixes`.
"""
return list(self._get_tld_extractor(session=session).tlds())
def _get_tld_extractor(
self, session: requests.Session | None = None
) -> _PublicSuffixListTLDExtractor:
"""Get or compute this object's TLDExtractor.
Looks up the TLDExtractor in roughly the following order, based on the
settings passed to __init__:
1. Memoized on `self`
2. Local system _cache file
3. Remote PSL, over HTTP
4. Bundled PSL snapshot file
"""
if self._extractor:
return self._extractor
public_tlds, private_tlds = get_suffix_lists(
cache=self._cache,
urls=self.suffix_list_urls,
cache_fetch_timeout=self.cache_fetch_timeout,
fallback_to_snapshot=self.fallback_to_snapshot,
session=session,
)
if not any([public_tlds, private_tlds, self.extra_suffixes]):
raise ValueError("No tlds set. Cannot proceed without tlds.")
self._extractor = _PublicSuffixListTLDExtractor(
public_tlds=public_tlds,
private_tlds=private_tlds,
extra_tlds=list(self.extra_suffixes),
include_psl_private_domains=self.include_psl_private_domains,
)
return self._extractor
def _suffix_list_urls_from_env() -> Sequence[str]:
"""Resolve the default suffix list URLs from the environment."""
raw = os.environ.get("TLDEXTRACT_PUBLIC_SUFFIX_LIST_URLS")
if raw is None:
return PUBLIC_SUFFIX_LIST_URLS
urls = []
for line in raw.splitlines():
source = line.strip()
if not source:
continue
if os.path.isfile(source):
source = pathlib.Path(os.path.abspath(source)).as_uri()
urls.append(source)
return urls
TLD_EXTRACTOR = TLDExtract()
class Trie:
"""Trie for storing eTLDs with their labels in reverse-order."""
def __init__(
self,
matches: dict[str, Trie] | None = None,
end: bool = False,
is_private: bool = False,
) -> None:
"""TODO."""
self.matches = matches if matches else {}
self.end = end
self.is_private = is_private
@staticmethod
def create(
public_suffixes: Collection[str],
private_suffixes: Collection[str] | None = None,
) -> Trie:
"""Create a Trie from a list of suffixes and return its root node."""
root_node = Trie()
for suffix in public_suffixes:
root_node.add_suffix(suffix)
if private_suffixes is None:
private_suffixes = []
for suffix in private_suffixes:
root_node.add_suffix(suffix, True)
return root_node
def add_suffix(self, suffix: str, is_private: bool = False) -> None:
"""Append a suffix's labels to this Trie node."""
node = self
labels = suffix.split(".")
labels.reverse()
for label in labels:
if label not in node.matches:
node.matches[label] = Trie()
node = node.matches[label]
node.end = True
node.is_private = is_private
@wraps(TLD_EXTRACTOR.__call__)
def extract( # noqa: D103
url: str,
include_psl_private_domains: bool | None = False,
session: requests.Session | None = None,
) -> ExtractResult:
return TLD_EXTRACTOR(
url, include_psl_private_domains=include_psl_private_domains, session=session
)
@wraps(TLD_EXTRACTOR.update)
def update(*args, **kwargs): # type: ignore[no-untyped-def] # noqa: D103
return TLD_EXTRACTOR.update(*args, **kwargs)
class _PublicSuffixListTLDExtractor:
"""Wrapper around this project's main algo for PSL lookups."""
def __init__(
self,
public_tlds: list[str],
private_tlds: list[str],
extra_tlds: list[str],
include_psl_private_domains: bool = False,
):
# set the default value
self.include_psl_private_domains = include_psl_private_domains
self.public_tlds = public_tlds
self.private_tlds = private_tlds
self.tlds_incl_private = frozenset(public_tlds + private_tlds + extra_tlds)
self.tlds_excl_private = frozenset(public_tlds + extra_tlds)
self.tlds_incl_private_trie = Trie.create(
self.tlds_excl_private, frozenset(private_tlds)
)
self.tlds_excl_private_trie = Trie.create(self.tlds_excl_private)
def tlds(self, include_psl_private_domains: bool | None = None) -> frozenset[str]:
"""Get the currently filtered list of suffixes."""
if include_psl_private_domains is None:
include_psl_private_domains = self.include_psl_private_domains
return (
self.tlds_incl_private
if include_psl_private_domains
else self.tlds_excl_private
)
def suffix_index(
self, spl: list[str], include_psl_private_domains: bool | None = None
) -> tuple[tuple[int, Trie], tuple[int, Trie]] | None:
"""Return the index of the first public suffix label, the index of the first registry suffix label, and their corresponding trie nodes.
Returns `None` if no suffix is found.
"""
if include_psl_private_domains is None:
include_psl_private_domains = self.include_psl_private_domains
node = reg_node = suffix_node = (
self.tlds_incl_private_trie
if include_psl_private_domains
else self.tlds_excl_private_trie
)
suffix_idx = reg_idx = label_idx = len(spl)
for label in reversed(spl):
decoded_label = _decode_punycode(label)
if decoded_label in node.matches:
label_idx -= 1
node = node.matches[decoded_label]
if node.end:
suffix_idx = label_idx
suffix_node = node
if not node.is_private:
reg_node = node
reg_idx = label_idx
continue
is_wildcard = "*" in node.matches
if is_wildcard:
is_wildcard_exception = "!" + decoded_label in node.matches
return (
label_idx if is_wildcard_exception else label_idx - 1,
node.matches["*"],
), (
reg_idx,
reg_node,
)
break
if suffix_idx == len(spl):
return None
return ((suffix_idx, suffix_node), (reg_idx, reg_node))
def _decode_punycode(label: str) -> str:
lowered = label.lower()
looks_like_puny = lowered.startswith("xn--")
if looks_like_puny:
try:
return idna.decode(lowered)
except (UnicodeError, IndexError):
pass
return lowered