Initial commit
This commit is contained in:
commit
57fc3525eb
7311 files changed
+1291398
No files matched your search
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,14 @@
|
||||
"""Export tldextract's public interface."""
|
||||
|
||||
from . import _version
|
||||
from .tldextract import ExtractResult, TLDExtract, extract, update
|
||||
|
||||
__version__: str = _version.version
|
||||
|
||||
__all__ = [
|
||||
"__version__",
|
||||
"extract",
|
||||
"ExtractResult",
|
||||
"TLDExtract",
|
||||
"update",
|
||||
]
|
||||
@@ -0,0 +1,6 @@
|
||||
"""tldextract __main__."""
|
||||
|
||||
from .cli import main
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -0,0 +1,24 @@
|
||||
# file generated by vcs-versioning
|
||||
# don't change, don't track in version control
|
||||
from __future__ import annotations
|
||||
|
||||
__all__ = [
|
||||
"__version__",
|
||||
"__version_tuple__",
|
||||
"version",
|
||||
"version_tuple",
|
||||
"__commit_id__",
|
||||
"commit_id",
|
||||
]
|
||||
|
||||
version: str
|
||||
__version__: str
|
||||
__version_tuple__: tuple[int | str, ...]
|
||||
version_tuple: tuple[int | str, ...]
|
||||
commit_id: str | None
|
||||
__commit_id__: str | None
|
||||
|
||||
__version__ = version = '5.4.0'
|
||||
__version_tuple__ = version_tuple = (5, 4, 0)
|
||||
|
||||
__commit_id__ = commit_id = 'j6937495ce'
|
||||
@@ -0,0 +1,238 @@
|
||||
"""Helpers."""
|
||||
|
||||
import errno
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
from collections.abc import Callable, Hashable, Iterable
|
||||
from pathlib import Path
|
||||
from typing import (
|
||||
TypeVar,
|
||||
cast,
|
||||
)
|
||||
|
||||
import requests
|
||||
from filelock import FileLock
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
_DID_LOG_UNABLE_TO_CACHE = False
|
||||
|
||||
T = TypeVar("T")
|
||||
|
||||
|
||||
def get_pkg_unique_identifier() -> str:
|
||||
"""Generate an identifier unique to the python version, tldextract version, and python instance.
|
||||
|
||||
This will prevent interference between virtualenvs and issues that might arise when installing
|
||||
a new version of tldextract
|
||||
"""
|
||||
try:
|
||||
from tldextract._version import version
|
||||
except ImportError:
|
||||
version = "dev"
|
||||
|
||||
tldextract_version = "tldextract-" + version
|
||||
python_env_name = os.path.basename(sys.prefix)
|
||||
# just to handle the edge case of two identically named python environments
|
||||
python_binary_path_short_hash = hashlib.md5(
|
||||
sys.prefix.encode("utf-8"), usedforsecurity=False
|
||||
).hexdigest()[:6]
|
||||
python_version = ".".join([str(v) for v in sys.version_info[:-1]])
|
||||
identifier_parts = [
|
||||
python_version,
|
||||
python_env_name,
|
||||
python_binary_path_short_hash,
|
||||
tldextract_version,
|
||||
]
|
||||
pkg_identifier = "__".join(identifier_parts)
|
||||
|
||||
return pkg_identifier
|
||||
|
||||
|
||||
def get_cache_dir() -> str:
|
||||
"""Get a cache dir that we have permission to write to.
|
||||
|
||||
Try to follow the XDG standard, but if that doesn't work fallback to the package directory
|
||||
http://specifications.freedesktop.org/basedir-spec/basedir-spec-latest.html
|
||||
"""
|
||||
cache_dir = os.environ.get("TLDEXTRACT_CACHE", None)
|
||||
if cache_dir is not None:
|
||||
return cache_dir
|
||||
|
||||
xdg_cache_home = os.getenv("XDG_CACHE_HOME", None)
|
||||
if xdg_cache_home is None:
|
||||
user_home = os.getenv("HOME", None)
|
||||
if user_home:
|
||||
xdg_cache_home = str(Path(user_home, ".cache"))
|
||||
|
||||
if xdg_cache_home is not None:
|
||||
return str(
|
||||
Path(xdg_cache_home, "python-tldextract", get_pkg_unique_identifier())
|
||||
)
|
||||
|
||||
# fallback to trying to use package directory itself
|
||||
return str(Path(os.path.dirname(__file__), ".suffix_cache"))
|
||||
|
||||
|
||||
class DiskCache:
|
||||
"""Disk _cache that only works for jsonable values."""
|
||||
|
||||
def __init__(self, cache_dir: str | None, lock_timeout: int = 20):
|
||||
"""Construct a disk cache in the given directory."""
|
||||
self.enabled = bool(cache_dir)
|
||||
self.cache_dir = os.path.expanduser(str(cache_dir) or "")
|
||||
self.lock_timeout = lock_timeout
|
||||
# using a unique extension provides some safety that an incorrectly set cache_dir
|
||||
# combined with a call to `.clear()` wont wipe someones hard drive
|
||||
self.file_ext = ".tldextract.json"
|
||||
|
||||
def get(self, namespace: str, key: str | dict[str, Hashable]) -> object:
|
||||
"""Retrieve a value from the disk cache."""
|
||||
if not self.enabled:
|
||||
raise KeyError("Cache is disabled")
|
||||
cache_filepath = self._key_to_cachefile_path(namespace, key)
|
||||
|
||||
if not os.path.isfile(cache_filepath):
|
||||
raise KeyError("namespace: " + namespace + " key: " + repr(key))
|
||||
try:
|
||||
with open(cache_filepath) as cache_file:
|
||||
return json.load(cache_file)
|
||||
except (OSError, ValueError) as exc:
|
||||
raise KeyError("namespace: " + namespace + " key: " + repr(key)) from exc
|
||||
|
||||
def set( # noqa: A003
|
||||
self, namespace: str, key: str | dict[str, Hashable], value: object
|
||||
) -> None:
|
||||
"""Set a value in the disk cache."""
|
||||
if not self.enabled:
|
||||
return
|
||||
|
||||
cache_filepath = self._key_to_cachefile_path(namespace, key)
|
||||
|
||||
try:
|
||||
_make_dir(cache_filepath)
|
||||
with open(cache_filepath, "w") as cache_file:
|
||||
json.dump(value, cache_file)
|
||||
except OSError as ioe:
|
||||
global _DID_LOG_UNABLE_TO_CACHE
|
||||
if not _DID_LOG_UNABLE_TO_CACHE:
|
||||
LOG.warning(
|
||||
"unable to cache %s.%s in %s. This could refresh the "
|
||||
"Public Suffix List over HTTP every app startup. "
|
||||
"Construct your `TLDExtract` with a writable `cache_dir` or "
|
||||
"set `cache_dir=None` to silence this warning. %s",
|
||||
namespace,
|
||||
key,
|
||||
cache_filepath,
|
||||
ioe,
|
||||
)
|
||||
_DID_LOG_UNABLE_TO_CACHE = True
|
||||
|
||||
def clear(self) -> None:
|
||||
"""Clear the disk cache."""
|
||||
for root, _, files in os.walk(self.cache_dir):
|
||||
for filename in files:
|
||||
if filename.endswith(self.file_ext) or filename.endswith(
|
||||
self.file_ext + ".lock"
|
||||
):
|
||||
try:
|
||||
os.unlink(str(Path(root, filename)))
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
except OSError as exc:
|
||||
# errno.ENOENT == "No such file or directory"
|
||||
# https://docs.python.org/2/library/errno.html#errno.ENOENT
|
||||
if exc.errno != errno.ENOENT:
|
||||
raise
|
||||
|
||||
def _key_to_cachefile_path(
|
||||
self, namespace: str, key: str | dict[str, Hashable]
|
||||
) -> str:
|
||||
namespace_path = str(Path(self.cache_dir, namespace))
|
||||
hashed_key = _make_cache_key(key)
|
||||
|
||||
cache_path = str(Path(namespace_path, hashed_key + self.file_ext))
|
||||
|
||||
return cache_path
|
||||
|
||||
def run_and_cache(
|
||||
self,
|
||||
func: Callable[..., T],
|
||||
namespace: str,
|
||||
kwargs: dict[str, Hashable],
|
||||
hashed_argnames: Iterable[str],
|
||||
) -> T:
|
||||
"""Get a url but cache the response."""
|
||||
if not self.enabled:
|
||||
return func(**kwargs)
|
||||
|
||||
key_args = {k: v for k, v in kwargs.items() if k in hashed_argnames}
|
||||
cache_filepath = self._key_to_cachefile_path(namespace, key_args)
|
||||
lock_path = cache_filepath + ".lock"
|
||||
try:
|
||||
_make_dir(cache_filepath)
|
||||
except OSError as ioe:
|
||||
global _DID_LOG_UNABLE_TO_CACHE
|
||||
if not _DID_LOG_UNABLE_TO_CACHE:
|
||||
LOG.warning(
|
||||
"unable to cache %s.%s in %s. This could refresh the "
|
||||
"Public Suffix List over HTTP every app startup. "
|
||||
"Construct your `TLDExtract` with a writable `cache_dir` or "
|
||||
"set `cache_dir=None` to silence this warning. %s",
|
||||
namespace,
|
||||
key_args,
|
||||
cache_filepath,
|
||||
ioe,
|
||||
)
|
||||
_DID_LOG_UNABLE_TO_CACHE = True
|
||||
|
||||
return func(**kwargs)
|
||||
|
||||
with FileLock(lock_path, timeout=self.lock_timeout):
|
||||
try:
|
||||
result = cast(T, self.get(namespace=namespace, key=key_args))
|
||||
except KeyError:
|
||||
result = func(**kwargs)
|
||||
self.set(namespace=namespace, key=key_args, value=result)
|
||||
|
||||
return result
|
||||
|
||||
def cached_fetch_url(
|
||||
self, session: requests.Session, url: str, timeout: float | int | None
|
||||
) -> str:
|
||||
"""Get a url but cache the response."""
|
||||
return self.run_and_cache(
|
||||
func=_fetch_url,
|
||||
namespace="urls",
|
||||
kwargs={"session": session, "url": url, "timeout": timeout},
|
||||
hashed_argnames=["url"],
|
||||
)
|
||||
|
||||
|
||||
def _fetch_url(session: requests.Session, url: str, timeout: int | None) -> str:
|
||||
response = session.get(url, timeout=timeout)
|
||||
response.raise_for_status()
|
||||
text = response.text
|
||||
|
||||
if not isinstance(text, str):
|
||||
text = str(text, "utf-8")
|
||||
|
||||
return text
|
||||
|
||||
|
||||
def _make_cache_key(inputs: str | dict[str, Hashable]) -> str:
|
||||
key = repr(inputs)
|
||||
return hashlib.md5(key.encode("utf8"), usedforsecurity=False).hexdigest()
|
||||
|
||||
|
||||
def _make_dir(filename: str) -> None:
|
||||
"""Make a directory if it doesn't already exist."""
|
||||
if not os.path.exists(os.path.dirname(filename)):
|
||||
try:
|
||||
os.makedirs(os.path.dirname(filename))
|
||||
except OSError as exc: # Guard against race condition
|
||||
if exc.errno != errno.EEXIST:
|
||||
raise
|
||||
@@ -0,0 +1,119 @@
|
||||
"""tldextract CLI."""
|
||||
|
||||
import argparse
|
||||
import dataclasses
|
||||
import json
|
||||
import logging
|
||||
import os.path
|
||||
import pathlib
|
||||
import sys
|
||||
|
||||
from ._version import version as __version__
|
||||
from .tldextract import TLDExtract
|
||||
|
||||
|
||||
def main() -> None:
|
||||
"""Tldextract CLI main command."""
|
||||
logging.basicConfig()
|
||||
|
||||
parser = argparse.ArgumentParser(
|
||||
prog="tldextract", description="Parse hostname from a url or fqdn"
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"--version", action="version", version="%(prog)s " + __version__
|
||||
)
|
||||
parser.add_argument(
|
||||
"-j",
|
||||
"--json",
|
||||
default=False,
|
||||
action="store_true",
|
||||
help="output in json format",
|
||||
)
|
||||
parser.add_argument(
|
||||
"input", metavar="fqdn|url", type=str, nargs="*", help="fqdn or url"
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"-u",
|
||||
"--update",
|
||||
default=False,
|
||||
action="store_true",
|
||||
help="force fetch the latest TLD definitions",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--suffix_list_url",
|
||||
action="append",
|
||||
required=False,
|
||||
help="use an alternate URL or local file for TLD definitions",
|
||||
)
|
||||
parser.add_argument(
|
||||
"-c", "--cache_dir", help="use an alternate TLD definition caching folder"
|
||||
)
|
||||
parser.add_argument(
|
||||
"-p",
|
||||
"--include_psl_private_domains",
|
||||
"--private_domains",
|
||||
default=False,
|
||||
action="store_true",
|
||||
help="Include private domains",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--no_fallback_to_snapshot",
|
||||
default=True,
|
||||
action="store_false",
|
||||
dest="fallback_to_snapshot",
|
||||
help="Don't fall back to the package's snapshot of the suffix list",
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
obj_kwargs = {
|
||||
"include_psl_private_domains": args.include_psl_private_domains,
|
||||
"fallback_to_snapshot": args.fallback_to_snapshot,
|
||||
}
|
||||
|
||||
if args.cache_dir:
|
||||
obj_kwargs["cache_dir"] = args.cache_dir
|
||||
|
||||
if args.suffix_list_url is not None:
|
||||
suffix_list_urls = []
|
||||
for source in args.suffix_list_url:
|
||||
if os.path.isfile(source):
|
||||
as_path_uri = pathlib.Path(os.path.abspath(source)).as_uri()
|
||||
suffix_list_urls.append(as_path_uri)
|
||||
else:
|
||||
suffix_list_urls.append(source)
|
||||
|
||||
obj_kwargs["suffix_list_urls"] = suffix_list_urls
|
||||
|
||||
tld_extract = TLDExtract(**obj_kwargs)
|
||||
|
||||
if args.update:
|
||||
tld_extract.update(True)
|
||||
elif not args.input:
|
||||
parser.print_usage()
|
||||
sys.exit(1)
|
||||
|
||||
for i in args.input:
|
||||
ext = tld_extract(i)
|
||||
if args.json:
|
||||
properties = (
|
||||
"fqdn",
|
||||
"ipv4",
|
||||
"ipv6",
|
||||
"registered_domain",
|
||||
"reverse_domain_name",
|
||||
"top_domain_under_public_suffix",
|
||||
"top_domain_under_registry_suffix",
|
||||
)
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
**dataclasses.asdict(ext),
|
||||
**{prop: getattr(ext, prop) for prop in properties},
|
||||
}
|
||||
)
|
||||
)
|
||||
else:
|
||||
print(f"{ext.subdomain} {ext.domain} {ext.suffix}")
|
||||
Whitespace-only changes.
@@ -0,0 +1,69 @@
|
||||
"""tldextract helpers for testing and fetching remote resources."""
|
||||
|
||||
import re
|
||||
from ipaddress import AddressValueError, IPv6Address
|
||||
from urllib.parse import scheme_chars
|
||||
|
||||
IP_RE = re.compile(
|
||||
r"^(?:(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.)"
|
||||
r"{3}(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])$",
|
||||
re.ASCII,
|
||||
)
|
||||
|
||||
scheme_chars_set = set(scheme_chars)
|
||||
|
||||
|
||||
def lenient_netloc(url: str) -> str:
|
||||
"""Extract the host portion of a URL-like string.
|
||||
|
||||
Parse more leniently than urllib.parse.{urlparse,urlsplit}, preserving
|
||||
casing and brackets around IPv6 addresses.
|
||||
"""
|
||||
authority = (
|
||||
_schemeless_url(url).partition("/")[0].partition("?")[0].partition("#")[0]
|
||||
)
|
||||
return _host_from_authority(authority)
|
||||
|
||||
|
||||
def _host_from_authority(authority: str) -> str:
|
||||
"""Extract a case-preserving host from a parsed URL's authority."""
|
||||
after_userinfo = authority.rpartition("@")[-1]
|
||||
|
||||
if after_userinfo and after_userinfo[0] == "[":
|
||||
maybe_ipv6 = after_userinfo.partition("]")
|
||||
if maybe_ipv6[1] == "]":
|
||||
return f"{maybe_ipv6[0]}]"
|
||||
|
||||
hostname = after_userinfo.partition(":")[0].strip()
|
||||
without_root_label = hostname.rstrip(".\u3002\uff0e\uff61")
|
||||
return without_root_label
|
||||
|
||||
|
||||
def _schemeless_url(url: str) -> str:
|
||||
double_slashes_start = url.find("//")
|
||||
if double_slashes_start == 0:
|
||||
return url[2:]
|
||||
if (
|
||||
double_slashes_start < 2
|
||||
or url[double_slashes_start - 1] != ":"
|
||||
or set(url[: double_slashes_start - 1]) - scheme_chars_set
|
||||
):
|
||||
return url
|
||||
return url[double_slashes_start + 2 :]
|
||||
|
||||
|
||||
def looks_like_ip(maybe_ip: str) -> bool:
|
||||
"""Check whether the given str looks like an IPv4 address."""
|
||||
if not maybe_ip[0].isdecimal():
|
||||
return False
|
||||
|
||||
return IP_RE.fullmatch(maybe_ip) is not None
|
||||
|
||||
|
||||
def looks_like_ipv6(maybe_ip: str) -> bool:
|
||||
"""Check whether the given str looks like an IPv6 address."""
|
||||
try:
|
||||
IPv6Address(maybe_ip)
|
||||
except AddressValueError:
|
||||
return False
|
||||
return True
|
||||
@@ -0,0 +1,118 @@
|
||||
"""tldextract helpers for testing and fetching remote resources."""
|
||||
|
||||
import logging
|
||||
import pkgutil
|
||||
import re
|
||||
from collections.abc import Sequence
|
||||
from typing import cast
|
||||
|
||||
import requests
|
||||
from requests_file import FileAdapter
|
||||
|
||||
from .cache import DiskCache
|
||||
|
||||
LOG = logging.getLogger("tldextract")
|
||||
|
||||
PUBLIC_SUFFIX_RE = re.compile(r"^(?P<suffix>[.*!]*\w[\S]*)", re.UNICODE | re.MULTILINE)
|
||||
PUBLIC_PRIVATE_SUFFIX_SEPARATOR = "// ===BEGIN PRIVATE DOMAINS==="
|
||||
|
||||
|
||||
class SuffixListNotFound(LookupError): # noqa: N818
|
||||
"""A recoverable error while looking up a suffix list.
|
||||
|
||||
Recoverable because you can specify backups, or use this library's bundled
|
||||
snapshot.
|
||||
"""
|
||||
|
||||
|
||||
def find_first_response(
|
||||
cache: DiskCache,
|
||||
urls: Sequence[str],
|
||||
cache_fetch_timeout: float | int | None = None,
|
||||
session: requests.Session | None = None,
|
||||
) -> str:
|
||||
"""Decode the first successfully fetched URL, from UTF-8 encoding to Python unicode."""
|
||||
session_created = False
|
||||
if session is None:
|
||||
session = requests.Session()
|
||||
session.mount("file://", FileAdapter())
|
||||
session_created = True
|
||||
|
||||
try:
|
||||
for url in urls:
|
||||
try:
|
||||
return cache.cached_fetch_url(
|
||||
session=session, url=url, timeout=cache_fetch_timeout
|
||||
)
|
||||
except requests.exceptions.RequestException:
|
||||
LOG.warning(
|
||||
"Exception reading Public Suffix List url %s", url, exc_info=True
|
||||
)
|
||||
finally:
|
||||
# Ensure the session is always closed if it's constructed in the method
|
||||
if session_created:
|
||||
session.close()
|
||||
|
||||
raise SuffixListNotFound(
|
||||
"No remote Public Suffix List found. Consider using a mirror, or avoid this"
|
||||
" fetch by constructing your TLDExtract with `suffix_list_urls=()`."
|
||||
)
|
||||
|
||||
|
||||
def extract_tlds_from_suffix_list(suffix_list_text: str) -> tuple[list[str], list[str]]:
|
||||
"""Parse the raw suffix list text for its different designations of suffixes."""
|
||||
public_text, _, private_text = suffix_list_text.partition(
|
||||
PUBLIC_PRIVATE_SUFFIX_SEPARATOR
|
||||
)
|
||||
|
||||
public_tlds = [m.group("suffix") for m in PUBLIC_SUFFIX_RE.finditer(public_text)]
|
||||
private_tlds = [m.group("suffix") for m in PUBLIC_SUFFIX_RE.finditer(private_text)]
|
||||
return public_tlds, private_tlds
|
||||
|
||||
|
||||
def get_suffix_lists(
|
||||
cache: DiskCache,
|
||||
urls: Sequence[str],
|
||||
cache_fetch_timeout: float | int | None,
|
||||
fallback_to_snapshot: bool,
|
||||
session: requests.Session | None = None,
|
||||
) -> tuple[list[str], list[str]]:
|
||||
"""Fetch, parse, and cache the suffix lists."""
|
||||
return cache.run_and_cache(
|
||||
func=_get_suffix_lists,
|
||||
namespace="publicsuffix.org-tlds",
|
||||
kwargs={
|
||||
"cache": cache,
|
||||
"urls": urls,
|
||||
"cache_fetch_timeout": cache_fetch_timeout,
|
||||
"fallback_to_snapshot": fallback_to_snapshot,
|
||||
"session": session,
|
||||
},
|
||||
hashed_argnames=["urls", "fallback_to_snapshot"],
|
||||
)
|
||||
|
||||
|
||||
def _get_suffix_lists(
|
||||
cache: DiskCache,
|
||||
urls: Sequence[str],
|
||||
cache_fetch_timeout: float | int | None,
|
||||
fallback_to_snapshot: bool,
|
||||
session: requests.Session | None = None,
|
||||
) -> tuple[list[str], list[str]]:
|
||||
"""Fetch, parse, and cache the suffix lists."""
|
||||
try:
|
||||
text = find_first_response(
|
||||
cache, urls, cache_fetch_timeout=cache_fetch_timeout, session=session
|
||||
)
|
||||
except SuffixListNotFound as exc:
|
||||
if fallback_to_snapshot:
|
||||
maybe_pkg_data = pkgutil.get_data("tldextract", ".tld_set_snapshot")
|
||||
# package maintainers guarantee file is included
|
||||
pkg_data = cast(bytes, maybe_pkg_data)
|
||||
text = pkg_data.decode("utf-8")
|
||||
else:
|
||||
raise exc
|
||||
|
||||
public_tlds, private_tlds = extract_tlds_from_suffix_list(text)
|
||||
|
||||
return public_tlds, private_tlds
|
||||
@@ -0,0 +1,801 @@
|
||||
"""`tldextract` accurately separates a URL's subdomain, domain, and public suffix.
|
||||
|
||||
It does this via the Public Suffix List (PSL).
|
||||
|
||||
>>> import tldextract
|
||||
|
||||
>>> tldextract.extract("http://forums.news.cnn.com/")
|
||||
ExtractResult(subdomain='forums.news', domain='cnn', suffix='com', is_private=False)
|
||||
|
||||
>>> tldextract.extract("http://forums.bbc.co.uk/") # United Kingdom
|
||||
ExtractResult(subdomain='forums', domain='bbc', suffix='co.uk', is_private=False)
|
||||
|
||||
>>> tldextract.extract("http://www.worldbank.org.kg/") # Kyrgyzstan
|
||||
ExtractResult(subdomain='www', domain='worldbank', suffix='org.kg', is_private=False)
|
||||
|
||||
Note subdomain and suffix are _optional_. Not all URL-like inputs have a
|
||||
subdomain or a valid suffix.
|
||||
|
||||
>>> tldextract.extract("google.com")
|
||||
ExtractResult(subdomain='', domain='google', suffix='com', is_private=False)
|
||||
|
||||
>>> tldextract.extract("google.notavalidsuffix")
|
||||
ExtractResult(subdomain='google', domain='notavalidsuffix', suffix='', is_private=False)
|
||||
|
||||
>>> tldextract.extract("http://127.0.0.1:8080/deployed/")
|
||||
ExtractResult(subdomain='', domain='127.0.0.1', suffix='', is_private=False)
|
||||
|
||||
To rejoin the original hostname, if it was indeed a valid, registered hostname:
|
||||
|
||||
>>> ext = tldextract.extract("http://forums.bbc.co.uk")
|
||||
>>> ext.top_domain_under_public_suffix
|
||||
'bbc.co.uk'
|
||||
>>> ext.fqdn
|
||||
'forums.bbc.co.uk'
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import pathlib
|
||||
import urllib.parse
|
||||
import warnings
|
||||
from collections.abc import Collection, Sequence
|
||||
from dataclasses import dataclass, field
|
||||
from functools import wraps
|
||||
|
||||
import idna
|
||||
import requests
|
||||
|
||||
from .cache import DiskCache, get_cache_dir
|
||||
from .remote import (
|
||||
_host_from_authority,
|
||||
lenient_netloc,
|
||||
looks_like_ip,
|
||||
looks_like_ipv6,
|
||||
)
|
||||
from .suffix_list import get_suffix_lists
|
||||
|
||||
|
||||
def _get_default_cache_fetch_timeout() -> str | None:
|
||||
"""Read the default PSL fetch timeout from the environment."""
|
||||
return os.environ.get(
|
||||
"TLDEXTRACT_DEFAULT_FETCH_TIMEOUT",
|
||||
os.environ.get("TLDEXTRACT_CACHE_TIMEOUT"),
|
||||
)
|
||||
|
||||
|
||||
CACHE_TIMEOUT = _get_default_cache_fetch_timeout()
|
||||
|
||||
PUBLIC_SUFFIX_LIST_URLS = (
|
||||
"https://publicsuffix.org/list/public_suffix_list.dat",
|
||||
"https://raw.githubusercontent.com/publicsuffix/list/master/public_suffix_list.dat",
|
||||
)
|
||||
|
||||
|
||||
@dataclass(order=True)
|
||||
class ExtractResult:
|
||||
"""A URL's extracted subdomain, domain, and suffix.
|
||||
|
||||
These first 3 fields are what most users of this library will care about.
|
||||
They are the split, non-overlapping hostname components of the input URL.
|
||||
They can be used to rebuild the original URL's hostname.
|
||||
|
||||
The extracted strings generally preserve the input's casing and Unicode or
|
||||
ASCII-compatible encoding (ACE, commonly called Punycode). They are not
|
||||
normalized for comparison or validated as real hostnames.
|
||||
|
||||
Beyond the first 3 fields, the class contains metadata fields, like a flag
|
||||
that indicates if the input URL's suffix is from a private domain.
|
||||
"""
|
||||
|
||||
subdomain: str
|
||||
"""All subdomains beneath the domain of the input URL, if it contained any such subdomains, or else the empty string."""
|
||||
|
||||
domain: str
|
||||
"""The topmost domain of the input URL, if it contained a domain name, or else everything hostname-like in the input.
|
||||
|
||||
If the input URL didn't contain a real domain name, the `suffix` field will
|
||||
be empty, and this field will catch values like an IP address, or
|
||||
private network hostnames like "localhost".
|
||||
"""
|
||||
|
||||
suffix: str
|
||||
"""The public suffix of the input URL, if it contained one, or else the empty string.
|
||||
|
||||
If `include_psl_private_domains` was set to `False`, this field is the same
|
||||
as `registry_suffix`, i.e. a domain under which people can register
|
||||
subdomains through a registrar. If `include_psl_private_domains` was set to
|
||||
`True`, this field may be a PSL private domain, like "blogspot.com".
|
||||
"""
|
||||
|
||||
is_private: bool
|
||||
"""Whether the input URL belongs in the Public Suffix List's private domains.
|
||||
|
||||
If `include_psl_private_domains` was set to `False`, this field is always
|
||||
`False`.
|
||||
"""
|
||||
|
||||
registry_suffix: str = field(repr=False)
|
||||
"""The registry suffix of the input URL, if it contained one, or else the empty string.
|
||||
|
||||
This field is a domain under which people can register subdomains through a
|
||||
registar.
|
||||
|
||||
This field is unaffected by the `include_psl_private_domains` setting. If
|
||||
`include_psl_private_domains` was set to `False`, this field is always the
|
||||
same as `suffix`.
|
||||
|
||||
.. versionadded:: 5.3.0
|
||||
"""
|
||||
|
||||
@property
|
||||
def fqdn(self) -> str:
|
||||
"""The Fully Qualified Domain Name (FQDN), if there is a proper `domain` and `suffix`, or else the empty string.
|
||||
|
||||
>>> extract("http://forums.bbc.co.uk/path/to/file").fqdn
|
||||
'forums.bbc.co.uk'
|
||||
>>> extract("http://localhost:8080").fqdn
|
||||
''
|
||||
"""
|
||||
if self.suffix and (self.domain or self.is_private):
|
||||
return ".".join(i for i in (self.subdomain, self.domain, self.suffix) if i)
|
||||
return ""
|
||||
|
||||
@property
|
||||
def ipv4(self) -> str:
|
||||
"""The IPv4 address, if that is what the input domain/URL was, or else the empty string.
|
||||
|
||||
>>> extract("http://127.0.0.1/path/to/file").ipv4
|
||||
'127.0.0.1'
|
||||
>>> extract("http://127.0.0.1.1/path/to/file").ipv4
|
||||
''
|
||||
>>> extract("http://256.1.1.1").ipv4
|
||||
''
|
||||
"""
|
||||
if (
|
||||
self.domain
|
||||
and not (self.suffix or self.subdomain)
|
||||
and looks_like_ip(self.domain)
|
||||
):
|
||||
return self.domain
|
||||
return ""
|
||||
|
||||
@property
|
||||
def ipv6(self) -> str:
|
||||
"""The IPv6 address, if that is what the input domain/URL was, or else the empty string.
|
||||
|
||||
>>> extract(
|
||||
... "http://[aBcD:ef01:2345:6789:aBcD:ef01:127.0.0.1]/path/to/file"
|
||||
... ).ipv6
|
||||
'aBcD:ef01:2345:6789:aBcD:ef01:127.0.0.1'
|
||||
>>> extract(
|
||||
... "http://[aBcD:ef01:2345:6789:aBcD:ef01:127.0.0.1.1]/path/to/file"
|
||||
... ).ipv6
|
||||
''
|
||||
>>> extract("http://[aBcD:ef01:2345:6789:aBcD:ef01:256.0.0.1]").ipv6
|
||||
''
|
||||
"""
|
||||
min_num_ipv6_chars = 4
|
||||
if (
|
||||
len(self.domain) >= min_num_ipv6_chars
|
||||
and self.domain[0] == "["
|
||||
and self.domain[-1] == "]"
|
||||
and not (self.suffix or self.subdomain)
|
||||
):
|
||||
debracketed = self.domain[1:-1]
|
||||
if looks_like_ipv6(debracketed):
|
||||
return debracketed
|
||||
return ""
|
||||
|
||||
@property
|
||||
def registered_domain(self) -> str:
|
||||
"""The `domain` and `suffix` fields joined with a dot, if they're both set, or else the empty string.
|
||||
|
||||
>>> extract("http://forums.bbc.co.uk").registered_domain
|
||||
'bbc.co.uk'
|
||||
>>> extract("http://localhost:8080").registered_domain
|
||||
''
|
||||
|
||||
.. deprecated:: 5.3.0
|
||||
Use `top_domain_under_public_suffix` instead, which has the same
|
||||
behavior but a more accurate name.
|
||||
|
||||
.. versionremoved:: 6.0.0
|
||||
This property will be removed in the next major version.
|
||||
|
||||
This is an alias for the `top_domain_under_public_suffix` property.
|
||||
`registered_domain` is so called because is roughly the domain the
|
||||
owner paid to register with a registrar or, in the case of a private
|
||||
domain, "registered" with the domain owner. If the input was not
|
||||
something one could register, this property returns the empty string.
|
||||
|
||||
To distinguish the case of private domains, consider Blogspot, which is
|
||||
in the PSL's private domains. If `include_psl_private_domains` was set
|
||||
to `False`, the `registered_domain` property of a Blogspot URL
|
||||
represents the domain the owner of Blogspot registered with a
|
||||
registrar, i.e. Google registered "blogspot.com". If
|
||||
`include_psl_private_domains=True`, the `registered_domain` property
|
||||
represents the "blogspot.com" _subdomain_ the owner of a blog
|
||||
"registered" with Blogspot.
|
||||
|
||||
>>> extract(
|
||||
... "http://waiterrant.blogspot.com", include_psl_private_domains=False
|
||||
... ).registered_domain
|
||||
'blogspot.com'
|
||||
>>> extract(
|
||||
... "http://waiterrant.blogspot.com", include_psl_private_domains=True
|
||||
... ).registered_domain
|
||||
'waiterrant.blogspot.com'
|
||||
|
||||
To always get the same joined string, regardless of the
|
||||
`include_psl_private_domains` setting, consider the
|
||||
`top_domain_under_registry_suffix` property.
|
||||
"""
|
||||
warnings.warn(
|
||||
"The 'registered_domain' property is deprecated and will be removed in the next major version. "
|
||||
"Use 'top_domain_under_public_suffix' instead, which has the same behavior but a more accurate name.",
|
||||
DeprecationWarning,
|
||||
stacklevel=2,
|
||||
)
|
||||
return self.top_domain_under_public_suffix
|
||||
|
||||
@property
|
||||
def reverse_domain_name(self) -> str:
|
||||
"""The domain name in Reverse Domain Name Notation.
|
||||
|
||||
Joins extracted components of the input URL in reverse domain name
|
||||
notation. The suffix is used as the leftmost component, followed by the
|
||||
domain, then followed by the subdomain with its parts reversed.
|
||||
|
||||
Reverse Domain Name Notation is typically used to organize namespaces
|
||||
for packages and plugins. Technically, a full reversal would reverse
|
||||
the parts of the suffix, e.g. "co.uk" would become "uk.co", but this is
|
||||
not done in practice when Reverse Domain Name Notation is called for.
|
||||
So this property leaves the `suffix` part in its original order.
|
||||
|
||||
>>> extract("login.example.com").reverse_domain_name
|
||||
'com.example.login'
|
||||
|
||||
>>> extract("login.example.co.uk").reverse_domain_name
|
||||
'co.uk.example.login'
|
||||
|
||||
>>> extract("localhost").reverse_domain_name
|
||||
'localhost'
|
||||
"""
|
||||
stack = [i for i in (self.suffix, self.domain) if i]
|
||||
if self.subdomain:
|
||||
stack.extend(reversed(self.subdomain.split(".")))
|
||||
return ".".join(stack)
|
||||
|
||||
@property
|
||||
def top_domain_under_registry_suffix(self) -> str:
|
||||
"""The rightmost domain label and `registry_suffix` joined with a dot, if such a domain is available and `registry_suffix` is set, or else the empty string.
|
||||
|
||||
The rightmost domain label might be in the `domain` field, or, if the
|
||||
input URL's suffix is a PSL private domain, in the public suffix
|
||||
`suffix` field.
|
||||
|
||||
If the input was not in the PSL's private domains, this property is
|
||||
equivalent to `top_domain_under_public_suffix`.
|
||||
|
||||
>>> extract(
|
||||
... "http://waiterrant.blogspot.com", include_psl_private_domains=False
|
||||
... ).top_domain_under_registry_suffix
|
||||
'blogspot.com'
|
||||
>>> extract(
|
||||
... "http://waiterrant.blogspot.com", include_psl_private_domains=True
|
||||
... ).top_domain_under_registry_suffix
|
||||
'blogspot.com'
|
||||
>>> extract("http://localhost:8080").top_domain_under_registry_suffix
|
||||
''
|
||||
|
||||
.. versionadded:: 5.3.0
|
||||
"""
|
||||
top_domain_under_public_suffix = self.top_domain_under_public_suffix
|
||||
if not top_domain_under_public_suffix or not self.is_private:
|
||||
return top_domain_under_public_suffix
|
||||
|
||||
num_labels = self.registry_suffix.count(".") + 2
|
||||
return ".".join(top_domain_under_public_suffix.split(".")[-num_labels:])
|
||||
|
||||
@property
|
||||
def top_domain_under_public_suffix(self) -> str:
|
||||
"""The `domain` and `suffix` fields joined with a dot, if they're both set, or else the empty string.
|
||||
|
||||
>>> extract("http://forums.bbc.co.uk").top_domain_under_public_suffix
|
||||
'bbc.co.uk'
|
||||
>>> extract("http://localhost:8080").top_domain_under_public_suffix
|
||||
''
|
||||
|
||||
.. versionadded:: 5.3.0
|
||||
"""
|
||||
if self.suffix and self.domain:
|
||||
return f"{self.domain}.{self.suffix}"
|
||||
return ""
|
||||
|
||||
|
||||
class _UseEnvironmentUrls:
|
||||
"""Mark omitted URLs so the constructor can read the environment."""
|
||||
|
||||
|
||||
_USE_ENVIRONMENT_URLS = _UseEnvironmentUrls()
|
||||
|
||||
|
||||
class TLDExtract:
|
||||
"""A callable for extracting, subdomain, domain, and suffix components from a URL."""
|
||||
|
||||
# TODO: too-many-arguments
|
||||
def __init__(
|
||||
self,
|
||||
cache_dir: str | None = get_cache_dir(),
|
||||
suffix_list_urls: Sequence[str] | _UseEnvironmentUrls = _USE_ENVIRONMENT_URLS,
|
||||
fallback_to_snapshot: bool = True,
|
||||
include_psl_private_domains: bool = False,
|
||||
extra_suffixes: Sequence[str] = (),
|
||||
cache_fetch_timeout: str | float | None = CACHE_TIMEOUT,
|
||||
) -> None:
|
||||
"""Construct a callable for extracting subdomain, domain, and suffix components from a URL.
|
||||
|
||||
Upon calling it, it first checks for a JSON in `cache_dir`. By default,
|
||||
the `cache_dir` follows the XDG Base Directory standard, e.g.
|
||||
`$HOME/.cache/python-tldextract`, falling back to the tldextract package
|
||||
directory only when no suitable user cache location can be determined.
|
||||
You can disable the caching functionality of this module by setting
|
||||
`cache_dir` to `None`.
|
||||
|
||||
If the cached version does not exist, such as on the first run, HTTP
|
||||
request the URLs in `suffix_list_urls` in order, and use the first
|
||||
successful response for public suffix definitions. Subsequent, untried
|
||||
URLs are ignored. The default URLs are the latest version of the
|
||||
Mozilla Public Suffix List and its mirror, but any similar document URL
|
||||
could be specified. Local files can be specified by using the `file://`
|
||||
protocol (see `urllib2` documentation). To disable HTTP requests, set
|
||||
this to an empty sequence. The default can also be set with the
|
||||
newline-delimited environment variable `TLDEXTRACT_PUBLIC_SUFFIX_LIST_URLS`.
|
||||
An empty value disables HTTP requests. Entries naming existing local
|
||||
files are converted to `file://` URLs, as with the `--suffix_list_url`
|
||||
CLI option. New instances read the environment when constructed; an
|
||||
explicit `suffix_list_urls` argument takes precedence.
|
||||
|
||||
If there is no cached version loaded and no data is found from the `suffix_list_urls`,
|
||||
the module will fall back to the included TLD set snapshot. If you do not want
|
||||
this behavior, you may set `fallback_to_snapshot` to `False`, and an exception will be
|
||||
raised instead.
|
||||
|
||||
The Public Suffix List includes a list of "private domains" as TLDs,
|
||||
such as blogspot.com. These do not fit `tldextract`'s definition of a
|
||||
suffix, so these domains are excluded by default. If you'd like them
|
||||
included instead, set `include_psl_private_domains` to `True`.
|
||||
|
||||
You can specify additional suffixes in the `extra_suffixes` argument.
|
||||
These will be merged into whatever public suffix definitions are
|
||||
already in use by `tldextract`, above.
|
||||
|
||||
`cache_fetch_timeout` is passed unmodified to the underlying request object
|
||||
per the requests documentation here:
|
||||
http://docs.python-requests.org/en/master/user/advanced/#timeouts
|
||||
|
||||
`cache_fetch_timeout` can also be set to a single value with the
|
||||
environment variable `TLDEXTRACT_DEFAULT_FETCH_TIMEOUT`, like so:
|
||||
|
||||
TLDEXTRACT_DEFAULT_FETCH_TIMEOUT="1.2"
|
||||
|
||||
When set this way, the same timeout value will be used for both connect
|
||||
and read timeouts. The older `TLDEXTRACT_CACHE_TIMEOUT` is also supported.
|
||||
"""
|
||||
if isinstance(suffix_list_urls, _UseEnvironmentUrls):
|
||||
suffix_list_urls = _suffix_list_urls_from_env()
|
||||
suffix_list_urls = suffix_list_urls or ()
|
||||
self.suffix_list_urls = tuple(
|
||||
url.strip() for url in suffix_list_urls if url.strip()
|
||||
)
|
||||
|
||||
self.fallback_to_snapshot = fallback_to_snapshot
|
||||
if not (self.suffix_list_urls or cache_dir or self.fallback_to_snapshot):
|
||||
raise ValueError(
|
||||
"The arguments you have provided disable all ways for tldextract "
|
||||
"to obtain data. Please provide a suffix list data, a cache_dir, "
|
||||
"or set `fallback_to_snapshot` to `True`."
|
||||
)
|
||||
|
||||
self.include_psl_private_domains = include_psl_private_domains
|
||||
self.extra_suffixes = extra_suffixes
|
||||
self._extractor: _PublicSuffixListTLDExtractor | None = None
|
||||
|
||||
self.cache_fetch_timeout = (
|
||||
float(cache_fetch_timeout)
|
||||
if isinstance(cache_fetch_timeout, str)
|
||||
else cache_fetch_timeout
|
||||
)
|
||||
self._cache = DiskCache(cache_dir)
|
||||
|
||||
def __call__(
|
||||
self,
|
||||
url: str,
|
||||
include_psl_private_domains: bool | None = None,
|
||||
session: requests.Session | None = None,
|
||||
) -> ExtractResult:
|
||||
"""Alias for `extract_str`."""
|
||||
return self.extract_str(url, include_psl_private_domains, session=session)
|
||||
|
||||
def extract_str(
|
||||
self,
|
||||
url: str,
|
||||
include_psl_private_domains: bool | None = None,
|
||||
session: requests.Session | None = None,
|
||||
) -> ExtractResult:
|
||||
"""Take a string URL and splits it into its subdomain, domain, and suffix components.
|
||||
|
||||
Args:
|
||||
url: The URL string to extract components from
|
||||
include_psl_private_domains: Whether to treat PSL private domains as suffixes.
|
||||
If None, uses the instance default.
|
||||
session: Optional requests.Session for HTTP configuration (e.g., proxies)
|
||||
|
||||
Returns:
|
||||
ExtractResult: Named tuple containing subdomain, domain, suffix, and metadata
|
||||
|
||||
Examples:
|
||||
Basic extraction:
|
||||
>>> extractor = TLDExtract()
|
||||
>>> extractor.extract_str("http://forums.news.cnn.com/")
|
||||
ExtractResult(subdomain='forums.news', domain='cnn', suffix='com', is_private=False)
|
||||
>>> extractor.extract_str("http://forums.bbc.co.uk/")
|
||||
ExtractResult(subdomain='forums', domain='bbc', suffix='co.uk', is_private=False)
|
||||
|
||||
Using a custom session:
|
||||
>>> import requests
|
||||
>>> session = requests.Session()
|
||||
>>> # customize your session here
|
||||
>>> with session:
|
||||
... extractor.extract_str(
|
||||
... "http://forums.news.cnn.com/", session=session
|
||||
... )
|
||||
ExtractResult(subdomain='forums.news', domain='cnn', suffix='com', is_private=False)
|
||||
"""
|
||||
return self._extract_netloc(
|
||||
lenient_netloc(url), include_psl_private_domains, session=session
|
||||
)
|
||||
|
||||
def extract_urllib(
|
||||
self,
|
||||
url: urllib.parse.ParseResult | urllib.parse.SplitResult,
|
||||
include_psl_private_domains: bool | None = None,
|
||||
session: requests.Session | None = None,
|
||||
) -> ExtractResult:
|
||||
"""Extract components from a pre-parsed URL object.
|
||||
|
||||
Args:
|
||||
url: ParseResult or SplitResult from urllib.parse methods
|
||||
include_psl_private_domains: Whether to treat PSL private domains as suffixes.
|
||||
If None, uses the instance default.
|
||||
session: Optional requests.Session for HTTP configuration
|
||||
|
||||
Returns:
|
||||
ExtractResult: Named tuple containing subdomain, domain, suffix, and metadata
|
||||
|
||||
Note:
|
||||
This method is faster than `extract_str` since the URL is already parsed.
|
||||
|
||||
Examples:
|
||||
>>> extractor = TLDExtract()
|
||||
>>> extractor.extract_urllib(
|
||||
... urllib.parse.urlsplit("http://forums.news.cnn.com/")
|
||||
... )
|
||||
ExtractResult(subdomain='forums.news', domain='cnn', suffix='com', is_private=False)
|
||||
>>> extractor.extract_urllib(
|
||||
... urllib.parse.urlsplit("http://forums.bbc.co.uk/")
|
||||
... )
|
||||
ExtractResult(subdomain='forums', domain='bbc', suffix='co.uk', is_private=False)
|
||||
"""
|
||||
return self._extract_netloc(
|
||||
_host_from_authority(url.netloc),
|
||||
include_psl_private_domains,
|
||||
session=session,
|
||||
)
|
||||
|
||||
def _extract_netloc(
|
||||
self,
|
||||
netloc: str,
|
||||
include_psl_private_domains: bool | None,
|
||||
session: requests.Session | None = None,
|
||||
) -> ExtractResult:
|
||||
netloc_with_ascii_dots = (
|
||||
netloc.replace("\u3002", "\u002e")
|
||||
.replace("\uff0e", "\u002e")
|
||||
.replace("\uff61", "\u002e")
|
||||
)
|
||||
|
||||
min_num_ipv6_chars = 4
|
||||
if (
|
||||
len(netloc_with_ascii_dots) >= min_num_ipv6_chars
|
||||
and netloc_with_ascii_dots[0] == "["
|
||||
and netloc_with_ascii_dots[-1] == "]"
|
||||
and looks_like_ipv6(netloc_with_ascii_dots[1:-1])
|
||||
):
|
||||
return ExtractResult(
|
||||
"", netloc_with_ascii_dots, "", is_private=False, registry_suffix=""
|
||||
)
|
||||
|
||||
labels = netloc_with_ascii_dots.split(".")
|
||||
|
||||
maybe_indexes = self._get_tld_extractor(session).suffix_index(
|
||||
labels, include_psl_private_domains=include_psl_private_domains
|
||||
)
|
||||
|
||||
num_ipv4_labels = 4
|
||||
if (
|
||||
not maybe_indexes
|
||||
and len(labels) == num_ipv4_labels
|
||||
and looks_like_ip(netloc_with_ascii_dots)
|
||||
):
|
||||
return ExtractResult(
|
||||
"", netloc_with_ascii_dots, "", is_private=False, registry_suffix=""
|
||||
)
|
||||
elif not maybe_indexes:
|
||||
return ExtractResult(
|
||||
subdomain=".".join(labels[:-1]),
|
||||
domain=labels[-1],
|
||||
suffix="",
|
||||
is_private=False,
|
||||
registry_suffix="",
|
||||
)
|
||||
|
||||
(
|
||||
(public_suffix_index, public_suffix_node),
|
||||
(registry_suffix_index, registry_suffix_node),
|
||||
) = maybe_indexes
|
||||
|
||||
subdomain = (
|
||||
".".join(labels[: public_suffix_index - 1])
|
||||
if public_suffix_index >= 2
|
||||
else ""
|
||||
)
|
||||
domain = labels[public_suffix_index - 1] if public_suffix_index > 0 else ""
|
||||
public_suffix = ".".join(labels[public_suffix_index:])
|
||||
registry_suffix = (
|
||||
".".join(labels[registry_suffix_index:])
|
||||
if public_suffix_node.is_private
|
||||
else public_suffix
|
||||
)
|
||||
return ExtractResult(
|
||||
subdomain=subdomain,
|
||||
domain=domain,
|
||||
suffix=public_suffix,
|
||||
is_private=public_suffix_node.is_private,
|
||||
registry_suffix=registry_suffix,
|
||||
)
|
||||
|
||||
def update(
|
||||
self, fetch_now: bool = False, session: requests.Session | None = None
|
||||
) -> None:
|
||||
"""Clear cache and force fresh suffix list fetch on next extraction.
|
||||
|
||||
Args:
|
||||
fetch_now: If True, immediately fetch updated suffix lists
|
||||
session: Optional requests.Session for HTTP configuration
|
||||
"""
|
||||
self._extractor = None
|
||||
self._cache.clear()
|
||||
if fetch_now:
|
||||
self._get_tld_extractor(session=session)
|
||||
|
||||
@property
|
||||
def tlds(self, session: requests.Session | None = None) -> list[str]:
|
||||
"""The list of TLDs used by default.
|
||||
|
||||
This will vary based on `include_psl_private_domains` and `extra_suffixes`.
|
||||
"""
|
||||
return list(self._get_tld_extractor(session=session).tlds())
|
||||
|
||||
def _get_tld_extractor(
|
||||
self, session: requests.Session | None = None
|
||||
) -> _PublicSuffixListTLDExtractor:
|
||||
"""Get or compute this object's TLDExtractor.
|
||||
|
||||
Looks up the TLDExtractor in roughly the following order, based on the
|
||||
settings passed to __init__:
|
||||
|
||||
1. Memoized on `self`
|
||||
2. Local system _cache file
|
||||
3. Remote PSL, over HTTP
|
||||
4. Bundled PSL snapshot file
|
||||
"""
|
||||
if self._extractor:
|
||||
return self._extractor
|
||||
|
||||
public_tlds, private_tlds = get_suffix_lists(
|
||||
cache=self._cache,
|
||||
urls=self.suffix_list_urls,
|
||||
cache_fetch_timeout=self.cache_fetch_timeout,
|
||||
fallback_to_snapshot=self.fallback_to_snapshot,
|
||||
session=session,
|
||||
)
|
||||
|
||||
if not any([public_tlds, private_tlds, self.extra_suffixes]):
|
||||
raise ValueError("No tlds set. Cannot proceed without tlds.")
|
||||
|
||||
self._extractor = _PublicSuffixListTLDExtractor(
|
||||
public_tlds=public_tlds,
|
||||
private_tlds=private_tlds,
|
||||
extra_tlds=list(self.extra_suffixes),
|
||||
include_psl_private_domains=self.include_psl_private_domains,
|
||||
)
|
||||
return self._extractor
|
||||
|
||||
|
||||
def _suffix_list_urls_from_env() -> Sequence[str]:
|
||||
"""Resolve the default suffix list URLs from the environment."""
|
||||
raw = os.environ.get("TLDEXTRACT_PUBLIC_SUFFIX_LIST_URLS")
|
||||
if raw is None:
|
||||
return PUBLIC_SUFFIX_LIST_URLS
|
||||
|
||||
urls = []
|
||||
for line in raw.splitlines():
|
||||
source = line.strip()
|
||||
if not source:
|
||||
continue
|
||||
if os.path.isfile(source):
|
||||
source = pathlib.Path(os.path.abspath(source)).as_uri()
|
||||
urls.append(source)
|
||||
return urls
|
||||
|
||||
|
||||
TLD_EXTRACTOR = TLDExtract()
|
||||
|
||||
|
||||
class Trie:
|
||||
"""Trie for storing eTLDs with their labels in reverse-order."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
matches: dict[str, Trie] | None = None,
|
||||
end: bool = False,
|
||||
is_private: bool = False,
|
||||
) -> None:
|
||||
"""TODO."""
|
||||
self.matches = matches if matches else {}
|
||||
self.end = end
|
||||
self.is_private = is_private
|
||||
|
||||
@staticmethod
|
||||
def create(
|
||||
public_suffixes: Collection[str],
|
||||
private_suffixes: Collection[str] | None = None,
|
||||
) -> Trie:
|
||||
"""Create a Trie from a list of suffixes and return its root node."""
|
||||
root_node = Trie()
|
||||
|
||||
for suffix in public_suffixes:
|
||||
root_node.add_suffix(suffix)
|
||||
|
||||
if private_suffixes is None:
|
||||
private_suffixes = []
|
||||
|
||||
for suffix in private_suffixes:
|
||||
root_node.add_suffix(suffix, True)
|
||||
|
||||
return root_node
|
||||
|
||||
def add_suffix(self, suffix: str, is_private: bool = False) -> None:
|
||||
"""Append a suffix's labels to this Trie node."""
|
||||
node = self
|
||||
|
||||
labels = suffix.split(".")
|
||||
labels.reverse()
|
||||
|
||||
for label in labels:
|
||||
if label not in node.matches:
|
||||
node.matches[label] = Trie()
|
||||
node = node.matches[label]
|
||||
|
||||
node.end = True
|
||||
node.is_private = is_private
|
||||
|
||||
|
||||
@wraps(TLD_EXTRACTOR.__call__)
|
||||
def extract( # noqa: D103
|
||||
url: str,
|
||||
include_psl_private_domains: bool | None = False,
|
||||
session: requests.Session | None = None,
|
||||
) -> ExtractResult:
|
||||
return TLD_EXTRACTOR(
|
||||
url, include_psl_private_domains=include_psl_private_domains, session=session
|
||||
)
|
||||
|
||||
|
||||
@wraps(TLD_EXTRACTOR.update)
|
||||
def update(*args, **kwargs): # type: ignore[no-untyped-def] # noqa: D103
|
||||
return TLD_EXTRACTOR.update(*args, **kwargs)
|
||||
|
||||
|
||||
class _PublicSuffixListTLDExtractor:
|
||||
"""Wrapper around this project's main algo for PSL lookups."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
public_tlds: list[str],
|
||||
private_tlds: list[str],
|
||||
extra_tlds: list[str],
|
||||
include_psl_private_domains: bool = False,
|
||||
):
|
||||
# set the default value
|
||||
self.include_psl_private_domains = include_psl_private_domains
|
||||
self.public_tlds = public_tlds
|
||||
self.private_tlds = private_tlds
|
||||
self.tlds_incl_private = frozenset(public_tlds + private_tlds + extra_tlds)
|
||||
self.tlds_excl_private = frozenset(public_tlds + extra_tlds)
|
||||
self.tlds_incl_private_trie = Trie.create(
|
||||
self.tlds_excl_private, frozenset(private_tlds)
|
||||
)
|
||||
self.tlds_excl_private_trie = Trie.create(self.tlds_excl_private)
|
||||
|
||||
def tlds(self, include_psl_private_domains: bool | None = None) -> frozenset[str]:
|
||||
"""Get the currently filtered list of suffixes."""
|
||||
if include_psl_private_domains is None:
|
||||
include_psl_private_domains = self.include_psl_private_domains
|
||||
|
||||
return (
|
||||
self.tlds_incl_private
|
||||
if include_psl_private_domains
|
||||
else self.tlds_excl_private
|
||||
)
|
||||
|
||||
def suffix_index(
|
||||
self, spl: list[str], include_psl_private_domains: bool | None = None
|
||||
) -> tuple[tuple[int, Trie], tuple[int, Trie]] | None:
|
||||
"""Return the index of the first public suffix label, the index of the first registry suffix label, and their corresponding trie nodes.
|
||||
|
||||
Returns `None` if no suffix is found.
|
||||
"""
|
||||
if include_psl_private_domains is None:
|
||||
include_psl_private_domains = self.include_psl_private_domains
|
||||
|
||||
node = reg_node = suffix_node = (
|
||||
self.tlds_incl_private_trie
|
||||
if include_psl_private_domains
|
||||
else self.tlds_excl_private_trie
|
||||
)
|
||||
suffix_idx = reg_idx = label_idx = len(spl)
|
||||
for label in reversed(spl):
|
||||
decoded_label = _decode_punycode(label)
|
||||
if decoded_label in node.matches:
|
||||
label_idx -= 1
|
||||
node = node.matches[decoded_label]
|
||||
if node.end:
|
||||
suffix_idx = label_idx
|
||||
suffix_node = node
|
||||
if not node.is_private:
|
||||
reg_node = node
|
||||
reg_idx = label_idx
|
||||
continue
|
||||
|
||||
is_wildcard = "*" in node.matches
|
||||
if is_wildcard:
|
||||
is_wildcard_exception = "!" + decoded_label in node.matches
|
||||
return (
|
||||
label_idx if is_wildcard_exception else label_idx - 1,
|
||||
node.matches["*"],
|
||||
), (
|
||||
reg_idx,
|
||||
reg_node,
|
||||
)
|
||||
|
||||
break
|
||||
|
||||
if suffix_idx == len(spl):
|
||||
return None
|
||||
|
||||
return ((suffix_idx, suffix_node), (reg_idx, reg_node))
|
||||
|
||||
|
||||
def _decode_punycode(label: str) -> str:
|
||||
lowered = label.lower()
|
||||
looks_like_puny = lowered.startswith("xn--")
|
||||
if looks_like_puny:
|
||||
try:
|
||||
return idna.decode(lowered)
|
||||
except (UnicodeError, IndexError):
|
||||
pass
|
||||
return lowered
|
||||
Reference in new issue
Block a user