1015 lines
38 KiB
Python
1015 lines
38 KiB
Python
"""
|
||
cssselect.xpath
|
||
===============
|
||
|
||
Translation of parsed CSS selectors to XPath expressions.
|
||
|
||
|
||
:copyright: (c) 2007-2012 Ian Bicking and contributors.
|
||
See AUTHORS for more details.
|
||
:license: BSD, see LICENSE for more details.
|
||
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import re
|
||
from string import ascii_lowercase, ascii_uppercase
|
||
from typing import TYPE_CHECKING, Self, cast
|
||
|
||
from cssselect.parser import (
|
||
Attrib,
|
||
Class,
|
||
CombinedSelector,
|
||
Element,
|
||
Function,
|
||
Hash,
|
||
Matching,
|
||
Negation,
|
||
Pseudo,
|
||
PseudoElement,
|
||
Relation,
|
||
Selector,
|
||
SelectorError,
|
||
SpecificityAdjustment,
|
||
Tree,
|
||
ascii_lower,
|
||
parse,
|
||
parse_series,
|
||
)
|
||
|
||
if TYPE_CHECKING:
|
||
from collections.abc import Callable, Iterable
|
||
|
||
|
||
class ExpressionError(SelectorError, RuntimeError):
|
||
"""Unknown or unsupported selector (eg. pseudo-class)."""
|
||
|
||
|
||
#### XPath Helpers
|
||
|
||
|
||
class XPathExpr:
|
||
def __init__(
|
||
self,
|
||
path: str = "",
|
||
element: str = "*",
|
||
condition: str = "",
|
||
star_prefix: bool = False,
|
||
) -> None:
|
||
self.path = path
|
||
self.element = element
|
||
self.condition = condition
|
||
|
||
def __str__(self) -> str:
|
||
path = str(self.path) + str(self.element)
|
||
if self.condition:
|
||
path += f"[{self.condition}]"
|
||
return path
|
||
|
||
def __repr__(self) -> str:
|
||
return f"{self.__class__.__name__}[{self}]"
|
||
|
||
def add_condition(self, condition: str, conjuction: str = "and") -> Self:
|
||
if self.condition:
|
||
self.condition = f"({self.condition}) {conjuction} ({condition})"
|
||
else:
|
||
self.condition = condition
|
||
return self
|
||
|
||
def add_name_test(self) -> None:
|
||
if self.element == "*":
|
||
# We weren't doing a test anyway
|
||
return
|
||
prefix, colon, local = self.element.partition(":")
|
||
if is_safe_name(prefix) and (not colon or local == "*" or is_safe_name(local)):
|
||
# A node test, not a name() comparison: name() returns the
|
||
# qualified name as written in the document, which would bypass
|
||
# the XPath prefix mapping for a prefixed name like "ns:f" or
|
||
# "ns:*" (from the CSS "ns|f" or "ns|*"), and would match
|
||
# elements in a default namespace for an unprefixed name (which
|
||
# the node test emitted for a bare "f" selector does not).
|
||
self.add_condition(f"self::{self.element}")
|
||
else:
|
||
self.add_condition(
|
||
f"name() = {GenericTranslator.xpath_literal(self.element)}"
|
||
)
|
||
self.element = "*"
|
||
|
||
def add_star_prefix(self) -> None:
|
||
"""
|
||
Append '*/' to the path to keep the context constrained
|
||
to a single parent.
|
||
"""
|
||
self.path += "*/"
|
||
|
||
def join(self, combiner: str, other: XPathExpr) -> Self:
|
||
path = str(self) + combiner
|
||
# Any "star prefix" is redundant when joining.
|
||
if other.path != "*/":
|
||
path += other.path
|
||
self.path = path
|
||
self.element = other.element
|
||
self.condition = other.condition
|
||
return self
|
||
|
||
|
||
split_at_single_quotes = re.compile("('+)").split
|
||
|
||
# The spec is actually more permissive than that, but don’t bother.
|
||
# This is just for the fast path.
|
||
# http://www.w3.org/TR/REC-xml/#NT-NameStartChar
|
||
is_safe_name = re.compile("^[a-zA-Z_][a-zA-Z0-9_.-]*$").match
|
||
|
||
# Test that the string is not empty and does not contain whitespace
|
||
is_non_whitespace = re.compile(r"^[^ \t\r\n\f]+$").match
|
||
|
||
|
||
#### Translation
|
||
|
||
|
||
class GenericTranslator:
|
||
"""
|
||
Translator for "generic" XML documents.
|
||
|
||
Everything is case-sensitive, no assumption is made on the meaning
|
||
of element names and attribute names.
|
||
|
||
"""
|
||
|
||
####
|
||
#### HERE BE DRAGONS
|
||
####
|
||
#### You are welcome to hook into this to change some behavior,
|
||
#### but do so at your own risks.
|
||
#### Until it has received a lot more work and review,
|
||
#### I reserve the right to change this API in backward-incompatible ways
|
||
#### with any minor version of cssselect.
|
||
#### See https://github.com/scrapy/cssselect/pull/22
|
||
#### -- Simon Sapin.
|
||
####
|
||
|
||
combinator_mapping = {
|
||
" ": "descendant",
|
||
">": "child",
|
||
"+": "direct_adjacent",
|
||
"~": "indirect_adjacent",
|
||
}
|
||
|
||
# Used to match a combinator against the context node, in :not().
|
||
_reverse_combinator_mapping = {
|
||
" ": "ancestor::*",
|
||
">": "parent::*",
|
||
"+": "preceding-sibling::*[1]",
|
||
"~": "preceding-sibling::*",
|
||
}
|
||
|
||
attribute_operator_mapping = {
|
||
"exists": "exists",
|
||
"=": "equals",
|
||
"~=": "includes",
|
||
"|=": "dashmatch",
|
||
"^=": "prefixmatch",
|
||
"$=": "suffixmatch",
|
||
"*=": "substringmatch",
|
||
"!=": "different", # not part of Selectors Level 3, but widely supported
|
||
}
|
||
|
||
#: The attribute used for ID selectors depends on the document language:
|
||
#: http://www.w3.org/TR/selectors/#id-selectors
|
||
id_attribute = "id"
|
||
|
||
#: The attribute used for ``:lang()`` depends on the document language:
|
||
#: http://www.w3.org/TR/selectors/#lang-pseudo
|
||
lang_attribute = "xml:lang"
|
||
|
||
#: The case sensitivity of document language element names,
|
||
#: attribute names, and attribute values in selectors depends
|
||
#: on the document language.
|
||
#: http://www.w3.org/TR/selectors/#casesens
|
||
#:
|
||
#: When a document language defines one of these as case-insensitive,
|
||
#: cssselect assumes that the document parser makes the parsed values
|
||
#: lower-case. Making the selector lower-case too makes the comparaison
|
||
#: case-insensitive.
|
||
#:
|
||
#: In HTML, element names and attributes names (but not attribute values)
|
||
#: are case-insensitive. All of lxml.html, html5lib, BeautifulSoup4
|
||
#: and HTMLParser make them lower-case in their parse result, so
|
||
#: the assumption holds.
|
||
lower_case_element_names = False
|
||
lower_case_attribute_names = False
|
||
lower_case_attribute_values = False
|
||
|
||
# class used to represent and xpath expression
|
||
xpathexpr_cls = XPathExpr
|
||
|
||
def css_to_xpath(self, css: str, prefix: str = "descendant-or-self::") -> str:
|
||
"""Translate a *group of selectors* to XPath.
|
||
|
||
Pseudo-elements are not supported here since XPath only knows
|
||
about "real" elements.
|
||
|
||
:param css:
|
||
A *group of selectors* as a string.
|
||
:param prefix:
|
||
This string is prepended to the XPath expression for each selector.
|
||
The default makes selectors scoped to the context node’s subtree.
|
||
:raises:
|
||
:class:`~cssselect.SelectorSyntaxError` on invalid selectors,
|
||
:class:`ExpressionError` on unknown/unsupported selectors,
|
||
including pseudo-elements.
|
||
:returns:
|
||
The equivalent XPath 1.0 expression as a string.
|
||
|
||
"""
|
||
return " | ".join(
|
||
self.selector_to_xpath(selector, prefix, translate_pseudo_elements=True)
|
||
for selector in parse(css)
|
||
)
|
||
|
||
def selector_to_xpath(
|
||
self,
|
||
selector: Selector,
|
||
prefix: str = "descendant-or-self::",
|
||
translate_pseudo_elements: bool = False,
|
||
) -> str:
|
||
"""Translate a parsed selector to XPath.
|
||
|
||
|
||
:param selector:
|
||
A parsed :class:`Selector` object.
|
||
:param prefix:
|
||
This string is prepended to the resulting XPath expression.
|
||
The default makes selectors scoped to the context node’s subtree.
|
||
:param translate_pseudo_elements:
|
||
Unless this is set to ``True`` (as :meth:`css_to_xpath` does),
|
||
the :attr:`~Selector.pseudo_element` attribute of the selector
|
||
is ignored.
|
||
It is the caller's responsibility to reject selectors
|
||
with pseudo-elements, or to account for them somehow.
|
||
:raises:
|
||
:class:`ExpressionError` on unknown/unsupported selectors.
|
||
:returns:
|
||
The equivalent XPath 1.0 expression as a string.
|
||
|
||
"""
|
||
tree = getattr(selector, "parsed_tree", None)
|
||
if not tree:
|
||
raise TypeError(f"Expected a parsed selector, got {selector!r}")
|
||
xpath = self.xpath(tree)
|
||
assert isinstance(xpath, self.xpathexpr_cls) # help debug a missing 'return'
|
||
if translate_pseudo_elements and selector.pseudo_element:
|
||
xpath = self.xpath_pseudo_element(xpath, selector.pseudo_element)
|
||
return (prefix or "") + str(xpath)
|
||
|
||
def xpath_pseudo_element(
|
||
self, xpath: XPathExpr, pseudo_element: PseudoElement
|
||
) -> XPathExpr:
|
||
"""Translate a pseudo-element.
|
||
|
||
Defaults to not supporting pseudo-elements at all,
|
||
but can be overridden by sub-classes.
|
||
|
||
"""
|
||
raise ExpressionError("Pseudo-elements are not supported.")
|
||
|
||
@staticmethod
|
||
def xpath_literal(s: str) -> str:
|
||
s = str(s)
|
||
if "'" not in s:
|
||
s = f"'{s}'"
|
||
elif '"' not in s:
|
||
s = f'"{s}"'
|
||
else:
|
||
parts_quoted = [
|
||
f'"{part}"' if "'" in part else f"'{part}'"
|
||
for part in split_at_single_quotes(s)
|
||
if part
|
||
]
|
||
s = "concat({})".format(",".join(parts_quoted))
|
||
return s
|
||
|
||
def xpath(self, parsed_selector: Tree) -> XPathExpr:
|
||
"""Translate any parsed selector object."""
|
||
type_name = type(parsed_selector).__name__
|
||
method = cast(
|
||
"Callable[[Tree], XPathExpr] | None",
|
||
getattr(self, f"xpath_{type_name.lower()}", None),
|
||
)
|
||
if method is None:
|
||
raise ExpressionError(f"{type_name} is not supported.")
|
||
return method(parsed_selector)
|
||
|
||
# Dispatched by parsed object type
|
||
|
||
def xpath_combinedselector(self, combined: CombinedSelector) -> XPathExpr:
|
||
"""Translate a combined selector."""
|
||
combinator = self.combinator_mapping[combined.combinator]
|
||
method = cast(
|
||
"Callable[[XPathExpr, XPathExpr], XPathExpr]",
|
||
getattr(self, f"xpath_{combinator}_combinator"),
|
||
)
|
||
return method(self.xpath(combined.selector), self.xpath(combined.subselector))
|
||
|
||
def xpath_negation(self, negation: Negation) -> XPathExpr:
|
||
xpath = self.xpath(negation.selector)
|
||
condition = self._xpath_match_condition(negation.subselector)
|
||
if condition is None:
|
||
# The argument matches every element, so :not() matches none.
|
||
return xpath.add_condition("0")
|
||
return xpath.add_condition(f"not({condition})")
|
||
|
||
def _xpath_match_condition(self, selector: Tree) -> str | None:
|
||
"""Return a condition that holds for the elements matching *selector*,
|
||
or None if that is every element.
|
||
|
||
Unlike xpath(), which walks a selector from left to right, this
|
||
matches the whole selector against the context node, using reverse
|
||
axes for combinators.
|
||
"""
|
||
if isinstance(selector, CombinedSelector):
|
||
axis = self._reverse_combinator_mapping[selector.combinator]
|
||
left = self._xpath_match_condition(selector.selector)
|
||
if left is not None:
|
||
axis = f"{axis}[{left}]"
|
||
right = self._xpath_match_condition(selector.subselector)
|
||
return axis if right is None else f"{right} and {axis}"
|
||
sub_xpath = self.xpath(selector)
|
||
sub_xpath.add_name_test()
|
||
return sub_xpath.condition or None
|
||
|
||
def xpath_relation(self, relation: Relation) -> XPathExpr:
|
||
condition = ""
|
||
for combinator, subselector in relation.arguments:
|
||
method = cast(
|
||
"Callable[[XPathExpr], str]",
|
||
getattr(
|
||
self,
|
||
f"xpath_relation_{self.combinator_mapping[cast('str', combinator.value)]}_combinator",
|
||
),
|
||
)
|
||
argument = method(self.xpath(subselector.parsed_tree))
|
||
condition = f"({condition}) or ({argument})" if condition else argument
|
||
return self.xpath(relation.selector).add_condition(condition)
|
||
|
||
def xpath_matching(self, matching: Matching) -> XPathExpr:
|
||
return self._xpath_add_selector_list_condition(
|
||
self.xpath(matching.selector), matching.selector_list
|
||
)
|
||
|
||
def xpath_specificityadjustment(self, matching: SpecificityAdjustment) -> XPathExpr:
|
||
return self._xpath_add_selector_list_condition(
|
||
self.xpath(matching.selector), matching.selector_list
|
||
)
|
||
|
||
def _xpath_add_selector_list_condition(
|
||
self, xpath: XPathExpr, selector_list: Iterable[Tree]
|
||
) -> XPathExpr:
|
||
"""Add a condition matching any selector of the list
|
||
(for :is() and :where())."""
|
||
condition = ""
|
||
for e in (self.xpath(selector) for selector in selector_list):
|
||
if e.path:
|
||
# Only a combined selector (e.g. "a b") translates to a path,
|
||
# which cannot be embedded into a predicate of the outer
|
||
# expression. The parser rejects combinators in these arguments,
|
||
# so this is only reachable through a hand-built Matching or
|
||
# SpecificityAdjustment node.
|
||
raise ExpressionError(
|
||
"Combined selectors are not supported inside "
|
||
":is(), :where() and :matches()"
|
||
)
|
||
e.add_name_test()
|
||
if not e.condition:
|
||
# This argument matches any element, so the whole selector
|
||
# list does too: it adds no condition.
|
||
return xpath
|
||
condition = (
|
||
f"({condition}) or ({e.condition})" if condition else e.condition
|
||
)
|
||
return xpath.add_condition(condition)
|
||
|
||
def xpath_function(self, function: Function) -> XPathExpr:
|
||
"""Translate a functional pseudo-class."""
|
||
method_name = "xpath_{}_function".format(function.name.replace("-", "_"))
|
||
method = cast(
|
||
"Callable[[XPathExpr, Function], XPathExpr] | None",
|
||
getattr(self, method_name, None),
|
||
)
|
||
if not method:
|
||
raise ExpressionError(f"The pseudo-class :{function.name}() is unknown")
|
||
return method(self.xpath(function.selector), function)
|
||
|
||
def xpath_pseudo(self, pseudo: Pseudo) -> XPathExpr:
|
||
"""Translate a pseudo-class."""
|
||
method_name = "xpath_{}_pseudo".format(pseudo.ident.replace("-", "_"))
|
||
method = cast(
|
||
"Callable[[XPathExpr], XPathExpr] | None",
|
||
getattr(self, method_name, None),
|
||
)
|
||
if not method:
|
||
raise ExpressionError(f"The pseudo-class :{pseudo.ident} is unknown")
|
||
return method(self.xpath(pseudo.selector))
|
||
|
||
def xpath_attrib(self, selector: Attrib) -> XPathExpr:
|
||
"""Translate an attribute selector."""
|
||
operator = self.attribute_operator_mapping[selector.operator]
|
||
method = cast(
|
||
"Callable[[XPathExpr, str, str | None], XPathExpr]",
|
||
getattr(self, f"xpath_attrib_{operator}"),
|
||
)
|
||
if self.lower_case_attribute_names:
|
||
name = selector.attrib.lower()
|
||
else:
|
||
name = selector.attrib
|
||
if selector.namespace == "*":
|
||
# Namespace wildcard, e.g. "[*|href]": any namespace, including
|
||
# none. XPath 1.0 has no "*:name" name test for this, so match
|
||
# by local name instead.
|
||
attrib = f"attribute::*[local-name() = {self.xpath_literal(name)}]"
|
||
else:
|
||
safe = is_safe_name(name)
|
||
if selector.namespace:
|
||
name = f"{selector.namespace}:{name}"
|
||
safe = safe and is_safe_name(selector.namespace)
|
||
if safe:
|
||
attrib = "@" + name
|
||
else:
|
||
attrib = f"attribute::*[name() = {self.xpath_literal(name)}]"
|
||
if selector.value is None:
|
||
value = None
|
||
elif self.lower_case_attribute_values:
|
||
value = cast("str", selector.value.value).lower()
|
||
else:
|
||
value = selector.value.value
|
||
if selector.flag == "i" and value:
|
||
# ASCII-lowering both sides is what the specification defines a
|
||
# case-insensitive match as, and all XPath 1.0 can express.
|
||
attrib = f"translate({attrib}, {self.xpath_literal(ascii_uppercase)}, {self.xpath_literal(ascii_lowercase)})"
|
||
value = ascii_lower(value)
|
||
return method(self.xpath(selector.selector), attrib, value)
|
||
|
||
def xpath_class(self, class_selector: Class) -> XPathExpr:
|
||
"""Translate a class selector."""
|
||
# .foo is defined as [class~=foo] in the spec.
|
||
xpath = self.xpath(class_selector.selector)
|
||
return self.xpath_attrib_includes(xpath, "@class", class_selector.class_name)
|
||
|
||
def xpath_hash(self, id_selector: Hash) -> XPathExpr:
|
||
"""Translate an ID selector."""
|
||
xpath = self.xpath(id_selector.selector)
|
||
return self.xpath_attrib_equals(xpath, "@id", id_selector.id)
|
||
|
||
def xpath_element(self, selector: Element) -> XPathExpr:
|
||
"""Translate a type or universal selector."""
|
||
element = selector.element
|
||
if not element:
|
||
element = "*"
|
||
safe = True
|
||
else:
|
||
safe = bool(is_safe_name(element))
|
||
if self.lower_case_element_names:
|
||
element = element.lower()
|
||
if selector.namespace == "*":
|
||
# Namespace wildcard, e.g. "*|div": any namespace,
|
||
# including none. XPath 1.0 has no "*:name" name test
|
||
# for this, so match by local name instead.
|
||
xpath = self.xpathexpr_cls(element="*")
|
||
xpath.add_condition(f"local-name() = {self.xpath_literal(element)}")
|
||
return xpath
|
||
if selector.namespace and selector.namespace != "*":
|
||
# Namespace prefixes are case-sensitive.
|
||
# http://www.w3.org/TR/css3-namespace/#prefixes
|
||
element = f"{selector.namespace}:{element}"
|
||
safe = safe and bool(is_safe_name(selector.namespace))
|
||
xpath = self.xpathexpr_cls(element=element)
|
||
if not safe:
|
||
# Not usable as an XPath name test (e.g. an escaped identifier
|
||
# like di\a0 v): compare the serialized name instead. Done here
|
||
# rather than through add_name_test(), which would mistake a ":"
|
||
# inside such a name for a namespace prefix separator.
|
||
xpath.add_condition(f"name() = {self.xpath_literal(element)}")
|
||
xpath.element = "*"
|
||
return xpath
|
||
|
||
# CombinedSelector: dispatch by combinator
|
||
|
||
def xpath_descendant_combinator(
|
||
self, left: XPathExpr, right: XPathExpr
|
||
) -> XPathExpr:
|
||
"""right is a child, grand-child or further descendant of left"""
|
||
if right.element == "*" and not right.condition and not right.path:
|
||
# A subclass that maps a pseudo-element to a different node test
|
||
# rewrites the trailing "descendant-or-self::*/*" of the
|
||
# expression, so an unconstrained right-hand side keeps that shape.
|
||
return left.join("/descendant-or-self::*/", right)
|
||
return left.join("/descendant::", right)
|
||
|
||
def xpath_child_combinator(self, left: XPathExpr, right: XPathExpr) -> XPathExpr:
|
||
"""right is an immediate child of left"""
|
||
return left.join("/", right)
|
||
|
||
def xpath_direct_adjacent_combinator(
|
||
self, left: XPathExpr, right: XPathExpr
|
||
) -> XPathExpr:
|
||
"""right is a sibling immediately after left"""
|
||
xpath = left.join("/following-sibling::", right)
|
||
xpath.add_name_test()
|
||
return xpath.add_condition("position() = 1")
|
||
|
||
def xpath_indirect_adjacent_combinator(
|
||
self, left: XPathExpr, right: XPathExpr
|
||
) -> XPathExpr:
|
||
"""right is a sibling after left, immediately or not"""
|
||
return left.join("/following-sibling::", right)
|
||
|
||
# The relative selector is kept in `condition` (instead of being folded
|
||
# into `path`/`element`) so that `element` stays a plain element name:
|
||
# later steps such as :first-of-type or :not() read and rewrite it.
|
||
|
||
def xpath_relation_descendant_combinator(self, right: XPathExpr) -> str:
|
||
"""right is a child, grand-child or further descendant of the element"""
|
||
return f"descendant::{right}"
|
||
|
||
def xpath_relation_child_combinator(self, right: XPathExpr) -> str:
|
||
"""right is an immediate child of the element"""
|
||
return f"./{right}"
|
||
|
||
def xpath_relation_direct_adjacent_combinator(self, right: XPathExpr) -> str:
|
||
"""right is a sibling immediately after the element"""
|
||
# Test the first step of right against that sibling itself, so that
|
||
# any further step of right is walked from there.
|
||
if right.path:
|
||
head, sep, tail = right.path.partition("/")
|
||
right.path = f"self::{head}{sep}{tail}"
|
||
else:
|
||
right.element = f"self::{right.element}"
|
||
return f"following-sibling::*[1]/{right}"
|
||
|
||
def xpath_relation_indirect_adjacent_combinator(self, right: XPathExpr) -> str:
|
||
"""right is a sibling after the element, immediately or not"""
|
||
return f"following-sibling::{right}"
|
||
|
||
# Function: dispatch by function/pseudo-class name
|
||
|
||
def xpath_nth_child_function(
|
||
self,
|
||
xpath: XPathExpr,
|
||
function: Function,
|
||
last: bool = False,
|
||
add_name_test: bool = True,
|
||
) -> XPathExpr:
|
||
try:
|
||
a, b = parse_series(function.arguments)
|
||
except ValueError as ex:
|
||
raise ExpressionError(f"Invalid series: '{function.arguments!r}'") from ex
|
||
|
||
# From https://www.w3.org/TR/css3-selectors/#structural-pseudos:
|
||
#
|
||
# :nth-child(an+b)
|
||
# an+b-1 siblings before
|
||
#
|
||
# :nth-last-child(an+b)
|
||
# an+b-1 siblings after
|
||
#
|
||
# :nth-of-type(an+b)
|
||
# an+b-1 siblings with the same expanded element name before
|
||
#
|
||
# :nth-last-of-type(an+b)
|
||
# an+b-1 siblings with the same expanded element name after
|
||
#
|
||
# So,
|
||
# for :nth-child and :nth-of-type
|
||
#
|
||
# count(preceding-sibling::<nodetest>) = an+b-1
|
||
#
|
||
# for :nth-last-child and :nth-last-of-type
|
||
#
|
||
# count(following-sibling::<nodetest>) = an+b-1
|
||
#
|
||
# therefore,
|
||
# count(...) - (b-1) ≡ 0 (mod a)
|
||
#
|
||
# if a == 0:
|
||
# ~~~~~~~~~~
|
||
# count(...) = b-1
|
||
#
|
||
# if a < 0:
|
||
# ~~~~~~~~~
|
||
# count(...) - b +1 <= 0
|
||
# -> count(...) <= b-1
|
||
#
|
||
# if a > 0:
|
||
# ~~~~~~~~~
|
||
# count(...) - b +1 >= 0
|
||
# -> count(...) >= b-1
|
||
|
||
# work with b-1 instead
|
||
b_min_1 = b - 1
|
||
|
||
# early-exit condition 1:
|
||
# ~~~~~~~~~~~~~~~~~~~~~~~
|
||
# for a == 1, nth-*(an+b) means n+b-1 siblings before/after,
|
||
# and since n ∈ {0, 1, 2, ...}, if b-1<=0,
|
||
# there is always an "n" matching any number of siblings (maybe none)
|
||
if a == 1 and b_min_1 <= 0:
|
||
return xpath
|
||
|
||
# early-exit condition 2:
|
||
# ~~~~~~~~~~~~~~~~~~~~~~~
|
||
# an+b-1 siblings with a<0 and (b-1)<0 is not possible
|
||
if a < 0 and b_min_1 < 0:
|
||
return xpath.add_condition("0")
|
||
|
||
# `add_name_test` boolean is inverted and somewhat counter-intuitive:
|
||
#
|
||
# nth_of_type() calls nth_child(add_name_test=False)
|
||
nodetest = "*" if add_name_test else f"{xpath.element}"
|
||
|
||
# count siblings before or after the element
|
||
if not last:
|
||
siblings_count = f"count(preceding-sibling::{nodetest})"
|
||
else:
|
||
siblings_count = f"count(following-sibling::{nodetest})"
|
||
|
||
# special case of fixed position: nth-*(0n+b)
|
||
# if a == 0:
|
||
# ~~~~~~~~~~
|
||
# count(***-sibling::***) = b-1
|
||
if a == 0:
|
||
return xpath.add_condition(f"{siblings_count} = {b_min_1}")
|
||
|
||
expressions = []
|
||
|
||
if a > 0:
|
||
# siblings count, an+b-1, is always >= 0,
|
||
# so if a>0, and (b-1)<=0, an "n" exists to satisfy this,
|
||
# therefore, the predicate is only interesting if (b-1)>0
|
||
if b_min_1 > 0:
|
||
expressions.append(f"{siblings_count} >= {b_min_1}")
|
||
else:
|
||
# if a<0, and (b-1)<0, no "n" satisfies this,
|
||
# this is tested above as an early exit condition
|
||
# otherwise,
|
||
expressions.append(f"{siblings_count} <= {b_min_1}")
|
||
|
||
# operations modulo 1 or -1 are simpler, one only needs to verify:
|
||
#
|
||
# - either:
|
||
# count(***-sibling::***) - (b-1) = n = 0, 1, 2, 3, etc.,
|
||
# i.e. count(***-sibling::***) >= (b-1)
|
||
#
|
||
# - or:
|
||
# count(***-sibling::***) - (b-1) = -n = 0, -1, -2, -3, etc.,
|
||
# i.e. count(***-sibling::***) <= (b-1)
|
||
# we just did above.
|
||
#
|
||
if abs(a) != 1:
|
||
# count(***-sibling::***) - (b-1) ≡ 0 (mod a)
|
||
left = siblings_count
|
||
|
||
# apply "modulo a" on 2nd term, -(b-1),
|
||
# to simplify things like "(... +6) % -3",
|
||
# and also make it positive with |a|
|
||
b_neg = (-b_min_1) % abs(a)
|
||
|
||
if b_neg != 0:
|
||
left = f"({left} +{b_neg})"
|
||
|
||
expressions.append(f"{left} mod {a} = 0")
|
||
|
||
template = "(%s)" if len(expressions) > 1 else "%s"
|
||
xpath.add_condition(
|
||
" and ".join(template % expression for expression in expressions)
|
||
)
|
||
return xpath
|
||
|
||
def xpath_nth_last_child_function(
|
||
self, xpath: XPathExpr, function: Function
|
||
) -> XPathExpr:
|
||
return self.xpath_nth_child_function(xpath, function, last=True)
|
||
|
||
@staticmethod
|
||
def _check_of_type_element(xpath: XPathExpr, pseudo: str) -> None:
|
||
"""Raise an exception if an -of-type pseudo-class can't be used with
|
||
the given element.
|
||
|
||
For "*" and namespace wildcards like "ns:*" the type of the element is
|
||
not known, and counting same-type siblings cannot be expressed as an
|
||
XPath 1.0 node test.
|
||
"""
|
||
element = xpath.element
|
||
if element == "*" or element.endswith(":*"):
|
||
css_element = element.replace(":", "|")
|
||
raise ExpressionError(f"{css_element}:{pseudo} is not implemented")
|
||
|
||
def xpath_nth_of_type_function(
|
||
self, xpath: XPathExpr, function: Function
|
||
) -> XPathExpr:
|
||
self._check_of_type_element(xpath, "nth-of-type()")
|
||
return self.xpath_nth_child_function(xpath, function, add_name_test=False)
|
||
|
||
def xpath_nth_last_of_type_function(
|
||
self, xpath: XPathExpr, function: Function
|
||
) -> XPathExpr:
|
||
self._check_of_type_element(xpath, "nth-last-of-type()")
|
||
return self.xpath_nth_child_function(
|
||
xpath, function, last=True, add_name_test=False
|
||
)
|
||
|
||
def xpath_contains_function(
|
||
self, xpath: XPathExpr, function: Function
|
||
) -> XPathExpr:
|
||
# Defined there, removed in later drafts:
|
||
# http://www.w3.org/TR/2001/CR-css3-selectors-20011113/#content-selectors
|
||
if function.argument_types() not in (["STRING"], ["IDENT"]):
|
||
raise ExpressionError(
|
||
f"Expected a single string or ident for :contains(), got {function.arguments!r}"
|
||
)
|
||
value = cast("str", function.arguments[0].value)
|
||
return xpath.add_condition(f"contains(., {self.xpath_literal(value)})")
|
||
|
||
def xpath_lang_function(self, xpath: XPathExpr, function: Function) -> XPathExpr:
|
||
if function.argument_types() not in (["STRING"], ["IDENT"]):
|
||
raise ExpressionError(
|
||
f"Expected a single string or ident for :lang(), got {function.arguments!r}"
|
||
)
|
||
value = cast("str", function.arguments[0].value)
|
||
return xpath.add_condition(f"lang({self.xpath_literal(value)})")
|
||
|
||
# Pseudo: dispatch by pseudo-class name
|
||
|
||
def xpath_root_pseudo(self, xpath: XPathExpr) -> XPathExpr:
|
||
return xpath.add_condition("not(parent::*)")
|
||
|
||
# CSS immediate children (CSS ":scope > div" to XPath "child::div" or "./div")
|
||
# Works only at the start of a selector
|
||
# Needed to get immediate children of a processed selector in Scrapy
|
||
# for product in response.css('.product'):
|
||
# description = product.css(':scope > div::text').get()
|
||
def xpath_scope_pseudo(self, xpath: XPathExpr) -> XPathExpr:
|
||
xpath.add_name_test()
|
||
return xpath.add_condition("position() = 1")
|
||
|
||
def xpath_first_child_pseudo(self, xpath: XPathExpr) -> XPathExpr:
|
||
return xpath.add_condition("count(preceding-sibling::*) = 0")
|
||
|
||
def xpath_last_child_pseudo(self, xpath: XPathExpr) -> XPathExpr:
|
||
return xpath.add_condition("count(following-sibling::*) = 0")
|
||
|
||
def xpath_first_of_type_pseudo(self, xpath: XPathExpr) -> XPathExpr:
|
||
self._check_of_type_element(xpath, "first-of-type")
|
||
return xpath.add_condition(f"count(preceding-sibling::{xpath.element}) = 0")
|
||
|
||
def xpath_last_of_type_pseudo(self, xpath: XPathExpr) -> XPathExpr:
|
||
self._check_of_type_element(xpath, "last-of-type")
|
||
return xpath.add_condition(f"count(following-sibling::{xpath.element}) = 0")
|
||
|
||
def xpath_only_child_pseudo(self, xpath: XPathExpr) -> XPathExpr:
|
||
# Count siblings, not the parent's children: the root element has
|
||
# no parent, but it has no siblings either, so it must match.
|
||
return xpath.add_condition(
|
||
"count(preceding-sibling::*) = 0 and count(following-sibling::*) = 0"
|
||
)
|
||
|
||
def xpath_only_of_type_pseudo(self, xpath: XPathExpr) -> XPathExpr:
|
||
self._check_of_type_element(xpath, "only-of-type")
|
||
return xpath.add_condition(
|
||
f"count(preceding-sibling::{xpath.element}) = 0 "
|
||
f"and count(following-sibling::{xpath.element}) = 0"
|
||
)
|
||
|
||
def xpath_empty_pseudo(self, xpath: XPathExpr) -> XPathExpr:
|
||
return xpath.add_condition("not(*) and not(string-length())")
|
||
|
||
def pseudo_never_matches(self, xpath: XPathExpr) -> XPathExpr:
|
||
"""Common implementation for pseudo-classes that never match."""
|
||
return xpath.add_condition("0")
|
||
|
||
xpath_link_pseudo = pseudo_never_matches
|
||
xpath_visited_pseudo = pseudo_never_matches
|
||
xpath_hover_pseudo = pseudo_never_matches
|
||
xpath_active_pseudo = pseudo_never_matches
|
||
xpath_focus_pseudo = pseudo_never_matches
|
||
xpath_target_pseudo = pseudo_never_matches
|
||
xpath_enabled_pseudo = pseudo_never_matches
|
||
xpath_disabled_pseudo = pseudo_never_matches
|
||
xpath_checked_pseudo = pseudo_never_matches
|
||
|
||
# Attrib: dispatch by attribute operator
|
||
|
||
def xpath_attrib_exists(
|
||
self, xpath: XPathExpr, name: str, value: str | None
|
||
) -> XPathExpr:
|
||
assert not value
|
||
xpath.add_condition(name)
|
||
return xpath
|
||
|
||
def xpath_attrib_equals(
|
||
self, xpath: XPathExpr, name: str, value: str | None
|
||
) -> XPathExpr:
|
||
assert value is not None
|
||
xpath.add_condition(f"{name} = {self.xpath_literal(value)}")
|
||
return xpath
|
||
|
||
def xpath_attrib_different(
|
||
self, xpath: XPathExpr, name: str, value: str | None
|
||
) -> XPathExpr:
|
||
assert value is not None
|
||
# FIXME: this seems like a weird hack...
|
||
if value:
|
||
xpath.add_condition(f"not({name}) or {name} != {self.xpath_literal(value)}")
|
||
else:
|
||
xpath.add_condition(f"{name} != {self.xpath_literal(value)}")
|
||
return xpath
|
||
|
||
def xpath_attrib_includes(
|
||
self, xpath: XPathExpr, name: str, value: str | None
|
||
) -> XPathExpr:
|
||
if value and is_non_whitespace(value):
|
||
arg = self.xpath_literal(" " + value + " ")
|
||
xpath.add_condition(
|
||
f"{name} and contains({name}, {self.xpath_literal(value)}) "
|
||
f"and contains(concat(' ', normalize-space({name}), ' '), {arg})"
|
||
)
|
||
else:
|
||
xpath.add_condition("0")
|
||
return xpath
|
||
|
||
def xpath_attrib_dashmatch(
|
||
self, xpath: XPathExpr, name: str, value: str | None
|
||
) -> XPathExpr:
|
||
assert value is not None
|
||
arg = self.xpath_literal(value)
|
||
arg_dash = self.xpath_literal(value + "-")
|
||
# Weird, but true...
|
||
xpath.add_condition(
|
||
f"{name} and ({name} = {arg} or starts-with({name}, {arg_dash}))"
|
||
)
|
||
return xpath
|
||
|
||
def xpath_attrib_prefixmatch(
|
||
self, xpath: XPathExpr, name: str, value: str | None
|
||
) -> XPathExpr:
|
||
if value:
|
||
xpath.add_condition(
|
||
f"{name} and starts-with({name}, {self.xpath_literal(value)})"
|
||
)
|
||
else:
|
||
xpath.add_condition("0")
|
||
return xpath
|
||
|
||
def xpath_attrib_suffixmatch(
|
||
self, xpath: XPathExpr, name: str, value: str | None
|
||
) -> XPathExpr:
|
||
if value:
|
||
# Oddly there is a starts-with in XPath 1.0, but not ends-with
|
||
xpath.add_condition(
|
||
f"{name} and substring({name}, string-length({name})-{len(value) - 1}) = {self.xpath_literal(value)}"
|
||
)
|
||
else:
|
||
xpath.add_condition("0")
|
||
return xpath
|
||
|
||
def xpath_attrib_substringmatch(
|
||
self, xpath: XPathExpr, name: str, value: str | None
|
||
) -> XPathExpr:
|
||
if value:
|
||
# Attribute selectors are case sensitive
|
||
xpath.add_condition(
|
||
f"{name} and contains({name}, {self.xpath_literal(value)})"
|
||
)
|
||
else:
|
||
xpath.add_condition("0")
|
||
return xpath
|
||
|
||
|
||
class HTMLTranslator(GenericTranslator):
|
||
"""
|
||
Translator for (X)HTML documents.
|
||
|
||
Has a more useful implementation of some pseudo-classes based on
|
||
HTML-specific element names and attribute names, as described in
|
||
the `HTML specification`_. It assumes no-quirks mode.
|
||
The API is the same as :class:`GenericTranslator`.
|
||
|
||
.. _HTML specification: https://html.spec.whatwg.org/multipage/semantics-other.html#pseudo-classes
|
||
|
||
:param xhtml:
|
||
If false (the default), element names and attribute names
|
||
are case-insensitive.
|
||
|
||
"""
|
||
|
||
lang_attribute = "lang"
|
||
|
||
def __init__(self, xhtml: bool = False) -> None:
|
||
self.xhtml = xhtml # Might be useful for sub-classes?
|
||
if not xhtml:
|
||
# See their definition in GenericTranslator.
|
||
self.lower_case_element_names = True
|
||
self.lower_case_attribute_names = True
|
||
|
||
def xpath_checked_pseudo(self, xpath: XPathExpr) -> XPathExpr:
|
||
# FIXME: is this really all the elements?
|
||
return xpath.add_condition(
|
||
"(@selected and name(.) = 'option') or "
|
||
"(@checked "
|
||
"and (name(.) = 'input' or name(.) = 'command')"
|
||
"and (@type = 'checkbox' or @type = 'radio'))"
|
||
)
|
||
|
||
def xpath_lang_function(self, xpath: XPathExpr, function: Function) -> XPathExpr:
|
||
if function.argument_types() not in (["STRING"], ["IDENT"]):
|
||
raise ExpressionError(
|
||
f"Expected a single string or ident for :lang(), got {function.arguments!r}"
|
||
)
|
||
value = function.arguments[0].value
|
||
assert value
|
||
arg = self.xpath_literal(value.lower() + "-")
|
||
return xpath.add_condition(
|
||
"ancestor-or-self::*[@lang][1][starts-with(concat("
|
||
# XPath 1.0 has no lower-case function...
|
||
f"translate(@{self.lang_attribute}, 'ABCDEFGHIJKLMNOPQRSTUVWXYZ', "
|
||
"'abcdefghijklmnopqrstuvwxyz'), "
|
||
f"'-'), {arg})]"
|
||
)
|
||
|
||
def xpath_link_pseudo(self, xpath: XPathExpr) -> XPathExpr:
|
||
return xpath.add_condition(
|
||
"@href and (name(.) = 'a' or name(.) = 'link' or name(.) = 'area')"
|
||
)
|
||
|
||
# Links are never visited, the implementation for :visited is the same
|
||
# as in GenericTranslator
|
||
|
||
# An element is a descendant of a disabled fieldset ancestor unless it is
|
||
# a descendant of that fieldset's own first legend element child. lxml's
|
||
# XPath evaluator has no generate-id() or current(), so the exemption is
|
||
# tested by checking whether any ancestor legend is also the first legend
|
||
# child of a disabled ancestor fieldset; with several nested disabled
|
||
# fieldsets this can exempt an element that a fully spec-accurate
|
||
# algorithm would still treat as disabled.
|
||
_disabled_by_fieldset = (
|
||
"ancestor::fieldset[@disabled] and not("
|
||
"count(ancestor::legend | ancestor::fieldset[@disabled]/legend[1]) < "
|
||
"count(ancestor::legend) + count(ancestor::fieldset[@disabled]/legend[1])"
|
||
")"
|
||
)
|
||
|
||
def xpath_disabled_pseudo(self, xpath: XPathExpr) -> XPathExpr:
|
||
# https://html.spec.whatwg.org/multipage/semantics-other.html#concept-element-disabled
|
||
return xpath.add_condition(
|
||
f"""
|
||
(
|
||
@disabled and
|
||
(
|
||
name(.) = 'input' or
|
||
name(.) = 'button' or
|
||
name(.) = 'select' or
|
||
name(.) = 'textarea' or
|
||
name(.) = 'fieldset' or
|
||
name(.) = 'optgroup' or
|
||
name(.) = 'option'
|
||
)
|
||
) or (
|
||
(
|
||
name(.) = 'input' or
|
||
name(.) = 'button' or
|
||
name(.) = 'select' or
|
||
name(.) = 'textarea'
|
||
)
|
||
and {self._disabled_by_fieldset}
|
||
) or (
|
||
name(.) = 'option' and ancestor::optgroup[@disabled]
|
||
)
|
||
"""
|
||
)
|
||
|
||
def xpath_enabled_pseudo(self, xpath: XPathExpr) -> XPathExpr:
|
||
# https://html.spec.whatwg.org/multipage/semantics-other.html#concept-element-disabled
|
||
return xpath.add_condition(
|
||
f"""
|
||
(
|
||
(
|
||
name(.) = 'fieldset' or
|
||
name(.) = 'optgroup'
|
||
)
|
||
and not(@disabled)
|
||
) or (
|
||
(
|
||
name(.) = 'input' or
|
||
name(.) = 'button' or
|
||
name(.) = 'select' or
|
||
name(.) = 'textarea'
|
||
)
|
||
and not(@disabled) and not({self._disabled_by_fieldset})
|
||
) or (
|
||
name(.) = 'option' and not(
|
||
@disabled or ancestor::optgroup[@disabled]
|
||
)
|
||
)
|
||
"""
|
||
)
|