""" cssselect.xpath =============== Translation of parsed CSS selectors to XPath expressions. :copyright: (c) 2007-2012 Ian Bicking and contributors. See AUTHORS for more details. :license: BSD, see LICENSE for more details. """ from __future__ import annotations import re from string import ascii_lowercase, ascii_uppercase from typing import TYPE_CHECKING, Self, cast from cssselect.parser import ( Attrib, Class, CombinedSelector, Element, Function, Hash, Matching, Negation, Pseudo, PseudoElement, Relation, Selector, SelectorError, SpecificityAdjustment, Tree, ascii_lower, parse, parse_series, ) if TYPE_CHECKING: from collections.abc import Callable, Iterable class ExpressionError(SelectorError, RuntimeError): """Unknown or unsupported selector (eg. pseudo-class).""" #### XPath Helpers class XPathExpr: def __init__( self, path: str = "", element: str = "*", condition: str = "", star_prefix: bool = False, ) -> None: self.path = path self.element = element self.condition = condition def __str__(self) -> str: path = str(self.path) + str(self.element) if self.condition: path += f"[{self.condition}]" return path def __repr__(self) -> str: return f"{self.__class__.__name__}[{self}]" def add_condition(self, condition: str, conjuction: str = "and") -> Self: if self.condition: self.condition = f"({self.condition}) {conjuction} ({condition})" else: self.condition = condition return self def add_name_test(self) -> None: if self.element == "*": # We weren't doing a test anyway return prefix, colon, local = self.element.partition(":") if is_safe_name(prefix) and (not colon or local == "*" or is_safe_name(local)): # A node test, not a name() comparison: name() returns the # qualified name as written in the document, which would bypass # the XPath prefix mapping for a prefixed name like "ns:f" or # "ns:*" (from the CSS "ns|f" or "ns|*"), and would match # elements in a default namespace for an unprefixed name (which # the node test emitted for a bare "f" selector does not). self.add_condition(f"self::{self.element}") else: self.add_condition( f"name() = {GenericTranslator.xpath_literal(self.element)}" ) self.element = "*" def add_star_prefix(self) -> None: """ Append '*/' to the path to keep the context constrained to a single parent. """ self.path += "*/" def join(self, combiner: str, other: XPathExpr) -> Self: path = str(self) + combiner # Any "star prefix" is redundant when joining. if other.path != "*/": path += other.path self.path = path self.element = other.element self.condition = other.condition return self split_at_single_quotes = re.compile("('+)").split # The spec is actually more permissive than that, but don’t bother. # This is just for the fast path. # http://www.w3.org/TR/REC-xml/#NT-NameStartChar is_safe_name = re.compile("^[a-zA-Z_][a-zA-Z0-9_.-]*$").match # Test that the string is not empty and does not contain whitespace is_non_whitespace = re.compile(r"^[^ \t\r\n\f]+$").match #### Translation class GenericTranslator: """ Translator for "generic" XML documents. Everything is case-sensitive, no assumption is made on the meaning of element names and attribute names. """ #### #### HERE BE DRAGONS #### #### You are welcome to hook into this to change some behavior, #### but do so at your own risks. #### Until it has received a lot more work and review, #### I reserve the right to change this API in backward-incompatible ways #### with any minor version of cssselect. #### See https://github.com/scrapy/cssselect/pull/22 #### -- Simon Sapin. #### combinator_mapping = { " ": "descendant", ">": "child", "+": "direct_adjacent", "~": "indirect_adjacent", } # Used to match a combinator against the context node, in :not(). _reverse_combinator_mapping = { " ": "ancestor::*", ">": "parent::*", "+": "preceding-sibling::*[1]", "~": "preceding-sibling::*", } attribute_operator_mapping = { "exists": "exists", "=": "equals", "~=": "includes", "|=": "dashmatch", "^=": "prefixmatch", "$=": "suffixmatch", "*=": "substringmatch", "!=": "different", # not part of Selectors Level 3, but widely supported } #: The attribute used for ID selectors depends on the document language: #: http://www.w3.org/TR/selectors/#id-selectors id_attribute = "id" #: The attribute used for ``:lang()`` depends on the document language: #: http://www.w3.org/TR/selectors/#lang-pseudo lang_attribute = "xml:lang" #: The case sensitivity of document language element names, #: attribute names, and attribute values in selectors depends #: on the document language. #: http://www.w3.org/TR/selectors/#casesens #: #: When a document language defines one of these as case-insensitive, #: cssselect assumes that the document parser makes the parsed values #: lower-case. Making the selector lower-case too makes the comparaison #: case-insensitive. #: #: In HTML, element names and attributes names (but not attribute values) #: are case-insensitive. All of lxml.html, html5lib, BeautifulSoup4 #: and HTMLParser make them lower-case in their parse result, so #: the assumption holds. lower_case_element_names = False lower_case_attribute_names = False lower_case_attribute_values = False # class used to represent and xpath expression xpathexpr_cls = XPathExpr def css_to_xpath(self, css: str, prefix: str = "descendant-or-self::") -> str: """Translate a *group of selectors* to XPath. Pseudo-elements are not supported here since XPath only knows about "real" elements. :param css: A *group of selectors* as a string. :param prefix: This string is prepended to the XPath expression for each selector. The default makes selectors scoped to the context node’s subtree. :raises: :class:`~cssselect.SelectorSyntaxError` on invalid selectors, :class:`ExpressionError` on unknown/unsupported selectors, including pseudo-elements. :returns: The equivalent XPath 1.0 expression as a string. """ return " | ".join( self.selector_to_xpath(selector, prefix, translate_pseudo_elements=True) for selector in parse(css) ) def selector_to_xpath( self, selector: Selector, prefix: str = "descendant-or-self::", translate_pseudo_elements: bool = False, ) -> str: """Translate a parsed selector to XPath. :param selector: A parsed :class:`Selector` object. :param prefix: This string is prepended to the resulting XPath expression. The default makes selectors scoped to the context node’s subtree. :param translate_pseudo_elements: Unless this is set to ``True`` (as :meth:`css_to_xpath` does), the :attr:`~Selector.pseudo_element` attribute of the selector is ignored. It is the caller's responsibility to reject selectors with pseudo-elements, or to account for them somehow. :raises: :class:`ExpressionError` on unknown/unsupported selectors. :returns: The equivalent XPath 1.0 expression as a string. """ tree = getattr(selector, "parsed_tree", None) if not tree: raise TypeError(f"Expected a parsed selector, got {selector!r}") xpath = self.xpath(tree) assert isinstance(xpath, self.xpathexpr_cls) # help debug a missing 'return' if translate_pseudo_elements and selector.pseudo_element: xpath = self.xpath_pseudo_element(xpath, selector.pseudo_element) return (prefix or "") + str(xpath) def xpath_pseudo_element( self, xpath: XPathExpr, pseudo_element: PseudoElement ) -> XPathExpr: """Translate a pseudo-element. Defaults to not supporting pseudo-elements at all, but can be overridden by sub-classes. """ raise ExpressionError("Pseudo-elements are not supported.") @staticmethod def xpath_literal(s: str) -> str: s = str(s) if "'" not in s: s = f"'{s}'" elif '"' not in s: s = f'"{s}"' else: parts_quoted = [ f'"{part}"' if "'" in part else f"'{part}'" for part in split_at_single_quotes(s) if part ] s = "concat({})".format(",".join(parts_quoted)) return s def xpath(self, parsed_selector: Tree) -> XPathExpr: """Translate any parsed selector object.""" type_name = type(parsed_selector).__name__ method = cast( "Callable[[Tree], XPathExpr] | None", getattr(self, f"xpath_{type_name.lower()}", None), ) if method is None: raise ExpressionError(f"{type_name} is not supported.") return method(parsed_selector) # Dispatched by parsed object type def xpath_combinedselector(self, combined: CombinedSelector) -> XPathExpr: """Translate a combined selector.""" combinator = self.combinator_mapping[combined.combinator] method = cast( "Callable[[XPathExpr, XPathExpr], XPathExpr]", getattr(self, f"xpath_{combinator}_combinator"), ) return method(self.xpath(combined.selector), self.xpath(combined.subselector)) def xpath_negation(self, negation: Negation) -> XPathExpr: xpath = self.xpath(negation.selector) condition = self._xpath_match_condition(negation.subselector) if condition is None: # The argument matches every element, so :not() matches none. return xpath.add_condition("0") return xpath.add_condition(f"not({condition})") def _xpath_match_condition(self, selector: Tree) -> str | None: """Return a condition that holds for the elements matching *selector*, or None if that is every element. Unlike xpath(), which walks a selector from left to right, this matches the whole selector against the context node, using reverse axes for combinators. """ if isinstance(selector, CombinedSelector): axis = self._reverse_combinator_mapping[selector.combinator] left = self._xpath_match_condition(selector.selector) if left is not None: axis = f"{axis}[{left}]" right = self._xpath_match_condition(selector.subselector) return axis if right is None else f"{right} and {axis}" sub_xpath = self.xpath(selector) sub_xpath.add_name_test() return sub_xpath.condition or None def xpath_relation(self, relation: Relation) -> XPathExpr: condition = "" for combinator, subselector in relation.arguments: method = cast( "Callable[[XPathExpr], str]", getattr( self, f"xpath_relation_{self.combinator_mapping[cast('str', combinator.value)]}_combinator", ), ) argument = method(self.xpath(subselector.parsed_tree)) condition = f"({condition}) or ({argument})" if condition else argument return self.xpath(relation.selector).add_condition(condition) def xpath_matching(self, matching: Matching) -> XPathExpr: return self._xpath_add_selector_list_condition( self.xpath(matching.selector), matching.selector_list ) def xpath_specificityadjustment(self, matching: SpecificityAdjustment) -> XPathExpr: return self._xpath_add_selector_list_condition( self.xpath(matching.selector), matching.selector_list ) def _xpath_add_selector_list_condition( self, xpath: XPathExpr, selector_list: Iterable[Tree] ) -> XPathExpr: """Add a condition matching any selector of the list (for :is() and :where()).""" condition = "" for e in (self.xpath(selector) for selector in selector_list): if e.path: # Only a combined selector (e.g. "a b") translates to a path, # which cannot be embedded into a predicate of the outer # expression. The parser rejects combinators in these arguments, # so this is only reachable through a hand-built Matching or # SpecificityAdjustment node. raise ExpressionError( "Combined selectors are not supported inside " ":is(), :where() and :matches()" ) e.add_name_test() if not e.condition: # This argument matches any element, so the whole selector # list does too: it adds no condition. return xpath condition = ( f"({condition}) or ({e.condition})" if condition else e.condition ) return xpath.add_condition(condition) def xpath_function(self, function: Function) -> XPathExpr: """Translate a functional pseudo-class.""" method_name = "xpath_{}_function".format(function.name.replace("-", "_")) method = cast( "Callable[[XPathExpr, Function], XPathExpr] | None", getattr(self, method_name, None), ) if not method: raise ExpressionError(f"The pseudo-class :{function.name}() is unknown") return method(self.xpath(function.selector), function) def xpath_pseudo(self, pseudo: Pseudo) -> XPathExpr: """Translate a pseudo-class.""" method_name = "xpath_{}_pseudo".format(pseudo.ident.replace("-", "_")) method = cast( "Callable[[XPathExpr], XPathExpr] | None", getattr(self, method_name, None), ) if not method: raise ExpressionError(f"The pseudo-class :{pseudo.ident} is unknown") return method(self.xpath(pseudo.selector)) def xpath_attrib(self, selector: Attrib) -> XPathExpr: """Translate an attribute selector.""" operator = self.attribute_operator_mapping[selector.operator] method = cast( "Callable[[XPathExpr, str, str | None], XPathExpr]", getattr(self, f"xpath_attrib_{operator}"), ) if self.lower_case_attribute_names: name = selector.attrib.lower() else: name = selector.attrib if selector.namespace == "*": # Namespace wildcard, e.g. "[*|href]": any namespace, including # none. XPath 1.0 has no "*:name" name test for this, so match # by local name instead. attrib = f"attribute::*[local-name() = {self.xpath_literal(name)}]" else: safe = is_safe_name(name) if selector.namespace: name = f"{selector.namespace}:{name}" safe = safe and is_safe_name(selector.namespace) if safe: attrib = "@" + name else: attrib = f"attribute::*[name() = {self.xpath_literal(name)}]" if selector.value is None: value = None elif self.lower_case_attribute_values: value = cast("str", selector.value.value).lower() else: value = selector.value.value if selector.flag == "i" and value: # ASCII-lowering both sides is what the specification defines a # case-insensitive match as, and all XPath 1.0 can express. attrib = f"translate({attrib}, {self.xpath_literal(ascii_uppercase)}, {self.xpath_literal(ascii_lowercase)})" value = ascii_lower(value) return method(self.xpath(selector.selector), attrib, value) def xpath_class(self, class_selector: Class) -> XPathExpr: """Translate a class selector.""" # .foo is defined as [class~=foo] in the spec. xpath = self.xpath(class_selector.selector) return self.xpath_attrib_includes(xpath, "@class", class_selector.class_name) def xpath_hash(self, id_selector: Hash) -> XPathExpr: """Translate an ID selector.""" xpath = self.xpath(id_selector.selector) return self.xpath_attrib_equals(xpath, "@id", id_selector.id) def xpath_element(self, selector: Element) -> XPathExpr: """Translate a type or universal selector.""" element = selector.element if not element: element = "*" safe = True else: safe = bool(is_safe_name(element)) if self.lower_case_element_names: element = element.lower() if selector.namespace == "*": # Namespace wildcard, e.g. "*|div": any namespace, # including none. XPath 1.0 has no "*:name" name test # for this, so match by local name instead. xpath = self.xpathexpr_cls(element="*") xpath.add_condition(f"local-name() = {self.xpath_literal(element)}") return xpath if selector.namespace and selector.namespace != "*": # Namespace prefixes are case-sensitive. # http://www.w3.org/TR/css3-namespace/#prefixes element = f"{selector.namespace}:{element}" safe = safe and bool(is_safe_name(selector.namespace)) xpath = self.xpathexpr_cls(element=element) if not safe: # Not usable as an XPath name test (e.g. an escaped identifier # like di\a0 v): compare the serialized name instead. Done here # rather than through add_name_test(), which would mistake a ":" # inside such a name for a namespace prefix separator. xpath.add_condition(f"name() = {self.xpath_literal(element)}") xpath.element = "*" return xpath # CombinedSelector: dispatch by combinator def xpath_descendant_combinator( self, left: XPathExpr, right: XPathExpr ) -> XPathExpr: """right is a child, grand-child or further descendant of left""" if right.element == "*" and not right.condition and not right.path: # A subclass that maps a pseudo-element to a different node test # rewrites the trailing "descendant-or-self::*/*" of the # expression, so an unconstrained right-hand side keeps that shape. return left.join("/descendant-or-self::*/", right) return left.join("/descendant::", right) def xpath_child_combinator(self, left: XPathExpr, right: XPathExpr) -> XPathExpr: """right is an immediate child of left""" return left.join("/", right) def xpath_direct_adjacent_combinator( self, left: XPathExpr, right: XPathExpr ) -> XPathExpr: """right is a sibling immediately after left""" xpath = left.join("/following-sibling::", right) xpath.add_name_test() return xpath.add_condition("position() = 1") def xpath_indirect_adjacent_combinator( self, left: XPathExpr, right: XPathExpr ) -> XPathExpr: """right is a sibling after left, immediately or not""" return left.join("/following-sibling::", right) # The relative selector is kept in `condition` (instead of being folded # into `path`/`element`) so that `element` stays a plain element name: # later steps such as :first-of-type or :not() read and rewrite it. def xpath_relation_descendant_combinator(self, right: XPathExpr) -> str: """right is a child, grand-child or further descendant of the element""" return f"descendant::{right}" def xpath_relation_child_combinator(self, right: XPathExpr) -> str: """right is an immediate child of the element""" return f"./{right}" def xpath_relation_direct_adjacent_combinator(self, right: XPathExpr) -> str: """right is a sibling immediately after the element""" # Test the first step of right against that sibling itself, so that # any further step of right is walked from there. if right.path: head, sep, tail = right.path.partition("/") right.path = f"self::{head}{sep}{tail}" else: right.element = f"self::{right.element}" return f"following-sibling::*[1]/{right}" def xpath_relation_indirect_adjacent_combinator(self, right: XPathExpr) -> str: """right is a sibling after the element, immediately or not""" return f"following-sibling::{right}" # Function: dispatch by function/pseudo-class name def xpath_nth_child_function( self, xpath: XPathExpr, function: Function, last: bool = False, add_name_test: bool = True, ) -> XPathExpr: try: a, b = parse_series(function.arguments) except ValueError as ex: raise ExpressionError(f"Invalid series: '{function.arguments!r}'") from ex # From https://www.w3.org/TR/css3-selectors/#structural-pseudos: # # :nth-child(an+b) # an+b-1 siblings before # # :nth-last-child(an+b) # an+b-1 siblings after # # :nth-of-type(an+b) # an+b-1 siblings with the same expanded element name before # # :nth-last-of-type(an+b) # an+b-1 siblings with the same expanded element name after # # So, # for :nth-child and :nth-of-type # # count(preceding-sibling::) = an+b-1 # # for :nth-last-child and :nth-last-of-type # # count(following-sibling::) = an+b-1 # # therefore, # count(...) - (b-1) ≡ 0 (mod a) # # if a == 0: # ~~~~~~~~~~ # count(...) = b-1 # # if a < 0: # ~~~~~~~~~ # count(...) - b +1 <= 0 # -> count(...) <= b-1 # # if a > 0: # ~~~~~~~~~ # count(...) - b +1 >= 0 # -> count(...) >= b-1 # work with b-1 instead b_min_1 = b - 1 # early-exit condition 1: # ~~~~~~~~~~~~~~~~~~~~~~~ # for a == 1, nth-*(an+b) means n+b-1 siblings before/after, # and since n ∈ {0, 1, 2, ...}, if b-1<=0, # there is always an "n" matching any number of siblings (maybe none) if a == 1 and b_min_1 <= 0: return xpath # early-exit condition 2: # ~~~~~~~~~~~~~~~~~~~~~~~ # an+b-1 siblings with a<0 and (b-1)<0 is not possible if a < 0 and b_min_1 < 0: return xpath.add_condition("0") # `add_name_test` boolean is inverted and somewhat counter-intuitive: # # nth_of_type() calls nth_child(add_name_test=False) nodetest = "*" if add_name_test else f"{xpath.element}" # count siblings before or after the element if not last: siblings_count = f"count(preceding-sibling::{nodetest})" else: siblings_count = f"count(following-sibling::{nodetest})" # special case of fixed position: nth-*(0n+b) # if a == 0: # ~~~~~~~~~~ # count(***-sibling::***) = b-1 if a == 0: return xpath.add_condition(f"{siblings_count} = {b_min_1}") expressions = [] if a > 0: # siblings count, an+b-1, is always >= 0, # so if a>0, and (b-1)<=0, an "n" exists to satisfy this, # therefore, the predicate is only interesting if (b-1)>0 if b_min_1 > 0: expressions.append(f"{siblings_count} >= {b_min_1}") else: # if a<0, and (b-1)<0, no "n" satisfies this, # this is tested above as an early exit condition # otherwise, expressions.append(f"{siblings_count} <= {b_min_1}") # operations modulo 1 or -1 are simpler, one only needs to verify: # # - either: # count(***-sibling::***) - (b-1) = n = 0, 1, 2, 3, etc., # i.e. count(***-sibling::***) >= (b-1) # # - or: # count(***-sibling::***) - (b-1) = -n = 0, -1, -2, -3, etc., # i.e. count(***-sibling::***) <= (b-1) # we just did above. # if abs(a) != 1: # count(***-sibling::***) - (b-1) ≡ 0 (mod a) left = siblings_count # apply "modulo a" on 2nd term, -(b-1), # to simplify things like "(... +6) % -3", # and also make it positive with |a| b_neg = (-b_min_1) % abs(a) if b_neg != 0: left = f"({left} +{b_neg})" expressions.append(f"{left} mod {a} = 0") template = "(%s)" if len(expressions) > 1 else "%s" xpath.add_condition( " and ".join(template % expression for expression in expressions) ) return xpath def xpath_nth_last_child_function( self, xpath: XPathExpr, function: Function ) -> XPathExpr: return self.xpath_nth_child_function(xpath, function, last=True) @staticmethod def _check_of_type_element(xpath: XPathExpr, pseudo: str) -> None: """Raise an exception if an -of-type pseudo-class can't be used with the given element. For "*" and namespace wildcards like "ns:*" the type of the element is not known, and counting same-type siblings cannot be expressed as an XPath 1.0 node test. """ element = xpath.element if element == "*" or element.endswith(":*"): css_element = element.replace(":", "|") raise ExpressionError(f"{css_element}:{pseudo} is not implemented") def xpath_nth_of_type_function( self, xpath: XPathExpr, function: Function ) -> XPathExpr: self._check_of_type_element(xpath, "nth-of-type()") return self.xpath_nth_child_function(xpath, function, add_name_test=False) def xpath_nth_last_of_type_function( self, xpath: XPathExpr, function: Function ) -> XPathExpr: self._check_of_type_element(xpath, "nth-last-of-type()") return self.xpath_nth_child_function( xpath, function, last=True, add_name_test=False ) def xpath_contains_function( self, xpath: XPathExpr, function: Function ) -> XPathExpr: # Defined there, removed in later drafts: # http://www.w3.org/TR/2001/CR-css3-selectors-20011113/#content-selectors if function.argument_types() not in (["STRING"], ["IDENT"]): raise ExpressionError( f"Expected a single string or ident for :contains(), got {function.arguments!r}" ) value = cast("str", function.arguments[0].value) return xpath.add_condition(f"contains(., {self.xpath_literal(value)})") def xpath_lang_function(self, xpath: XPathExpr, function: Function) -> XPathExpr: if function.argument_types() not in (["STRING"], ["IDENT"]): raise ExpressionError( f"Expected a single string or ident for :lang(), got {function.arguments!r}" ) value = cast("str", function.arguments[0].value) return xpath.add_condition(f"lang({self.xpath_literal(value)})") # Pseudo: dispatch by pseudo-class name def xpath_root_pseudo(self, xpath: XPathExpr) -> XPathExpr: return xpath.add_condition("not(parent::*)") # CSS immediate children (CSS ":scope > div" to XPath "child::div" or "./div") # Works only at the start of a selector # Needed to get immediate children of a processed selector in Scrapy # for product in response.css('.product'): # description = product.css(':scope > div::text').get() def xpath_scope_pseudo(self, xpath: XPathExpr) -> XPathExpr: xpath.add_name_test() return xpath.add_condition("position() = 1") def xpath_first_child_pseudo(self, xpath: XPathExpr) -> XPathExpr: return xpath.add_condition("count(preceding-sibling::*) = 0") def xpath_last_child_pseudo(self, xpath: XPathExpr) -> XPathExpr: return xpath.add_condition("count(following-sibling::*) = 0") def xpath_first_of_type_pseudo(self, xpath: XPathExpr) -> XPathExpr: self._check_of_type_element(xpath, "first-of-type") return xpath.add_condition(f"count(preceding-sibling::{xpath.element}) = 0") def xpath_last_of_type_pseudo(self, xpath: XPathExpr) -> XPathExpr: self._check_of_type_element(xpath, "last-of-type") return xpath.add_condition(f"count(following-sibling::{xpath.element}) = 0") def xpath_only_child_pseudo(self, xpath: XPathExpr) -> XPathExpr: # Count siblings, not the parent's children: the root element has # no parent, but it has no siblings either, so it must match. return xpath.add_condition( "count(preceding-sibling::*) = 0 and count(following-sibling::*) = 0" ) def xpath_only_of_type_pseudo(self, xpath: XPathExpr) -> XPathExpr: self._check_of_type_element(xpath, "only-of-type") return xpath.add_condition( f"count(preceding-sibling::{xpath.element}) = 0 " f"and count(following-sibling::{xpath.element}) = 0" ) def xpath_empty_pseudo(self, xpath: XPathExpr) -> XPathExpr: return xpath.add_condition("not(*) and not(string-length())") def pseudo_never_matches(self, xpath: XPathExpr) -> XPathExpr: """Common implementation for pseudo-classes that never match.""" return xpath.add_condition("0") xpath_link_pseudo = pseudo_never_matches xpath_visited_pseudo = pseudo_never_matches xpath_hover_pseudo = pseudo_never_matches xpath_active_pseudo = pseudo_never_matches xpath_focus_pseudo = pseudo_never_matches xpath_target_pseudo = pseudo_never_matches xpath_enabled_pseudo = pseudo_never_matches xpath_disabled_pseudo = pseudo_never_matches xpath_checked_pseudo = pseudo_never_matches # Attrib: dispatch by attribute operator def xpath_attrib_exists( self, xpath: XPathExpr, name: str, value: str | None ) -> XPathExpr: assert not value xpath.add_condition(name) return xpath def xpath_attrib_equals( self, xpath: XPathExpr, name: str, value: str | None ) -> XPathExpr: assert value is not None xpath.add_condition(f"{name} = {self.xpath_literal(value)}") return xpath def xpath_attrib_different( self, xpath: XPathExpr, name: str, value: str | None ) -> XPathExpr: assert value is not None # FIXME: this seems like a weird hack... if value: xpath.add_condition(f"not({name}) or {name} != {self.xpath_literal(value)}") else: xpath.add_condition(f"{name} != {self.xpath_literal(value)}") return xpath def xpath_attrib_includes( self, xpath: XPathExpr, name: str, value: str | None ) -> XPathExpr: if value and is_non_whitespace(value): arg = self.xpath_literal(" " + value + " ") xpath.add_condition( f"{name} and contains({name}, {self.xpath_literal(value)}) " f"and contains(concat(' ', normalize-space({name}), ' '), {arg})" ) else: xpath.add_condition("0") return xpath def xpath_attrib_dashmatch( self, xpath: XPathExpr, name: str, value: str | None ) -> XPathExpr: assert value is not None arg = self.xpath_literal(value) arg_dash = self.xpath_literal(value + "-") # Weird, but true... xpath.add_condition( f"{name} and ({name} = {arg} or starts-with({name}, {arg_dash}))" ) return xpath def xpath_attrib_prefixmatch( self, xpath: XPathExpr, name: str, value: str | None ) -> XPathExpr: if value: xpath.add_condition( f"{name} and starts-with({name}, {self.xpath_literal(value)})" ) else: xpath.add_condition("0") return xpath def xpath_attrib_suffixmatch( self, xpath: XPathExpr, name: str, value: str | None ) -> XPathExpr: if value: # Oddly there is a starts-with in XPath 1.0, but not ends-with xpath.add_condition( f"{name} and substring({name}, string-length({name})-{len(value) - 1}) = {self.xpath_literal(value)}" ) else: xpath.add_condition("0") return xpath def xpath_attrib_substringmatch( self, xpath: XPathExpr, name: str, value: str | None ) -> XPathExpr: if value: # Attribute selectors are case sensitive xpath.add_condition( f"{name} and contains({name}, {self.xpath_literal(value)})" ) else: xpath.add_condition("0") return xpath class HTMLTranslator(GenericTranslator): """ Translator for (X)HTML documents. Has a more useful implementation of some pseudo-classes based on HTML-specific element names and attribute names, as described in the `HTML specification`_. It assumes no-quirks mode. The API is the same as :class:`GenericTranslator`. .. _HTML specification: https://html.spec.whatwg.org/multipage/semantics-other.html#pseudo-classes :param xhtml: If false (the default), element names and attribute names are case-insensitive. """ lang_attribute = "lang" def __init__(self, xhtml: bool = False) -> None: self.xhtml = xhtml # Might be useful for sub-classes? if not xhtml: # See their definition in GenericTranslator. self.lower_case_element_names = True self.lower_case_attribute_names = True def xpath_checked_pseudo(self, xpath: XPathExpr) -> XPathExpr: # FIXME: is this really all the elements? return xpath.add_condition( "(@selected and name(.) = 'option') or " "(@checked " "and (name(.) = 'input' or name(.) = 'command')" "and (@type = 'checkbox' or @type = 'radio'))" ) def xpath_lang_function(self, xpath: XPathExpr, function: Function) -> XPathExpr: if function.argument_types() not in (["STRING"], ["IDENT"]): raise ExpressionError( f"Expected a single string or ident for :lang(), got {function.arguments!r}" ) value = function.arguments[0].value assert value arg = self.xpath_literal(value.lower() + "-") return xpath.add_condition( "ancestor-or-self::*[@lang][1][starts-with(concat(" # XPath 1.0 has no lower-case function... f"translate(@{self.lang_attribute}, 'ABCDEFGHIJKLMNOPQRSTUVWXYZ', " "'abcdefghijklmnopqrstuvwxyz'), " f"'-'), {arg})]" ) def xpath_link_pseudo(self, xpath: XPathExpr) -> XPathExpr: return xpath.add_condition( "@href and (name(.) = 'a' or name(.) = 'link' or name(.) = 'area')" ) # Links are never visited, the implementation for :visited is the same # as in GenericTranslator # An element is a descendant of a disabled fieldset ancestor unless it is # a descendant of that fieldset's own first legend element child. lxml's # XPath evaluator has no generate-id() or current(), so the exemption is # tested by checking whether any ancestor legend is also the first legend # child of a disabled ancestor fieldset; with several nested disabled # fieldsets this can exempt an element that a fully spec-accurate # algorithm would still treat as disabled. _disabled_by_fieldset = ( "ancestor::fieldset[@disabled] and not(" "count(ancestor::legend | ancestor::fieldset[@disabled]/legend[1]) < " "count(ancestor::legend) + count(ancestor::fieldset[@disabled]/legend[1])" ")" ) def xpath_disabled_pseudo(self, xpath: XPathExpr) -> XPathExpr: # https://html.spec.whatwg.org/multipage/semantics-other.html#concept-element-disabled return xpath.add_condition( f""" ( @disabled and ( name(.) = 'input' or name(.) = 'button' or name(.) = 'select' or name(.) = 'textarea' or name(.) = 'fieldset' or name(.) = 'optgroup' or name(.) = 'option' ) ) or ( ( name(.) = 'input' or name(.) = 'button' or name(.) = 'select' or name(.) = 'textarea' ) and {self._disabled_by_fieldset} ) or ( name(.) = 'option' and ancestor::optgroup[@disabled] ) """ ) def xpath_enabled_pseudo(self, xpath: XPathExpr) -> XPathExpr: # https://html.spec.whatwg.org/multipage/semantics-other.html#concept-element-disabled return xpath.add_condition( f""" ( ( name(.) = 'fieldset' or name(.) = 'optgroup' ) and not(@disabled) ) or ( ( name(.) = 'input' or name(.) = 'button' or name(.) = 'select' or name(.) = 'textarea' ) and not(@disabled) and not({self._disabled_by_fieldset}) ) or ( name(.) = 'option' and not( @disabled or ancestor::optgroup[@disabled] ) ) """ )