# -*- coding:utf-8 -*- """ @Author : g1879 @Contact : g1879@qq.com @Website : https://DrissionPage.cn @Copyright: (c) 2020 by g1879, Inc. All Rights Reserved. """ from abc import abstractmethod from copy import copy from pathlib import Path from re import sub from urllib.parse import quote from DrissionGet import DrissionGet from requests import Session from .._configs.session_options import SessionOptions from .._elements.none_element import NoneElement from .._functions.elements import get_frame, get_eles from .._functions.locator import get_loc from .._functions.settings import Settings as _S from .._functions.web import format_html from ..errors import ElementNotFoundError, LocatorError class BaseParser(object): def __call__(self, locator): return self.ele(locator) def ele(self, locator, index=1, timeout=None): return self._ele(locator, timeout, index=index, method='ele()') def eles(self, locator, timeout=None): return self._ele(locator, timeout, index=None) def find(self, locators, any_one=True, first_ele=True, timeout=None): if 'Session' in self._type: timeout = 0 if timeout is None: timeout = self.timeout r = get_eles(locators, self, any_one, first_ele, timeout) if any_one: for ele in r: if r[ele]: return ele, r[ele] return None, None return r # ----------------以下属性或方法待后代实现---------------- @property def html(self): return '' def s_ele(self, locator=None): pass def s_eles(self, locator): pass def _ele(self, locator, timeout=None, index=1, raise_err=None, method=None): pass def _find_elements(self, locator, timeout, index=1, relative=False, raise_err=None): pass class BaseElement(BaseParser): def __init__(self, owner=None): self.owner = owner self._type = 'BaseElement' def get_frame(self, loc_or_ind, timeout=None): if not isinstance(loc_or_ind, (int, str, tuple)): raise ValueError(_S._lang.join(_S._lang.INCORRECT_TYPE_, 'loc_or_ind', ALLOW_TYPE=_S._lang.LOC_OR_IND, CURR_VAL=loc_or_ind)) return get_frame(self, loc_ind_ele=loc_or_ind, timeout=timeout) def _ele(self, locator, timeout=None, index=1, relative=False, raise_err=None, method=None): if hasattr(locator, '_type'): return locator if timeout is None: timeout = self.timeout r = self._find_elements(locator, timeout=timeout, index=index, relative=relative, raise_err=raise_err) if r or isinstance(r, list): return r if raise_err is True or (_S.raise_when_ele_not_found and raise_err is None): raise ElementNotFoundError(METHOD=method, ARGS={'locator': locator, 'index': index, 'timeout': timeout}) r.method = method r.args = {'locator': locator, 'index': index, 'timeout': timeout} return r @property def timeout(self): return self.owner.timeout if self.owner else 10 @property def child_count(self): return int(self._ele('xpath:count(./*)')) # ----------------以下属性或方法由后代实现---------------- @property def tag(self): return def parent(self, level_or_loc=1): pass def next(self, index=1): pass def nexts(self): pass class DrissionElement(BaseElement): @property def link(self): return self.attr('href') or self.attr('src') @property def css_path(self): return self._get_ele_path(xpath=False) @property def xpath(self): return self._get_ele_path() @property def comments(self): return self.eles('xpath:.//comment()') def texts(self, text_node_only=False): texts = self.eles('xpath:/text()') if text_node_only else [x if isinstance(x, str) else x.text for x in self.eles('xpath:./text() | *')] return [format_html(x.strip(' ').rstrip('\n')) for x in texts if x and sub('[\r\n\t ]', '', x) != ''] def parent(self, level_or_loc=1, index=1, timeout=None): if isinstance(level_or_loc, int): loc = f'xpath:./ancestor::*[{level_or_loc}]' elif isinstance(level_or_loc, (tuple, str)): loc = get_loc(level_or_loc, True) if loc[0] == 'css selector': raise LocatorError(_S._lang.UNSUPPORTED_CSS_SYNTAX) loc = f'xpath:./ancestor::{loc[1].lstrip(". / ")}[{index}]' else: raise ValueError(_S._lang.join(_S._lang.INCORRECT_TYPE_, 'level_or_loc', ALLOW_TYPE='tuple, int, str', CURR_VAL=level_or_loc)) return self._ele(loc, timeout=timeout, relative=True, raise_err=False, method='parent()') def child(self, locator='', index=1, timeout=None, ele_only=True): if isinstance(locator, int): index = locator locator = '' if not locator: loc = '*' if ele_only else 'node()' else: loc = get_loc(locator, True) # 把定位符转换为xpath if loc[0] == 'css selector': raise LocatorError(_S._lang.UNSUPPORTED_CSS_SYNTAX) loc = loc[1].lstrip('./') node = self._ele(f'xpath:./{loc}', timeout=timeout, index=index, relative=True, raise_err=False) return node if node else NoneElement(self.owner, 'child()', {'locator': locator, 'index': index, 'ele_only': ele_only}) def prev(self, locator='', index=1, timeout=None, ele_only=True): return self._get_relative('prev()', 'preceding', True, locator, index, timeout, ele_only) def next(self, locator='', index=1, timeout=None, ele_only=True): return self._get_relative('next()', 'following', True, locator, index, timeout, ele_only) def before(self, locator='', index=1, timeout=None, ele_only=True): return self._get_relative('before()', 'preceding', False, locator, index, timeout, ele_only) def after(self, locator='', index=1, timeout=None, ele_only=True): return self._get_relative('after()', 'following', False, locator, index, timeout, ele_only) def children(self, locator='', timeout=None, ele_only=True): if not locator: loc = '*' if ele_only else 'node()' else: loc = get_loc(locator, True) # 把定位符转换为xpath if loc[0] == 'css selector': raise LocatorError(_S._lang.UNSUPPORTED_CSS_SYNTAX) loc = loc[1].lstrip('./') loc = f'xpath:./{loc}' nodes = self._ele(loc, timeout=timeout, index=None, relative=True) return [e for e in nodes if not (isinstance(e, str) and sub('[ \n\t\r]', '', e) == '')] def prevs(self, locator='', timeout=None, ele_only=True): return self._get_relatives(locator=locator, direction='preceding', timeout=timeout, ele_only=ele_only) def nexts(self, locator='', timeout=None, ele_only=True): return self._get_relatives(locator=locator, direction='following', timeout=timeout, ele_only=ele_only) def befores(self, locator='', timeout=None, ele_only=True): return self._get_relatives(locator=locator, direction='preceding', brother=False, timeout=timeout, ele_only=ele_only) def afters(self, locator='', timeout=None, ele_only=True): return self._get_relatives(locator=locator, direction='following', brother=False, timeout=timeout, ele_only=ele_only) def _get_relative(self, func, direction, brother, locator='', index=1, timeout=None, ele_only=True): if isinstance(locator, int): index = locator locator = '' node = self._get_relatives(index, locator, direction, brother, timeout, ele_only) return node if node else NoneElement(self.owner, func, {'locator': locator, 'index': index, 'ele_only': ele_only}) def _get_relatives(self, index=None, locator='', direction='following', brother=True, timeout=.5, ele_only=True): brother = '-sibling' if brother else '' if not locator: loc = '*' if ele_only else 'node()' else: loc = get_loc(locator, True) # 把定位符转换为xpath if loc[0] == 'css selector': raise LocatorError(_S._lang.UNSUPPORTED_CSS_SYNTAX) loc = loc[1].lstrip('./') loc = f'xpath:./{direction}{brother}::{loc}' if index is not None: index = index if direction == 'following' else -index nodes = self._ele(loc, timeout=timeout, index=index, relative=True, raise_err=False) if isinstance(nodes, list): nodes = [e for e in nodes if not (isinstance(e, str) and sub('[ \n\t\r]', '', e) == '')] return nodes # ----------------以下属性或方法由后代实现---------------- @property def attrs(self): return @property def text(self): return @property def raw_text(self): return @abstractmethod def attr(self, name): return '' def _get_ele_path(self, xpath=True): return '' def _find_elements(self, locator, timeout, index=1, relative=False, raise_err=None): pass class BasePage(BaseParser): def __init__(self): self._url = None self._url_available = None self.retry_times = 3 self.retry_interval = 2 self._DrissionGet = None self._download_path = None self._none_ele_return_value = False self._none_ele_value = None self._session = None self._headers = None self._session_options = None self._type = 'BasePage' @property def title(self): ele = self._ele('xpath://title', raise_err=False, method='title') return ele.text if ele else None @property def url_available(self): return self._url_available @property def download_path(self): return self._download_path @property def download(self): if self._DrissionGet is None: if not self._session: self._create_session() self._DrissionGet = DrissionGet(driver=self, save_path=self.download_path) return self._DrissionGet def _before_connect(self, url, retry, interval): is_file = False if isinstance(url, Path) or ('://' not in url and ':\\\\' not in url): p = Path(url) if p.exists(): url = str(p.absolute()) is_file = True self._url = url if is_file else quote(url, safe='-_.~!*\'"();:@&=+$,/\\?#[]%') retry = retry if retry is not None else self.retry_times interval = interval if interval is not None else self.retry_interval return retry, interval, is_file def _set_session_options(self, session_or_options=None): if not session_or_options: self._session_options = SessionOptions(session_or_options) elif isinstance(session_or_options, SessionOptions): self._session_options = session_or_options elif isinstance(session_or_options, Session): self._session_options = SessionOptions() self._session = copy(session_or_options) self._headers = self._session.headers self._session.headers = None def _create_session(self): if not self._session_options: self._set_session_options() self._session, self._headers = self._session_options.make_session() # ----------------以下属性或方法由后代实现---------------- @property def url(self): return @property def json(self): return @property def user_agent(self): return @abstractmethod def get(self, url, show_errmsg=False, retry=None, interval=None): pass def _ele(self, locator, timeout=None, index=1, raise_err=None, method=None): if not locator: raise ElementNotFoundError(METHOD=method, ARGS={'locator': locator, 'index': index, 'timeout': timeout}) if timeout is None: timeout = self.timeout r = self._find_elements(locator, timeout=timeout, index=index, raise_err=raise_err) if r or isinstance(r, list): return r if raise_err is True or (_S.raise_when_ele_not_found and raise_err is None): raise ElementNotFoundError(METHOD=method, ARGS={'locator': locator, 'index': index, 'timeout': timeout}) r.method = method r.args = {'locator': locator, 'index': index, 'timeout': timeout} return r