Initial commit
This commit is contained in:
commit
57fc3525eb
7311 files changed
+1291398
No files matched your search
@@ -0,0 +1,362 @@
|
||||
# -*- coding:utf-8 -*-
|
||||
"""
|
||||
@Author : g1879
|
||||
@Contact : g1879@qq.com
|
||||
@Website : https://DrissionPage.cn
|
||||
@Copyright: (c) 2020 by g1879, Inc. All Rights Reserved.
|
||||
"""
|
||||
from abc import abstractmethod
|
||||
from copy import copy
|
||||
from pathlib import Path
|
||||
from re import sub
|
||||
from urllib.parse import quote
|
||||
|
||||
from DrissionGet import DrissionGet
|
||||
from requests import Session
|
||||
|
||||
from .._configs.session_options import SessionOptions
|
||||
from .._elements.none_element import NoneElement
|
||||
from .._functions.elements import get_frame, get_eles
|
||||
from .._functions.locator import get_loc
|
||||
from .._functions.settings import Settings as _S
|
||||
from .._functions.web import format_html
|
||||
from ..errors import ElementNotFoundError, LocatorError
|
||||
|
||||
|
||||
class BaseParser(object):
|
||||
def __call__(self, locator):
|
||||
return self.ele(locator)
|
||||
|
||||
def ele(self, locator, index=1, timeout=None):
|
||||
return self._ele(locator, timeout, index=index, method='ele()')
|
||||
|
||||
def eles(self, locator, timeout=None):
|
||||
return self._ele(locator, timeout, index=None)
|
||||
|
||||
def find(self, locators, any_one=True, first_ele=True, timeout=None):
|
||||
if 'Session' in self._type:
|
||||
timeout = 0
|
||||
if timeout is None:
|
||||
timeout = self.timeout
|
||||
r = get_eles(locators, self, any_one, first_ele, timeout)
|
||||
if any_one:
|
||||
for ele in r:
|
||||
if r[ele]:
|
||||
return ele, r[ele]
|
||||
return None, None
|
||||
return r
|
||||
|
||||
# ----------------以下属性或方法待后代实现----------------
|
||||
@property
|
||||
def html(self):
|
||||
return ''
|
||||
|
||||
def s_ele(self, locator=None):
|
||||
pass
|
||||
|
||||
def s_eles(self, locator):
|
||||
pass
|
||||
|
||||
def _ele(self, locator, timeout=None, index=1, raise_err=None, method=None):
|
||||
pass
|
||||
|
||||
def _find_elements(self, locator, timeout, index=1, relative=False, raise_err=None):
|
||||
pass
|
||||
|
||||
|
||||
class BaseElement(BaseParser):
|
||||
def __init__(self, owner=None):
|
||||
self.owner = owner
|
||||
self._type = 'BaseElement'
|
||||
|
||||
def get_frame(self, loc_or_ind, timeout=None):
|
||||
if not isinstance(loc_or_ind, (int, str, tuple)):
|
||||
raise ValueError(_S._lang.join(_S._lang.INCORRECT_TYPE_, 'loc_or_ind',
|
||||
ALLOW_TYPE=_S._lang.LOC_OR_IND, CURR_VAL=loc_or_ind))
|
||||
return get_frame(self, loc_ind_ele=loc_or_ind, timeout=timeout)
|
||||
|
||||
def _ele(self, locator, timeout=None, index=1, relative=False, raise_err=None, method=None):
|
||||
if hasattr(locator, '_type'):
|
||||
return locator
|
||||
if timeout is None:
|
||||
timeout = self.timeout
|
||||
r = self._find_elements(locator, timeout=timeout, index=index, relative=relative, raise_err=raise_err)
|
||||
if r or isinstance(r, list):
|
||||
return r
|
||||
if raise_err is True or (_S.raise_when_ele_not_found and raise_err is None):
|
||||
raise ElementNotFoundError(METHOD=method, ARGS={'locator': locator, 'index': index, 'timeout': timeout})
|
||||
|
||||
r.method = method
|
||||
r.args = {'locator': locator, 'index': index, 'timeout': timeout}
|
||||
return r
|
||||
|
||||
@property
|
||||
def timeout(self):
|
||||
return self.owner.timeout if self.owner else 10
|
||||
|
||||
@property
|
||||
def child_count(self):
|
||||
return int(self._ele('xpath:count(./*)'))
|
||||
|
||||
# ----------------以下属性或方法由后代实现----------------
|
||||
@property
|
||||
def tag(self):
|
||||
return
|
||||
|
||||
def parent(self, level_or_loc=1):
|
||||
pass
|
||||
|
||||
def next(self, index=1):
|
||||
pass
|
||||
|
||||
def nexts(self):
|
||||
pass
|
||||
|
||||
|
||||
class DrissionElement(BaseElement):
|
||||
|
||||
@property
|
||||
def link(self):
|
||||
return self.attr('href') or self.attr('src')
|
||||
|
||||
@property
|
||||
def css_path(self):
|
||||
return self._get_ele_path(xpath=False)
|
||||
|
||||
@property
|
||||
def xpath(self):
|
||||
return self._get_ele_path()
|
||||
|
||||
@property
|
||||
def comments(self):
|
||||
return self.eles('xpath:.//comment()')
|
||||
|
||||
def texts(self, text_node_only=False):
|
||||
texts = self.eles('xpath:/text()') if text_node_only else [x if isinstance(x, str) else x.text
|
||||
for x in self.eles('xpath:./text() | *')]
|
||||
return [format_html(x.strip(' ').rstrip('\n')) for x in texts if x and sub('[\r\n\t ]', '', x) != '']
|
||||
|
||||
def parent(self, level_or_loc=1, index=1, timeout=None):
|
||||
if isinstance(level_or_loc, int):
|
||||
loc = f'xpath:./ancestor::*[{level_or_loc}]'
|
||||
|
||||
elif isinstance(level_or_loc, (tuple, str)):
|
||||
loc = get_loc(level_or_loc, True)
|
||||
if loc[0] == 'css selector':
|
||||
raise LocatorError(_S._lang.UNSUPPORTED_CSS_SYNTAX)
|
||||
loc = f'xpath:./ancestor::{loc[1].lstrip(". / ")}[{index}]'
|
||||
|
||||
else:
|
||||
raise ValueError(_S._lang.join(_S._lang.INCORRECT_TYPE_, 'level_or_loc', ALLOW_TYPE='tuple, int, str',
|
||||
CURR_VAL=level_or_loc))
|
||||
|
||||
return self._ele(loc, timeout=timeout, relative=True, raise_err=False, method='parent()')
|
||||
|
||||
def child(self, locator='', index=1, timeout=None, ele_only=True):
|
||||
if isinstance(locator, int):
|
||||
index = locator
|
||||
locator = ''
|
||||
if not locator:
|
||||
loc = '*' if ele_only else 'node()'
|
||||
else:
|
||||
loc = get_loc(locator, True) # 把定位符转换为xpath
|
||||
if loc[0] == 'css selector':
|
||||
raise LocatorError(_S._lang.UNSUPPORTED_CSS_SYNTAX)
|
||||
loc = loc[1].lstrip('./')
|
||||
|
||||
node = self._ele(f'xpath:./{loc}', timeout=timeout, index=index, relative=True, raise_err=False)
|
||||
return node if node else NoneElement(self.owner, 'child()',
|
||||
{'locator': locator, 'index': index, 'ele_only': ele_only})
|
||||
|
||||
def prev(self, locator='', index=1, timeout=None, ele_only=True):
|
||||
return self._get_relative('prev()', 'preceding', True, locator, index, timeout, ele_only)
|
||||
|
||||
def next(self, locator='', index=1, timeout=None, ele_only=True):
|
||||
return self._get_relative('next()', 'following', True, locator, index, timeout, ele_only)
|
||||
|
||||
def before(self, locator='', index=1, timeout=None, ele_only=True):
|
||||
return self._get_relative('before()', 'preceding', False, locator, index, timeout, ele_only)
|
||||
|
||||
def after(self, locator='', index=1, timeout=None, ele_only=True):
|
||||
return self._get_relative('after()', 'following', False, locator, index, timeout, ele_only)
|
||||
|
||||
def children(self, locator='', timeout=None, ele_only=True):
|
||||
if not locator:
|
||||
loc = '*' if ele_only else 'node()'
|
||||
else:
|
||||
loc = get_loc(locator, True) # 把定位符转换为xpath
|
||||
if loc[0] == 'css selector':
|
||||
raise LocatorError(_S._lang.UNSUPPORTED_CSS_SYNTAX)
|
||||
loc = loc[1].lstrip('./')
|
||||
|
||||
loc = f'xpath:./{loc}'
|
||||
nodes = self._ele(loc, timeout=timeout, index=None, relative=True)
|
||||
return [e for e in nodes if not (isinstance(e, str) and sub('[ \n\t\r]', '', e) == '')]
|
||||
|
||||
def prevs(self, locator='', timeout=None, ele_only=True):
|
||||
return self._get_relatives(locator=locator, direction='preceding', timeout=timeout, ele_only=ele_only)
|
||||
|
||||
def nexts(self, locator='', timeout=None, ele_only=True):
|
||||
return self._get_relatives(locator=locator, direction='following', timeout=timeout, ele_only=ele_only)
|
||||
|
||||
def befores(self, locator='', timeout=None, ele_only=True):
|
||||
return self._get_relatives(locator=locator, direction='preceding',
|
||||
brother=False, timeout=timeout, ele_only=ele_only)
|
||||
|
||||
def afters(self, locator='', timeout=None, ele_only=True):
|
||||
return self._get_relatives(locator=locator, direction='following',
|
||||
brother=False, timeout=timeout, ele_only=ele_only)
|
||||
|
||||
def _get_relative(self, func, direction, brother, locator='', index=1, timeout=None, ele_only=True):
|
||||
if isinstance(locator, int):
|
||||
index = locator
|
||||
locator = ''
|
||||
node = self._get_relatives(index, locator, direction, brother, timeout, ele_only)
|
||||
return node if node else NoneElement(self.owner, func,
|
||||
{'locator': locator, 'index': index, 'ele_only': ele_only})
|
||||
|
||||
def _get_relatives(self, index=None, locator='', direction='following', brother=True, timeout=.5, ele_only=True):
|
||||
brother = '-sibling' if brother else ''
|
||||
|
||||
if not locator:
|
||||
loc = '*' if ele_only else 'node()'
|
||||
|
||||
else:
|
||||
loc = get_loc(locator, True) # 把定位符转换为xpath
|
||||
if loc[0] == 'css selector':
|
||||
raise LocatorError(_S._lang.UNSUPPORTED_CSS_SYNTAX)
|
||||
loc = loc[1].lstrip('./')
|
||||
|
||||
loc = f'xpath:./{direction}{brother}::{loc}'
|
||||
|
||||
if index is not None:
|
||||
index = index if direction == 'following' else -index
|
||||
nodes = self._ele(loc, timeout=timeout, index=index, relative=True, raise_err=False)
|
||||
if isinstance(nodes, list):
|
||||
nodes = [e for e in nodes if not (isinstance(e, str) and sub('[ \n\t\r]', '', e) == '')]
|
||||
return nodes
|
||||
|
||||
# ----------------以下属性或方法由后代实现----------------
|
||||
@property
|
||||
def attrs(self):
|
||||
return
|
||||
|
||||
@property
|
||||
def text(self):
|
||||
return
|
||||
|
||||
@property
|
||||
def raw_text(self):
|
||||
return
|
||||
|
||||
@abstractmethod
|
||||
def attr(self, name):
|
||||
return ''
|
||||
|
||||
def _get_ele_path(self, xpath=True):
|
||||
return ''
|
||||
|
||||
def _find_elements(self, locator, timeout, index=1, relative=False, raise_err=None):
|
||||
pass
|
||||
|
||||
|
||||
class BasePage(BaseParser):
|
||||
|
||||
def __init__(self):
|
||||
self._url = None
|
||||
self._url_available = None
|
||||
self.retry_times = 3
|
||||
self.retry_interval = 2
|
||||
self._DrissionGet = None
|
||||
self._download_path = None
|
||||
self._none_ele_return_value = False
|
||||
self._none_ele_value = None
|
||||
self._session = None
|
||||
self._headers = None
|
||||
self._session_options = None
|
||||
self._type = 'BasePage'
|
||||
|
||||
@property
|
||||
def title(self):
|
||||
ele = self._ele('xpath://title', raise_err=False, method='title')
|
||||
return ele.text if ele else None
|
||||
|
||||
@property
|
||||
def url_available(self):
|
||||
return self._url_available
|
||||
|
||||
@property
|
||||
def download_path(self):
|
||||
return self._download_path
|
||||
|
||||
@property
|
||||
def download(self):
|
||||
if self._DrissionGet is None:
|
||||
if not self._session:
|
||||
self._create_session()
|
||||
self._DrissionGet = DrissionGet(driver=self, save_path=self.download_path)
|
||||
return self._DrissionGet
|
||||
|
||||
def _before_connect(self, url, retry, interval):
|
||||
is_file = False
|
||||
if isinstance(url, Path) or ('://' not in url and ':\\\\' not in url):
|
||||
p = Path(url)
|
||||
if p.exists():
|
||||
url = str(p.absolute())
|
||||
is_file = True
|
||||
|
||||
self._url = url if is_file else quote(url, safe='-_.~!*\'"();:@&=+$,/\\?#[]%')
|
||||
retry = retry if retry is not None else self.retry_times
|
||||
interval = interval if interval is not None else self.retry_interval
|
||||
return retry, interval, is_file
|
||||
|
||||
def _set_session_options(self, session_or_options=None):
|
||||
if not session_or_options:
|
||||
self._session_options = SessionOptions(session_or_options)
|
||||
|
||||
elif isinstance(session_or_options, SessionOptions):
|
||||
self._session_options = session_or_options
|
||||
|
||||
elif isinstance(session_or_options, Session):
|
||||
self._session_options = SessionOptions()
|
||||
self._session = copy(session_or_options)
|
||||
self._headers = self._session.headers
|
||||
self._session.headers = None
|
||||
|
||||
def _create_session(self):
|
||||
if not self._session_options:
|
||||
self._set_session_options()
|
||||
self._session, self._headers = self._session_options.make_session()
|
||||
|
||||
# ----------------以下属性或方法由后代实现----------------
|
||||
@property
|
||||
def url(self):
|
||||
return
|
||||
|
||||
@property
|
||||
def json(self):
|
||||
return
|
||||
|
||||
@property
|
||||
def user_agent(self):
|
||||
return
|
||||
|
||||
@abstractmethod
|
||||
def get(self, url, show_errmsg=False, retry=None, interval=None):
|
||||
pass
|
||||
|
||||
def _ele(self, locator, timeout=None, index=1, raise_err=None, method=None):
|
||||
if not locator:
|
||||
raise ElementNotFoundError(METHOD=method, ARGS={'locator': locator, 'index': index, 'timeout': timeout})
|
||||
if timeout is None:
|
||||
timeout = self.timeout
|
||||
|
||||
r = self._find_elements(locator, timeout=timeout, index=index, raise_err=raise_err)
|
||||
if r or isinstance(r, list):
|
||||
return r
|
||||
if raise_err is True or (_S.raise_when_ele_not_found and raise_err is None):
|
||||
raise ElementNotFoundError(METHOD=method, ARGS={'locator': locator, 'index': index, 'timeout': timeout})
|
||||
|
||||
r.method = method
|
||||
r.args = {'locator': locator, 'index': index, 'timeout': timeout}
|
||||
return r
|
||||
Reference in new issue
Block a user