Initial commit
This commit is contained in:
commit
57fc3525eb
7311 files changed
+1291398
No files matched your search
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -0,0 +1,362 @@
|
||||
# -*- coding:utf-8 -*-
|
||||
"""
|
||||
@Author : g1879
|
||||
@Contact : g1879@qq.com
|
||||
@Website : https://DrissionPage.cn
|
||||
@Copyright: (c) 2020 by g1879, Inc. All Rights Reserved.
|
||||
"""
|
||||
from abc import abstractmethod
|
||||
from copy import copy
|
||||
from pathlib import Path
|
||||
from re import sub
|
||||
from urllib.parse import quote
|
||||
|
||||
from DrissionGet import DrissionGet
|
||||
from requests import Session
|
||||
|
||||
from .._configs.session_options import SessionOptions
|
||||
from .._elements.none_element import NoneElement
|
||||
from .._functions.elements import get_frame, get_eles
|
||||
from .._functions.locator import get_loc
|
||||
from .._functions.settings import Settings as _S
|
||||
from .._functions.web import format_html
|
||||
from ..errors import ElementNotFoundError, LocatorError
|
||||
|
||||
|
||||
class BaseParser(object):
|
||||
def __call__(self, locator):
|
||||
return self.ele(locator)
|
||||
|
||||
def ele(self, locator, index=1, timeout=None):
|
||||
return self._ele(locator, timeout, index=index, method='ele()')
|
||||
|
||||
def eles(self, locator, timeout=None):
|
||||
return self._ele(locator, timeout, index=None)
|
||||
|
||||
def find(self, locators, any_one=True, first_ele=True, timeout=None):
|
||||
if 'Session' in self._type:
|
||||
timeout = 0
|
||||
if timeout is None:
|
||||
timeout = self.timeout
|
||||
r = get_eles(locators, self, any_one, first_ele, timeout)
|
||||
if any_one:
|
||||
for ele in r:
|
||||
if r[ele]:
|
||||
return ele, r[ele]
|
||||
return None, None
|
||||
return r
|
||||
|
||||
# ----------------以下属性或方法待后代实现----------------
|
||||
@property
|
||||
def html(self):
|
||||
return ''
|
||||
|
||||
def s_ele(self, locator=None):
|
||||
pass
|
||||
|
||||
def s_eles(self, locator):
|
||||
pass
|
||||
|
||||
def _ele(self, locator, timeout=None, index=1, raise_err=None, method=None):
|
||||
pass
|
||||
|
||||
def _find_elements(self, locator, timeout, index=1, relative=False, raise_err=None):
|
||||
pass
|
||||
|
||||
|
||||
class BaseElement(BaseParser):
|
||||
def __init__(self, owner=None):
|
||||
self.owner = owner
|
||||
self._type = 'BaseElement'
|
||||
|
||||
def get_frame(self, loc_or_ind, timeout=None):
|
||||
if not isinstance(loc_or_ind, (int, str, tuple)):
|
||||
raise ValueError(_S._lang.join(_S._lang.INCORRECT_TYPE_, 'loc_or_ind',
|
||||
ALLOW_TYPE=_S._lang.LOC_OR_IND, CURR_VAL=loc_or_ind))
|
||||
return get_frame(self, loc_ind_ele=loc_or_ind, timeout=timeout)
|
||||
|
||||
def _ele(self, locator, timeout=None, index=1, relative=False, raise_err=None, method=None):
|
||||
if hasattr(locator, '_type'):
|
||||
return locator
|
||||
if timeout is None:
|
||||
timeout = self.timeout
|
||||
r = self._find_elements(locator, timeout=timeout, index=index, relative=relative, raise_err=raise_err)
|
||||
if r or isinstance(r, list):
|
||||
return r
|
||||
if raise_err is True or (_S.raise_when_ele_not_found and raise_err is None):
|
||||
raise ElementNotFoundError(METHOD=method, ARGS={'locator': locator, 'index': index, 'timeout': timeout})
|
||||
|
||||
r.method = method
|
||||
r.args = {'locator': locator, 'index': index, 'timeout': timeout}
|
||||
return r
|
||||
|
||||
@property
|
||||
def timeout(self):
|
||||
return self.owner.timeout if self.owner else 10
|
||||
|
||||
@property
|
||||
def child_count(self):
|
||||
return int(self._ele('xpath:count(./*)'))
|
||||
|
||||
# ----------------以下属性或方法由后代实现----------------
|
||||
@property
|
||||
def tag(self):
|
||||
return
|
||||
|
||||
def parent(self, level_or_loc=1):
|
||||
pass
|
||||
|
||||
def next(self, index=1):
|
||||
pass
|
||||
|
||||
def nexts(self):
|
||||
pass
|
||||
|
||||
|
||||
class DrissionElement(BaseElement):
|
||||
|
||||
@property
|
||||
def link(self):
|
||||
return self.attr('href') or self.attr('src')
|
||||
|
||||
@property
|
||||
def css_path(self):
|
||||
return self._get_ele_path(xpath=False)
|
||||
|
||||
@property
|
||||
def xpath(self):
|
||||
return self._get_ele_path()
|
||||
|
||||
@property
|
||||
def comments(self):
|
||||
return self.eles('xpath:.//comment()')
|
||||
|
||||
def texts(self, text_node_only=False):
|
||||
texts = self.eles('xpath:/text()') if text_node_only else [x if isinstance(x, str) else x.text
|
||||
for x in self.eles('xpath:./text() | *')]
|
||||
return [format_html(x.strip(' ').rstrip('\n')) for x in texts if x and sub('[\r\n\t ]', '', x) != '']
|
||||
|
||||
def parent(self, level_or_loc=1, index=1, timeout=None):
|
||||
if isinstance(level_or_loc, int):
|
||||
loc = f'xpath:./ancestor::*[{level_or_loc}]'
|
||||
|
||||
elif isinstance(level_or_loc, (tuple, str)):
|
||||
loc = get_loc(level_or_loc, True)
|
||||
if loc[0] == 'css selector':
|
||||
raise LocatorError(_S._lang.UNSUPPORTED_CSS_SYNTAX)
|
||||
loc = f'xpath:./ancestor::{loc[1].lstrip(". / ")}[{index}]'
|
||||
|
||||
else:
|
||||
raise ValueError(_S._lang.join(_S._lang.INCORRECT_TYPE_, 'level_or_loc', ALLOW_TYPE='tuple, int, str',
|
||||
CURR_VAL=level_or_loc))
|
||||
|
||||
return self._ele(loc, timeout=timeout, relative=True, raise_err=False, method='parent()')
|
||||
|
||||
def child(self, locator='', index=1, timeout=None, ele_only=True):
|
||||
if isinstance(locator, int):
|
||||
index = locator
|
||||
locator = ''
|
||||
if not locator:
|
||||
loc = '*' if ele_only else 'node()'
|
||||
else:
|
||||
loc = get_loc(locator, True) # 把定位符转换为xpath
|
||||
if loc[0] == 'css selector':
|
||||
raise LocatorError(_S._lang.UNSUPPORTED_CSS_SYNTAX)
|
||||
loc = loc[1].lstrip('./')
|
||||
|
||||
node = self._ele(f'xpath:./{loc}', timeout=timeout, index=index, relative=True, raise_err=False)
|
||||
return node if node else NoneElement(self.owner, 'child()',
|
||||
{'locator': locator, 'index': index, 'ele_only': ele_only})
|
||||
|
||||
def prev(self, locator='', index=1, timeout=None, ele_only=True):
|
||||
return self._get_relative('prev()', 'preceding', True, locator, index, timeout, ele_only)
|
||||
|
||||
def next(self, locator='', index=1, timeout=None, ele_only=True):
|
||||
return self._get_relative('next()', 'following', True, locator, index, timeout, ele_only)
|
||||
|
||||
def before(self, locator='', index=1, timeout=None, ele_only=True):
|
||||
return self._get_relative('before()', 'preceding', False, locator, index, timeout, ele_only)
|
||||
|
||||
def after(self, locator='', index=1, timeout=None, ele_only=True):
|
||||
return self._get_relative('after()', 'following', False, locator, index, timeout, ele_only)
|
||||
|
||||
def children(self, locator='', timeout=None, ele_only=True):
|
||||
if not locator:
|
||||
loc = '*' if ele_only else 'node()'
|
||||
else:
|
||||
loc = get_loc(locator, True) # 把定位符转换为xpath
|
||||
if loc[0] == 'css selector':
|
||||
raise LocatorError(_S._lang.UNSUPPORTED_CSS_SYNTAX)
|
||||
loc = loc[1].lstrip('./')
|
||||
|
||||
loc = f'xpath:./{loc}'
|
||||
nodes = self._ele(loc, timeout=timeout, index=None, relative=True)
|
||||
return [e for e in nodes if not (isinstance(e, str) and sub('[ \n\t\r]', '', e) == '')]
|
||||
|
||||
def prevs(self, locator='', timeout=None, ele_only=True):
|
||||
return self._get_relatives(locator=locator, direction='preceding', timeout=timeout, ele_only=ele_only)
|
||||
|
||||
def nexts(self, locator='', timeout=None, ele_only=True):
|
||||
return self._get_relatives(locator=locator, direction='following', timeout=timeout, ele_only=ele_only)
|
||||
|
||||
def befores(self, locator='', timeout=None, ele_only=True):
|
||||
return self._get_relatives(locator=locator, direction='preceding',
|
||||
brother=False, timeout=timeout, ele_only=ele_only)
|
||||
|
||||
def afters(self, locator='', timeout=None, ele_only=True):
|
||||
return self._get_relatives(locator=locator, direction='following',
|
||||
brother=False, timeout=timeout, ele_only=ele_only)
|
||||
|
||||
def _get_relative(self, func, direction, brother, locator='', index=1, timeout=None, ele_only=True):
|
||||
if isinstance(locator, int):
|
||||
index = locator
|
||||
locator = ''
|
||||
node = self._get_relatives(index, locator, direction, brother, timeout, ele_only)
|
||||
return node if node else NoneElement(self.owner, func,
|
||||
{'locator': locator, 'index': index, 'ele_only': ele_only})
|
||||
|
||||
def _get_relatives(self, index=None, locator='', direction='following', brother=True, timeout=.5, ele_only=True):
|
||||
brother = '-sibling' if brother else ''
|
||||
|
||||
if not locator:
|
||||
loc = '*' if ele_only else 'node()'
|
||||
|
||||
else:
|
||||
loc = get_loc(locator, True) # 把定位符转换为xpath
|
||||
if loc[0] == 'css selector':
|
||||
raise LocatorError(_S._lang.UNSUPPORTED_CSS_SYNTAX)
|
||||
loc = loc[1].lstrip('./')
|
||||
|
||||
loc = f'xpath:./{direction}{brother}::{loc}'
|
||||
|
||||
if index is not None:
|
||||
index = index if direction == 'following' else -index
|
||||
nodes = self._ele(loc, timeout=timeout, index=index, relative=True, raise_err=False)
|
||||
if isinstance(nodes, list):
|
||||
nodes = [e for e in nodes if not (isinstance(e, str) and sub('[ \n\t\r]', '', e) == '')]
|
||||
return nodes
|
||||
|
||||
# ----------------以下属性或方法由后代实现----------------
|
||||
@property
|
||||
def attrs(self):
|
||||
return
|
||||
|
||||
@property
|
||||
def text(self):
|
||||
return
|
||||
|
||||
@property
|
||||
def raw_text(self):
|
||||
return
|
||||
|
||||
@abstractmethod
|
||||
def attr(self, name):
|
||||
return ''
|
||||
|
||||
def _get_ele_path(self, xpath=True):
|
||||
return ''
|
||||
|
||||
def _find_elements(self, locator, timeout, index=1, relative=False, raise_err=None):
|
||||
pass
|
||||
|
||||
|
||||
class BasePage(BaseParser):
|
||||
|
||||
def __init__(self):
|
||||
self._url = None
|
||||
self._url_available = None
|
||||
self.retry_times = 3
|
||||
self.retry_interval = 2
|
||||
self._DrissionGet = None
|
||||
self._download_path = None
|
||||
self._none_ele_return_value = False
|
||||
self._none_ele_value = None
|
||||
self._session = None
|
||||
self._headers = None
|
||||
self._session_options = None
|
||||
self._type = 'BasePage'
|
||||
|
||||
@property
|
||||
def title(self):
|
||||
ele = self._ele('xpath://title', raise_err=False, method='title')
|
||||
return ele.text if ele else None
|
||||
|
||||
@property
|
||||
def url_available(self):
|
||||
return self._url_available
|
||||
|
||||
@property
|
||||
def download_path(self):
|
||||
return self._download_path
|
||||
|
||||
@property
|
||||
def download(self):
|
||||
if self._DrissionGet is None:
|
||||
if not self._session:
|
||||
self._create_session()
|
||||
self._DrissionGet = DrissionGet(driver=self, save_path=self.download_path)
|
||||
return self._DrissionGet
|
||||
|
||||
def _before_connect(self, url, retry, interval):
|
||||
is_file = False
|
||||
if isinstance(url, Path) or ('://' not in url and ':\\\\' not in url):
|
||||
p = Path(url)
|
||||
if p.exists():
|
||||
url = str(p.absolute())
|
||||
is_file = True
|
||||
|
||||
self._url = url if is_file else quote(url, safe='-_.~!*\'"();:@&=+$,/\\?#[]%')
|
||||
retry = retry if retry is not None else self.retry_times
|
||||
interval = interval if interval is not None else self.retry_interval
|
||||
return retry, interval, is_file
|
||||
|
||||
def _set_session_options(self, session_or_options=None):
|
||||
if not session_or_options:
|
||||
self._session_options = SessionOptions(session_or_options)
|
||||
|
||||
elif isinstance(session_or_options, SessionOptions):
|
||||
self._session_options = session_or_options
|
||||
|
||||
elif isinstance(session_or_options, Session):
|
||||
self._session_options = SessionOptions()
|
||||
self._session = copy(session_or_options)
|
||||
self._headers = self._session.headers
|
||||
self._session.headers = None
|
||||
|
||||
def _create_session(self):
|
||||
if not self._session_options:
|
||||
self._set_session_options()
|
||||
self._session, self._headers = self._session_options.make_session()
|
||||
|
||||
# ----------------以下属性或方法由后代实现----------------
|
||||
@property
|
||||
def url(self):
|
||||
return
|
||||
|
||||
@property
|
||||
def json(self):
|
||||
return
|
||||
|
||||
@property
|
||||
def user_agent(self):
|
||||
return
|
||||
|
||||
@abstractmethod
|
||||
def get(self, url, show_errmsg=False, retry=None, interval=None):
|
||||
pass
|
||||
|
||||
def _ele(self, locator, timeout=None, index=1, raise_err=None, method=None):
|
||||
if not locator:
|
||||
raise ElementNotFoundError(METHOD=method, ARGS={'locator': locator, 'index': index, 'timeout': timeout})
|
||||
if timeout is None:
|
||||
timeout = self.timeout
|
||||
|
||||
r = self._find_elements(locator, timeout=timeout, index=index, raise_err=raise_err)
|
||||
if r or isinstance(r, list):
|
||||
return r
|
||||
if raise_err is True or (_S.raise_when_ele_not_found and raise_err is None):
|
||||
raise ElementNotFoundError(METHOD=method, ARGS={'locator': locator, 'index': index, 'timeout': timeout})
|
||||
|
||||
r.method = method
|
||||
r.args = {'locator': locator, 'index': index, 'timeout': timeout}
|
||||
return r
|
||||
@@ -0,0 +1,449 @@
|
||||
# -*- coding:utf-8 -*-
|
||||
"""
|
||||
@Author : g1879
|
||||
@Contact : g1879@qq.com
|
||||
@Website : https://DrissionPage.cn
|
||||
@Copyright: (c) 2020 by g1879, Inc. All Rights Reserved.
|
||||
"""
|
||||
from abc import abstractmethod
|
||||
from typing import Union, Tuple, List, Any, Optional, Dict
|
||||
|
||||
from DrissionGet import DrissionGet
|
||||
from requests import Session
|
||||
from requests.structures import CaseInsensitiveDict
|
||||
|
||||
from .._configs.session_options import SessionOptions
|
||||
from .._elements.chromium_element import ChromiumElement
|
||||
from .._elements.none_element import NoneElement
|
||||
from .._elements.session_element import SessionElement
|
||||
from .._functions.elements import SessionElementsList
|
||||
from .._pages.chromium_frame import ChromiumFrame
|
||||
from .._pages.chromium_page import ChromiumPage
|
||||
from .._pages.session_page import SessionPage
|
||||
from .._pages.web_page import WebPage
|
||||
|
||||
|
||||
class BaseParser(object):
|
||||
"""所有页面、元素类的基类"""
|
||||
_type: str
|
||||
timeout: float
|
||||
|
||||
def __call__(self, locator: Union[Tuple[str, str], str], index: int = 1): ...
|
||||
|
||||
def ele(self,
|
||||
locator: Union[Tuple[str, str], str, BaseElement],
|
||||
index: int = 1,
|
||||
timeout: float = None): ...
|
||||
|
||||
def eles(self, locator: Union[Tuple[str, str], str], timeout=None): ...
|
||||
|
||||
def find(self,
|
||||
locators: Union[str, List[str], tuple],
|
||||
any_one: bool = True,
|
||||
first_ele: bool = True,
|
||||
timeout: float = None) -> Union[Dict[str, ChromiumElement], Dict[str, SessionElement],
|
||||
Dict[str, List[ChromiumElement]], Dict[str, List[SessionElement]], Tuple[str, SessionElement],
|
||||
Tuple[str, ChromiumElement]]:
|
||||
"""传入多个定位符,获取多个ele
|
||||
:param locators: 定位符组成的列表
|
||||
:param any_one: 是否任何一个定位符找到结果即返回
|
||||
:param first_ele: 每个定位符是否只获取第一个元素
|
||||
:param timeout: 超时时间(秒)
|
||||
:return: any_one为True时,返回一个找到的元素定位符和对象组成的元组,格式:(loc, ele),全都没找到返回(None, None)
|
||||
any_one为False时,返回dict格式,key为定位符,value为找到的元素或列表
|
||||
"""
|
||||
...
|
||||
|
||||
# ----------------以下属性或方法待后代实现----------------
|
||||
@property
|
||||
def html(self) -> str: ...
|
||||
|
||||
def s_ele(self,
|
||||
locator: Union[Tuple[str, str], str, BaseElement, None] = None,
|
||||
index: int = 1) -> SessionElement: ...
|
||||
|
||||
def s_eles(self, locator: Union[Tuple[str, str], str]) -> SessionElementsList: ...
|
||||
|
||||
def _ele(self,
|
||||
locator: Union[Tuple[str, str], str],
|
||||
timeout: float = None,
|
||||
index: Optional[int] = 1,
|
||||
raise_err: bool = None,
|
||||
method: str = None): ...
|
||||
|
||||
def _find_elements(self,
|
||||
locator: Union[Tuple[str, str], str],
|
||||
timeout: float,
|
||||
index: Optional[int] = 1,
|
||||
relative: bool = False,
|
||||
raise_err: bool = None): ...
|
||||
|
||||
|
||||
class BaseElement(BaseParser):
|
||||
"""各元素类的基类"""
|
||||
owner: BasePage = ...
|
||||
|
||||
def __init__(self, owner: BasePage = None): ...
|
||||
|
||||
@property
|
||||
def timeout(self) -> float:
|
||||
"""返回其查找元素时超时时间"""
|
||||
...
|
||||
|
||||
@property
|
||||
def child_count(self) -> int:
|
||||
"""返回直接子元素的个数"""
|
||||
...
|
||||
|
||||
# ----------------以下属性或方法由后代实现----------------
|
||||
@property
|
||||
def tag(self) -> str: ...
|
||||
|
||||
def parent(self, level_or_loc: Union[tuple, str, int] = 1): ...
|
||||
|
||||
def prev(self, index: int = 1) -> None: ...
|
||||
|
||||
def prevs(self) -> None: ...
|
||||
|
||||
def next(self, index: int = 1): ...
|
||||
|
||||
def nexts(self): ...
|
||||
|
||||
def get_frame(self, loc_or_ind, timeout=None) -> ChromiumFrame:
|
||||
"""获取元素中一个frame对象
|
||||
:param loc_or_ind: 定位符、iframe序号,序号从1开始,可传入负数获取倒数第几个
|
||||
:param timeout: 查找元素超时时间(秒)
|
||||
:return: ChromiumFrame对象
|
||||
"""
|
||||
...
|
||||
|
||||
def _ele(self,
|
||||
locator: Union[Tuple[str, str], str],
|
||||
timeout: float = None,
|
||||
index: Optional[int] = 1,
|
||||
relative: bool = False,
|
||||
raise_err: bool = None,
|
||||
method: str = None):
|
||||
"""调用获取元素的方法
|
||||
:param locator: 定位符
|
||||
:param timeout: 超时时间(秒)
|
||||
:param index: 获取第几个,从1开始,可传入负数获取倒数第几个
|
||||
:param relative: 是否相对定位
|
||||
:param raise_err: 找不到时是否抛出异常
|
||||
:param method: 调用的方法名
|
||||
:return: 元素对象或它们组成的列表
|
||||
"""
|
||||
...
|
||||
|
||||
|
||||
class DrissionElement(BaseElement):
|
||||
"""ChromiumElement 和 SessionElement的基类,但不是ShadowRoot的基类"""
|
||||
|
||||
def __init__(self, owner: BasePage = None): ...
|
||||
|
||||
@property
|
||||
def link(self) -> str:
|
||||
"""返回href或src绝对url"""
|
||||
...
|
||||
|
||||
@property
|
||||
def css_path(self) -> str:
|
||||
"""返回css path路径"""
|
||||
...
|
||||
|
||||
@property
|
||||
def xpath(self) -> str:
|
||||
"""返回xpath路径"""
|
||||
...
|
||||
|
||||
@property
|
||||
def comments(self) -> list:
|
||||
"""返回元素注释文本组成的列表"""
|
||||
...
|
||||
|
||||
def texts(self, text_node_only: bool = False) -> list:
|
||||
"""返回元素内所有直接子节点的文本,包括元素和文本节点
|
||||
:param text_node_only: 是否只返回文本节点
|
||||
:return: 文本列表
|
||||
"""
|
||||
...
|
||||
|
||||
def parent(self,
|
||||
level_or_loc: Union[tuple, str, int] = 1,
|
||||
index: int = 1,
|
||||
timeout: float = None) -> Union[DrissionElement, None]:
|
||||
"""返回上面某一级父元素,可指定层数或用查询语法定位
|
||||
:param level_or_loc: 第几级父元素,1开始,或定位符
|
||||
:param index: 当level_or_loc传入定位符,使用此参数选择第几个结果,1开始
|
||||
:param timeout: 时间(秒)
|
||||
:return: 上级元素对象
|
||||
"""
|
||||
...
|
||||
|
||||
def child(self,
|
||||
locator: Union[Tuple[str, str], str, int] = '',
|
||||
index: int = 1,
|
||||
timeout: float = None,
|
||||
ele_only: bool = True) -> Union[DrissionElement, str, NoneElement]:
|
||||
"""返回直接子元素元素或节点组成的列表,可用查询语法筛选
|
||||
:param locator: 用于筛选的查询语法
|
||||
:param index: 第几个查询结果,1开始
|
||||
:param timeout: 查找节点的超时时间(秒)
|
||||
:param ele_only: 是否只获取元素,为False时把文本、注释节点也纳入
|
||||
:return: 直接子元素或节点文本组成的列表
|
||||
"""
|
||||
...
|
||||
|
||||
def prev(self,
|
||||
locator: Union[Tuple[str, str], str, int] = '',
|
||||
index: int = 1,
|
||||
timeout: float = None,
|
||||
ele_only: bool = True) -> Union[DrissionElement, str, NoneElement]:
|
||||
"""返回前面的一个兄弟元素,可用查询语法筛选,可指定返回筛选结果的第几个
|
||||
:param locator: 用于筛选的查询语法
|
||||
:param index: 前面第几个查询结果,1开始
|
||||
:param timeout: 查找节点的超时时间(秒)
|
||||
:param ele_only: 是否只获取元素,为False时把文本、注释节点也纳入
|
||||
:return: 兄弟元素
|
||||
"""
|
||||
...
|
||||
|
||||
def next(self,
|
||||
locator: Union[Tuple[str, str], str, int] = '',
|
||||
index: int = 1,
|
||||
timeout: float = None,
|
||||
ele_only: bool = True) -> Union[DrissionElement, str, NoneElement]:
|
||||
"""返回后面的一个兄弟元素,可用查询语法筛选,可指定返回筛选结果的第几个
|
||||
:param locator: 用于筛选的查询语法
|
||||
:param index: 后面第几个查询结果,1开始
|
||||
:param timeout: 查找节点的超时时间(秒)
|
||||
:param ele_only: 是否只获取元素,为False时把文本、注释节点也纳入
|
||||
:return: 兄弟元素
|
||||
"""
|
||||
...
|
||||
|
||||
def before(self,
|
||||
locator: Union[Tuple[str, str], str, int] = '',
|
||||
index: int = 1,
|
||||
timeout: float = None,
|
||||
ele_only: bool = True) -> Union[DrissionElement, str, NoneElement]:
|
||||
"""返回前面的一个兄弟元素,可用查询语法筛选,可指定返回筛选结果的第几个
|
||||
:param locator: 用于筛选的查询语法
|
||||
:param index: 前面第几个查询结果,1开始
|
||||
:param timeout: 查找节点的超时时间(秒)
|
||||
:param ele_only: 是否只获取元素,为False时把文本、注释节点也纳入
|
||||
:return: 本元素前面的某个元素或节点
|
||||
"""
|
||||
...
|
||||
|
||||
def after(self,
|
||||
locator: Union[Tuple[str, str], str, int] = '',
|
||||
index: int = 1,
|
||||
timeout: float = None,
|
||||
ele_only: bool = True) -> Union[DrissionElement, str, NoneElement]:
|
||||
"""返回后面的一个兄弟元素,可用查询语法筛选,可指定返回筛选结果的第几个
|
||||
:param locator: 用于筛选的查询语法
|
||||
:param index: 后面第几个查询结果,1开始
|
||||
:param timeout: 查找节点的超时时间(秒)
|
||||
:param ele_only: 是否只获取元素,为False时把文本、注释节点也纳入
|
||||
:return: 本元素后面的某个元素或节点
|
||||
"""
|
||||
...
|
||||
|
||||
def children(self,
|
||||
locator: Union[Tuple[str, str], str] = '',
|
||||
timeout: float = None,
|
||||
ele_only: bool = True) -> List[Union[DrissionElement, str]]:
|
||||
"""返回直接子元素元素或节点组成的列表,可用查询语法筛选
|
||||
:param locator: 用于筛选的查询语法
|
||||
:param timeout: 查找节点的超时时间(秒)
|
||||
:param ele_only: 是否只获取元素,为False时把文本、注释节点也纳入
|
||||
:return: 直接子元素或节点文本组成的列表
|
||||
"""
|
||||
...
|
||||
|
||||
def prevs(self,
|
||||
locator: Union[Tuple[str, str], str] = '',
|
||||
timeout: float = None,
|
||||
ele_only: bool = True) -> List[Union[DrissionElement, str]]:
|
||||
"""返回前面全部兄弟元素或节点组成的列表,可用查询语法筛选
|
||||
:param locator: 用于筛选的查询语法
|
||||
:param timeout: 查找节点的超时时间(秒)
|
||||
:param ele_only: 是否只获取元素,为False时把文本、注释节点也纳入
|
||||
:return: 兄弟元素或节点文本组成的列表
|
||||
"""
|
||||
...
|
||||
|
||||
def nexts(self,
|
||||
locator: Union[Tuple[str, str], str] = '',
|
||||
timeout: float = None,
|
||||
ele_only: bool = True) -> List[Union[DrissionElement, str]]:
|
||||
"""返回后面全部兄弟元素或节点组成的列表,可用查询语法筛选
|
||||
:param locator: 用于筛选的查询语法
|
||||
:param timeout: 查找节点的超时时间(秒)
|
||||
:param ele_only: 是否只获取元素,为False时把文本、注释节点也纳入
|
||||
:return: 兄弟元素或节点文本组成的列表
|
||||
"""
|
||||
...
|
||||
|
||||
def befores(self,
|
||||
locator: Union[Tuple[str, str], str] = '',
|
||||
timeout: float = None,
|
||||
ele_only: bool = True) -> List[Union[DrissionElement, str]]:
|
||||
"""返回后面全部兄弟元素或节点组成的列表,可用查询语法筛选
|
||||
:param locator: 用于筛选的查询语法
|
||||
:param timeout: 查找节点的超时时间(秒)
|
||||
:param ele_only: 是否只获取元素,为False时把文本、注释节点也纳入
|
||||
:return: 本元素前面的元素或节点组成的列表
|
||||
"""
|
||||
...
|
||||
|
||||
def afters(self,
|
||||
locator: Union[Tuple[str, str], str] = '',
|
||||
timeout: float = None,
|
||||
ele_only: bool = True) -> List[Union[DrissionElement, str]]:
|
||||
"""返回前面全部兄弟元素或节点组成的列表,可用查询语法筛选
|
||||
:param locator: 用于筛选的查询语法
|
||||
:param timeout: 查找节点的超时时间(秒)
|
||||
:param ele_only: 是否只获取元素,为False时把文本、注释节点也纳入
|
||||
:return: 本元素后面的元素或节点组成的列表
|
||||
"""
|
||||
...
|
||||
|
||||
def _get_relative(self,
|
||||
func: str,
|
||||
direction: str,
|
||||
brother: bool,
|
||||
locator: Union[Tuple[str, str], str] = '',
|
||||
index: int = 1,
|
||||
timeout: float = None,
|
||||
ele_only: bool = True) -> DrissionElement:
|
||||
"""获取一个亲戚元素或节点,可用查询语法筛选,可指定返回筛选结果的第几个
|
||||
:param func: 方法名称
|
||||
:param direction: 方向,'following' 或 'preceding'
|
||||
:param locator: 用于筛选的查询语法
|
||||
:param index: 前面第几个查询结果,1开始
|
||||
:param timeout: 查找节点的超时时间(秒)
|
||||
:param ele_only: 是否只获取元素,为False时把文本、注释节点也纳入
|
||||
:return: 本元素前面的某个元素或节点
|
||||
"""
|
||||
...
|
||||
|
||||
def _get_relatives(self,
|
||||
index: int = None,
|
||||
locator: Union[Tuple[str, str], str] = '',
|
||||
direction: str = 'following',
|
||||
brother: bool = True,
|
||||
timeout: float = 0.5,
|
||||
ele_only: bool = True) -> List[Union[DrissionElement, str]]:
|
||||
"""按要求返回兄弟元素或节点组成的列表
|
||||
:param index: 获取第几个,该参数不为None时只获取该编号的元素
|
||||
:param locator: 用于筛选的查询语法
|
||||
:param direction: 'following' 或 'preceding',查找的方向
|
||||
:param brother: 查找范围,在同级查找还是整个dom前后查找
|
||||
:param timeout: 查找等待时间(秒)
|
||||
:return: 元素对象或字符串
|
||||
"""
|
||||
...
|
||||
|
||||
# ----------------以下属性或方法由后代实现----------------
|
||||
@property
|
||||
def attrs(self) -> dict: ...
|
||||
|
||||
@property
|
||||
def text(self) -> str: ...
|
||||
|
||||
@property
|
||||
def raw_text(self) -> str: ...
|
||||
|
||||
@abstractmethod
|
||||
def attr(self, name: str) -> str: ...
|
||||
|
||||
def _get_ele_path(self, xpath: bool = True) -> str: ...
|
||||
|
||||
|
||||
class BasePage(BaseParser):
|
||||
"""页面类的基类"""
|
||||
|
||||
_url_available: Optional[bool] = ...
|
||||
retry_times: int = ...
|
||||
retry_interval: float = ...
|
||||
_download_path: Optional[str] = ...
|
||||
_DrissionGet: Optional[DrissionGet] = ...
|
||||
_none_ele_return_value: bool = ...
|
||||
_none_ele_value: Any = ...
|
||||
_page: Union[ChromiumPage, SessionPage, WebPage] = ...
|
||||
_session: Optional[Session] = ...
|
||||
_headers: Optional[CaseInsensitiveDict] = ...
|
||||
_session_options: Optional[SessionOptions] = ...
|
||||
|
||||
def __init__(self): ...
|
||||
|
||||
@property
|
||||
def title(self) -> Union[str, None]:
|
||||
"""返回网页title"""
|
||||
...
|
||||
|
||||
@property
|
||||
def url_available(self) -> bool:
|
||||
"""返回当前访问的url有效性"""
|
||||
...
|
||||
|
||||
@property
|
||||
def download_path(self) -> str:
|
||||
"""返回默认下载路径"""
|
||||
...
|
||||
|
||||
@property
|
||||
def download(self) -> DrissionGet:
|
||||
"""返回下载器对象"""
|
||||
...
|
||||
|
||||
def _before_connect(self, url: str, retry: int, interval: float) -> tuple:
|
||||
"""连接前的准备
|
||||
:param url: 要访问的url
|
||||
:param retry: 重试次数
|
||||
:param interval: 重试间隔
|
||||
:return: 重试次数、间隔、是否文件组成的tuple
|
||||
"""
|
||||
...
|
||||
|
||||
def _set_session_options(self, session_or_options: Union[Session, SessionOptions] = None) -> None:
|
||||
"""启动配置
|
||||
:param session_or_options: Session、SessionOptions对象
|
||||
:return: None
|
||||
"""
|
||||
...
|
||||
|
||||
def _create_session(self) -> None:
|
||||
"""创建内建Session对象"""
|
||||
...
|
||||
|
||||
# ----------------以下属性或方法由后代实现----------------
|
||||
@property
|
||||
def url(self) -> str: ...
|
||||
|
||||
@property
|
||||
def json(self) -> dict: ...
|
||||
|
||||
@property
|
||||
def user_agent(self) -> str: ...
|
||||
|
||||
@abstractmethod
|
||||
def get(self, url: str, show_errmsg: bool = False, retry: int = None, interval: float = None): ...
|
||||
|
||||
def _ele(self,
|
||||
locator,
|
||||
timeout: float = None,
|
||||
index: Optional[int] = 1,
|
||||
raise_err: bool = None,
|
||||
method: str = None):
|
||||
"""调用获取元素的方法
|
||||
:param locator: 定位符
|
||||
:param timeout: 超时时间(秒)
|
||||
:param index: 获取第几个,从1开始,可传入负数获取倒数第几个
|
||||
:param raise_err: 找不到时是否抛出异常
|
||||
:param method: 调用的方法名
|
||||
:return: 元素对象或它们组成的列表
|
||||
"""
|
||||
...
|
||||
@@ -0,0 +1,537 @@
|
||||
# -*- coding:utf-8 -*-
|
||||
"""
|
||||
@Author : g1879
|
||||
@Contact : g1879@qq.com
|
||||
@Website : https://DrissionPage.cn
|
||||
@Copyright: (c) 2020 by g1879, Inc. All Rights Reserved.
|
||||
"""
|
||||
from pathlib import Path
|
||||
from re import match
|
||||
from shutil import rmtree
|
||||
from threading import Lock
|
||||
from time import sleep, perf_counter
|
||||
|
||||
from requests import Session
|
||||
from websocket import WebSocketBadStatusException
|
||||
|
||||
from .driver import BrowserDriver, Driver
|
||||
from .._configs.chromium_options import ChromiumOptions
|
||||
from .._functions.browser import connect_browser
|
||||
from .._functions.cookies import CookiesList
|
||||
from .._functions.settings import Settings as _S
|
||||
from .._functions.tools import PortFinder, raise_error
|
||||
from .._pages.chromium_base import Timeout
|
||||
from .._pages.chromium_tab import ChromiumTab
|
||||
from .._pages.mix_tab import MixTab
|
||||
from .._units.downloader import DownloadManager
|
||||
from .._units.setter import BrowserSetter
|
||||
from .._units.states import BrowserStates
|
||||
from .._units.waiter import BrowserWaiter
|
||||
from ..errors import BrowserConnectError, CDPError, PageDisconnectedError, IncorrectURLError
|
||||
|
||||
__ERROR__ = 'error'
|
||||
|
||||
|
||||
class Chromium(object):
|
||||
_BROWSERS = {}
|
||||
_lock = Lock()
|
||||
|
||||
def __new__(cls, addr_or_opts=None, session_options=None):
|
||||
opt = handle_options(addr_or_opts)
|
||||
is_headless, browser_id, is_exists, ws_only = run_browser(opt)
|
||||
with cls._lock:
|
||||
if browser_id in cls._BROWSERS:
|
||||
r = cls._BROWSERS[browser_id]
|
||||
while not hasattr(r, '_driver'):
|
||||
sleep(.05)
|
||||
return r
|
||||
r = object.__new__(cls)
|
||||
r._chromium_options = opt
|
||||
r._is_headless = is_headless
|
||||
r._is_exists = is_exists
|
||||
r._ws_only = ws_only
|
||||
r.id = browser_id
|
||||
cls._BROWSERS[browser_id] = r
|
||||
return r
|
||||
|
||||
def __init__(self, addr_or_opts=None, session_options=None):
|
||||
if hasattr(self, '_created'):
|
||||
return
|
||||
self._created = True
|
||||
|
||||
self._type = 'Chromium'
|
||||
self._frames = {}
|
||||
self._drivers = {}
|
||||
self._all_drivers = {}
|
||||
self._relation = {}
|
||||
self._newest_tab_id = None
|
||||
|
||||
self._set = None
|
||||
self._wait = None
|
||||
self._states = None
|
||||
self._timeouts = Timeout(**self._chromium_options.timeouts)
|
||||
self._load_mode = self._chromium_options.load_mode
|
||||
self._download_path = str(Path(self._chromium_options.download_path).absolute())
|
||||
self._auto_handle_alert = None
|
||||
self._none_ele_return_value = False
|
||||
self._none_ele_value = None
|
||||
self.retry_times = self._chromium_options.retry_times
|
||||
self.retry_interval = self._chromium_options.retry_interval
|
||||
self.address = self._chromium_options.address
|
||||
self._ws_address = (self._chromium_options.ws_address if self._chromium_options.ws_address
|
||||
else f'ws://{self.address}/devtools/browser/{self.id}')
|
||||
self._disconnect_flag = False
|
||||
self._driver = BrowserDriver(self.id, self._ws_address, self)
|
||||
|
||||
if (not self._ws_only
|
||||
and (not self._chromium_options._ua_set and self._is_headless != self._chromium_options.is_headless)
|
||||
or (self._is_exists and self._chromium_options._new_env)):
|
||||
self.quit(3, True)
|
||||
connect_browser(self._chromium_options)
|
||||
s = Session()
|
||||
s.trust_env = False
|
||||
s.keep_alive = False
|
||||
ws = s.get(f'http://{self.address}/json/version', headers={'Connection': 'close'})
|
||||
self.id = ws.json()['webSocketDebuggerUrl'].split('/')[-1]
|
||||
self._driver = BrowserDriver(self.id, self._ws_address, self)
|
||||
ws.close()
|
||||
s.close()
|
||||
self._is_exists = False
|
||||
self._frames = {}
|
||||
self._drivers = {}
|
||||
self._all_drivers = {}
|
||||
|
||||
self.version = self._run_cdp('Browser.getVersion')['product']
|
||||
|
||||
self._process_id = None
|
||||
try:
|
||||
r = self._run_cdp('SystemInfo.getProcessInfo')
|
||||
for i in r.get('processInfo', []):
|
||||
if i['type'] == 'browser':
|
||||
self._process_id = i['id']
|
||||
break
|
||||
except:
|
||||
pass
|
||||
|
||||
self._run_cdp('Target.setDiscoverTargets', discover=True)
|
||||
self._driver.set_callback('Target.targetDestroyed', self._onTargetDestroyed)
|
||||
self._driver.set_callback('Target.targetCreated', self._onTargetCreated)
|
||||
self._dl_mgr = DownloadManager(self)
|
||||
|
||||
self._session_options = session_options
|
||||
|
||||
@property
|
||||
def user_data_path(self):
|
||||
return self._chromium_options.user_data_path
|
||||
|
||||
@property
|
||||
def process_id(self):
|
||||
return self._process_id
|
||||
|
||||
@property
|
||||
def timeout(self):
|
||||
return self._timeouts.base
|
||||
|
||||
@property
|
||||
def timeouts(self):
|
||||
return self._timeouts
|
||||
|
||||
@property
|
||||
def load_mode(self):
|
||||
return self._load_mode
|
||||
|
||||
@property
|
||||
def download_path(self):
|
||||
return self._download_path
|
||||
|
||||
@property
|
||||
def set(self):
|
||||
if self._set is None:
|
||||
self._set = BrowserSetter(self)
|
||||
return self._set
|
||||
|
||||
@property
|
||||
def states(self):
|
||||
if self._states is None:
|
||||
self._states = BrowserStates(self)
|
||||
return self._states
|
||||
|
||||
@property
|
||||
def wait(self):
|
||||
if self._wait is None:
|
||||
self._wait = BrowserWaiter(self)
|
||||
return self._wait
|
||||
|
||||
@property
|
||||
def tabs_count(self):
|
||||
j = self._run_cdp('Target.getTargets')['targetInfos'] # 不要改用get,避免卡死
|
||||
return len([i for i in j if i['type'] in ('page', 'webview') and not i['url'].startswith('devtools://')])
|
||||
|
||||
@property
|
||||
def tab_ids(self):
|
||||
if self._ws_only:
|
||||
return [i['targetId'] for i in self._run_cdp('Target.getTargets')['targetInfos']
|
||||
if i['type'] in ('page', 'webview') and not i['url'].startswith('devtools://')]
|
||||
else:
|
||||
return [i['id'] for i in self._driver.get(f'http://{self.address}/json').json()
|
||||
if i['type'] in ('page', 'webview') and not i['url'].startswith('devtools://')]
|
||||
|
||||
@property
|
||||
def latest_tab(self):
|
||||
return self._get_tab(id_or_num=self.tab_ids[0], as_id=not _S.singleton_tab_obj)
|
||||
|
||||
def cookies(self, all_info=False):
|
||||
cks = self._run_cdp(f'Storage.getCookies')['cookies']
|
||||
r = cks if all_info else [{'name': c['name'], 'value': c['value'], 'domain': c['domain']} for c in cks]
|
||||
return CookiesList(r)
|
||||
|
||||
def new_tab(self, url=None, new_window=False, background=False, new_context=False):
|
||||
return self._new_tab(True, url=url, new_window=new_window, background=background, new_context=new_context)
|
||||
|
||||
def get_tab(self, id_or_num=None, title=None, url=None, tab_type='page'):
|
||||
t = self._get_tab(id_or_num=id_or_num, title=title, url=url, tab_type=tab_type, mix=True, as_id=False)
|
||||
if t._type != 'MixTab':
|
||||
raise RuntimeError(_S._lang.join(_S._lang.TAB_OBJ_EXISTS))
|
||||
return t
|
||||
|
||||
def get_tabs(self, title=None, url=None, tab_type='page'):
|
||||
return self._get_tabs(title=title, url=url, tab_type=tab_type, mix=True, as_id=False)
|
||||
|
||||
def close_tabs(self, tabs_or_ids, others=False):
|
||||
if isinstance(tabs_or_ids, str):
|
||||
tabs = {tabs_or_ids}
|
||||
elif isinstance(tabs_or_ids, ChromiumTab):
|
||||
tabs = {tabs_or_ids.tab_id}
|
||||
elif isinstance(tabs_or_ids, (list, tuple)):
|
||||
tabs = set(i.tab_id if isinstance(i, ChromiumTab) else i for i in tabs_or_ids)
|
||||
else:
|
||||
raise ValueError(_S._lang.join(_S._lang.INCORRECT_TYPE_, 'tabs_or_ids',
|
||||
ALLOW_TYPE=_S._lang.TAB_OR_ID, CURR_VAL=tabs_or_ids))
|
||||
|
||||
all_tabs = set(self.tab_ids)
|
||||
if others:
|
||||
tabs = all_tabs - tabs
|
||||
|
||||
if len(all_tabs - tabs) > 0:
|
||||
for tab in tabs:
|
||||
self._close_tab(tab=tab)
|
||||
else:
|
||||
self.quit()
|
||||
|
||||
def _close_tab(self, tab):
|
||||
if isinstance(tab, str):
|
||||
tab = self.get_tab(tab)
|
||||
tab._run_cdp('Target.closeTarget', targetId=tab.tab_id)
|
||||
while tab.driver.is_running and tab.tab_id in self._all_drivers:
|
||||
sleep(.01)
|
||||
|
||||
def activate_tab(self, id_ind_tab):
|
||||
if isinstance(id_ind_tab, int):
|
||||
id_ind_tab += -1 if id_ind_tab else 1
|
||||
id_ind_tab = self.tab_ids[id_ind_tab]
|
||||
elif isinstance(id_ind_tab, ChromiumTab):
|
||||
id_ind_tab = id_ind_tab.tab_id
|
||||
self._run_cdp('Target.activateTarget', targetId=id_ind_tab)
|
||||
|
||||
def reconnect(self):
|
||||
self._disconnect_flag = True
|
||||
self._driver.stop()
|
||||
BrowserDriver.BROWSERS.pop(self.id)
|
||||
self._driver = BrowserDriver(self.id, self._ws_address, self)
|
||||
self._run_cdp('Target.setDiscoverTargets', discover=True)
|
||||
self._driver.set_callback('Target.targetDestroyed', self._onTargetDestroyed)
|
||||
self._driver.set_callback('Target.targetCreated', self._onTargetCreated)
|
||||
self._disconnect_flag = False
|
||||
|
||||
def clear_cache(self, cache=True, cookies=True):
|
||||
if cache:
|
||||
self.latest_tab.run_cdp('Network.clearBrowserCache')
|
||||
|
||||
if cookies:
|
||||
self._run_cdp('Storage.clearCookies')
|
||||
|
||||
def quit(self, timeout=5, force=False, del_data=False):
|
||||
try:
|
||||
self._run_cdp('Browser.close')
|
||||
except PageDisconnectedError:
|
||||
pass
|
||||
self._driver.stop()
|
||||
|
||||
drivers = list(self._all_drivers.values())
|
||||
for tab in drivers:
|
||||
for driver in tab:
|
||||
driver.stop()
|
||||
if not self.address.startswith('127.0.0.1'):
|
||||
return
|
||||
|
||||
if force:
|
||||
pids = None
|
||||
try:
|
||||
pids = [pid['id'] for pid in self._run_cdp('SystemInfo.getProcessInfo')['processInfo']]
|
||||
except:
|
||||
pass
|
||||
|
||||
if pids:
|
||||
from psutil import Process
|
||||
for pid in pids:
|
||||
try:
|
||||
Process(pid).kill()
|
||||
except:
|
||||
pass
|
||||
|
||||
from os import popen
|
||||
from platform import system
|
||||
end_time = perf_counter() + timeout
|
||||
while perf_counter() < end_time:
|
||||
ok = True
|
||||
for pid in pids:
|
||||
txt = f'tasklist | findstr {pid}' if system().lower() == 'windows' else f'ps -ef | grep {pid}'
|
||||
p = popen(txt)
|
||||
sleep(.05)
|
||||
try:
|
||||
if f' {pid} ' in p.read():
|
||||
ok = False
|
||||
break
|
||||
except TypeError:
|
||||
pass
|
||||
|
||||
if ok:
|
||||
break
|
||||
|
||||
if del_data and not self._chromium_options.is_auto_port and self._chromium_options.user_data_path:
|
||||
path = Path(self._chromium_options.user_data_path)
|
||||
rmtree(path, True)
|
||||
|
||||
def _new_tab(self, mix=True, url=None, new_window=False, background=False, new_context=False):
|
||||
tab_type = MixTab if mix else ChromiumTab
|
||||
tab = None
|
||||
if new_context:
|
||||
tab = self._run_cdp('Target.createBrowserContext')['browserContextId']
|
||||
|
||||
kwargs = {'url': ''}
|
||||
if new_window:
|
||||
kwargs['newWindow'] = True
|
||||
if background:
|
||||
kwargs['background'] = True
|
||||
if tab:
|
||||
kwargs['browserContextId'] = tab
|
||||
|
||||
if self.states.is_incognito and not new_context:
|
||||
return _new_tab_by_js(self, url, tab_type, new_window)
|
||||
else:
|
||||
try:
|
||||
tab = self._run_cdp('Target.createTarget', **kwargs)['targetId']
|
||||
except CDPError:
|
||||
return _new_tab_by_js(self, url, tab_type, new_window)
|
||||
|
||||
while self.states.is_alive:
|
||||
if tab in self._drivers:
|
||||
break
|
||||
sleep(.01)
|
||||
else:
|
||||
raise BrowserConnectError(_S._lang.BROWSER_DISCONNECTED)
|
||||
tab = tab_type(self, tab)
|
||||
if url:
|
||||
tab.get(url)
|
||||
return tab
|
||||
|
||||
def _get_tab(self, id_or_num=None, title=None, url=None, tab_type='page', mix=True, as_id=False):
|
||||
if id_or_num is not None:
|
||||
if isinstance(id_or_num, int):
|
||||
id_or_num = self.tab_ids[id_or_num - 1 if id_or_num > 0 else id_or_num]
|
||||
elif isinstance(id_or_num, ChromiumTab):
|
||||
return id_or_num.tab_id if as_id else ChromiumTab(self, id_or_num.tab_id)
|
||||
else:
|
||||
j = self._run_cdp('Target.getTargets')['targetInfos']
|
||||
if id_or_num not in [i['targetId'] for i in j]:
|
||||
raise RuntimeError(_S._lang.join(_S._lang.NO_SUCH_TAB, ARG=id_or_num, ALL_TABS=self.tab_ids))
|
||||
|
||||
elif title == url is None and tab_type == 'page':
|
||||
id_or_num = self.tab_ids[0]
|
||||
|
||||
else:
|
||||
tabs = self._get_tabs(title=title, url=url, tab_type=tab_type, as_id=True)
|
||||
if tabs:
|
||||
id_or_num = tabs[0]
|
||||
else:
|
||||
raise RuntimeError(_S._lang.join(_S._lang.NO_SUCH_TAB,
|
||||
ARGS={'id_or_num': id_or_num, 'title': title, 'url': url,
|
||||
'tab_type': tab_type}))
|
||||
|
||||
if as_id:
|
||||
return id_or_num
|
||||
with self._lock:
|
||||
return MixTab(self, id_or_num) if mix else ChromiumTab(self, id_or_num)
|
||||
|
||||
def _get_tabs(self, title=None, url=None, tab_type='page', mix=True, as_id=False):
|
||||
if self._ws_only:
|
||||
tabs = self._run_cdp('Target.getTargets')['targetInfos']
|
||||
_id = 'targetId'
|
||||
else:
|
||||
tabs = self._driver.get(f'http://{self.address}/json').json() # 不要改用cdp
|
||||
_id = 'id'
|
||||
|
||||
if isinstance(tab_type, str):
|
||||
tab_type = {tab_type}
|
||||
elif isinstance(tab_type, (list, tuple, set)):
|
||||
tab_type = set(tab_type)
|
||||
elif tab_type is not None:
|
||||
raise ValueError(_S._lang.join(_S._lang.INCORRECT_TYPE_, 'tab_type',
|
||||
ALLOW_TYPE='set, list, tuple, str, None', CURR_VAL=tab_type))
|
||||
|
||||
tabs = [i for i in tabs if ((title is None or title in i['title']) and (url is None or url in i['url'])
|
||||
and (tab_type is None or i['type'] in tab_type)
|
||||
and i['title'] != 'chrome-extension://neajdppkdcdipfabeoofebfddakdcjhd/audio.html')]
|
||||
if as_id:
|
||||
return [tab[_id] for tab in tabs]
|
||||
with self._lock:
|
||||
if mix:
|
||||
return [MixTab(self, tab[_id]) for tab in tabs]
|
||||
else:
|
||||
return [ChromiumTab(self, tab[_id]) for tab in tabs]
|
||||
|
||||
def _run_cdp(self, cmd, **cmd_args):
|
||||
ignore = cmd_args.pop('_ignore', None)
|
||||
r = self._driver.run(cmd, **cmd_args)
|
||||
return r if __ERROR__ not in r else raise_error(r, self, ignore)
|
||||
|
||||
def _get_driver(self, tab_id, owner=None):
|
||||
d = self._drivers.pop(tab_id, None)
|
||||
if not d:
|
||||
if self._ws_only:
|
||||
d = Driver(tab_id, self._ws_address)
|
||||
d.session_id = d.run('Target.attachToTarget', targetId=tab_id, flatten=True)['sessionId']
|
||||
else:
|
||||
d = Driver(tab_id, f'ws://{self.address}/devtools/page/{tab_id}')
|
||||
d.owner = owner
|
||||
self._all_drivers.setdefault(tab_id, set()).add(d)
|
||||
return d
|
||||
|
||||
def _onTargetCreated(self, **kwargs):
|
||||
if (kwargs['targetInfo']['type'] in ('page', 'webview')
|
||||
and kwargs['targetInfo']['targetId'] not in self._all_drivers
|
||||
and not kwargs['targetInfo']['url'].startswith('devtools://')):
|
||||
try:
|
||||
tab_id = kwargs['targetInfo']['targetId']
|
||||
self._frames[tab_id] = tab_id
|
||||
if self._ws_only:
|
||||
d = Driver(tab_id, self._ws_address)
|
||||
d.session_id = d.run('Target.attachToTarget', targetId=tab_id, flatten=True)['sessionId']
|
||||
else:
|
||||
d = Driver(tab_id, f'ws://{self.address}/devtools/page/{tab_id}')
|
||||
self._relation[tab_id] = kwargs['targetInfo'].get('openerId', None)
|
||||
self._drivers[tab_id] = d
|
||||
self._all_drivers.setdefault(tab_id, set()).add(d)
|
||||
self._newest_tab_id = tab_id
|
||||
except WebSocketBadStatusException:
|
||||
pass
|
||||
|
||||
def _onTargetDestroyed(self, **kwargs):
|
||||
tab_id = kwargs['targetId']
|
||||
self._dl_mgr.clear_tab_info(tab_id)
|
||||
for key in [k for k, i in self._frames.items() if i == tab_id]:
|
||||
self._frames.pop(key, None)
|
||||
for d in self._all_drivers.get(tab_id, tuple()):
|
||||
d.stop()
|
||||
self._drivers.pop(tab_id, None)
|
||||
self._all_drivers.pop(tab_id, None)
|
||||
self._relation.pop(tab_id, None)
|
||||
|
||||
def _on_disconnect(self):
|
||||
if not self._disconnect_flag:
|
||||
Chromium._BROWSERS.pop(self.id, None)
|
||||
if self._chromium_options.is_auto_port and self._chromium_options.user_data_path:
|
||||
path = Path(self._chromium_options.user_data_path)
|
||||
end_time = perf_counter() + 7
|
||||
while perf_counter() < end_time:
|
||||
if not path.exists():
|
||||
break
|
||||
try:
|
||||
rmtree(path)
|
||||
break
|
||||
except (PermissionError, FileNotFoundError, OSError):
|
||||
pass
|
||||
sleep(.03)
|
||||
|
||||
|
||||
def handle_options(addr_or_opts):
|
||||
if not addr_or_opts:
|
||||
_chromium_options = ChromiumOptions(addr_or_opts)
|
||||
if _chromium_options.is_auto_port:
|
||||
port, path = PortFinder(_chromium_options.tmp_path).get_port(_chromium_options.is_auto_port)
|
||||
_chromium_options.set_address(f'127.0.0.1:{port}')
|
||||
_chromium_options.set_user_data_path(path)
|
||||
_chromium_options.auto_port(scope=_chromium_options.is_auto_port)
|
||||
|
||||
elif isinstance(addr_or_opts, ChromiumOptions):
|
||||
if addr_or_opts.is_auto_port:
|
||||
port, path = PortFinder(addr_or_opts.tmp_path).get_port(addr_or_opts.is_auto_port)
|
||||
addr_or_opts._address = f'127.0.0.1:{port}'
|
||||
addr_or_opts.set_user_data_path(path)
|
||||
_chromium_options = addr_or_opts
|
||||
|
||||
elif isinstance(addr_or_opts, str) and ':' in addr_or_opts:
|
||||
_chromium_options = ChromiumOptions()
|
||||
_chromium_options.set_address(addr_or_opts)
|
||||
|
||||
elif isinstance(addr_or_opts, int):
|
||||
_chromium_options = ChromiumOptions()
|
||||
_chromium_options.set_local_port(addr_or_opts)
|
||||
|
||||
else:
|
||||
raise ValueError(_S._lang.join(_S._lang.INCORRECT_VAL_, 'addr_or_opts',
|
||||
ALLOW_TYPE=_S._lang.IP_OR_OPTIONS, CURR_VAL=addr_or_opts))
|
||||
|
||||
return _chromium_options
|
||||
|
||||
|
||||
def run_browser(chromium_options):
|
||||
if chromium_options.ws_address:
|
||||
d = Driver('', chromium_options.ws_address)
|
||||
browser_id = d.run('Target.getTargets')
|
||||
if 'error' in browser_id:
|
||||
raise BrowserConnectError(_S._lang.join(_S._lang.BROWSER_CONNECT_ERR2, INFO=browser_id['error']))
|
||||
browser_id = browser_id['targetInfos'][0]['browserContextId']
|
||||
is_headless = 'headless' in d.run('Browser.getVersion')['userAgent'].lower()
|
||||
is_exists = True
|
||||
d.stop()
|
||||
if 'devtools/browser' not in chromium_options.ws_address:
|
||||
return is_headless, browser_id, is_exists, True
|
||||
else:
|
||||
is_exists = connect_browser(chromium_options)
|
||||
|
||||
try:
|
||||
s = Session()
|
||||
s.trust_env = False
|
||||
s.keep_alive = False
|
||||
ws = s.get(f'http://{chromium_options.address}/json/version', headers={'Connection': 'close'}, timeout=2)
|
||||
if not ws and not chromium_options.ws_address:
|
||||
raise BrowserConnectError(_S._lang.BROWSER_CONNECT_ERR2)
|
||||
json = ws.json()
|
||||
if not chromium_options.ws_address:
|
||||
browser_id = json['webSocketDebuggerUrl'].split('/')[-1]
|
||||
is_headless = 'headless' in json['User-Agent'].lower()
|
||||
ws.close()
|
||||
s.close()
|
||||
ws_only = False
|
||||
except KeyError:
|
||||
raise BrowserConnectError(_S._lang.BROWSER_NOT_FOR_CONTROL)
|
||||
except:
|
||||
if chromium_options.ws_address:
|
||||
ws_only = True
|
||||
else:
|
||||
raise BrowserConnectError(_S._lang.BROWSER_CONNECT_ERR2)
|
||||
return is_headless, browser_id, is_exists, ws_only
|
||||
|
||||
|
||||
def _new_tab_by_js(browser: Chromium, url, tab_type, new_window):
|
||||
mix = tab_type == MixTab
|
||||
tab = browser._get_tab(mix=mix)
|
||||
if url and not match(r'^.*?://.*', url):
|
||||
raise IncorrectURLError(_S._lang.INVALID_URL, url=url)
|
||||
url = f'"{url}"' if url else '""'
|
||||
new = 'target="_new"' if new_window else 'target="_blank"'
|
||||
tid = browser._newest_tab_id
|
||||
tab.run_js(f'window.open({url}, {new})')
|
||||
tid = browser.wait.new_tab(curr_tab=tid)
|
||||
return browser._get_tab(tid, mix=mix)
|
||||
@@ -0,0 +1,319 @@
|
||||
# -*- coding:utf-8 -*-
|
||||
"""
|
||||
@Author : g1879
|
||||
@Contact : g1879@qq.com
|
||||
@Website : https://DrissionPage.cn
|
||||
@Copyright: (c) 2020 by g1879, Inc. All Rights Reserved.
|
||||
"""
|
||||
from threading import Lock
|
||||
from typing import List, Optional, Set, Dict, Union, Tuple, Literal, Any
|
||||
|
||||
from .driver import BrowserDriver, Driver
|
||||
from .._configs.chromium_options import ChromiumOptions
|
||||
from .._configs.session_options import SessionOptions
|
||||
from .._functions.cookies import CookiesList
|
||||
from .._pages.chromium_base import Timeout, ChromiumBase
|
||||
from .._pages.chromium_tab import ChromiumTab
|
||||
from .._pages.mix_tab import MixTab
|
||||
from .._units.downloader import DownloadManager
|
||||
from .._units.setter import BrowserSetter
|
||||
from .._units.states import BrowserStates
|
||||
from .._units.waiter import BrowserWaiter
|
||||
|
||||
|
||||
class Chromium(object):
|
||||
_BROWSERS: dict = ...
|
||||
_lock: Lock = ...
|
||||
|
||||
id: str = ...
|
||||
address: str = ...
|
||||
_ws_address: str = ...
|
||||
version: str = ...
|
||||
retry_times: int = ...
|
||||
retry_interval: float = ...
|
||||
|
||||
_set: Optional[BrowserSetter] = ...
|
||||
_wait: Optional[BrowserWaiter] = ...
|
||||
_states: Optional[BrowserStates] = ...
|
||||
_chromium_options: ChromiumOptions = ...
|
||||
_session_options: SessionOptions = ...
|
||||
_driver: BrowserDriver = ...
|
||||
_frames: dict = ...
|
||||
_drivers: Dict[str, Driver] = ...
|
||||
_all_drivers: Dict[str, Set[Driver]] = ...
|
||||
_relation: Dict[str, Optional[str]] = ...
|
||||
_process_id: Optional[int] = ...
|
||||
_dl_mgr: DownloadManager = ...
|
||||
_timeouts: Timeout = ...
|
||||
_load_mode: str = ...
|
||||
_download_path: str = ...
|
||||
_auto_handle_alert: Optional[bool] = ...
|
||||
_is_exists: bool = ...
|
||||
_is_headless: bool = ...
|
||||
_ws_only: bool = ...
|
||||
_disconnect_flag: bool = ...
|
||||
_none_ele_return_value: bool = ...
|
||||
_none_ele_value: Any = ...
|
||||
_newest_tab_id: Optional[str] = ...
|
||||
|
||||
def __new__(cls,
|
||||
addr_or_opts: Union[str, int, ChromiumOptions] = None,
|
||||
session_options: Union[SessionOptions, None, False] = None):
|
||||
"""
|
||||
:param addr_or_opts: 浏览器地址:端口、ws地址、ChromiumOptions对象或端口数字(int)
|
||||
:param session_options: 使用双模Tab时使用的默认Session配置,为None使用ini文件配置,为False不从ini读取
|
||||
"""
|
||||
...
|
||||
|
||||
def __init__(self, addr_or_opts: Union[str, int, ChromiumOptions] = None,
|
||||
session_options: Union[SessionOptions, None, False] = None):
|
||||
"""
|
||||
:param addr_or_opts: 浏览器地址:端口、ws地址、ChromiumOptions对象或端口数字(int)
|
||||
:param session_options: 使用双模Tab时使用的默认Session配置,为None使用ini文件配置,为False不从ini读取
|
||||
"""
|
||||
...
|
||||
|
||||
@property
|
||||
def user_data_path(self) -> str:
|
||||
"""返回用户文件夹路径"""
|
||||
...
|
||||
|
||||
@property
|
||||
def process_id(self) -> Optional[int]:
|
||||
"""返回浏览器进程id"""
|
||||
...
|
||||
|
||||
@property
|
||||
def timeout(self) -> float:
|
||||
"""返回基础超时设置"""
|
||||
...
|
||||
|
||||
@property
|
||||
def timeouts(self) -> Timeout:
|
||||
"""返回所有超时设置"""
|
||||
...
|
||||
|
||||
@property
|
||||
def load_mode(self) -> Literal['none', 'normal', 'eager']:
|
||||
"""返回页面加载模式,包括 'none', 'normal', 'eager' 三种"""
|
||||
...
|
||||
|
||||
@property
|
||||
def download_path(self) -> str:
|
||||
"""返回默认下载路径"""
|
||||
...
|
||||
|
||||
@property
|
||||
def set(self) -> BrowserSetter:
|
||||
"""返回用于设置的对象"""
|
||||
...
|
||||
|
||||
@property
|
||||
def states(self) -> BrowserStates:
|
||||
"""返回用于获取状态的对象"""
|
||||
...
|
||||
|
||||
@property
|
||||
def wait(self) -> BrowserWaiter:
|
||||
"""返回用于等待的对象"""
|
||||
...
|
||||
|
||||
@property
|
||||
def tabs_count(self) -> int:
|
||||
"""返回标签页数量,只统计page、webview类型"""
|
||||
...
|
||||
|
||||
@property
|
||||
def tab_ids(self) -> List[str]:
|
||||
"""返回所有标签页id组成的列表,只统计page、webview类型"""
|
||||
...
|
||||
|
||||
@property
|
||||
def latest_tab(self) -> Union[MixTab, str]:
|
||||
"""返回最新的标签页,最新标签页指最后创建或最后被激活的
|
||||
当Settings.singleton_tab_obj==True时返回Tab对象,否则返回tab id"""
|
||||
...
|
||||
|
||||
def cookies(self, all_info: bool = False) -> CookiesList:
|
||||
"""以list格式返回所有域名的cookies
|
||||
:param all_info: 是否返回所有内容,False则只返回name, value, domain
|
||||
:return: cookies组成的列表
|
||||
"""
|
||||
...
|
||||
|
||||
def new_tab(self,
|
||||
url: str = None,
|
||||
new_window: bool = False,
|
||||
background: bool = False,
|
||||
new_context: bool = False) -> MixTab:
|
||||
"""新建一个标签页
|
||||
:param url: 新标签页跳转到的网址,为None时新建空标签页
|
||||
:param new_window: 是否在新窗口打开标签页,隐身模式下无效
|
||||
:param background: 是否不激活新标签页,隐身模式和访客模式及new_window为True时无效
|
||||
:param new_context: 是否创建独立环境,隐身模式和访客模式下无效
|
||||
:return: 新标签页对象
|
||||
"""
|
||||
...
|
||||
|
||||
def get_tab(self,
|
||||
id_or_num: Union[str, int] = None,
|
||||
title: str = None,
|
||||
url: str = None,
|
||||
tab_type: Union[str, list, tuple] = 'page',
|
||||
as_id: bool = False) -> Union[MixTab, str]:
|
||||
"""获取一个标签页对象,id_or_num不为None时,后面几个参数无效
|
||||
:param id_or_num: 要获取的标签页id或序号,序号从1开始,可传入负数获取倒数第几个,不是视觉排列顺序,而是激活顺序
|
||||
:param title: 要匹配title的文本,模糊匹配,为None则匹配所有
|
||||
:param url: 要匹配url的文本,模糊匹配,为None则匹配所有
|
||||
:param tab_type: tab类型,可用列表输入多个,如 'page', 'iframe' 等,为None则匹配所有
|
||||
:param as_id: 是否返回标签页id而不是标签页对象
|
||||
:return: Tab对象
|
||||
"""
|
||||
...
|
||||
|
||||
def get_tabs(self,
|
||||
title: str = None,
|
||||
url: str = None,
|
||||
tab_type: Union[str, list, tuple] = 'page',
|
||||
as_id: bool = False) -> List[MixTab, str]:
|
||||
"""查找符合条件的tab,返回它们组成的列表,title和url是与关系
|
||||
:param title: 要匹配title的文本
|
||||
:param url: 要匹配url的文本
|
||||
:param tab_type: tab类型,可用列表输入多个
|
||||
:param as_id: 是否返回标签页id而不是标签页对象
|
||||
:return: Tab对象列表
|
||||
"""
|
||||
...
|
||||
|
||||
def close_tabs(self,
|
||||
tabs_or_ids: Union[str, ChromiumTab, List[Union[str, ChromiumTab]],
|
||||
Tuple[Union[str, ChromiumTab]]],
|
||||
others: bool = False) -> None:
|
||||
"""关闭传入的标签页,可传入多个
|
||||
:param tabs_or_ids: 指定的标签页对象或id,可用列表或元组传入多个
|
||||
:param others: 是否关闭指定标签页之外的
|
||||
:return: None
|
||||
"""
|
||||
...
|
||||
|
||||
def _close_tab(self, tab: Union[ChromiumBase, str]):
|
||||
"""关闭一个标签页
|
||||
:param tab: 标签页对象或id
|
||||
:return: None
|
||||
"""
|
||||
|
||||
def activate_tab(self, id_ind_tab: Union[int, str, ChromiumTab]) -> None:
|
||||
"""使一个标签页显示到前端
|
||||
:param id_ind_tab: 标签页id(str)、Tab对象或标签页序号(int),序号从1开始
|
||||
:return: None
|
||||
"""
|
||||
...
|
||||
|
||||
def reconnect(self) -> None:
|
||||
"""断开重连"""
|
||||
...
|
||||
|
||||
def clear_cache(self, cache: bool = True, cookies: bool = True) -> None:
|
||||
"""清除缓存,可选要清除的项
|
||||
:param cache: 是否清除cache
|
||||
:param cookies: 是否清除cookies
|
||||
:return: None
|
||||
"""
|
||||
...
|
||||
|
||||
def quit(self, timeout: float = 5, force: bool = False, del_data: bool = False) -> None:
|
||||
"""关闭浏览器
|
||||
:param timeout: 等待浏览器关闭超时时间(秒)
|
||||
:param force: 是否立刻强制终止进程
|
||||
:param del_data: 是否删除用户文件夹
|
||||
:return: None
|
||||
"""
|
||||
...
|
||||
|
||||
def _new_tab(self,
|
||||
mix: bool = True,
|
||||
url: str = None,
|
||||
new_window: bool = False,
|
||||
background: bool = False,
|
||||
new_context: bool = False) -> Union[ChromiumTab, MixTab]:
|
||||
"""新建一个标签页
|
||||
:param mix: 是否创建MixTab
|
||||
:param url: 新标签页跳转到的网址
|
||||
:param new_window: 是否在新窗口打开标签页
|
||||
:param background: 是否不激活新标签页,如new_window为True则无效
|
||||
:param new_context: 是否创建新的上下文
|
||||
:return: 新标签页对象
|
||||
"""
|
||||
...
|
||||
|
||||
def _get_tab(self,
|
||||
id_or_num: Union[str, int] = None,
|
||||
title: str = None,
|
||||
url: str = None,
|
||||
tab_type: Union[str, list, tuple] = 'page',
|
||||
mix: bool = True,
|
||||
as_id: bool = False) -> Union[ChromiumTab, str]:
|
||||
"""获取一个标签页对象,id_or_num不为None时,后面几个参数无效
|
||||
:param id_or_num: 要获取的标签页id或序号,序号从1开始,可传入负数获取倒数第几个,不是视觉排列顺序,而是激活顺序
|
||||
:param title: 要匹配title的文本,模糊匹配,为None则匹配所有
|
||||
:param url: 要匹配url的文本,模糊匹配,为None则匹配所有
|
||||
:param tab_type: tab类型,可用列表输入多个,如 'page', 'iframe' 等,为None则匹配所有
|
||||
:param mix: 是否返回可切换模式的Tab对象
|
||||
:param as_id: 是否返回标签页id而不是标签页对象,mix=False时无效
|
||||
:return: Tab对象
|
||||
"""
|
||||
...
|
||||
|
||||
def _get_tabs(self,
|
||||
title: str = None,
|
||||
url: str = None,
|
||||
tab_type: Union[str, list, tuple] = 'page',
|
||||
mix: bool = True,
|
||||
as_id: bool = False) -> List[ChromiumTab, str]:
|
||||
"""查找符合条件的tab,返回它们组成的列表,title和url是与关系
|
||||
:param title: 要匹配title的文本
|
||||
:param url: 要匹配url的文本
|
||||
:param tab_type: tab类型,可用列表输入多个
|
||||
:param mix: 是否返回可切换模式的Tab对象
|
||||
:param as_id: 是否返回标签页id而不是标签页对象,mix=False时无效
|
||||
:return: Tab对象列表
|
||||
"""
|
||||
...
|
||||
|
||||
def _run_cdp(self, cmd, **cmd_args) -> dict:
|
||||
"""执行Chrome DevTools Protocol语句
|
||||
:param cmd: 协议项目
|
||||
:param cmd_args: 参数
|
||||
:return: 执行的结果
|
||||
"""
|
||||
...
|
||||
|
||||
def _get_driver(self, tab_id: str, owner=None) -> Driver:
|
||||
"""新建并返回指定tab id的Driver
|
||||
:param tab_id: 标签页id
|
||||
:param owner: 使用该驱动的对象
|
||||
:return: Driver对象
|
||||
"""
|
||||
...
|
||||
|
||||
def _onTargetCreated(self, **kwargs): ...
|
||||
|
||||
def _onTargetDestroyed(self, **kwargs): ...
|
||||
|
||||
def _on_disconnect(self): ...
|
||||
|
||||
|
||||
def handle_options(addr_or_opts):
|
||||
"""设置浏览器启动属性
|
||||
:param addr_or_opts: 'ip:port'、ChromiumOptions、Driver
|
||||
:return: 返回ChromiumOptions对象
|
||||
"""
|
||||
...
|
||||
|
||||
|
||||
def run_browser(chromium_options)->Tuple[bool, str, bool, bool]:
|
||||
"""连接浏览器
|
||||
:param chromium_options: ChromiumOptions对象
|
||||
:return: 返回(是否无头, 浏览器id, 是否接管已存在, 是否只能用ws连接)
|
||||
"""
|
||||
...
|
||||
@@ -0,0 +1,229 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
@Author : g1879
|
||||
@Contact : g1879@qq.com
|
||||
@Website : https://DrissionPage.cn
|
||||
@Copyright: (c) 2020 by g1879, Inc. All Rights Reserved.
|
||||
"""
|
||||
from json import dumps, loads, JSONDecodeError
|
||||
from queue import Queue, Empty
|
||||
from threading import Thread
|
||||
from time import perf_counter, sleep
|
||||
|
||||
from requests import Session
|
||||
from requests import adapters
|
||||
from websocket import (WebSocketTimeoutException, WebSocketConnectionClosedException, create_connection,
|
||||
WebSocketException, WebSocketBadStatusException)
|
||||
|
||||
from .._functions.settings import Settings as _S
|
||||
from ..errors import PageDisconnectedError, BrowserConnectError
|
||||
|
||||
adapters.DEFAULT_RETRIES = 5
|
||||
|
||||
|
||||
class Driver(object):
|
||||
def __init__(self, _id, address, owner=None):
|
||||
self.id = _id
|
||||
self.address = address
|
||||
self.owner = owner
|
||||
self.alert_flag = False # 标记alert出现,跳过一条请求后复原
|
||||
|
||||
self._cur_id = 0
|
||||
self._ws = None
|
||||
|
||||
self._recv_th = Thread(target=self._recv_loop)
|
||||
self._handle_event_th = Thread(target=self._handle_event_loop)
|
||||
self._recv_th.daemon = True
|
||||
self._handle_event_th.daemon = True
|
||||
self._handle_immediate_event_th = None
|
||||
|
||||
self.is_running = False
|
||||
self.session_id = None
|
||||
|
||||
self.event_handlers = {}
|
||||
self.immediate_event_handlers = {}
|
||||
self.method_results = {}
|
||||
self.event_queue = Queue()
|
||||
self.immediate_event_queue = Queue()
|
||||
|
||||
self.start()
|
||||
|
||||
def _send(self, message, timeout=None):
|
||||
self._cur_id += 1
|
||||
ws_id = self._cur_id
|
||||
message['id'] = ws_id
|
||||
message_json = dumps(message)
|
||||
|
||||
end_time = perf_counter() + timeout if timeout is not None else None
|
||||
self.method_results[ws_id] = Queue()
|
||||
try:
|
||||
self._ws.send(message_json)
|
||||
if timeout == 0:
|
||||
self.method_results.pop(ws_id, None)
|
||||
return {'id': ws_id, 'result': {}}
|
||||
|
||||
except (OSError, WebSocketConnectionClosedException):
|
||||
self.method_results.pop(ws_id, None)
|
||||
return {'error': {'message': 'connection disconnected'}, 'type': 'connection_error'}
|
||||
|
||||
while self.is_running:
|
||||
try:
|
||||
result = self.method_results[ws_id].get(timeout=.2)
|
||||
self.method_results.pop(ws_id, None)
|
||||
return result
|
||||
|
||||
except Empty:
|
||||
if self.alert_flag and message['method'].startswith(('Input.', 'Runtime.')):
|
||||
return {'error': {'message': 'alert exists.'}, 'type': 'alert_exists'}
|
||||
|
||||
if timeout is not None and perf_counter() > end_time:
|
||||
self.method_results.pop(ws_id, None)
|
||||
return {'error': {'message': 'alert exists.'}, 'type': 'alert_exists'} \
|
||||
if self.alert_flag else {'error': {'message': 'timeout'}, 'type': 'timeout'}
|
||||
|
||||
continue
|
||||
|
||||
return {'error': {'message': 'connection disconnected'}, 'type': 'connection_error'}
|
||||
|
||||
def _recv_loop(self):
|
||||
while self.is_running:
|
||||
try:
|
||||
# self._ws.settimeout(1)
|
||||
msg_json = self._ws.recv()
|
||||
msg = loads(msg_json)
|
||||
except WebSocketTimeoutException:
|
||||
continue
|
||||
except (WebSocketException, OSError, WebSocketConnectionClosedException, JSONDecodeError):
|
||||
self._stop()
|
||||
return
|
||||
|
||||
if 'method' in msg:
|
||||
if msg['method'].startswith('Page.javascriptDialog'):
|
||||
self.alert_flag = msg['method'].endswith('Opening')
|
||||
function = self.immediate_event_handlers.get(msg['method'])
|
||||
if function:
|
||||
self._handle_immediate_event(function, msg['params'])
|
||||
else:
|
||||
self.event_queue.put(msg)
|
||||
|
||||
elif msg.get('id') in self.method_results:
|
||||
self.method_results[msg['id']].put(msg)
|
||||
|
||||
def _handle_event_loop(self):
|
||||
while self.is_running:
|
||||
try:
|
||||
event = self.event_queue.get(timeout=1)
|
||||
except Empty:
|
||||
continue
|
||||
|
||||
function = self.event_handlers.get(event['method'])
|
||||
if function:
|
||||
function(**event['params'])
|
||||
|
||||
self.event_queue.task_done()
|
||||
|
||||
def _handle_immediate_event_loop(self):
|
||||
while not self.immediate_event_queue.empty():
|
||||
function, kwargs = self.immediate_event_queue.get(timeout=1)
|
||||
try:
|
||||
function(**kwargs)
|
||||
except PageDisconnectedError:
|
||||
pass
|
||||
|
||||
def _handle_immediate_event(self, function, kwargs):
|
||||
self.immediate_event_queue.put((function, kwargs))
|
||||
if self._handle_immediate_event_th is None or not self._handle_immediate_event_th.is_alive():
|
||||
self._handle_immediate_event_th = Thread(target=self._handle_immediate_event_loop)
|
||||
self._handle_immediate_event_th.daemon = True
|
||||
self._handle_immediate_event_th.start()
|
||||
|
||||
def run(self, _method, sessionId=None, **kwargs):
|
||||
if not self.is_running:
|
||||
return {'error': 'connection disconnected', 'type': 'connection_error'}
|
||||
|
||||
timeout = kwargs.pop('_timeout', _S.cdp_timeout)
|
||||
if self.session_id:
|
||||
result = self._send({'method': _method, 'params': kwargs, 'sessionId': self.session_id}, timeout=timeout)
|
||||
elif sessionId:
|
||||
result = self._send({'method': _method, 'params': kwargs, 'sessionId': sessionId}, timeout=timeout)
|
||||
else:
|
||||
result = self._send({'method': _method, 'params': kwargs}, timeout=timeout)
|
||||
if 'result' not in result and 'error' in result:
|
||||
kwargs['_timeout'] = timeout
|
||||
return {'error': result['error']['message'], 'type': result.get('type', 'call_method_error'),
|
||||
'method': _method, 'args': kwargs, 'data': result['error'].get('data')}
|
||||
else:
|
||||
return result['result']
|
||||
|
||||
def start(self):
|
||||
self.is_running = True
|
||||
try:
|
||||
self._ws = create_connection(self.address, enable_multithread=True, suppress_origin=True)
|
||||
except WebSocketBadStatusException as e:
|
||||
if 'Handshake status 403 Forbidden' in str(e):
|
||||
raise EnvironmentError(_S._lang.join(_S._lang.UPGRADE_WS))
|
||||
else:
|
||||
raise
|
||||
except ConnectionRefusedError:
|
||||
raise BrowserConnectError(_S._lang.BROWSER_NOT_EXIST)
|
||||
self._recv_th.start()
|
||||
self._handle_event_th.start()
|
||||
return True
|
||||
|
||||
def stop(self):
|
||||
self._stop()
|
||||
while self._handle_event_th.is_alive() or self._recv_th.is_alive():
|
||||
sleep(.01)
|
||||
return True
|
||||
|
||||
def _stop(self):
|
||||
if not self.is_running:
|
||||
return False
|
||||
|
||||
self.is_running = False
|
||||
if self._ws:
|
||||
self._ws.close()
|
||||
self._ws = None
|
||||
|
||||
self.event_handlers.clear()
|
||||
self.method_results.clear()
|
||||
self.event_queue.queue.clear()
|
||||
|
||||
if hasattr(self.owner, '_on_disconnect'):
|
||||
self.owner._on_disconnect()
|
||||
|
||||
def set_callback(self, event, callback, immediate=False):
|
||||
handler = self.immediate_event_handlers if immediate else self.event_handlers
|
||||
if callback:
|
||||
handler[event] = callback
|
||||
else:
|
||||
handler.pop(event, None)
|
||||
|
||||
|
||||
class BrowserDriver(Driver):
|
||||
BROWSERS = {}
|
||||
|
||||
def __new__(cls, _id, address, owner):
|
||||
if _id in cls.BROWSERS:
|
||||
return cls.BROWSERS[_id]
|
||||
return object.__new__(cls)
|
||||
|
||||
def __init__(self, _id, address, owner):
|
||||
if hasattr(self, '_created'):
|
||||
return
|
||||
self._created = True
|
||||
BrowserDriver.BROWSERS[_id] = self
|
||||
super().__init__(_id, address, owner)
|
||||
|
||||
def __repr__(self):
|
||||
return f'<BrowserDriver {self.id}>'
|
||||
|
||||
@staticmethod
|
||||
def get(url):
|
||||
s = Session()
|
||||
s.trust_env = False
|
||||
s.keep_alive = False
|
||||
r = s.get(url, headers={'Connection': 'close'})
|
||||
r.close()
|
||||
s.close()
|
||||
return r
|
||||
@@ -0,0 +1,114 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
@Author : g1879
|
||||
@Contact : g1879@qq.com
|
||||
@Website : https://DrissionPage.cn
|
||||
@Copyright: (c) 2020 by g1879, Inc. All Rights Reserved.
|
||||
"""
|
||||
from queue import Queue
|
||||
from threading import Thread
|
||||
from typing import Union, Callable, Dict, Optional
|
||||
|
||||
from requests import Response
|
||||
from websocket import WebSocket
|
||||
|
||||
from .._base.chromium import Chromium
|
||||
|
||||
|
||||
class Driver(object):
|
||||
id: str
|
||||
address: str
|
||||
owner = ...
|
||||
alert_flag: bool
|
||||
_cur_id: int
|
||||
_ws: Optional[WebSocket]
|
||||
_recv_th: Thread
|
||||
_handle_event_th: Thread
|
||||
_handle_immediate_event_th: Optional[Thread]
|
||||
session_id: Optional[str] = ...
|
||||
is_running: bool
|
||||
event_handlers: dict
|
||||
immediate_event_handlers: dict
|
||||
method_results: dict
|
||||
event_queue: Queue
|
||||
immediate_event_queue: Queue
|
||||
|
||||
def __init__(self, _id: str, address: str, owner=None):
|
||||
"""
|
||||
:param _id: 标签页id
|
||||
:param address: 浏览器连接地址
|
||||
:param owner: 创建这个驱动的对象
|
||||
"""
|
||||
...
|
||||
|
||||
def _send(self, message: dict, timeout: float = None) -> dict:
|
||||
"""发送信息到浏览器,并返回浏览器返回的信息
|
||||
:param message: 发送给浏览器的数据
|
||||
:param timeout: 超时时间,为None表示无限
|
||||
:return: 浏览器返回的数据
|
||||
"""
|
||||
...
|
||||
|
||||
def _recv_loop(self) -> None:
|
||||
"""接收浏览器信息的守护线程方法"""
|
||||
...
|
||||
|
||||
def _handle_event_loop(self) -> None:
|
||||
"""当接收到浏览器信息,执行已绑定的方法"""
|
||||
...
|
||||
|
||||
def _handle_immediate_event_loop(self): ...
|
||||
|
||||
def _handle_immediate_event(self, function: Callable, kwargs: dict):
|
||||
"""处理立即执行的动作
|
||||
:param function: 要运行下方法
|
||||
:param kwargs: 方法参数
|
||||
:return: None
|
||||
"""
|
||||
...
|
||||
|
||||
def run(self, _method: str, **kwargs) -> dict:
|
||||
"""执行cdp方法
|
||||
:param _method: cdp方法名
|
||||
:param kwargs: cdp参数
|
||||
:return: 执行结果
|
||||
"""
|
||||
...
|
||||
|
||||
def start(self) -> bool:
|
||||
"""启动连接"""
|
||||
...
|
||||
|
||||
def stop(self) -> bool:
|
||||
"""中断连接"""
|
||||
...
|
||||
|
||||
def _stop(self) -> None:
|
||||
"""中断连接"""
|
||||
...
|
||||
|
||||
def set_callback(self, event: str, callback: Union[Callable, None], immediate: bool = False) -> None:
|
||||
"""绑定cdp event和回调方法
|
||||
:param event: cdp event
|
||||
:param callback: 绑定到cdp event的回调方法
|
||||
:param immediate: 是否要立即处理的动作
|
||||
:return: None
|
||||
"""
|
||||
...
|
||||
|
||||
|
||||
class BrowserDriver(Driver):
|
||||
BROWSERS: Dict[str, Driver] = ...
|
||||
owner: Chromium = ...
|
||||
|
||||
def __new__(cls, _id: str, address: str, owner: Chromium): ...
|
||||
|
||||
def __init__(self, _id: str, address: str, owner: Chromium): ...
|
||||
|
||||
@staticmethod
|
||||
def get(url) -> Response:
|
||||
"""
|
||||
:param url: 要访问的链接
|
||||
:return: Response对象
|
||||
"""
|
||||
...
|
||||
Reference in new issue
Block a user