Initial commit

This commit is contained in:
Alex committed 2026-11-02 22:54:14 +08:00
commit 57fc3525eb
7311 files changed
+1291398

No files matched your search

@@ -0,0 +1,362 @@
# -*- coding:utf-8 -*-
"""
@Author : g1879
@Contact : g1879@qq.com
@Website : https://DrissionPage.cn
@Copyright: (c) 2020 by g1879, Inc. All Rights Reserved.
"""
from abc import abstractmethod
from copy import copy
from pathlib import Path
from re import sub
from urllib.parse import quote
from DrissionGet import DrissionGet
from requests import Session
from .._configs.session_options import SessionOptions
from .._elements.none_element import NoneElement
from .._functions.elements import get_frame, get_eles
from .._functions.locator import get_loc
from .._functions.settings import Settings as _S
from .._functions.web import format_html
from ..errors import ElementNotFoundError, LocatorError
class BaseParser(object):
def __call__(self, locator):
return self.ele(locator)
def ele(self, locator, index=1, timeout=None):
return self._ele(locator, timeout, index=index, method='ele()')
def eles(self, locator, timeout=None):
return self._ele(locator, timeout, index=None)
def find(self, locators, any_one=True, first_ele=True, timeout=None):
if 'Session' in self._type:
timeout = 0
if timeout is None:
timeout = self.timeout
r = get_eles(locators, self, any_one, first_ele, timeout)
if any_one:
for ele in r:
if r[ele]:
return ele, r[ele]
return None, None
return r
# ----------------以下属性或方法待后代实现----------------
@property
def html(self):
return ''
def s_ele(self, locator=None):
pass
def s_eles(self, locator):
pass
def _ele(self, locator, timeout=None, index=1, raise_err=None, method=None):
pass
def _find_elements(self, locator, timeout, index=1, relative=False, raise_err=None):
pass
class BaseElement(BaseParser):
def __init__(self, owner=None):
self.owner = owner
self._type = 'BaseElement'
def get_frame(self, loc_or_ind, timeout=None):
if not isinstance(loc_or_ind, (int, str, tuple)):
raise ValueError(_S._lang.join(_S._lang.INCORRECT_TYPE_, 'loc_or_ind',
ALLOW_TYPE=_S._lang.LOC_OR_IND, CURR_VAL=loc_or_ind))
return get_frame(self, loc_ind_ele=loc_or_ind, timeout=timeout)
def _ele(self, locator, timeout=None, index=1, relative=False, raise_err=None, method=None):
if hasattr(locator, '_type'):
return locator
if timeout is None:
timeout = self.timeout
r = self._find_elements(locator, timeout=timeout, index=index, relative=relative, raise_err=raise_err)
if r or isinstance(r, list):
return r
if raise_err is True or (_S.raise_when_ele_not_found and raise_err is None):
raise ElementNotFoundError(METHOD=method, ARGS={'locator': locator, 'index': index, 'timeout': timeout})
r.method = method
r.args = {'locator': locator, 'index': index, 'timeout': timeout}
return r
@property
def timeout(self):
return self.owner.timeout if self.owner else 10
@property
def child_count(self):
return int(self._ele('xpath:count(./*)'))
# ----------------以下属性或方法由后代实现----------------
@property
def tag(self):
return
def parent(self, level_or_loc=1):
pass
def next(self, index=1):
pass
def nexts(self):
pass
class DrissionElement(BaseElement):
@property
def link(self):
return self.attr('href') or self.attr('src')
@property
def css_path(self):
return self._get_ele_path(xpath=False)
@property
def xpath(self):
return self._get_ele_path()
@property
def comments(self):
return self.eles('xpath:.//comment()')
def texts(self, text_node_only=False):
texts = self.eles('xpath:/text()') if text_node_only else [x if isinstance(x, str) else x.text
for x in self.eles('xpath:./text() | *')]
return [format_html(x.strip(' ').rstrip('\n')) for x in texts if x and sub('[\r\n\t ]', '', x) != '']
def parent(self, level_or_loc=1, index=1, timeout=None):
if isinstance(level_or_loc, int):
loc = f'xpath:./ancestor::*[{level_or_loc}]'
elif isinstance(level_or_loc, (tuple, str)):
loc = get_loc(level_or_loc, True)
if loc[0] == 'css selector':
raise LocatorError(_S._lang.UNSUPPORTED_CSS_SYNTAX)
loc = f'xpath:./ancestor::{loc[1].lstrip(". / ")}[{index}]'
else:
raise ValueError(_S._lang.join(_S._lang.INCORRECT_TYPE_, 'level_or_loc', ALLOW_TYPE='tuple, int, str',
CURR_VAL=level_or_loc))
return self._ele(loc, timeout=timeout, relative=True, raise_err=False, method='parent()')
def child(self, locator='', index=1, timeout=None, ele_only=True):
if isinstance(locator, int):
index = locator
locator = ''
if not locator:
loc = '*' if ele_only else 'node()'
else:
loc = get_loc(locator, True) # 把定位符转换为xpath
if loc[0] == 'css selector':
raise LocatorError(_S._lang.UNSUPPORTED_CSS_SYNTAX)
loc = loc[1].lstrip('./')
node = self._ele(f'xpath:./{loc}', timeout=timeout, index=index, relative=True, raise_err=False)
return node if node else NoneElement(self.owner, 'child()',
{'locator': locator, 'index': index, 'ele_only': ele_only})
def prev(self, locator='', index=1, timeout=None, ele_only=True):
return self._get_relative('prev()', 'preceding', True, locator, index, timeout, ele_only)
def next(self, locator='', index=1, timeout=None, ele_only=True):
return self._get_relative('next()', 'following', True, locator, index, timeout, ele_only)
def before(self, locator='', index=1, timeout=None, ele_only=True):
return self._get_relative('before()', 'preceding', False, locator, index, timeout, ele_only)
def after(self, locator='', index=1, timeout=None, ele_only=True):
return self._get_relative('after()', 'following', False, locator, index, timeout, ele_only)
def children(self, locator='', timeout=None, ele_only=True):
if not locator:
loc = '*' if ele_only else 'node()'
else:
loc = get_loc(locator, True) # 把定位符转换为xpath
if loc[0] == 'css selector':
raise LocatorError(_S._lang.UNSUPPORTED_CSS_SYNTAX)
loc = loc[1].lstrip('./')
loc = f'xpath:./{loc}'
nodes = self._ele(loc, timeout=timeout, index=None, relative=True)
return [e for e in nodes if not (isinstance(e, str) and sub('[ \n\t\r]', '', e) == '')]
def prevs(self, locator='', timeout=None, ele_only=True):
return self._get_relatives(locator=locator, direction='preceding', timeout=timeout, ele_only=ele_only)
def nexts(self, locator='', timeout=None, ele_only=True):
return self._get_relatives(locator=locator, direction='following', timeout=timeout, ele_only=ele_only)
def befores(self, locator='', timeout=None, ele_only=True):
return self._get_relatives(locator=locator, direction='preceding',
brother=False, timeout=timeout, ele_only=ele_only)
def afters(self, locator='', timeout=None, ele_only=True):
return self._get_relatives(locator=locator, direction='following',
brother=False, timeout=timeout, ele_only=ele_only)
def _get_relative(self, func, direction, brother, locator='', index=1, timeout=None, ele_only=True):
if isinstance(locator, int):
index = locator
locator = ''
node = self._get_relatives(index, locator, direction, brother, timeout, ele_only)
return node if node else NoneElement(self.owner, func,
{'locator': locator, 'index': index, 'ele_only': ele_only})
def _get_relatives(self, index=None, locator='', direction='following', brother=True, timeout=.5, ele_only=True):
brother = '-sibling' if brother else ''
if not locator:
loc = '*' if ele_only else 'node()'
else:
loc = get_loc(locator, True) # 把定位符转换为xpath
if loc[0] == 'css selector':
raise LocatorError(_S._lang.UNSUPPORTED_CSS_SYNTAX)
loc = loc[1].lstrip('./')
loc = f'xpath:./{direction}{brother}::{loc}'
if index is not None:
index = index if direction == 'following' else -index
nodes = self._ele(loc, timeout=timeout, index=index, relative=True, raise_err=False)
if isinstance(nodes, list):
nodes = [e for e in nodes if not (isinstance(e, str) and sub('[ \n\t\r]', '', e) == '')]
return nodes
# ----------------以下属性或方法由后代实现----------------
@property
def attrs(self):
return
@property
def text(self):
return
@property
def raw_text(self):
return
@abstractmethod
def attr(self, name):
return ''
def _get_ele_path(self, xpath=True):
return ''
def _find_elements(self, locator, timeout, index=1, relative=False, raise_err=None):
pass
class BasePage(BaseParser):
def __init__(self):
self._url = None
self._url_available = None
self.retry_times = 3
self.retry_interval = 2
self._DrissionGet = None
self._download_path = None
self._none_ele_return_value = False
self._none_ele_value = None
self._session = None
self._headers = None
self._session_options = None
self._type = 'BasePage'
@property
def title(self):
ele = self._ele('xpath://title', raise_err=False, method='title')
return ele.text if ele else None
@property
def url_available(self):
return self._url_available
@property
def download_path(self):
return self._download_path
@property
def download(self):
if self._DrissionGet is None:
if not self._session:
self._create_session()
self._DrissionGet = DrissionGet(driver=self, save_path=self.download_path)
return self._DrissionGet
def _before_connect(self, url, retry, interval):
is_file = False
if isinstance(url, Path) or ('://' not in url and ':\\\\' not in url):
p = Path(url)
if p.exists():
url = str(p.absolute())
is_file = True
self._url = url if is_file else quote(url, safe='-_.~!*\'"();:@&=+$,/\\?#[]%')
retry = retry if retry is not None else self.retry_times
interval = interval if interval is not None else self.retry_interval
return retry, interval, is_file
def _set_session_options(self, session_or_options=None):
if not session_or_options:
self._session_options = SessionOptions(session_or_options)
elif isinstance(session_or_options, SessionOptions):
self._session_options = session_or_options
elif isinstance(session_or_options, Session):
self._session_options = SessionOptions()
self._session = copy(session_or_options)
self._headers = self._session.headers
self._session.headers = None
def _create_session(self):
if not self._session_options:
self._set_session_options()
self._session, self._headers = self._session_options.make_session()
# ----------------以下属性或方法由后代实现----------------
@property
def url(self):
return
@property
def json(self):
return
@property
def user_agent(self):
return
@abstractmethod
def get(self, url, show_errmsg=False, retry=None, interval=None):
pass
def _ele(self, locator, timeout=None, index=1, raise_err=None, method=None):
if not locator:
raise ElementNotFoundError(METHOD=method, ARGS={'locator': locator, 'index': index, 'timeout': timeout})
if timeout is None:
timeout = self.timeout
r = self._find_elements(locator, timeout=timeout, index=index, raise_err=raise_err)
if r or isinstance(r, list):
return r
if raise_err is True or (_S.raise_when_ele_not_found and raise_err is None):
raise ElementNotFoundError(METHOD=method, ARGS={'locator': locator, 'index': index, 'timeout': timeout})
r.method = method
r.args = {'locator': locator, 'index': index, 'timeout': timeout}
return r
@@ -0,0 +1,449 @@
# -*- coding:utf-8 -*-
"""
@Author : g1879
@Contact : g1879@qq.com
@Website : https://DrissionPage.cn
@Copyright: (c) 2020 by g1879, Inc. All Rights Reserved.
"""
from abc import abstractmethod
from typing import Union, Tuple, List, Any, Optional, Dict
from DrissionGet import DrissionGet
from requests import Session
from requests.structures import CaseInsensitiveDict
from .._configs.session_options import SessionOptions
from .._elements.chromium_element import ChromiumElement
from .._elements.none_element import NoneElement
from .._elements.session_element import SessionElement
from .._functions.elements import SessionElementsList
from .._pages.chromium_frame import ChromiumFrame
from .._pages.chromium_page import ChromiumPage
from .._pages.session_page import SessionPage
from .._pages.web_page import WebPage
class BaseParser(object):
"""所有页面、元素类的基类"""
_type: str
timeout: float
def __call__(self, locator: Union[Tuple[str, str], str], index: int = 1): ...
def ele(self,
locator: Union[Tuple[str, str], str, BaseElement],
index: int = 1,
timeout: float = None): ...
def eles(self, locator: Union[Tuple[str, str], str], timeout=None): ...
def find(self,
locators: Union[str, List[str], tuple],
any_one: bool = True,
first_ele: bool = True,
timeout: float = None) -> Union[Dict[str, ChromiumElement], Dict[str, SessionElement],
Dict[str, List[ChromiumElement]], Dict[str, List[SessionElement]], Tuple[str, SessionElement],
Tuple[str, ChromiumElement]]:
"""传入多个定位符,获取多个ele
:param locators: 定位符组成的列表
:param any_one: 是否任何一个定位符找到结果即返回
:param first_ele: 每个定位符是否只获取第一个元素
:param timeout: 超时时间(秒)
:return: any_one为True时,返回一个找到的元素定位符和对象组成的元组,格式:(loc, ele),全都没找到返回(None, None)
any_one为False时,返回dict格式,key为定位符,value为找到的元素或列表
"""
...
# ----------------以下属性或方法待后代实现----------------
@property
def html(self) -> str: ...
def s_ele(self,
locator: Union[Tuple[str, str], str, BaseElement, None] = None,
index: int = 1) -> SessionElement: ...
def s_eles(self, locator: Union[Tuple[str, str], str]) -> SessionElementsList: ...
def _ele(self,
locator: Union[Tuple[str, str], str],
timeout: float = None,
index: Optional[int] = 1,
raise_err: bool = None,
method: str = None): ...
def _find_elements(self,
locator: Union[Tuple[str, str], str],
timeout: float,
index: Optional[int] = 1,
relative: bool = False,
raise_err: bool = None): ...
class BaseElement(BaseParser):
"""各元素类的基类"""
owner: BasePage = ...
def __init__(self, owner: BasePage = None): ...
@property
def timeout(self) -> float:
"""返回其查找元素时超时时间"""
...
@property
def child_count(self) -> int:
"""返回直接子元素的个数"""
...
# ----------------以下属性或方法由后代实现----------------
@property
def tag(self) -> str: ...
def parent(self, level_or_loc: Union[tuple, str, int] = 1): ...
def prev(self, index: int = 1) -> None: ...
def prevs(self) -> None: ...
def next(self, index: int = 1): ...
def nexts(self): ...
def get_frame(self, loc_or_ind, timeout=None) -> ChromiumFrame:
"""获取元素中一个frame对象
:param loc_or_ind: 定位符、iframe序号,序号从1开始,可传入负数获取倒数第几个
:param timeout: 查找元素超时时间(秒)
:return: ChromiumFrame对象
"""
...
def _ele(self,
locator: Union[Tuple[str, str], str],
timeout: float = None,
index: Optional[int] = 1,
relative: bool = False,
raise_err: bool = None,
method: str = None):
"""调用获取元素的方法
:param locator: 定位符
:param timeout: 超时时间(秒)
:param index: 获取第几个,从1开始,可传入负数获取倒数第几个
:param relative: 是否相对定位
:param raise_err: 找不到时是否抛出异常
:param method: 调用的方法名
:return: 元素对象或它们组成的列表
"""
...
class DrissionElement(BaseElement):
"""ChromiumElement 和 SessionElement的基类,但不是ShadowRoot的基类"""
def __init__(self, owner: BasePage = None): ...
@property
def link(self) -> str:
"""返回href或src绝对url"""
...
@property
def css_path(self) -> str:
"""返回css path路径"""
...
@property
def xpath(self) -> str:
"""返回xpath路径"""
...
@property
def comments(self) -> list:
"""返回元素注释文本组成的列表"""
...
def texts(self, text_node_only: bool = False) -> list:
"""返回元素内所有直接子节点的文本,包括元素和文本节点
:param text_node_only: 是否只返回文本节点
:return: 文本列表
"""
...
def parent(self,
level_or_loc: Union[tuple, str, int] = 1,
index: int = 1,
timeout: float = None) -> Union[DrissionElement, None]:
"""返回上面某一级父元素,可指定层数或用查询语法定位
:param level_or_loc: 第几级父元素,1开始,或定位符
:param index: 当level_or_loc传入定位符,使用此参数选择第几个结果,1开始
:param timeout: 时间(秒)
:return: 上级元素对象
"""
...
def child(self,
locator: Union[Tuple[str, str], str, int] = '',
index: int = 1,
timeout: float = None,
ele_only: bool = True) -> Union[DrissionElement, str, NoneElement]:
"""返回直接子元素元素或节点组成的列表,可用查询语法筛选
:param locator: 用于筛选的查询语法
:param index: 第几个查询结果,1开始
:param timeout: 查找节点的超时时间(秒)
:param ele_only: 是否只获取元素,为False时把文本、注释节点也纳入
:return: 直接子元素或节点文本组成的列表
"""
...
def prev(self,
locator: Union[Tuple[str, str], str, int] = '',
index: int = 1,
timeout: float = None,
ele_only: bool = True) -> Union[DrissionElement, str, NoneElement]:
"""返回前面的一个兄弟元素,可用查询语法筛选,可指定返回筛选结果的第几个
:param locator: 用于筛选的查询语法
:param index: 前面第几个查询结果,1开始
:param timeout: 查找节点的超时时间(秒)
:param ele_only: 是否只获取元素,为False时把文本、注释节点也纳入
:return: 兄弟元素
"""
...
def next(self,
locator: Union[Tuple[str, str], str, int] = '',
index: int = 1,
timeout: float = None,
ele_only: bool = True) -> Union[DrissionElement, str, NoneElement]:
"""返回后面的一个兄弟元素,可用查询语法筛选,可指定返回筛选结果的第几个
:param locator: 用于筛选的查询语法
:param index: 后面第几个查询结果,1开始
:param timeout: 查找节点的超时时间(秒)
:param ele_only: 是否只获取元素,为False时把文本、注释节点也纳入
:return: 兄弟元素
"""
...
def before(self,
locator: Union[Tuple[str, str], str, int] = '',
index: int = 1,
timeout: float = None,
ele_only: bool = True) -> Union[DrissionElement, str, NoneElement]:
"""返回前面的一个兄弟元素,可用查询语法筛选,可指定返回筛选结果的第几个
:param locator: 用于筛选的查询语法
:param index: 前面第几个查询结果,1开始
:param timeout: 查找节点的超时时间(秒)
:param ele_only: 是否只获取元素,为False时把文本、注释节点也纳入
:return: 本元素前面的某个元素或节点
"""
...
def after(self,
locator: Union[Tuple[str, str], str, int] = '',
index: int = 1,
timeout: float = None,
ele_only: bool = True) -> Union[DrissionElement, str, NoneElement]:
"""返回后面的一个兄弟元素,可用查询语法筛选,可指定返回筛选结果的第几个
:param locator: 用于筛选的查询语法
:param index: 后面第几个查询结果,1开始
:param timeout: 查找节点的超时时间(秒)
:param ele_only: 是否只获取元素,为False时把文本、注释节点也纳入
:return: 本元素后面的某个元素或节点
"""
...
def children(self,
locator: Union[Tuple[str, str], str] = '',
timeout: float = None,
ele_only: bool = True) -> List[Union[DrissionElement, str]]:
"""返回直接子元素元素或节点组成的列表,可用查询语法筛选
:param locator: 用于筛选的查询语法
:param timeout: 查找节点的超时时间(秒)
:param ele_only: 是否只获取元素,为False时把文本、注释节点也纳入
:return: 直接子元素或节点文本组成的列表
"""
...
def prevs(self,
locator: Union[Tuple[str, str], str] = '',
timeout: float = None,
ele_only: bool = True) -> List[Union[DrissionElement, str]]:
"""返回前面全部兄弟元素或节点组成的列表,可用查询语法筛选
:param locator: 用于筛选的查询语法
:param timeout: 查找节点的超时时间(秒)
:param ele_only: 是否只获取元素,为False时把文本、注释节点也纳入
:return: 兄弟元素或节点文本组成的列表
"""
...
def nexts(self,
locator: Union[Tuple[str, str], str] = '',
timeout: float = None,
ele_only: bool = True) -> List[Union[DrissionElement, str]]:
"""返回后面全部兄弟元素或节点组成的列表,可用查询语法筛选
:param locator: 用于筛选的查询语法
:param timeout: 查找节点的超时时间(秒)
:param ele_only: 是否只获取元素,为False时把文本、注释节点也纳入
:return: 兄弟元素或节点文本组成的列表
"""
...
def befores(self,
locator: Union[Tuple[str, str], str] = '',
timeout: float = None,
ele_only: bool = True) -> List[Union[DrissionElement, str]]:
"""返回后面全部兄弟元素或节点组成的列表,可用查询语法筛选
:param locator: 用于筛选的查询语法
:param timeout: 查找节点的超时时间(秒)
:param ele_only: 是否只获取元素,为False时把文本、注释节点也纳入
:return: 本元素前面的元素或节点组成的列表
"""
...
def afters(self,
locator: Union[Tuple[str, str], str] = '',
timeout: float = None,
ele_only: bool = True) -> List[Union[DrissionElement, str]]:
"""返回前面全部兄弟元素或节点组成的列表,可用查询语法筛选
:param locator: 用于筛选的查询语法
:param timeout: 查找节点的超时时间(秒)
:param ele_only: 是否只获取元素,为False时把文本、注释节点也纳入
:return: 本元素后面的元素或节点组成的列表
"""
...
def _get_relative(self,
func: str,
direction: str,
brother: bool,
locator: Union[Tuple[str, str], str] = '',
index: int = 1,
timeout: float = None,
ele_only: bool = True) -> DrissionElement:
"""获取一个亲戚元素或节点,可用查询语法筛选,可指定返回筛选结果的第几个
:param func: 方法名称
:param direction: 方向,'following' 或 'preceding'
:param locator: 用于筛选的查询语法
:param index: 前面第几个查询结果,1开始
:param timeout: 查找节点的超时时间(秒)
:param ele_only: 是否只获取元素,为False时把文本、注释节点也纳入
:return: 本元素前面的某个元素或节点
"""
...
def _get_relatives(self,
index: int = None,
locator: Union[Tuple[str, str], str] = '',
direction: str = 'following',
brother: bool = True,
timeout: float = 0.5,
ele_only: bool = True) -> List[Union[DrissionElement, str]]:
"""按要求返回兄弟元素或节点组成的列表
:param index: 获取第几个,该参数不为None时只获取该编号的元素
:param locator: 用于筛选的查询语法
:param direction: 'following' 或 'preceding',查找的方向
:param brother: 查找范围,在同级查找还是整个dom前后查找
:param timeout: 查找等待时间(秒)
:return: 元素对象或字符串
"""
...
# ----------------以下属性或方法由后代实现----------------
@property
def attrs(self) -> dict: ...
@property
def text(self) -> str: ...
@property
def raw_text(self) -> str: ...
@abstractmethod
def attr(self, name: str) -> str: ...
def _get_ele_path(self, xpath: bool = True) -> str: ...
class BasePage(BaseParser):
"""页面类的基类"""
_url_available: Optional[bool] = ...
retry_times: int = ...
retry_interval: float = ...
_download_path: Optional[str] = ...
_DrissionGet: Optional[DrissionGet] = ...
_none_ele_return_value: bool = ...
_none_ele_value: Any = ...
_page: Union[ChromiumPage, SessionPage, WebPage] = ...
_session: Optional[Session] = ...
_headers: Optional[CaseInsensitiveDict] = ...
_session_options: Optional[SessionOptions] = ...
def __init__(self): ...
@property
def title(self) -> Union[str, None]:
"""返回网页title"""
...
@property
def url_available(self) -> bool:
"""返回当前访问的url有效性"""
...
@property
def download_path(self) -> str:
"""返回默认下载路径"""
...
@property
def download(self) -> DrissionGet:
"""返回下载器对象"""
...
def _before_connect(self, url: str, retry: int, interval: float) -> tuple:
"""连接前的准备
:param url: 要访问的url
:param retry: 重试次数
:param interval: 重试间隔
:return: 重试次数、间隔、是否文件组成的tuple
"""
...
def _set_session_options(self, session_or_options: Union[Session, SessionOptions] = None) -> None:
"""启动配置
:param session_or_options: Session、SessionOptions对象
:return: None
"""
...
def _create_session(self) -> None:
"""创建内建Session对象"""
...
# ----------------以下属性或方法由后代实现----------------
@property
def url(self) -> str: ...
@property
def json(self) -> dict: ...
@property
def user_agent(self) -> str: ...
@abstractmethod
def get(self, url: str, show_errmsg: bool = False, retry: int = None, interval: float = None): ...
def _ele(self,
locator,
timeout: float = None,
index: Optional[int] = 1,
raise_err: bool = None,
method: str = None):
"""调用获取元素的方法
:param locator: 定位符
:param timeout: 超时时间(秒)
:param index: 获取第几个,从1开始,可传入负数获取倒数第几个
:param raise_err: 找不到时是否抛出异常
:param method: 调用的方法名
:return: 元素对象或它们组成的列表
"""
...
@@ -0,0 +1,537 @@
# -*- coding:utf-8 -*-
"""
@Author : g1879
@Contact : g1879@qq.com
@Website : https://DrissionPage.cn
@Copyright: (c) 2020 by g1879, Inc. All Rights Reserved.
"""
from pathlib import Path
from re import match
from shutil import rmtree
from threading import Lock
from time import sleep, perf_counter
from requests import Session
from websocket import WebSocketBadStatusException
from .driver import BrowserDriver, Driver
from .._configs.chromium_options import ChromiumOptions
from .._functions.browser import connect_browser
from .._functions.cookies import CookiesList
from .._functions.settings import Settings as _S
from .._functions.tools import PortFinder, raise_error
from .._pages.chromium_base import Timeout
from .._pages.chromium_tab import ChromiumTab
from .._pages.mix_tab import MixTab
from .._units.downloader import DownloadManager
from .._units.setter import BrowserSetter
from .._units.states import BrowserStates
from .._units.waiter import BrowserWaiter
from ..errors import BrowserConnectError, CDPError, PageDisconnectedError, IncorrectURLError
__ERROR__ = 'error'
class Chromium(object):
_BROWSERS = {}
_lock = Lock()
def __new__(cls, addr_or_opts=None, session_options=None):
opt = handle_options(addr_or_opts)
is_headless, browser_id, is_exists, ws_only = run_browser(opt)
with cls._lock:
if browser_id in cls._BROWSERS:
r = cls._BROWSERS[browser_id]
while not hasattr(r, '_driver'):
sleep(.05)
return r
r = object.__new__(cls)
r._chromium_options = opt
r._is_headless = is_headless
r._is_exists = is_exists
r._ws_only = ws_only
r.id = browser_id
cls._BROWSERS[browser_id] = r
return r
def __init__(self, addr_or_opts=None, session_options=None):
if hasattr(self, '_created'):
return
self._created = True
self._type = 'Chromium'
self._frames = {}
self._drivers = {}
self._all_drivers = {}
self._relation = {}
self._newest_tab_id = None
self._set = None
self._wait = None
self._states = None
self._timeouts = Timeout(**self._chromium_options.timeouts)
self._load_mode = self._chromium_options.load_mode
self._download_path = str(Path(self._chromium_options.download_path).absolute())
self._auto_handle_alert = None
self._none_ele_return_value = False
self._none_ele_value = None
self.retry_times = self._chromium_options.retry_times
self.retry_interval = self._chromium_options.retry_interval
self.address = self._chromium_options.address
self._ws_address = (self._chromium_options.ws_address if self._chromium_options.ws_address
else f'ws://{self.address}/devtools/browser/{self.id}')
self._disconnect_flag = False
self._driver = BrowserDriver(self.id, self._ws_address, self)
if (not self._ws_only
and (not self._chromium_options._ua_set and self._is_headless != self._chromium_options.is_headless)
or (self._is_exists and self._chromium_options._new_env)):
self.quit(3, True)
connect_browser(self._chromium_options)
s = Session()
s.trust_env = False
s.keep_alive = False
ws = s.get(f'http://{self.address}/json/version', headers={'Connection': 'close'})
self.id = ws.json()['webSocketDebuggerUrl'].split('/')[-1]
self._driver = BrowserDriver(self.id, self._ws_address, self)
ws.close()
s.close()
self._is_exists = False
self._frames = {}
self._drivers = {}
self._all_drivers = {}
self.version = self._run_cdp('Browser.getVersion')['product']
self._process_id = None
try:
r = self._run_cdp('SystemInfo.getProcessInfo')
for i in r.get('processInfo', []):
if i['type'] == 'browser':
self._process_id = i['id']
break
except:
pass
self._run_cdp('Target.setDiscoverTargets', discover=True)
self._driver.set_callback('Target.targetDestroyed', self._onTargetDestroyed)
self._driver.set_callback('Target.targetCreated', self._onTargetCreated)
self._dl_mgr = DownloadManager(self)
self._session_options = session_options
@property
def user_data_path(self):
return self._chromium_options.user_data_path
@property
def process_id(self):
return self._process_id
@property
def timeout(self):
return self._timeouts.base
@property
def timeouts(self):
return self._timeouts
@property
def load_mode(self):
return self._load_mode
@property
def download_path(self):
return self._download_path
@property
def set(self):
if self._set is None:
self._set = BrowserSetter(self)
return self._set
@property
def states(self):
if self._states is None:
self._states = BrowserStates(self)
return self._states
@property
def wait(self):
if self._wait is None:
self._wait = BrowserWaiter(self)
return self._wait
@property
def tabs_count(self):
j = self._run_cdp('Target.getTargets')['targetInfos'] # 不要改用get,避免卡死
return len([i for i in j if i['type'] in ('page', 'webview') and not i['url'].startswith('devtools://')])
@property
def tab_ids(self):
if self._ws_only:
return [i['targetId'] for i in self._run_cdp('Target.getTargets')['targetInfos']
if i['type'] in ('page', 'webview') and not i['url'].startswith('devtools://')]
else:
return [i['id'] for i in self._driver.get(f'http://{self.address}/json').json()
if i['type'] in ('page', 'webview') and not i['url'].startswith('devtools://')]
@property
def latest_tab(self):
return self._get_tab(id_or_num=self.tab_ids[0], as_id=not _S.singleton_tab_obj)
def cookies(self, all_info=False):
cks = self._run_cdp(f'Storage.getCookies')['cookies']
r = cks if all_info else [{'name': c['name'], 'value': c['value'], 'domain': c['domain']} for c in cks]
return CookiesList(r)
def new_tab(self, url=None, new_window=False, background=False, new_context=False):
return self._new_tab(True, url=url, new_window=new_window, background=background, new_context=new_context)
def get_tab(self, id_or_num=None, title=None, url=None, tab_type='page'):
t = self._get_tab(id_or_num=id_or_num, title=title, url=url, tab_type=tab_type, mix=True, as_id=False)
if t._type != 'MixTab':
raise RuntimeError(_S._lang.join(_S._lang.TAB_OBJ_EXISTS))
return t
def get_tabs(self, title=None, url=None, tab_type='page'):
return self._get_tabs(title=title, url=url, tab_type=tab_type, mix=True, as_id=False)
def close_tabs(self, tabs_or_ids, others=False):
if isinstance(tabs_or_ids, str):
tabs = {tabs_or_ids}
elif isinstance(tabs_or_ids, ChromiumTab):
tabs = {tabs_or_ids.tab_id}
elif isinstance(tabs_or_ids, (list, tuple)):
tabs = set(i.tab_id if isinstance(i, ChromiumTab) else i for i in tabs_or_ids)
else:
raise ValueError(_S._lang.join(_S._lang.INCORRECT_TYPE_, 'tabs_or_ids',
ALLOW_TYPE=_S._lang.TAB_OR_ID, CURR_VAL=tabs_or_ids))
all_tabs = set(self.tab_ids)
if others:
tabs = all_tabs - tabs
if len(all_tabs - tabs) > 0:
for tab in tabs:
self._close_tab(tab=tab)
else:
self.quit()
def _close_tab(self, tab):
if isinstance(tab, str):
tab = self.get_tab(tab)
tab._run_cdp('Target.closeTarget', targetId=tab.tab_id)
while tab.driver.is_running and tab.tab_id in self._all_drivers:
sleep(.01)
def activate_tab(self, id_ind_tab):
if isinstance(id_ind_tab, int):
id_ind_tab += -1 if id_ind_tab else 1
id_ind_tab = self.tab_ids[id_ind_tab]
elif isinstance(id_ind_tab, ChromiumTab):
id_ind_tab = id_ind_tab.tab_id
self._run_cdp('Target.activateTarget', targetId=id_ind_tab)
def reconnect(self):
self._disconnect_flag = True
self._driver.stop()
BrowserDriver.BROWSERS.pop(self.id)
self._driver = BrowserDriver(self.id, self._ws_address, self)
self._run_cdp('Target.setDiscoverTargets', discover=True)
self._driver.set_callback('Target.targetDestroyed', self._onTargetDestroyed)
self._driver.set_callback('Target.targetCreated', self._onTargetCreated)
self._disconnect_flag = False
def clear_cache(self, cache=True, cookies=True):
if cache:
self.latest_tab.run_cdp('Network.clearBrowserCache')
if cookies:
self._run_cdp('Storage.clearCookies')
def quit(self, timeout=5, force=False, del_data=False):
try:
self._run_cdp('Browser.close')
except PageDisconnectedError:
pass
self._driver.stop()
drivers = list(self._all_drivers.values())
for tab in drivers:
for driver in tab:
driver.stop()
if not self.address.startswith('127.0.0.1'):
return
if force:
pids = None
try:
pids = [pid['id'] for pid in self._run_cdp('SystemInfo.getProcessInfo')['processInfo']]
except:
pass
if pids:
from psutil import Process
for pid in pids:
try:
Process(pid).kill()
except:
pass
from os import popen
from platform import system
end_time = perf_counter() + timeout
while perf_counter() < end_time:
ok = True
for pid in pids:
txt = f'tasklist | findstr {pid}' if system().lower() == 'windows' else f'ps -ef | grep {pid}'
p = popen(txt)
sleep(.05)
try:
if f' {pid} ' in p.read():
ok = False
break
except TypeError:
pass
if ok:
break
if del_data and not self._chromium_options.is_auto_port and self._chromium_options.user_data_path:
path = Path(self._chromium_options.user_data_path)
rmtree(path, True)
def _new_tab(self, mix=True, url=None, new_window=False, background=False, new_context=False):
tab_type = MixTab if mix else ChromiumTab
tab = None
if new_context:
tab = self._run_cdp('Target.createBrowserContext')['browserContextId']
kwargs = {'url': ''}
if new_window:
kwargs['newWindow'] = True
if background:
kwargs['background'] = True
if tab:
kwargs['browserContextId'] = tab
if self.states.is_incognito and not new_context:
return _new_tab_by_js(self, url, tab_type, new_window)
else:
try:
tab = self._run_cdp('Target.createTarget', **kwargs)['targetId']
except CDPError:
return _new_tab_by_js(self, url, tab_type, new_window)
while self.states.is_alive:
if tab in self._drivers:
break
sleep(.01)
else:
raise BrowserConnectError(_S._lang.BROWSER_DISCONNECTED)
tab = tab_type(self, tab)
if url:
tab.get(url)
return tab
def _get_tab(self, id_or_num=None, title=None, url=None, tab_type='page', mix=True, as_id=False):
if id_or_num is not None:
if isinstance(id_or_num, int):
id_or_num = self.tab_ids[id_or_num - 1 if id_or_num > 0 else id_or_num]
elif isinstance(id_or_num, ChromiumTab):
return id_or_num.tab_id if as_id else ChromiumTab(self, id_or_num.tab_id)
else:
j = self._run_cdp('Target.getTargets')['targetInfos']
if id_or_num not in [i['targetId'] for i in j]:
raise RuntimeError(_S._lang.join(_S._lang.NO_SUCH_TAB, ARG=id_or_num, ALL_TABS=self.tab_ids))
elif title == url is None and tab_type == 'page':
id_or_num = self.tab_ids[0]
else:
tabs = self._get_tabs(title=title, url=url, tab_type=tab_type, as_id=True)
if tabs:
id_or_num = tabs[0]
else:
raise RuntimeError(_S._lang.join(_S._lang.NO_SUCH_TAB,
ARGS={'id_or_num': id_or_num, 'title': title, 'url': url,
'tab_type': tab_type}))
if as_id:
return id_or_num
with self._lock:
return MixTab(self, id_or_num) if mix else ChromiumTab(self, id_or_num)
def _get_tabs(self, title=None, url=None, tab_type='page', mix=True, as_id=False):
if self._ws_only:
tabs = self._run_cdp('Target.getTargets')['targetInfos']
_id = 'targetId'
else:
tabs = self._driver.get(f'http://{self.address}/json').json() # 不要改用cdp
_id = 'id'
if isinstance(tab_type, str):
tab_type = {tab_type}
elif isinstance(tab_type, (list, tuple, set)):
tab_type = set(tab_type)
elif tab_type is not None:
raise ValueError(_S._lang.join(_S._lang.INCORRECT_TYPE_, 'tab_type',
ALLOW_TYPE='set, list, tuple, str, None', CURR_VAL=tab_type))
tabs = [i for i in tabs if ((title is None or title in i['title']) and (url is None or url in i['url'])
and (tab_type is None or i['type'] in tab_type)
and i['title'] != 'chrome-extension://neajdppkdcdipfabeoofebfddakdcjhd/audio.html')]
if as_id:
return [tab[_id] for tab in tabs]
with self._lock:
if mix:
return [MixTab(self, tab[_id]) for tab in tabs]
else:
return [ChromiumTab(self, tab[_id]) for tab in tabs]
def _run_cdp(self, cmd, **cmd_args):
ignore = cmd_args.pop('_ignore', None)
r = self._driver.run(cmd, **cmd_args)
return r if __ERROR__ not in r else raise_error(r, self, ignore)
def _get_driver(self, tab_id, owner=None):
d = self._drivers.pop(tab_id, None)
if not d:
if self._ws_only:
d = Driver(tab_id, self._ws_address)
d.session_id = d.run('Target.attachToTarget', targetId=tab_id, flatten=True)['sessionId']
else:
d = Driver(tab_id, f'ws://{self.address}/devtools/page/{tab_id}')
d.owner = owner
self._all_drivers.setdefault(tab_id, set()).add(d)
return d
def _onTargetCreated(self, **kwargs):
if (kwargs['targetInfo']['type'] in ('page', 'webview')
and kwargs['targetInfo']['targetId'] not in self._all_drivers
and not kwargs['targetInfo']['url'].startswith('devtools://')):
try:
tab_id = kwargs['targetInfo']['targetId']
self._frames[tab_id] = tab_id
if self._ws_only:
d = Driver(tab_id, self._ws_address)
d.session_id = d.run('Target.attachToTarget', targetId=tab_id, flatten=True)['sessionId']
else:
d = Driver(tab_id, f'ws://{self.address}/devtools/page/{tab_id}')
self._relation[tab_id] = kwargs['targetInfo'].get('openerId', None)
self._drivers[tab_id] = d
self._all_drivers.setdefault(tab_id, set()).add(d)
self._newest_tab_id = tab_id
except WebSocketBadStatusException:
pass
def _onTargetDestroyed(self, **kwargs):
tab_id = kwargs['targetId']
self._dl_mgr.clear_tab_info(tab_id)
for key in [k for k, i in self._frames.items() if i == tab_id]:
self._frames.pop(key, None)
for d in self._all_drivers.get(tab_id, tuple()):
d.stop()
self._drivers.pop(tab_id, None)
self._all_drivers.pop(tab_id, None)
self._relation.pop(tab_id, None)
def _on_disconnect(self):
if not self._disconnect_flag:
Chromium._BROWSERS.pop(self.id, None)
if self._chromium_options.is_auto_port and self._chromium_options.user_data_path:
path = Path(self._chromium_options.user_data_path)
end_time = perf_counter() + 7
while perf_counter() < end_time:
if not path.exists():
break
try:
rmtree(path)
break
except (PermissionError, FileNotFoundError, OSError):
pass
sleep(.03)
def handle_options(addr_or_opts):
if not addr_or_opts:
_chromium_options = ChromiumOptions(addr_or_opts)
if _chromium_options.is_auto_port:
port, path = PortFinder(_chromium_options.tmp_path).get_port(_chromium_options.is_auto_port)
_chromium_options.set_address(f'127.0.0.1:{port}')
_chromium_options.set_user_data_path(path)
_chromium_options.auto_port(scope=_chromium_options.is_auto_port)
elif isinstance(addr_or_opts, ChromiumOptions):
if addr_or_opts.is_auto_port:
port, path = PortFinder(addr_or_opts.tmp_path).get_port(addr_or_opts.is_auto_port)
addr_or_opts._address = f'127.0.0.1:{port}'
addr_or_opts.set_user_data_path(path)
_chromium_options = addr_or_opts
elif isinstance(addr_or_opts, str) and ':' in addr_or_opts:
_chromium_options = ChromiumOptions()
_chromium_options.set_address(addr_or_opts)
elif isinstance(addr_or_opts, int):
_chromium_options = ChromiumOptions()
_chromium_options.set_local_port(addr_or_opts)
else:
raise ValueError(_S._lang.join(_S._lang.INCORRECT_VAL_, 'addr_or_opts',
ALLOW_TYPE=_S._lang.IP_OR_OPTIONS, CURR_VAL=addr_or_opts))
return _chromium_options
def run_browser(chromium_options):
if chromium_options.ws_address:
d = Driver('', chromium_options.ws_address)
browser_id = d.run('Target.getTargets')
if 'error' in browser_id:
raise BrowserConnectError(_S._lang.join(_S._lang.BROWSER_CONNECT_ERR2, INFO=browser_id['error']))
browser_id = browser_id['targetInfos'][0]['browserContextId']
is_headless = 'headless' in d.run('Browser.getVersion')['userAgent'].lower()
is_exists = True
d.stop()
if 'devtools/browser' not in chromium_options.ws_address:
return is_headless, browser_id, is_exists, True
else:
is_exists = connect_browser(chromium_options)
try:
s = Session()
s.trust_env = False
s.keep_alive = False
ws = s.get(f'http://{chromium_options.address}/json/version', headers={'Connection': 'close'}, timeout=2)
if not ws and not chromium_options.ws_address:
raise BrowserConnectError(_S._lang.BROWSER_CONNECT_ERR2)
json = ws.json()
if not chromium_options.ws_address:
browser_id = json['webSocketDebuggerUrl'].split('/')[-1]
is_headless = 'headless' in json['User-Agent'].lower()
ws.close()
s.close()
ws_only = False
except KeyError:
raise BrowserConnectError(_S._lang.BROWSER_NOT_FOR_CONTROL)
except:
if chromium_options.ws_address:
ws_only = True
else:
raise BrowserConnectError(_S._lang.BROWSER_CONNECT_ERR2)
return is_headless, browser_id, is_exists, ws_only
def _new_tab_by_js(browser: Chromium, url, tab_type, new_window):
mix = tab_type == MixTab
tab = browser._get_tab(mix=mix)
if url and not match(r'^.*?://.*', url):
raise IncorrectURLError(_S._lang.INVALID_URL, url=url)
url = f'"{url}"' if url else '""'
new = 'target="_new"' if new_window else 'target="_blank"'
tid = browser._newest_tab_id
tab.run_js(f'window.open({url}, {new})')
tid = browser.wait.new_tab(curr_tab=tid)
return browser._get_tab(tid, mix=mix)
@@ -0,0 +1,319 @@
# -*- coding:utf-8 -*-
"""
@Author : g1879
@Contact : g1879@qq.com
@Website : https://DrissionPage.cn
@Copyright: (c) 2020 by g1879, Inc. All Rights Reserved.
"""
from threading import Lock
from typing import List, Optional, Set, Dict, Union, Tuple, Literal, Any
from .driver import BrowserDriver, Driver
from .._configs.chromium_options import ChromiumOptions
from .._configs.session_options import SessionOptions
from .._functions.cookies import CookiesList
from .._pages.chromium_base import Timeout, ChromiumBase
from .._pages.chromium_tab import ChromiumTab
from .._pages.mix_tab import MixTab
from .._units.downloader import DownloadManager
from .._units.setter import BrowserSetter
from .._units.states import BrowserStates
from .._units.waiter import BrowserWaiter
class Chromium(object):
_BROWSERS: dict = ...
_lock: Lock = ...
id: str = ...
address: str = ...
_ws_address: str = ...
version: str = ...
retry_times: int = ...
retry_interval: float = ...
_set: Optional[BrowserSetter] = ...
_wait: Optional[BrowserWaiter] = ...
_states: Optional[BrowserStates] = ...
_chromium_options: ChromiumOptions = ...
_session_options: SessionOptions = ...
_driver: BrowserDriver = ...
_frames: dict = ...
_drivers: Dict[str, Driver] = ...
_all_drivers: Dict[str, Set[Driver]] = ...
_relation: Dict[str, Optional[str]] = ...
_process_id: Optional[int] = ...
_dl_mgr: DownloadManager = ...
_timeouts: Timeout = ...
_load_mode: str = ...
_download_path: str = ...
_auto_handle_alert: Optional[bool] = ...
_is_exists: bool = ...
_is_headless: bool = ...
_ws_only: bool = ...
_disconnect_flag: bool = ...
_none_ele_return_value: bool = ...
_none_ele_value: Any = ...
_newest_tab_id: Optional[str] = ...
def __new__(cls,
addr_or_opts: Union[str, int, ChromiumOptions] = None,
session_options: Union[SessionOptions, None, False] = None):
"""
:param addr_or_opts: 浏览器地址:端口、ws地址、ChromiumOptions对象或端口数字(int)
:param session_options: 使用双模Tab时使用的默认Session配置,为None使用ini文件配置,为False不从ini读取
"""
...
def __init__(self, addr_or_opts: Union[str, int, ChromiumOptions] = None,
session_options: Union[SessionOptions, None, False] = None):
"""
:param addr_or_opts: 浏览器地址:端口、ws地址、ChromiumOptions对象或端口数字(int)
:param session_options: 使用双模Tab时使用的默认Session配置,为None使用ini文件配置,为False不从ini读取
"""
...
@property
def user_data_path(self) -> str:
"""返回用户文件夹路径"""
...
@property
def process_id(self) -> Optional[int]:
"""返回浏览器进程id"""
...
@property
def timeout(self) -> float:
"""返回基础超时设置"""
...
@property
def timeouts(self) -> Timeout:
"""返回所有超时设置"""
...
@property
def load_mode(self) -> Literal['none', 'normal', 'eager']:
"""返回页面加载模式,包括 'none', 'normal', 'eager' 三种"""
...
@property
def download_path(self) -> str:
"""返回默认下载路径"""
...
@property
def set(self) -> BrowserSetter:
"""返回用于设置的对象"""
...
@property
def states(self) -> BrowserStates:
"""返回用于获取状态的对象"""
...
@property
def wait(self) -> BrowserWaiter:
"""返回用于等待的对象"""
...
@property
def tabs_count(self) -> int:
"""返回标签页数量,只统计page、webview类型"""
...
@property
def tab_ids(self) -> List[str]:
"""返回所有标签页id组成的列表,只统计page、webview类型"""
...
@property
def latest_tab(self) -> Union[MixTab, str]:
"""返回最新的标签页,最新标签页指最后创建或最后被激活的
当Settings.singleton_tab_obj==True时返回Tab对象,否则返回tab id"""
...
def cookies(self, all_info: bool = False) -> CookiesList:
"""以list格式返回所有域名的cookies
:param all_info: 是否返回所有内容,False则只返回name, value, domain
:return: cookies组成的列表
"""
...
def new_tab(self,
url: str = None,
new_window: bool = False,
background: bool = False,
new_context: bool = False) -> MixTab:
"""新建一个标签页
:param url: 新标签页跳转到的网址,为None时新建空标签页
:param new_window: 是否在新窗口打开标签页,隐身模式下无效
:param background: 是否不激活新标签页,隐身模式和访客模式及new_window为True时无效
:param new_context: 是否创建独立环境,隐身模式和访客模式下无效
:return: 新标签页对象
"""
...
def get_tab(self,
id_or_num: Union[str, int] = None,
title: str = None,
url: str = None,
tab_type: Union[str, list, tuple] = 'page',
as_id: bool = False) -> Union[MixTab, str]:
"""获取一个标签页对象,id_or_num不为None时,后面几个参数无效
:param id_or_num: 要获取的标签页id或序号,序号从1开始,可传入负数获取倒数第几个,不是视觉排列顺序,而是激活顺序
:param title: 要匹配title的文本,模糊匹配,为None则匹配所有
:param url: 要匹配url的文本,模糊匹配,为None则匹配所有
:param tab_type: tab类型,可用列表输入多个,如 'page', 'iframe' 等,为None则匹配所有
:param as_id: 是否返回标签页id而不是标签页对象
:return: Tab对象
"""
...
def get_tabs(self,
title: str = None,
url: str = None,
tab_type: Union[str, list, tuple] = 'page',
as_id: bool = False) -> List[MixTab, str]:
"""查找符合条件的tab,返回它们组成的列表,title和url是与关系
:param title: 要匹配title的文本
:param url: 要匹配url的文本
:param tab_type: tab类型,可用列表输入多个
:param as_id: 是否返回标签页id而不是标签页对象
:return: Tab对象列表
"""
...
def close_tabs(self,
tabs_or_ids: Union[str, ChromiumTab, List[Union[str, ChromiumTab]],
Tuple[Union[str, ChromiumTab]]],
others: bool = False) -> None:
"""关闭传入的标签页,可传入多个
:param tabs_or_ids: 指定的标签页对象或id,可用列表或元组传入多个
:param others: 是否关闭指定标签页之外的
:return: None
"""
...
def _close_tab(self, tab: Union[ChromiumBase, str]):
"""关闭一个标签页
:param tab: 标签页对象或id
:return: None
"""
def activate_tab(self, id_ind_tab: Union[int, str, ChromiumTab]) -> None:
"""使一个标签页显示到前端
:param id_ind_tab: 标签页id(str)、Tab对象或标签页序号(int),序号从1开始
:return: None
"""
...
def reconnect(self) -> None:
"""断开重连"""
...
def clear_cache(self, cache: bool = True, cookies: bool = True) -> None:
"""清除缓存,可选要清除的项
:param cache: 是否清除cache
:param cookies: 是否清除cookies
:return: None
"""
...
def quit(self, timeout: float = 5, force: bool = False, del_data: bool = False) -> None:
"""关闭浏览器
:param timeout: 等待浏览器关闭超时时间(秒)
:param force: 是否立刻强制终止进程
:param del_data: 是否删除用户文件夹
:return: None
"""
...
def _new_tab(self,
mix: bool = True,
url: str = None,
new_window: bool = False,
background: bool = False,
new_context: bool = False) -> Union[ChromiumTab, MixTab]:
"""新建一个标签页
:param mix: 是否创建MixTab
:param url: 新标签页跳转到的网址
:param new_window: 是否在新窗口打开标签页
:param background: 是否不激活新标签页,如new_window为True则无效
:param new_context: 是否创建新的上下文
:return: 新标签页对象
"""
...
def _get_tab(self,
id_or_num: Union[str, int] = None,
title: str = None,
url: str = None,
tab_type: Union[str, list, tuple] = 'page',
mix: bool = True,
as_id: bool = False) -> Union[ChromiumTab, str]:
"""获取一个标签页对象,id_or_num不为None时,后面几个参数无效
:param id_or_num: 要获取的标签页id或序号,序号从1开始,可传入负数获取倒数第几个,不是视觉排列顺序,而是激活顺序
:param title: 要匹配title的文本,模糊匹配,为None则匹配所有
:param url: 要匹配url的文本,模糊匹配,为None则匹配所有
:param tab_type: tab类型,可用列表输入多个,如 'page', 'iframe' 等,为None则匹配所有
:param mix: 是否返回可切换模式的Tab对象
:param as_id: 是否返回标签页id而不是标签页对象,mix=False时无效
:return: Tab对象
"""
...
def _get_tabs(self,
title: str = None,
url: str = None,
tab_type: Union[str, list, tuple] = 'page',
mix: bool = True,
as_id: bool = False) -> List[ChromiumTab, str]:
"""查找符合条件的tab,返回它们组成的列表,title和url是与关系
:param title: 要匹配title的文本
:param url: 要匹配url的文本
:param tab_type: tab类型,可用列表输入多个
:param mix: 是否返回可切换模式的Tab对象
:param as_id: 是否返回标签页id而不是标签页对象,mix=False时无效
:return: Tab对象列表
"""
...
def _run_cdp(self, cmd, **cmd_args) -> dict:
"""执行Chrome DevTools Protocol语句
:param cmd: 协议项目
:param cmd_args: 参数
:return: 执行的结果
"""
...
def _get_driver(self, tab_id: str, owner=None) -> Driver:
"""新建并返回指定tab id的Driver
:param tab_id: 标签页id
:param owner: 使用该驱动的对象
:return: Driver对象
"""
...
def _onTargetCreated(self, **kwargs): ...
def _onTargetDestroyed(self, **kwargs): ...
def _on_disconnect(self): ...
def handle_options(addr_or_opts):
"""设置浏览器启动属性
:param addr_or_opts: 'ip:port'、ChromiumOptions、Driver
:return: 返回ChromiumOptions对象
"""
...
def run_browser(chromium_options)->Tuple[bool, str, bool, bool]:
"""连接浏览器
:param chromium_options: ChromiumOptions对象
:return: 返回(是否无头, 浏览器id, 是否接管已存在, 是否只能用ws连接)
"""
...
@@ -0,0 +1,229 @@
# -*- coding: utf-8 -*-
"""
@Author : g1879
@Contact : g1879@qq.com
@Website : https://DrissionPage.cn
@Copyright: (c) 2020 by g1879, Inc. All Rights Reserved.
"""
from json import dumps, loads, JSONDecodeError
from queue import Queue, Empty
from threading import Thread
from time import perf_counter, sleep
from requests import Session
from requests import adapters
from websocket import (WebSocketTimeoutException, WebSocketConnectionClosedException, create_connection,
WebSocketException, WebSocketBadStatusException)
from .._functions.settings import Settings as _S
from ..errors import PageDisconnectedError, BrowserConnectError
adapters.DEFAULT_RETRIES = 5
class Driver(object):
def __init__(self, _id, address, owner=None):
self.id = _id
self.address = address
self.owner = owner
self.alert_flag = False # 标记alert出现,跳过一条请求后复原
self._cur_id = 0
self._ws = None
self._recv_th = Thread(target=self._recv_loop)
self._handle_event_th = Thread(target=self._handle_event_loop)
self._recv_th.daemon = True
self._handle_event_th.daemon = True
self._handle_immediate_event_th = None
self.is_running = False
self.session_id = None
self.event_handlers = {}
self.immediate_event_handlers = {}
self.method_results = {}
self.event_queue = Queue()
self.immediate_event_queue = Queue()
self.start()
def _send(self, message, timeout=None):
self._cur_id += 1
ws_id = self._cur_id
message['id'] = ws_id
message_json = dumps(message)
end_time = perf_counter() + timeout if timeout is not None else None
self.method_results[ws_id] = Queue()
try:
self._ws.send(message_json)
if timeout == 0:
self.method_results.pop(ws_id, None)
return {'id': ws_id, 'result': {}}
except (OSError, WebSocketConnectionClosedException):
self.method_results.pop(ws_id, None)
return {'error': {'message': 'connection disconnected'}, 'type': 'connection_error'}
while self.is_running:
try:
result = self.method_results[ws_id].get(timeout=.2)
self.method_results.pop(ws_id, None)
return result
except Empty:
if self.alert_flag and message['method'].startswith(('Input.', 'Runtime.')):
return {'error': {'message': 'alert exists.'}, 'type': 'alert_exists'}
if timeout is not None and perf_counter() > end_time:
self.method_results.pop(ws_id, None)
return {'error': {'message': 'alert exists.'}, 'type': 'alert_exists'} \
if self.alert_flag else {'error': {'message': 'timeout'}, 'type': 'timeout'}
continue
return {'error': {'message': 'connection disconnected'}, 'type': 'connection_error'}
def _recv_loop(self):
while self.is_running:
try:
# self._ws.settimeout(1)
msg_json = self._ws.recv()
msg = loads(msg_json)
except WebSocketTimeoutException:
continue
except (WebSocketException, OSError, WebSocketConnectionClosedException, JSONDecodeError):
self._stop()
return
if 'method' in msg:
if msg['method'].startswith('Page.javascriptDialog'):
self.alert_flag = msg['method'].endswith('Opening')
function = self.immediate_event_handlers.get(msg['method'])
if function:
self._handle_immediate_event(function, msg['params'])
else:
self.event_queue.put(msg)
elif msg.get('id') in self.method_results:
self.method_results[msg['id']].put(msg)
def _handle_event_loop(self):
while self.is_running:
try:
event = self.event_queue.get(timeout=1)
except Empty:
continue
function = self.event_handlers.get(event['method'])
if function:
function(**event['params'])
self.event_queue.task_done()
def _handle_immediate_event_loop(self):
while not self.immediate_event_queue.empty():
function, kwargs = self.immediate_event_queue.get(timeout=1)
try:
function(**kwargs)
except PageDisconnectedError:
pass
def _handle_immediate_event(self, function, kwargs):
self.immediate_event_queue.put((function, kwargs))
if self._handle_immediate_event_th is None or not self._handle_immediate_event_th.is_alive():
self._handle_immediate_event_th = Thread(target=self._handle_immediate_event_loop)
self._handle_immediate_event_th.daemon = True
self._handle_immediate_event_th.start()
def run(self, _method, sessionId=None, **kwargs):
if not self.is_running:
return {'error': 'connection disconnected', 'type': 'connection_error'}
timeout = kwargs.pop('_timeout', _S.cdp_timeout)
if self.session_id:
result = self._send({'method': _method, 'params': kwargs, 'sessionId': self.session_id}, timeout=timeout)
elif sessionId:
result = self._send({'method': _method, 'params': kwargs, 'sessionId': sessionId}, timeout=timeout)
else:
result = self._send({'method': _method, 'params': kwargs}, timeout=timeout)
if 'result' not in result and 'error' in result:
kwargs['_timeout'] = timeout
return {'error': result['error']['message'], 'type': result.get('type', 'call_method_error'),
'method': _method, 'args': kwargs, 'data': result['error'].get('data')}
else:
return result['result']
def start(self):
self.is_running = True
try:
self._ws = create_connection(self.address, enable_multithread=True, suppress_origin=True)
except WebSocketBadStatusException as e:
if 'Handshake status 403 Forbidden' in str(e):
raise EnvironmentError(_S._lang.join(_S._lang.UPGRADE_WS))
else:
raise
except ConnectionRefusedError:
raise BrowserConnectError(_S._lang.BROWSER_NOT_EXIST)
self._recv_th.start()
self._handle_event_th.start()
return True
def stop(self):
self._stop()
while self._handle_event_th.is_alive() or self._recv_th.is_alive():
sleep(.01)
return True
def _stop(self):
if not self.is_running:
return False
self.is_running = False
if self._ws:
self._ws.close()
self._ws = None
self.event_handlers.clear()
self.method_results.clear()
self.event_queue.queue.clear()
if hasattr(self.owner, '_on_disconnect'):
self.owner._on_disconnect()
def set_callback(self, event, callback, immediate=False):
handler = self.immediate_event_handlers if immediate else self.event_handlers
if callback:
handler[event] = callback
else:
handler.pop(event, None)
class BrowserDriver(Driver):
BROWSERS = {}
def __new__(cls, _id, address, owner):
if _id in cls.BROWSERS:
return cls.BROWSERS[_id]
return object.__new__(cls)
def __init__(self, _id, address, owner):
if hasattr(self, '_created'):
return
self._created = True
BrowserDriver.BROWSERS[_id] = self
super().__init__(_id, address, owner)
def __repr__(self):
return f'<BrowserDriver {self.id}>'
@staticmethod
def get(url):
s = Session()
s.trust_env = False
s.keep_alive = False
r = s.get(url, headers={'Connection': 'close'})
r.close()
s.close()
return r
@@ -0,0 +1,114 @@
# -*- coding: utf-8 -*-
"""
@Author : g1879
@Contact : g1879@qq.com
@Website : https://DrissionPage.cn
@Copyright: (c) 2020 by g1879, Inc. All Rights Reserved.
"""
from queue import Queue
from threading import Thread
from typing import Union, Callable, Dict, Optional
from requests import Response
from websocket import WebSocket
from .._base.chromium import Chromium
class Driver(object):
id: str
address: str
owner = ...
alert_flag: bool
_cur_id: int
_ws: Optional[WebSocket]
_recv_th: Thread
_handle_event_th: Thread
_handle_immediate_event_th: Optional[Thread]
session_id: Optional[str] = ...
is_running: bool
event_handlers: dict
immediate_event_handlers: dict
method_results: dict
event_queue: Queue
immediate_event_queue: Queue
def __init__(self, _id: str, address: str, owner=None):
"""
:param _id: 标签页id
:param address: 浏览器连接地址
:param owner: 创建这个驱动的对象
"""
...
def _send(self, message: dict, timeout: float = None) -> dict:
"""发送信息到浏览器,并返回浏览器返回的信息
:param message: 发送给浏览器的数据
:param timeout: 超时时间,为None表示无限
:return: 浏览器返回的数据
"""
...
def _recv_loop(self) -> None:
"""接收浏览器信息的守护线程方法"""
...
def _handle_event_loop(self) -> None:
"""当接收到浏览器信息,执行已绑定的方法"""
...
def _handle_immediate_event_loop(self): ...
def _handle_immediate_event(self, function: Callable, kwargs: dict):
"""处理立即执行的动作
:param function: 要运行下方法
:param kwargs: 方法参数
:return: None
"""
...
def run(self, _method: str, **kwargs) -> dict:
"""执行cdp方法
:param _method: cdp方法名
:param kwargs: cdp参数
:return: 执行结果
"""
...
def start(self) -> bool:
"""启动连接"""
...
def stop(self) -> bool:
"""中断连接"""
...
def _stop(self) -> None:
"""中断连接"""
...
def set_callback(self, event: str, callback: Union[Callable, None], immediate: bool = False) -> None:
"""绑定cdp event和回调方法
:param event: cdp event
:param callback: 绑定到cdp event的回调方法
:param immediate: 是否要立即处理的动作
:return: None
"""
...
class BrowserDriver(Driver):
BROWSERS: Dict[str, Driver] = ...
owner: Chromium = ...
def __new__(cls, _id: str, address: str, owner: Chromium): ...
def __init__(self, _id: str, address: str, owner: Chromium): ...
@staticmethod
def get(url) -> Response:
"""
:param url: 要访问的链接
:return: Response对象
"""
...