Files
universal-web-api/venv/Lib/site-packages/DrissionPage/_base/chromium.py
T
2026-11-02 22:54:14 +08:00

538 lines
21 KiB
Python

# -*- coding:utf-8 -*-
"""
@Author : g1879
@Contact : g1879@qq.com
@Website : https://DrissionPage.cn
@Copyright: (c) 2020 by g1879, Inc. All Rights Reserved.
"""
from pathlib import Path
from re import match
from shutil import rmtree
from threading import Lock
from time import sleep, perf_counter
from requests import Session
from websocket import WebSocketBadStatusException
from .driver import BrowserDriver, Driver
from .._configs.chromium_options import ChromiumOptions
from .._functions.browser import connect_browser
from .._functions.cookies import CookiesList
from .._functions.settings import Settings as _S
from .._functions.tools import PortFinder, raise_error
from .._pages.chromium_base import Timeout
from .._pages.chromium_tab import ChromiumTab
from .._pages.mix_tab import MixTab
from .._units.downloader import DownloadManager
from .._units.setter import BrowserSetter
from .._units.states import BrowserStates
from .._units.waiter import BrowserWaiter
from ..errors import BrowserConnectError, CDPError, PageDisconnectedError, IncorrectURLError
__ERROR__ = 'error'
class Chromium(object):
_BROWSERS = {}
_lock = Lock()
def __new__(cls, addr_or_opts=None, session_options=None):
opt = handle_options(addr_or_opts)
is_headless, browser_id, is_exists, ws_only = run_browser(opt)
with cls._lock:
if browser_id in cls._BROWSERS:
r = cls._BROWSERS[browser_id]
while not hasattr(r, '_driver'):
sleep(.05)
return r
r = object.__new__(cls)
r._chromium_options = opt
r._is_headless = is_headless
r._is_exists = is_exists
r._ws_only = ws_only
r.id = browser_id
cls._BROWSERS[browser_id] = r
return r
def __init__(self, addr_or_opts=None, session_options=None):
if hasattr(self, '_created'):
return
self._created = True
self._type = 'Chromium'
self._frames = {}
self._drivers = {}
self._all_drivers = {}
self._relation = {}
self._newest_tab_id = None
self._set = None
self._wait = None
self._states = None
self._timeouts = Timeout(**self._chromium_options.timeouts)
self._load_mode = self._chromium_options.load_mode
self._download_path = str(Path(self._chromium_options.download_path).absolute())
self._auto_handle_alert = None
self._none_ele_return_value = False
self._none_ele_value = None
self.retry_times = self._chromium_options.retry_times
self.retry_interval = self._chromium_options.retry_interval
self.address = self._chromium_options.address
self._ws_address = (self._chromium_options.ws_address if self._chromium_options.ws_address
else f'ws://{self.address}/devtools/browser/{self.id}')
self._disconnect_flag = False
self._driver = BrowserDriver(self.id, self._ws_address, self)
if (not self._ws_only
and (not self._chromium_options._ua_set and self._is_headless != self._chromium_options.is_headless)
or (self._is_exists and self._chromium_options._new_env)):
self.quit(3, True)
connect_browser(self._chromium_options)
s = Session()
s.trust_env = False
s.keep_alive = False
ws = s.get(f'http://{self.address}/json/version', headers={'Connection': 'close'})
self.id = ws.json()['webSocketDebuggerUrl'].split('/')[-1]
self._driver = BrowserDriver(self.id, self._ws_address, self)
ws.close()
s.close()
self._is_exists = False
self._frames = {}
self._drivers = {}
self._all_drivers = {}
self.version = self._run_cdp('Browser.getVersion')['product']
self._process_id = None
try:
r = self._run_cdp('SystemInfo.getProcessInfo')
for i in r.get('processInfo', []):
if i['type'] == 'browser':
self._process_id = i['id']
break
except:
pass
self._run_cdp('Target.setDiscoverTargets', discover=True)
self._driver.set_callback('Target.targetDestroyed', self._onTargetDestroyed)
self._driver.set_callback('Target.targetCreated', self._onTargetCreated)
self._dl_mgr = DownloadManager(self)
self._session_options = session_options
@property
def user_data_path(self):
return self._chromium_options.user_data_path
@property
def process_id(self):
return self._process_id
@property
def timeout(self):
return self._timeouts.base
@property
def timeouts(self):
return self._timeouts
@property
def load_mode(self):
return self._load_mode
@property
def download_path(self):
return self._download_path
@property
def set(self):
if self._set is None:
self._set = BrowserSetter(self)
return self._set
@property
def states(self):
if self._states is None:
self._states = BrowserStates(self)
return self._states
@property
def wait(self):
if self._wait is None:
self._wait = BrowserWaiter(self)
return self._wait
@property
def tabs_count(self):
j = self._run_cdp('Target.getTargets')['targetInfos'] # 不要改用get,避免卡死
return len([i for i in j if i['type'] in ('page', 'webview') and not i['url'].startswith('devtools://')])
@property
def tab_ids(self):
if self._ws_only:
return [i['targetId'] for i in self._run_cdp('Target.getTargets')['targetInfos']
if i['type'] in ('page', 'webview') and not i['url'].startswith('devtools://')]
else:
return [i['id'] for i in self._driver.get(f'http://{self.address}/json').json()
if i['type'] in ('page', 'webview') and not i['url'].startswith('devtools://')]
@property
def latest_tab(self):
return self._get_tab(id_or_num=self.tab_ids[0], as_id=not _S.singleton_tab_obj)
def cookies(self, all_info=False):
cks = self._run_cdp(f'Storage.getCookies')['cookies']
r = cks if all_info else [{'name': c['name'], 'value': c['value'], 'domain': c['domain']} for c in cks]
return CookiesList(r)
def new_tab(self, url=None, new_window=False, background=False, new_context=False):
return self._new_tab(True, url=url, new_window=new_window, background=background, new_context=new_context)
def get_tab(self, id_or_num=None, title=None, url=None, tab_type='page'):
t = self._get_tab(id_or_num=id_or_num, title=title, url=url, tab_type=tab_type, mix=True, as_id=False)
if t._type != 'MixTab':
raise RuntimeError(_S._lang.join(_S._lang.TAB_OBJ_EXISTS))
return t
def get_tabs(self, title=None, url=None, tab_type='page'):
return self._get_tabs(title=title, url=url, tab_type=tab_type, mix=True, as_id=False)
def close_tabs(self, tabs_or_ids, others=False):
if isinstance(tabs_or_ids, str):
tabs = {tabs_or_ids}
elif isinstance(tabs_or_ids, ChromiumTab):
tabs = {tabs_or_ids.tab_id}
elif isinstance(tabs_or_ids, (list, tuple)):
tabs = set(i.tab_id if isinstance(i, ChromiumTab) else i for i in tabs_or_ids)
else:
raise ValueError(_S._lang.join(_S._lang.INCORRECT_TYPE_, 'tabs_or_ids',
ALLOW_TYPE=_S._lang.TAB_OR_ID, CURR_VAL=tabs_or_ids))
all_tabs = set(self.tab_ids)
if others:
tabs = all_tabs - tabs
if len(all_tabs - tabs) > 0:
for tab in tabs:
self._close_tab(tab=tab)
else:
self.quit()
def _close_tab(self, tab):
if isinstance(tab, str):
tab = self.get_tab(tab)
tab._run_cdp('Target.closeTarget', targetId=tab.tab_id)
while tab.driver.is_running and tab.tab_id in self._all_drivers:
sleep(.01)
def activate_tab(self, id_ind_tab):
if isinstance(id_ind_tab, int):
id_ind_tab += -1 if id_ind_tab else 1
id_ind_tab = self.tab_ids[id_ind_tab]
elif isinstance(id_ind_tab, ChromiumTab):
id_ind_tab = id_ind_tab.tab_id
self._run_cdp('Target.activateTarget', targetId=id_ind_tab)
def reconnect(self):
self._disconnect_flag = True
self._driver.stop()
BrowserDriver.BROWSERS.pop(self.id)
self._driver = BrowserDriver(self.id, self._ws_address, self)
self._run_cdp('Target.setDiscoverTargets', discover=True)
self._driver.set_callback('Target.targetDestroyed', self._onTargetDestroyed)
self._driver.set_callback('Target.targetCreated', self._onTargetCreated)
self._disconnect_flag = False
def clear_cache(self, cache=True, cookies=True):
if cache:
self.latest_tab.run_cdp('Network.clearBrowserCache')
if cookies:
self._run_cdp('Storage.clearCookies')
def quit(self, timeout=5, force=False, del_data=False):
try:
self._run_cdp('Browser.close')
except PageDisconnectedError:
pass
self._driver.stop()
drivers = list(self._all_drivers.values())
for tab in drivers:
for driver in tab:
driver.stop()
if not self.address.startswith('127.0.0.1'):
return
if force:
pids = None
try:
pids = [pid['id'] for pid in self._run_cdp('SystemInfo.getProcessInfo')['processInfo']]
except:
pass
if pids:
from psutil import Process
for pid in pids:
try:
Process(pid).kill()
except:
pass
from os import popen
from platform import system
end_time = perf_counter() + timeout
while perf_counter() < end_time:
ok = True
for pid in pids:
txt = f'tasklist | findstr {pid}' if system().lower() == 'windows' else f'ps -ef | grep {pid}'
p = popen(txt)
sleep(.05)
try:
if f' {pid} ' in p.read():
ok = False
break
except TypeError:
pass
if ok:
break
if del_data and not self._chromium_options.is_auto_port and self._chromium_options.user_data_path:
path = Path(self._chromium_options.user_data_path)
rmtree(path, True)
def _new_tab(self, mix=True, url=None, new_window=False, background=False, new_context=False):
tab_type = MixTab if mix else ChromiumTab
tab = None
if new_context:
tab = self._run_cdp('Target.createBrowserContext')['browserContextId']
kwargs = {'url': ''}
if new_window:
kwargs['newWindow'] = True
if background:
kwargs['background'] = True
if tab:
kwargs['browserContextId'] = tab
if self.states.is_incognito and not new_context:
return _new_tab_by_js(self, url, tab_type, new_window)
else:
try:
tab = self._run_cdp('Target.createTarget', **kwargs)['targetId']
except CDPError:
return _new_tab_by_js(self, url, tab_type, new_window)
while self.states.is_alive:
if tab in self._drivers:
break
sleep(.01)
else:
raise BrowserConnectError(_S._lang.BROWSER_DISCONNECTED)
tab = tab_type(self, tab)
if url:
tab.get(url)
return tab
def _get_tab(self, id_or_num=None, title=None, url=None, tab_type='page', mix=True, as_id=False):
if id_or_num is not None:
if isinstance(id_or_num, int):
id_or_num = self.tab_ids[id_or_num - 1 if id_or_num > 0 else id_or_num]
elif isinstance(id_or_num, ChromiumTab):
return id_or_num.tab_id if as_id else ChromiumTab(self, id_or_num.tab_id)
else:
j = self._run_cdp('Target.getTargets')['targetInfos']
if id_or_num not in [i['targetId'] for i in j]:
raise RuntimeError(_S._lang.join(_S._lang.NO_SUCH_TAB, ARG=id_or_num, ALL_TABS=self.tab_ids))
elif title == url is None and tab_type == 'page':
id_or_num = self.tab_ids[0]
else:
tabs = self._get_tabs(title=title, url=url, tab_type=tab_type, as_id=True)
if tabs:
id_or_num = tabs[0]
else:
raise RuntimeError(_S._lang.join(_S._lang.NO_SUCH_TAB,
ARGS={'id_or_num': id_or_num, 'title': title, 'url': url,
'tab_type': tab_type}))
if as_id:
return id_or_num
with self._lock:
return MixTab(self, id_or_num) if mix else ChromiumTab(self, id_or_num)
def _get_tabs(self, title=None, url=None, tab_type='page', mix=True, as_id=False):
if self._ws_only:
tabs = self._run_cdp('Target.getTargets')['targetInfos']
_id = 'targetId'
else:
tabs = self._driver.get(f'http://{self.address}/json').json() # 不要改用cdp
_id = 'id'
if isinstance(tab_type, str):
tab_type = {tab_type}
elif isinstance(tab_type, (list, tuple, set)):
tab_type = set(tab_type)
elif tab_type is not None:
raise ValueError(_S._lang.join(_S._lang.INCORRECT_TYPE_, 'tab_type',
ALLOW_TYPE='set, list, tuple, str, None', CURR_VAL=tab_type))
tabs = [i for i in tabs if ((title is None or title in i['title']) and (url is None or url in i['url'])
and (tab_type is None or i['type'] in tab_type)
and i['title'] != 'chrome-extension://neajdppkdcdipfabeoofebfddakdcjhd/audio.html')]
if as_id:
return [tab[_id] for tab in tabs]
with self._lock:
if mix:
return [MixTab(self, tab[_id]) for tab in tabs]
else:
return [ChromiumTab(self, tab[_id]) for tab in tabs]
def _run_cdp(self, cmd, **cmd_args):
ignore = cmd_args.pop('_ignore', None)
r = self._driver.run(cmd, **cmd_args)
return r if __ERROR__ not in r else raise_error(r, self, ignore)
def _get_driver(self, tab_id, owner=None):
d = self._drivers.pop(tab_id, None)
if not d:
if self._ws_only:
d = Driver(tab_id, self._ws_address)
d.session_id = d.run('Target.attachToTarget', targetId=tab_id, flatten=True)['sessionId']
else:
d = Driver(tab_id, f'ws://{self.address}/devtools/page/{tab_id}')
d.owner = owner
self._all_drivers.setdefault(tab_id, set()).add(d)
return d
def _onTargetCreated(self, **kwargs):
if (kwargs['targetInfo']['type'] in ('page', 'webview')
and kwargs['targetInfo']['targetId'] not in self._all_drivers
and not kwargs['targetInfo']['url'].startswith('devtools://')):
try:
tab_id = kwargs['targetInfo']['targetId']
self._frames[tab_id] = tab_id
if self._ws_only:
d = Driver(tab_id, self._ws_address)
d.session_id = d.run('Target.attachToTarget', targetId=tab_id, flatten=True)['sessionId']
else:
d = Driver(tab_id, f'ws://{self.address}/devtools/page/{tab_id}')
self._relation[tab_id] = kwargs['targetInfo'].get('openerId', None)
self._drivers[tab_id] = d
self._all_drivers.setdefault(tab_id, set()).add(d)
self._newest_tab_id = tab_id
except WebSocketBadStatusException:
pass
def _onTargetDestroyed(self, **kwargs):
tab_id = kwargs['targetId']
self._dl_mgr.clear_tab_info(tab_id)
for key in [k for k, i in self._frames.items() if i == tab_id]:
self._frames.pop(key, None)
for d in self._all_drivers.get(tab_id, tuple()):
d.stop()
self._drivers.pop(tab_id, None)
self._all_drivers.pop(tab_id, None)
self._relation.pop(tab_id, None)
def _on_disconnect(self):
if not self._disconnect_flag:
Chromium._BROWSERS.pop(self.id, None)
if self._chromium_options.is_auto_port and self._chromium_options.user_data_path:
path = Path(self._chromium_options.user_data_path)
end_time = perf_counter() + 7
while perf_counter() < end_time:
if not path.exists():
break
try:
rmtree(path)
break
except (PermissionError, FileNotFoundError, OSError):
pass
sleep(.03)
def handle_options(addr_or_opts):
if not addr_or_opts:
_chromium_options = ChromiumOptions(addr_or_opts)
if _chromium_options.is_auto_port:
port, path = PortFinder(_chromium_options.tmp_path).get_port(_chromium_options.is_auto_port)
_chromium_options.set_address(f'127.0.0.1:{port}')
_chromium_options.set_user_data_path(path)
_chromium_options.auto_port(scope=_chromium_options.is_auto_port)
elif isinstance(addr_or_opts, ChromiumOptions):
if addr_or_opts.is_auto_port:
port, path = PortFinder(addr_or_opts.tmp_path).get_port(addr_or_opts.is_auto_port)
addr_or_opts._address = f'127.0.0.1:{port}'
addr_or_opts.set_user_data_path(path)
_chromium_options = addr_or_opts
elif isinstance(addr_or_opts, str) and ':' in addr_or_opts:
_chromium_options = ChromiumOptions()
_chromium_options.set_address(addr_or_opts)
elif isinstance(addr_or_opts, int):
_chromium_options = ChromiumOptions()
_chromium_options.set_local_port(addr_or_opts)
else:
raise ValueError(_S._lang.join(_S._lang.INCORRECT_VAL_, 'addr_or_opts',
ALLOW_TYPE=_S._lang.IP_OR_OPTIONS, CURR_VAL=addr_or_opts))
return _chromium_options
def run_browser(chromium_options):
if chromium_options.ws_address:
d = Driver('', chromium_options.ws_address)
browser_id = d.run('Target.getTargets')
if 'error' in browser_id:
raise BrowserConnectError(_S._lang.join(_S._lang.BROWSER_CONNECT_ERR2, INFO=browser_id['error']))
browser_id = browser_id['targetInfos'][0]['browserContextId']
is_headless = 'headless' in d.run('Browser.getVersion')['userAgent'].lower()
is_exists = True
d.stop()
if 'devtools/browser' not in chromium_options.ws_address:
return is_headless, browser_id, is_exists, True
else:
is_exists = connect_browser(chromium_options)
try:
s = Session()
s.trust_env = False
s.keep_alive = False
ws = s.get(f'http://{chromium_options.address}/json/version', headers={'Connection': 'close'}, timeout=2)
if not ws and not chromium_options.ws_address:
raise BrowserConnectError(_S._lang.BROWSER_CONNECT_ERR2)
json = ws.json()
if not chromium_options.ws_address:
browser_id = json['webSocketDebuggerUrl'].split('/')[-1]
is_headless = 'headless' in json['User-Agent'].lower()
ws.close()
s.close()
ws_only = False
except KeyError:
raise BrowserConnectError(_S._lang.BROWSER_NOT_FOR_CONTROL)
except:
if chromium_options.ws_address:
ws_only = True
else:
raise BrowserConnectError(_S._lang.BROWSER_CONNECT_ERR2)
return is_headless, browser_id, is_exists, ws_only
def _new_tab_by_js(browser: Chromium, url, tab_type, new_window):
mix = tab_type == MixTab
tab = browser._get_tab(mix=mix)
if url and not match(r'^.*?://.*', url):
raise IncorrectURLError(_S._lang.INVALID_URL, url=url)
url = f'"{url}"' if url else '""'
new = 'target="_new"' if new_window else 'target="_blank"'
tid = browser._newest_tab_id
tab.run_js(f'window.open({url}, {new})')
tid = browser.wait.new_tab(curr_tab=tid)
return browser._get_tab(tid, mix=mix)