Initial commit

This commit is contained in:
Alex committed 2026-11-02 22:54:14 +08:00
commit 57fc3525eb
7311 files changed
+1291398

No files matched your search

@@ -0,0 +1,439 @@
# -*- coding:utf-8 -*-
"""
@Author : g1879
@Contact : g1879@qq.com
@Website : https://DrissionPage.cn
@Copyright: (c) 2020 by g1879, Inc. All Rights Reserved.
"""
from pathlib import Path
from re import search
from urllib.parse import urlparse
from .options_manage import OptionsManager
from .._functions.settings import Settings as _S
class ChromiumOptions(object):
def __init__(self, read_file=True, ini_path=None):
self._user_data_path = None
self._user = 'Default'
self._prefs_to_del = []
self.clear_file_flags = False
self._is_headless = False
self._ua_set = False
self.ws_address = ''
if read_file is False:
ini_path = False
self.ini_path = None
elif ini_path:
ini_path = Path(ini_path).absolute()
if not ini_path.exists():
raise FileNotFoundError(_S._lang.join(_S._lang.INI_NOT_FOUND, PATH=ini_path))
self.ini_path = str(ini_path)
else:
self.ini_path = str(Path(__file__).parent / 'configs.ini')
om = OptionsManager(ini_path)
options = om.chromium_options
self._download_path = om.paths.get('download_path', '.') or '.'
self._tmp_path = om.paths.get('tmp_path', None) or None
self._arguments = options.get('arguments', [])
self._browser_path = options.get('browser_path', '')
self._extensions = options.get('extensions', [])
self._prefs = options.get('prefs', {})
self._flags = options.get('flags', {})
self._address = options.get('address', '')
self._load_mode = options.get('load_mode', 'normal')
self._system_user_path = options.get('system_user_path', False)
self._existing_only = options.get('existing_only', False)
self._new_env = options.get('new_env', False)
for i in self._arguments:
if i.startswith('--headless'):
self._is_headless = True
break
self._proxy = om.proxies.get('http', None) or om.proxies.get('https', None)
user_path = user = False
for arg in self._arguments:
if arg.startswith('--user-data-dir='):
self.set_user_data_path(arg[16:])
user_path = True
if arg.startswith('--profile-directory='):
self.set_user(arg[20:])
user = True
if user and user_path:
break
timeouts = om.timeouts
self._timeouts = {'base': timeouts['base'],
'page_load': timeouts['page_load'],
'script': timeouts['script']}
self._auto_port = options.get('auto_port', False)
others = om.others
self._retry_times = others.get('retry_times', 3)
self._retry_interval = others.get('retry_interval', 2)
return
def __repr__(self):
return f'<ChromiumOptions at {id(self)}>'
@property
def download_path(self):
return self._download_path
@property
def browser_path(self):
return self._browser_path
@property
def user_data_path(self):
return self._user_data_path
@property
def tmp_path(self):
return self._tmp_path
@property
def user(self):
return self._user
@property
def load_mode(self):
return self._load_mode
@property
def timeouts(self):
return self._timeouts
@property
def proxy(self):
return self._proxy
@property
def address(self):
return self._address
@property
def arguments(self):
return self._arguments
@property
def extensions(self):
return self._extensions
@property
def preferences(self):
return self._prefs
@property
def flags(self):
return self._flags
@property
def system_user_path(self):
return self._system_user_path
@property
def is_existing_only(self):
return self._existing_only
@property
def is_auto_port(self):
return self._auto_port
@property
def retry_times(self):
return self._retry_times
@property
def retry_interval(self):
return self._retry_interval
@property
def is_headless(self):
return self._is_headless
def set_retry(self, times=None, interval=None):
if times is not None:
self._retry_times = times
if interval is not None:
self._retry_interval = interval
return self
def set_argument(self, arg, value=None):
self.remove_argument(arg)
if value is not False:
if arg == '--headless':
if value == 'false':
self._is_headless = False
else:
if value is None:
value = 'new'
self._arguments.append(f'--headless={value}')
self._is_headless = True
else:
arg_str = arg if value is None else f'{arg}={value}'
self._arguments.append(arg_str)
elif arg == '--headless':
self._is_headless = False
return self
def remove_argument(self, value):
elements_to_delete = [arg for arg in self._arguments if arg == value or arg.startswith(f'{value}=')]
if not elements_to_delete:
return self
if len(elements_to_delete) == 1:
self._arguments.remove(elements_to_delete[0])
else:
self._arguments = [arg for arg in self._arguments if arg not in elements_to_delete]
return self
def add_extension(self, path):
self._extensions.append(path)
return self
def remove_extensions(self):
self._extensions = []
return self
def set_pref(self, arg, value):
self._prefs[arg] = value
return self
def remove_pref(self, arg):
self._prefs.pop(arg, None)
return self
def remove_pref_from_file(self, arg):
self._prefs_to_del.append(arg)
return self
def set_flag(self, flag, value=None):
if value is False:
self._flags.pop(flag, None)
else:
self._flags[flag] = value
return self
def clear_flags_in_file(self):
self.clear_file_flags = True
return self
def clear_flags(self):
self._flags = {}
return self
def clear_arguments(self):
self._arguments = []
return self
def clear_prefs(self):
self._prefs = {}
return self
def set_timeouts(self, base=None, page_load=None, script=None):
if base is not None:
self._timeouts['base'] = base
if page_load is not None:
self._timeouts['page_load'] = page_load
if script is not None:
self._timeouts['script'] = script
return self
def set_user(self, user='Default'):
self.set_argument('--profile-directory', user)
self._user = user
return self
def headless(self, on_off=True):
on_off = 'new' if on_off else on_off
return self.set_argument('--headless', on_off)
def no_imgs(self, on_off=True):
on_off = None if on_off else False
return self.set_argument('--blink-settings=imagesEnabled=false', on_off)
def no_js(self, on_off=True):
on_off = None if on_off else False
return self.set_argument('--disable-javascript', on_off)
def mute(self, on_off=True):
on_off = None if on_off else False
return self.set_argument('--mute-audio', on_off)
def incognito(self, on_off=True):
on_off = None if on_off else False
self.set_argument('--incognito', on_off)
return self.set_argument('--inprivate', on_off) # edge
def new_env(self, on_off=True):
self._new_env = on_off
return self
def ignore_certificate_errors(self, on_off=True):
on_off = None if on_off else False
return self.set_argument('--ignore-certificate-errors', on_off)
def set_user_agent(self, user_agent):
return self.set_argument('--user-agent', user_agent)
def set_proxy(self, proxy):
if search(r'.*?:.*?@.*?\..*', proxy):
print(_S._lang.UNSUPPORTED_USER_PROXY)
if proxy.lower().startswith('socks'):
print(_S._lang.UNSUPPORTED_SOCKS_PROXY)
self._proxy = proxy
return self.set_argument('--proxy-server', proxy)
def set_load_mode(self, value):
if value not in ('normal', 'eager', 'none'):
raise ValueError(_S._lang.join(_S._lang.INCORRECT_VAL_, 'value',
ALLOW_VAL="'normal', 'eager', 'none'", CURR_VAL=value))
self._load_mode = value.lower()
return self
def set_paths(self, browser_path=None, local_port=None, address=None, download_path=None,
user_data_path=None, cache_path=None):
"""快捷的路径设置函数,即将废弃
:param browser_path: 浏览器可执行文件路径
:param local_port: 本地端口号
:param address: 调试浏览器地址,例:127.0.0.1:9222
:param download_path: 下载文件路径
:param user_data_path: 用户数据路径
:param cache_path: 缓存路径
:return: 当前对象
"""
if browser_path is not None:
self.set_browser_path(browser_path)
if local_port is not None:
self.set_local_port(local_port)
if address is not None:
self.set_address(address)
if download_path is not None:
self.set_download_path(download_path)
if user_data_path is not None:
self.set_user_data_path(user_data_path)
if cache_path is not None:
self.set_cache_path(cache_path)
return self
def set_local_port(self, port):
self._address = f'127.0.0.1:{port}'
self._auto_port = False
self.ws_address = ''
return self
def set_address(self, address):
address = address.replace('localhost', '127.0.0.1')
self.ws_address = ''
if address.startswith('http'):
address = address.lstrip('htps:/')
elif address.startswith(('ws:', 'wss:')):
self.ws_address = address
address = urlparse(address).netloc
self._address = address
self._auto_port = False
return self
def set_browser_path(self, path):
self._browser_path = str(path)
return self
def set_download_path(self, path):
self._download_path = '.' if path is None else str(path)
return self
def set_tmp_path(self, path):
self._tmp_path = str(path)
return self
def set_user_data_path(self, path):
u = str(path)
self.set_argument('--user-data-dir', u)
self._user_data_path = u
self._auto_port = False
return self
def set_cache_path(self, path):
self.set_argument('--disk-cache-dir', str(path))
return self
def use_system_user_path(self, on_off=True):
self._system_user_path = on_off
return self
def auto_port(self, on_off=True, scope=None):
if on_off:
self._auto_port = scope if scope else (9600, 59600)
else:
self._auto_port = False
self._address = ''
self.ws_address = ''
return self
def existing_only(self, on_off=True):
self._existing_only = on_off
return self
def save(self, path=None):
if path == 'default':
path = (Path(__file__).parent / 'configs.ini').absolute()
elif path is None:
if self.ini_path:
path = Path(self.ini_path).absolute()
else:
path = (Path(__file__).parent / 'configs.ini').absolute()
else:
path = Path(path).absolute()
path = path / 'config.ini' if path.is_dir() else path
if path.exists():
om = OptionsManager(path)
else:
om = OptionsManager(self.ini_path or (Path(__file__).parent / 'configs.ini'))
# 设置chromium_options
attrs = ('address', 'browser_path', 'arguments', 'extensions', 'user', 'load_mode',
'auto_port', 'system_user_path', 'existing_only', 'flags', 'new_env')
for i in attrs:
om.set_item('chromium_options', i, self.__getattribute__(f'_{i}'))
# 设置代理
om.set_item('proxies', 'http', self._proxy or '')
om.set_item('proxies', 'https', self._proxy or '')
# 设置路径
om.set_item('paths', 'download_path', self._download_path or '')
om.set_item('paths', 'tmp_path', self._tmp_path or '')
# 设置timeout
om.set_item('timeouts', 'base', self._timeouts['base'])
om.set_item('timeouts', 'page_load', self._timeouts['page_load'])
om.set_item('timeouts', 'script', self._timeouts['script'])
# 设置重试
om.set_item('others', 'retry_times', self.retry_times)
om.set_item('others', 'retry_interval', self.retry_interval)
# 设置prefs
om.set_item('chromium_options', 'prefs', self._prefs)
path = str(path)
om.save(path)
return path
def save_to_default(self):
return self.save('default')
@@ -0,0 +1,399 @@
# -*- coding:utf-8 -*-
"""
@Author : g1879
@Contact : g1879@qq.com
@Website : https://DrissionPage.cn
@Copyright: (c) 2020 by g1879, Inc. All Rights Reserved.
"""
from pathlib import Path
from typing import Union, Any, Literal, Optional, Tuple
class ChromiumOptions(object):
ini_path: Optional[str] = ...
_user_data_path: Optional[str] = ...
_download_path: str = ...
_tmp_path: str = ...
_arguments: list = ...
_browser_path: str = ...
_user: str = ...
_load_mode: str = ...
_timeouts: dict = ...
_proxy: str = ...
_address: str = ...
_extensions: list = ...
_prefs: dict = ...
_flags: dict = ...
_prefs_to_del: list = ...
_new_env: bool = ...
clear_file_flags: bool = ...
_auto_port: Union[Tuple[int, int], False] = ...
_system_user_path: bool = ...
_existing_only: bool = ...
_retry_times: int = ...
_retry_interval: float = ...
_is_headless: bool = ...
_ua_set: bool = ...
ws_address: Optional[str] = ...
def __init__(self,
read_file: [bool, None] = True,
ini_path: Union[str, Path] = None):
"""
:param read_file: 是否从默认ini文件中读取配置信息
:param ini_path: ini文件路径,为None则读取默认ini文件
"""
...
@property
def download_path(self) -> str:
"""默认下载路径文件路径"""
...
@property
def browser_path(self) -> str:
"""浏览器启动文件路径"""
...
@property
def user_data_path(self) -> str:
"""返回用户数据文件夹路径"""
...
@property
def tmp_path(self) -> Optional[str]:
"""返回临时文件夹路径"""
...
@property
def user(self) -> str:
"""返回用户配置文件夹名称"""
...
@property
def load_mode(self) -> str:
"""返回页面加载策略,'normal', 'eager', 'none'"""
...
@property
def timeouts(self) -> dict:
"""返回timeouts设置"""
...
@property
def proxy(self) -> str:
"""返回代理设置"""
...
@property
def address(self) -> str:
"""返回浏览器地址,ip:port"""
...
@property
def arguments(self) -> list:
"""返回浏览器命令行设置列表"""
...
@property
def extensions(self) -> list:
"""以list形式返回要加载的插件路径"""
...
@property
def preferences(self) -> dict:
"""返回用户首选项配置"""
...
@property
def flags(self) -> dict:
"""返回实验项配置"""
...
@property
def system_user_path(self) -> bool:
"""返回是否使用系统安装的浏览器所使用的用户数据文件夹"""
...
@property
def is_existing_only(self) -> bool:
"""返回是否只接管现有浏览器方式"""
...
@property
def is_auto_port(self) -> Union[bool, Tuple[int, int]]:
"""返回是否使用自动端口和用户文件,如指定范围则返回范围tuple"""
...
@property
def retry_times(self) -> int:
"""返回连接失败时的重试次数"""
...
@property
def retry_interval(self) -> float:
"""返回连接失败时的重试间隔(秒)"""
...
@property
def is_headless(self) -> bool:
"""返回是否无头模式"""
...
def set_retry(self, times: int = None, interval: float = None) -> ChromiumOptions:
"""设置连接失败时的重试操作
:param times: 重试次数
:param interval: 重试间隔
:return: 当前对象
"""
...
def set_argument(self, arg: str, value: Union[str, None, bool] = None) -> ChromiumOptions:
"""设置浏览器配置的argument属性
:param arg: 属性名
:param value: 属性值,有值的属性传入值,没有的传入None,如传入False,删除该项
:return: 当前对象
"""
...
def remove_argument(self, value: str) -> ChromiumOptions:
"""移除一个argument项
:param value: 设置项名,有值的设置项传入设置名称即可
:return: 当前对象
"""
...
def add_extension(self, path: Union[str, Path]) -> ChromiumOptions:
"""添加插件
:param path: 插件路径,可指向文件夹
:return: 当前对象
"""
...
def remove_extensions(self) -> ChromiumOptions:
"""移除所有插件
:return: 当前对象
"""
...
def set_pref(self, arg: str, value: Any) -> ChromiumOptions:
"""设置Preferences文件中的用户设置项
:param arg: 设置项名称
:param value: 设置项值
:return: 当前对象
"""
...
def remove_pref(self, arg: str) -> ChromiumOptions:
"""删除用户首选项设置,不能删除已设置到文件中的项
:param arg: 设置项名称
:return: 当前对象
"""
...
def remove_pref_from_file(self, arg: str) -> ChromiumOptions:
"""删除用户配置文件中已设置的项
:param arg: 设置项名称
:return: 当前对象
"""
...
def set_flag(self, flag: str, value: Union[int, str, bool] = None) -> ChromiumOptions:
"""设置实验项
:param flag: 设置项名称
:param value: 设置项的值,为False则删除该项
:return: 当前对象
"""
...
def clear_flags_in_file(self) -> ChromiumOptions:
"""删除浏览器配置文件中已设置的实验项"""
...
def clear_flags(self) -> ChromiumOptions:
"""清空本对象已设置的flag参数"""
...
def clear_arguments(self) -> ChromiumOptions:
"""清空本对象已设置的argument参数"""
...
def clear_prefs(self) -> ChromiumOptions:
"""清空本对象已设置的pref参数"""
...
def set_timeouts(self,
base: float = None,
page_load: float = None,
script: float = None) -> ChromiumOptions:
"""设置超时时间,单位为秒
:param base: 默认超时时间
:param page_load: 页面加载超时时间
:param script: 脚本运行超时时间
:return: 当前对象
"""
...
def set_user(self, user: str = 'Default') -> ChromiumOptions:
"""设置使用哪个用户配置文件夹
:param user: 用户文件夹名称
:return: 当前对象
"""
...
def headless(self, on_off: bool = True) -> ChromiumOptions:
"""设置是否隐藏浏览器界面
:param on_off: 开或关
:return: 当前对象
"""
...
def no_imgs(self, on_off: bool = True) -> ChromiumOptions:
"""设置是否加载图片
:param on_off: 开或关
:return: 当前对象
"""
...
def no_js(self, on_off: bool = True) -> ChromiumOptions:
"""设置是否禁用js
:param on_off: 开或关
:return: 当前对象
"""
...
def mute(self, on_off: bool = True) -> ChromiumOptions:
"""设置是否静音
:param on_off: 开或关
:return: 当前对象
"""
...
def incognito(self, on_off: bool = True) -> ChromiumOptions:
"""设置是否使用无痕模式启动
:param on_off: 开或关
:return: 当前对象
"""
...
def new_env(self, on_off: bool = True) -> ChromiumOptions:
"""设置是否使用全新浏览器环境
:param on_off: 开或关
:return: 当前对象
"""
...
def ignore_certificate_errors(self, on_off=True) -> ChromiumOptions:
"""设置是否忽略证书错误
:param on_off: 开或关
:return: 当前对象
"""
...
def set_user_agent(self, user_agent: str) -> ChromiumOptions:
"""设置user agent
:param user_agent: user agent文本
:return: 当前对象
"""
...
def set_proxy(self, proxy: str) -> ChromiumOptions:
"""设置代理
:param proxy: 代理url和端口
:return: 当前对象
"""
...
def set_load_mode(self, value: Literal['normal', 'eager', 'none']) -> ChromiumOptions:
"""设置load_mode,可接收 'normal', 'eager', 'none'
normal:默认情况下使用, 等待所有资源下载完成
eager:DOM访问已准备就绪, 但其他资源 (如图像) 可能仍在加载中
none:完全不阻塞
:param value: 可接收 'normal', 'eager', 'none'
:return: 当前对象
"""
...
def set_local_port(self, port: Union[str, int]) -> ChromiumOptions:
"""设置本地启动端口,与set_address()和auto_port()互斥
:param port: 端口号
:return: 当前对象
"""
...
def set_address(self, address: str) -> ChromiumOptions:
"""设置浏览器地址,格式'ip:port',与auto_port()和set_local_port()互斥
:param address: 浏览器地址
:return: 当前对象
"""
...
def set_browser_path(self, path: Union[str, Path]) -> ChromiumOptions:
"""设置浏览器可执行文件路径
:param path: 浏览器路径
:return: 当前对象
"""
...
def set_download_path(self, path: Union[str, Path]) -> ChromiumOptions:
"""设置下载文件保存路径
:param path: 下载路径
:return: 当前对象
"""
...
def set_tmp_path(self, path: Union[str, Path]) -> ChromiumOptions:
"""设置临时文件文件保存路径
:param path: 下载路径
:return: 当前对象
"""
...
def set_user_data_path(self, path: Union[str, Path]) -> ChromiumOptions:
"""设置用户文件夹路径
:param path: 用户文件夹路径
:return: 当前对象
"""
...
def set_cache_path(self, path: Union[str, Path]) -> ChromiumOptions:
"""设置缓存路径
:param path: 缓存路径
:return: 当前对象
"""
...
def use_system_user_path(self, on_off: bool = True) -> ChromiumOptions:
"""设置是否使用系统安装的浏览器默认用户文件夹
:param on_off: 开或关
:return: 当前对象
"""
...
def auto_port(self,
on_off: bool = True,
scope: Tuple[int, int] = None) -> ChromiumOptions:
"""自动获取可用端口,与set_address()和set_local_port()互斥
:param on_off: 是否开启自动获取端口号
:param scope: 指定端口范围,不含最后的数字,为None则使用[9600-59600)
:return: 当前对象
"""
...
def existing_only(self, on_off: bool = True) -> ChromiumOptions:
"""设置只接管已有浏览器,不自动启动新的
:param on_off: 是否开启自动获取端口号
:return: 当前对象
"""
...
def save(self, path: Union[str, Path] = None) -> str:
"""保存设置到文件
:param path: ini文件的路径, None 保存到当前读取的配置文件,传入 'default' 保存到默认ini文件
:return: 保存文件的绝对路径
"""
...
def save_to_default(self) -> str:
"""保存当前配置到默认ini文件"""
...
@@ -0,0 +1,33 @@
[paths]
download_path =
tmp_path =
[chromium_options]
address = 127.0.0.1:9222
browser_path = chrome
arguments = ['--no-default-browser-check', '--disable-suggestions-ui', '--no-first-run', '--disable-infobars', '--disable-popup-blocking', '--hide-crash-restore-bubble', '--disable-features=PrivacySandboxSettings4']
extensions = []
prefs = {'profile.default_content_settings.popups': 0, 'profile.default_content_setting_values': {'notifications': 2}}
flags = {}
load_mode = normal
user = Default
auto_port = False
system_user_path = False
existing_only = False
new_env = False
[session_options]
headers = {'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/603.3.8 (KHTML, like Gecko) Version/10.1.2 Safari/603.3.8', 'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', 'connection': 'keep-alive', 'accept-charset': 'GB2312,utf-8;q=0.7,*;q=0.7'}
[timeouts]
base = 10
page_load = 30
script = 30
[proxies]
http =
https =
[others]
retry_times = 3
retry_interval = 2
@@ -0,0 +1,141 @@
# -*- coding:utf-8 -*-
"""
@Author : g1879
@Contact : g1879@qq.com
@Website : https://DrissionPage.cn
@Copyright: (c) 2020 by g1879, Inc. All Rights Reserved.
"""
from configparser import RawConfigParser, NoSectionError, NoOptionError
from pathlib import Path
from pprint import pprint
from .._functions.settings import Settings as _S
class OptionsManager(object):
def __init__(self, path=None):
if path is False:
self.ini_path = None
else:
default_configs = Path(__file__).parent / 'configs.ini'
if path is None:
dp_configs = Path('dp_configs.ini')
if dp_configs.exists():
self.ini_path = dp_configs
else:
self.ini_path = default_configs
elif path == 'default':
self.ini_path = default_configs
elif isinstance(path, Path):
self.ini_path = path
else:
self.ini_path = Path(path)
self._conf = RawConfigParser()
if path is not False and self.ini_path.exists():
self.file_exists = True
self._conf.read(self.ini_path, encoding='utf-8')
else:
self.file_exists = False
self._conf.add_section('paths')
self._conf.add_section('chromium_options')
self._conf.add_section('session_options')
self._conf.add_section('timeouts')
self._conf.add_section('proxies')
self._conf.add_section('others')
self.set_item('paths', 'download_path', '')
self.set_item('paths', 'tmp_path', '')
self.set_item('chromium_options', 'address', '127.0.0.1:9222')
self.set_item('chromium_options', 'browser_path', 'chrome')
self.set_item('chromium_options', 'arguments', "['--no-default-browser-check', '--disable-suggestions-ui', "
"'--no-first-run', '--disable-infobars', "
"'--disable-popup-blocking', '--hide-crash-restore-bubble', "
"'--disable-features=PrivacySandboxSettings4']")
self.set_item('chromium_options', 'extensions', '[]')
self.set_item('chromium_options', 'prefs', "{'profile.default_content_settings.popups': 0, "
"'profile.default_content_setting_values': "
"{'notifications': 2}}")
self.set_item('chromium_options', 'flags', '{}')
self.set_item('chromium_options', 'load_mode', 'normal')
self.set_item('chromium_options', 'user', 'Default')
self.set_item('chromium_options', 'auto_port', 'False')
self.set_item('chromium_options', 'system_user_path', 'False')
self.set_item('chromium_options', 'existing_only', 'False')
self.set_item('chromium_options', 'new_env', 'False')
self.set_item('session_options', 'headers', "{'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X "
"10_12_6) AppleWebKit/603.3.8 (KHTML, like Gecko) Version/10."
"1.2 Safari/603.3.8', 'accept': 'text/html,application/xhtml"
"+xml,application/xml;q=0.9,*/*;q=0.8', 'connection': "
"'keep-alive', 'accept-charset': 'GB2312,utf-8;q=0.7,*;q=0.7'}")
self.set_item('timeouts', 'base', '10')
self.set_item('timeouts', 'page_load', '30')
self.set_item('timeouts', 'script', '30')
self.set_item('proxies', 'http', '')
self.set_item('proxies', 'https', '')
self.set_item('others', 'retry_times', '3')
self.set_item('others', 'retry_interval', '2')
def __getattr__(self, item):
return self.get_option(item)
def get_value(self, section, item):
try:
return eval(self._conf.get(section, item))
except (SyntaxError, NameError):
return self._conf.get(section, item)
except NoSectionError and NoOptionError:
return None
def get_option(self, section):
items = self._conf.items(section)
option = dict()
for j in items:
try:
option[j[0]] = eval(self._conf.get(section, j[0]))
except Exception:
option[j[0]] = self._conf.get(section, j[0])
return option
def set_item(self, section, item, value):
self._conf.set(section, item, str(value))
self.__setattr__(f'_{section}', None)
return self
def remove_item(self, section, item):
self._conf.remove_option(section, item)
return self
def save(self, path=None):
default_path = (Path(__file__).parent / 'configs.ini').absolute()
if path == 'default':
path = default_path
elif path is None:
if self.ini_path is None:
raise RuntimeError(_S._lang.join(_S._lang.INI_NOT_SET))
path = self.ini_path.absolute()
else:
path = Path(path).absolute()
path = path / 'config.ini' if path.is_dir() else path
path.parent.mkdir(exist_ok=True, parents=True)
path = str(path)
self._conf.write(open(path, 'w', encoding='utf-8'))
print(f'{_S._lang.OPTIONS_HAVE_SAVED}: {path}')
if path == str(default_path):
print(_S._lang.AUTO_LOAD_TIP)
self.file_exists = True
return path
def save_to_default(self):
return self.save('default')
def show(self):
for i in self._conf.sections():
print(f'[{i}]')
pprint(self.get_option(i))
print()
@@ -0,0 +1,77 @@
# -*- coding:utf-8 -*-
"""
@Author : g1879
@Contact : g1879@qq.com
@Website : https://DrissionPage.cn
@Copyright: (c) 2020 by g1879, Inc. All Rights Reserved.
"""
from configparser import RawConfigParser
from pathlib import Path
from typing import Any, Optional, Union
class OptionsManager(object):
"""管理配置文件内容的类"""
ini_path: Optional[Path] = ...
file_exists: bool = ...
_conf: RawConfigParser = ...
def __init__(self, path: Union[Path, str] = None):
"""初始化,读取配置文件,如没有设置临时文件夹,则设置并新建
:param path: ini文件的路径,为None则找项目文件夹下的,找不到则读取模块文件夹下的
"""
...
def __getattr__(self, item) -> dict:
"""以dict形似返回获取大项信息
:param item: 项名
:return: None
"""
...
def get_value(self, section: str, item: str) -> Any:
"""获取配置的值
:param section: 段名
:param item: 项名
:return: 项值
"""
...
def get_option(self, section: str) -> dict:
"""把section内容以字典方式返回
:param section: 段名
:return: 段内容生成的字典
"""
...
def set_item(self, section: str, item: str, value: Any) -> None:
"""设置配置值
:param section: 段名
:param item: 项名
:param value: 项值
:return: None
"""
...
def remove_item(self, section: str, item: str) -> None:
"""删除配置值
:param section: 段名
:param item: 项名
:return: None
"""
...
def save(self, path: str = None) -> str:
"""保存配置文件
:param path: ini文件的路径,传入 'default' 保存到默认ini文件
:return: 保存路径
"""
...
def save_to_default(self) -> str:
"""保存当前配置到默认ini文件"""
...
def show(self) -> None:
"""打印所有设置信息"""
...
@@ -0,0 +1,371 @@
# -*- coding:utf-8 -*-
"""
@Author : g1879
@Contact : g1879@qq.com
@Website : https://DrissionPage.cn
@Copyright: (c) 2020 by g1879, Inc. All Rights Reserved.
"""
from copy import copy
from pathlib import Path
from requests import Session
from requests.structures import CaseInsensitiveDict
from .options_manage import OptionsManager
from .._functions.cookies import cookies_to_tuple, set_session_cookies
from .._functions.settings import Settings as _S
from .._functions.web import format_headers
class SessionOptions(object):
def __init__(self, read_file=True, ini_path=None):
self.ini_path = None
self._download_path = '.'
self._timeout = 10
self._del_set = set() # 记录要从ini文件删除的参数
if read_file is False:
ini_path = False
self.ini_path = None
elif ini_path:
ini_path = Path(ini_path).absolute()
if not ini_path.exists():
raise FileNotFoundError(_S._lang.join(_S._lang.INI_NOT_FOUND, PATH=ini_path))
self.ini_path = str(ini_path)
else:
self.ini_path = str(Path(__file__).parent / 'configs.ini')
om = OptionsManager(ini_path)
self._headers = None
self._cookies = None
self._auth = None
self._proxies = None
self._hooks = None
self._params = None
self._verify = None
self._cert = None
self._adapters = None
self._stream = None
self._trust_env = None
self._max_redirects = None
options = om.session_options
if options.get('headers', None) is not None:
self.set_headers(options['headers'])
if options.get('cookies', None) is not None:
self.set_cookies(options['cookies'])
if options.get('auth', None) is not None:
self._auth = options['auth']
if options.get('params', None) is not None:
self._params = options['params']
if options.get('verify', None) is not None:
self._verify = options['verify']
if options.get('cert', None) is not None:
self._cert = options['cert']
if options.get('stream', None) is not None:
self._stream = options['stream']
if options.get('trust_env', None) is not None:
self._trust_env = options['trust_env']
if options.get('max_redirects', None) is not None:
self._max_redirects = options['max_redirects']
self.set_proxies(om.proxies.get('http', None), om.proxies.get('https', None))
self._timeout = om.timeouts.get('base', 10)
self._download_path = om.paths.get('download_path', '.') or '.'
others = om.others
self._retry_times = others.get('retry_times', 3)
self._retry_interval = others.get('retry_interval', 2)
def __repr__(self):
return f'<SessionOptions at {id(self)}>'
# ===========须独立处理的项开始============
@property
def download_path(self):
return self._download_path
def set_download_path(self, path):
self._download_path = '.' if path is None else str(path)
return self
@property
def timeout(self):
return self._timeout
def set_timeout(self, second):
self._timeout = second
return self
@property
def proxies(self):
if self._proxies is None:
self._proxies = {}
return self._proxies
def set_proxies(self, http=None, https=None):
self._sets('proxies', {'http': http, 'https': https})
return self
@property
def retry_times(self):
return self._retry_times
@property
def retry_interval(self):
return self._retry_interval
def set_retry(self, times=None, interval=None):
if times is not None:
self._retry_times = times
if interval is not None:
self._retry_interval = interval
return self
# ===========须独立处理的项结束============
@property
def headers(self):
if self._headers is None:
self._headers = {}
return self._headers
def set_headers(self, headers):
if headers is None:
self._headers = None
self._del_set.add('headers')
else:
headers = format_headers(headers)
self._headers = {key.lower(): headers[key] for key in headers}
return self
def set_a_header(self, name, value):
if self._headers is None:
self._headers = {}
self._headers[name.lower()] = value
return self
def remove_a_header(self, name):
if self._headers is None:
return self
self._headers.pop(name.lower(), None)
return self
def clear_headers(self):
self._headers = None
self._del_set.add('headers')
@property
def cookies(self):
if self._cookies is None:
self._cookies = []
return self._cookies
def set_cookies(self, cookies):
cookies = cookies if cookies is None else list(cookies_to_tuple(cookies))
self._sets('cookies', cookies)
return self
@property
def auth(self):
return self._auth
def set_auth(self, auth):
self._sets('auth', auth)
return self
@property
def hooks(self):
if self._hooks is None:
self._hooks = {}
return self._hooks
def set_hooks(self, hooks):
self._hooks = hooks
return self
@property
def params(self):
if self._params is None:
self._params = {}
return self._params
def set_params(self, params):
self._sets('params', params)
return self
@property
def verify(self):
return self._verify
def set_verify(self, on_off):
self._sets('verify', on_off)
return self
@property
def cert(self):
return self._cert
def set_cert(self, cert):
self._sets('cert', cert)
return self
@property
def adapters(self):
if self._adapters is None:
self._adapters = []
return self._adapters
def add_adapter(self, url, adapter):
self._adapters.append((url, adapter))
return self
@property
def stream(self):
return self._stream
def set_stream(self, on_off):
self._sets('stream', on_off)
return self
@property
def trust_env(self):
return self._trust_env
def set_trust_env(self, on_off):
self._sets('trust_env', on_off)
return self
@property
def max_redirects(self):
return self._max_redirects
def set_max_redirects(self, times):
self._sets('max_redirects', times)
return self
def _sets(self, arg, val):
if val is None:
self.__setattr__(f'_{arg}', None)
self._del_set.add(arg)
else:
self.__setattr__(f'_{arg}', val)
if arg in self._del_set:
self._del_set.remove(arg)
def save(self, path=None):
if path == 'default':
path = (Path(__file__).parent / 'configs.ini').absolute()
elif path is None:
if self.ini_path:
path = Path(self.ini_path).absolute()
else:
path = (Path(__file__).parent / 'configs.ini').absolute()
else:
path = Path(path).absolute()
path = path / 'config.ini' if path.is_dir() else path
if path.exists():
om = OptionsManager(path)
else:
om = OptionsManager(self.ini_path or (Path(__file__).parent / 'configs.ini'))
options = session_options_to_dict(self)
for i in options:
if i not in ('download_path', 'timeout', 'proxies'):
om.set_item('session_options', i, options[i])
om.set_item('paths', 'download_path', self.download_path or '')
om.set_item('timeouts', 'base', self.timeout)
om.set_item('proxies', 'http', self.proxies.get('http', ''))
om.set_item('proxies', 'https', self.proxies.get('https', ''))
om.set_item('others', 'retry_times', self.retry_times)
om.set_item('others', 'retry_interval', self.retry_interval)
for i in self._del_set:
if i == 'download_path':
om.set_item('paths', 'download_path', '')
elif i == 'proxies':
om.set_item('proxies', 'http', '')
om.set_item('proxies', 'https', '')
else:
om.remove_item('session_options', i)
path = str(path)
om.save(path)
return path
def save_to_default(self):
return self.save('default')
def as_dict(self):
return session_options_to_dict(self)
def make_session(self):
s = Session()
h = CaseInsensitiveDict(self.headers) if self.headers else CaseInsensitiveDict()
if self.cookies:
set_session_cookies(s, self.cookies)
if self.adapters:
for url, adapter in self.adapters:
s.mount(url, adapter)
for i in ['auth', 'proxies', 'hooks', 'params', 'verify', 'cert', 'stream', 'trust_env', 'max_redirects']:
attr = self.__getattribute__(i)
if attr:
s.__setattr__(i, attr)
return s, h
def from_session(self, session, headers=None):
self._headers = CaseInsensitiveDict(copy(session.headers).update(headers)) if headers else session.headers
self._cookies = session.cookies
self._auth = session.auth
self._proxies = session.proxies
self._hooks = session.hooks
self._params = session.params
self._verify = session.verify
self._cert = session.cert
self._stream = session.stream
self._trust_env = session.trust_env
self._max_redirects = session.max_redirects
if session.adapters:
self._adapters = [(k, i) for k, i in session.adapters.items()]
return self
def session_options_to_dict(options):
if options in (False, None):
return SessionOptions(read_file=False).as_dict()
if isinstance(options, dict):
return options
re_dict = dict()
attrs = ['headers', 'cookies', 'proxies', 'params', 'verify', 'stream', 'trust_env', 'cert',
'max_redirects', 'timeout', 'download_path']
for attr in attrs:
val = options.__getattribute__(f'_{attr}')
if val is not None:
re_dict[attr] = val
return re_dict
@@ -0,0 +1,298 @@
# -*- coding:utf-8 -*-
"""
@Author : g1879
@Contact : g1879@qq.com
@Website : https://DrissionPage.cn
@Copyright: (c) 2020 by g1879, Inc. All Rights Reserved.
"""
from http.cookiejar import CookieJar, Cookie
from pathlib import Path
from typing import Any, Union, Tuple, Optional
from requests import Session
from requests.adapters import HTTPAdapter
from requests.auth import HTTPBasicAuth
from requests.cookies import RequestsCookieJar
from requests.structures import CaseInsensitiveDict
class SessionOptions(object):
"""requests的Session对象配置类"""
ini_path: Optional[str] = ...
_download_path: str = ...
_headers: Union[dict, CaseInsensitiveDict, None] = ...
_cookies: Union[list, RequestsCookieJar, None] = ...
_auth: Optional[tuple] = ...
_proxies: Optional[dict] = ...
_hooks: Optional[dict] = ...
_params: Union[dict, None] = ...
_verify: Optional[bool] = ...
_cert: Union[str, tuple, None] = ...
_adapters: Optional[list] = ...
_stream: Optional[bool] = ...
_trust_env: Optional[bool] = ...
_max_redirects: Optional[int] = ...
_timeout: float = ...
_del_set: set = ...
_retry_times: int = ...
_retry_interval: float = ...
def __init__(self,
read_file: [bool, None] = True,
ini_path: Union[str, Path] = None):
"""
:param read_file: 是否从文件读取配置
:param ini_path: ini文件路径
"""
...
@property
def download_path(self) -> str:
"""返回默认下载路径属性信息"""
...
def set_download_path(self, path: Union[str, Path]) -> SessionOptions:
"""设置默认下载路径
:param path: 下载路径
:return: 返回当前对象
"""
...
@property
def timeout(self) -> float:
"""返回timeout属性信息"""
...
def set_timeout(self, second: float) -> SessionOptions:
"""设置超时信息
:param second: 秒数
:return: 返回当前对象
"""
...
@property
def proxies(self) -> dict:
"""返回proxies设置信息"""
...
def set_proxies(self, http: Union[str, None], https: Union[str, None] = None) -> SessionOptions:
"""设置proxies参数
:param http: http代理地址
:param https: https代理地址
:return: 返回当前对象
"""
...
@property
def retry_times(self) -> int:
"""返回连接失败时的重试次数"""
...
@property
def retry_interval(self) -> float:
"""返回连接失败时的重试间隔(秒)"""
...
def set_retry(self, times: int = None, interval: float = None) -> SessionOptions:
"""设置连接失败时的重试操作
:param times: 重试次数
:param interval: 重试间隔
:return: 当前对象
"""
...
@property
def headers(self) -> dict:
"""返回headers设置信息"""
...
def set_headers(self, headers: Union[dict, str, None]) -> SessionOptions:
"""设置headers参数
:param headers: 参数值,传入None可在ini文件标记删除
:return: 返回当前对象
"""
...
def set_a_header(self, name: str, value: str) -> SessionOptions:
"""设置headers中一个项
:param name: 设置名称
:param value: 设置值
:return: 返回当前对象
"""
...
def remove_a_header(self, name: str) -> SessionOptions:
"""从headers中删除一个设置
:param name: 要删除的设置
:return: 返回当前对象
"""
...
def clear_headers(self) -> SessionOptions:
"""清空已设置的header参数"""
...
@property
def cookies(self) -> list:
"""以list形式返回cookies"""
...
def set_cookies(self, cookies: Union[Cookie, CookieJar, list, tuple, str, dict, None]) -> SessionOptions:
"""设置一个或多个cookies信息
:param cookies: cookies,可为Cookie, CookieJar, list, tuple, str, dict,传入None可在ini文件标记删除
:return: 返回当前对象
"""
...
@property
def auth(self) -> Union[Tuple[str, str], HTTPBasicAuth]:
"""返回认证设置信息"""
...
def set_auth(self, auth: Union[Tuple[str, str], HTTPBasicAuth, None]) -> SessionOptions:
"""设置认证元组或对象
:param auth: 认证元组或对象
:return: 返回当前对象
"""
...
@property
def hooks(self) -> dict:
"""返回回调方法"""
...
def set_hooks(self, hooks: Union[dict, None]) -> SessionOptions:
"""设置回调方法
:param hooks: 回调方法
:return: 返回当前对象
"""
...
@property
def params(self) -> dict:
"""返回连接参数设置信息"""
...
def set_params(self, params: Union[dict, None]) -> SessionOptions:
"""设置查询参数字典
:param params: 查询参数字典
:return: 返回当前对象
"""
...
@property
def verify(self) -> bool:
"""返回是否验证SSL证书设置"""
...
def set_verify(self, on_off: Union[bool, None]) -> SessionOptions:
"""设置是否验证SSL证书
:param on_off: 是否验证 SSL 证书
:return: 返回当前对象
"""
...
@property
def cert(self) -> Union[str, tuple]:
"""返回SSL证书设置信息"""
...
def set_cert(self, cert: Union[str, Tuple[str, str], None]) -> SessionOptions:
"""SSL客户端证书文件的路径(.pem格式),或('cert', 'key')元组
:param cert: 证书路径或元组
:return: 返回当前对象
"""
...
@property
def adapters(self) -> list:
"""返回适配器设置信息"""
...
def add_adapter(self, url: str, adapter: HTTPAdapter) -> SessionOptions:
"""添加适配器
:param url: 适配器对应url
:param adapter: 适配器对象
:return: 返回当前对象
"""
...
@property
def stream(self) -> bool:
"""返回是否使用流式响应内容设置信息"""
...
def set_stream(self, on_off: Union[bool, None]) -> SessionOptions:
"""设置是否使用流式响应内容
:param on_off: 是否使用流式响应内容
:return: 返回当前对象
"""
...
@property
def trust_env(self) -> bool:
"""返回是否信任环境设置信息"""
...
def set_trust_env(self, on_off: Union[bool, None]) -> SessionOptions:
"""设置是否信任环境
:param on_off: 是否信任环境
:return: 返回当前对象
"""
...
@property
def max_redirects(self) -> int:
"""返回最大重定向次数"""
...
def set_max_redirects(self, times: Union[int, None]) -> SessionOptions:
"""设置最大重定向次数
:param times: 最大重定向次数
:return: 返回当前对象
"""
...
def _sets(self, arg: str, val: Any) -> None:
"""给属性赋值或标记删除
:param arg: 属性名称
:param val: 参数值
:return: None
"""
...
def save(self, path: str = None) -> str:
"""保存设置到文件
:param path: ini文件的路径,传入 'default' 保存到默认ini文件
:return: 保存文件的绝对路径
"""
...
def save_to_default(self) -> str:
"""保存当前配置到默认ini文件"""
...
def as_dict(self) -> dict:
"""以字典形式返回本对象"""
...
def make_session(self) -> Tuple[Session, Optional[CaseInsensitiveDict]]:
"""根据内在的配置生成Session对象,headers从对象中分离"""
...
def from_session(self, session: Session, headers: CaseInsensitiveDict = None) -> SessionOptions:
"""从Session对象中读取配置
:param session: Session对象
:param headers: headers
:return: 当前对象
"""
...
def session_options_to_dict(options: Union[dict, SessionOptions, None]) -> Union[dict, None]:
"""把session配置对象转换为字典
:param options: session配置对象或字典
:return: 配置字典
"""
...