Initial commit

This commit is contained in:
Alex committed 2026-11-02 22:54:14 +08:00
commit 57fc3525eb
7311 files changed
+1291398

No files matched your search

@@ -0,0 +1,8 @@
# -*- coding:utf-8 -*-
"""
@Author : g1879
@Contact : g1879@qq.com
"""
from .drissionGet import DrissionGet
__version__ = '1.2.1'
@@ -0,0 +1,218 @@
# -*- coding:utf-8 -*-
from copy import copy
from os import path as os_PATH
from os.path import sep
from pathlib import Path
from random import randint
from re import search, sub
from time import time
from urllib.parse import unquote
from DrissionRecord.tools import get_usable_path, make_valid_name
from requests import Session
FILE_EXISTS_MODE = {'rename': 'rename', 'overwrite': 'overwrite', 'skip': 'skip', 'add': 'add', 'r': 'rename',
'o': 'overwrite', 's': 'skip', 'a': 'add'}
def copy_session(session):
new = Session()
new.headers = session.headers.copy()
new.cookies = session.cookies.copy()
new.stream = True
new.auth = session.auth
new.proxies = dict(session.proxies).copy()
new.params = copy(session.params) #
new.cert = session.cert
new.max_redirects = session.max_redirects
new.trust_env = session.trust_env
new.verify = session.verify
return new
class BlockSizeSetter(object):
def __set__(self, block_size, val):
if isinstance(val, int) and val > 0:
size = val
elif isinstance(val, str):
units = {'b': 1, 'k': 1024, 'm': 1048576, 'g': 21474836480}
num = int(val[:-1])
unit = units.get(val[-1].lower(), None)
if unit and num > 0:
size = num * unit
else:
raise ValueError('单位只支持B、K、M、G,数字必须为大于0的整数。')
else:
raise TypeError('split_size只能传入int或str,数字必须为大于0的整数。')
block_size._block_size = size
def __get__(self, block_size, objtype=None) -> int:
return block_size._block_size
class PathSetter(object):
def __set__(self, save_path, val):
if val is not None and not isinstance(val, (str, Path)):
raise TypeError('路径只能是str或Path类型。')
save_path._save_path = get_save_path(val)
def __get__(self, save_path, objtype=None):
return save_path._save_path
class FileExistsSetter(object):
def __set__(self, file_exists, mode):
file_exists._file_exists = get_file_exists_mode(mode)
def __get__(self, file_exists, objtype=None):
return file_exists._file_exists
def get_save_path(path):
path = Path(path or '.').absolute()
return Path(path.anchor + sub(r'[*:|<>?"]', '', sep.join(path.parts[1:])).strip())
def get_file_exists_mode(mode):
mode = FILE_EXISTS_MODE.get(mode, mode)
if mode not in FILE_EXISTS_MODE:
raise ValueError(f'''mode参数只能是 '{"', '".join(FILE_EXISTS_MODE.keys())}' 之一,现在是:{mode}''')
return mode
def set_charset(response, encoding):
if encoding:
response.encoding = encoding
return response
# 在headers中获取编码
content_type = response.headers.get('content-type', '').lower()
if not content_type.endswith(';'):
content_type += ';'
charset = search(r'charset[=: ]*(.*)?;?', content_type)
if charset:
response.encoding = charset.group(1)
# 在headers中获取不到编码,且如果是网页
elif content_type.replace(' ', '').startswith('text/html'):
re_result = search(b'<meta.*?charset=[ \\\'"]*([^"\\\' />]+).*?>', response.content)
if re_result:
charset = re_result.group(1).decode()
else:
charset = response.apparent_encoding
response.encoding = charset
return response
def get_file_info(response, save_path, rename=None, suffix=None, file_exists=None, encoding=None, lock=None):
# ------------获取文件大小------------
file_size = response.headers.get('Content-Length', None)
file_size = None if file_size is None else int(file_size)
# ------------获取网络文件名------------
file_name = get_file_name(response, encoding)
# ------------获取保存文件名------------
# -------------------重命名-------------------
if rename:
if suffix is not None:
full_name = f'{rename}.{suffix}' if suffix else rename
else:
tmp = file_name.rsplit('.', 1)
ext_name = f'.{tmp[-1]}' if len(tmp) > 1 else ''
tmp = rename.rsplit('.', 1)
ext_rename = f'.{tmp[-1]}' if len(tmp) > 1 else ''
full_name = rename if ext_rename == ext_name else f'{rename}{ext_name}'
elif suffix is not None:
full_name = file_name.rsplit(".", 1)[0]
if suffix:
full_name = f'{full_name}.{suffix}'
else:
full_name = file_name
full_name = make_valid_name(full_name)
# -------------------生成路径-------------------
skip = False
overwrite = False
full_path = save_path / full_name
with lock:
if full_path.exists():
if file_exists == 'rename':
full_path = get_usable_path(full_path)
elif file_exists == 'skip':
skip = True
elif file_exists == 'overwrite':
overwrite = True
full_path.unlink()
return {'size': file_size,
'path': full_path,
'skip': skip,
'overwrite': overwrite}
def get_file_name(response, encoding):
file_name = ''
charset = ''
content_disposition = response.headers.get('content-disposition', '').replace(' ', '')
# 使用header里的文件名
if content_disposition:
txt = search(r'filename\*="?([^";]+)', content_disposition)
if txt: # 文件名自带编码方式
txt = txt.group(1).split("''", 1)
if len(txt) == 2:
charset, file_name = txt
else:
file_name = txt[0]
else: # 文件名没带编码方式
txt = search(r'filename="?([^";]+)', content_disposition)
if txt:
file_name = txt.group(1)
# 获取编码(如有)
charset = encoding or response.encoding
file_name = file_name.strip("'")
# 在url里获取文件名
if not file_name and os_PATH.basename(response.url):
file_name = os_PATH.basename(response.url).split("?")[0]
# 找不到则用时间和随机数生成文件名
if not file_name:
file_name = f'untitled_{time()}_{randint(0, 100)}'
# 去除非法字符
charset = charset or 'utf-8'
return unquote(file_name, charset)
def set_session_cookies(session, cookies):
# cookies = cookies_to_tuple(cookies)
for cookie in cookies:
if cookie['value'] is None:
cookie['value'] = ''
kwargs = {x: cookie[x] for x in cookie
if x.lower() in ('version', 'port', 'domain', 'path', 'secure',
'expires', 'discard', 'comment', 'comment_url', 'rest')}
if 'expiry' in cookie:
kwargs['expires'] = cookie['expiry']
session.cookies.set(cookie['name'], cookie['value'], **kwargs)
@@ -0,0 +1,93 @@
# -*- coding:utf-8 -*-
from pathlib import Path
from threading import Lock
from typing import Union, Optional
from requests import Session, Response
FILE_EXISTS_MODE: dict = ...
def copy_session(session: Session) -> Session:
"""复制输入Session对象,返回一个新的
:param session: 被复制的Session对象
:return: 新Session对象
"""
...
class BlockSizeSetter(object):
def __set__(self, block_size, val: Union[str, int]): ...
def __get__(self, block_size, objtype=None) -> int: ...
class PathSetter(object):
def __set__(self, save_path, val: Union[str, Path]): ...
def __get__(self, save_path, objtype=None): ...
class FileExistsSetter(object):
def __set__(self, file_exists, mode: str): ...
def __get__(self, file_exists, objtype=None): ...
def get_save_path(path: Union[str, Path]) -> Path:
"""获取绝对路径,去除Windows系统非法字符
:param path: 输入的路径
:return: Path格式路径
"""
...
def get_file_exists_mode(mode: str) -> str:
"""获取文件重名时处理策略名称
:param mode: 输入
:return: 标准字符串
"""
...
def set_charset(response: Response, encoding: Optional[str]) -> Response:
"""设置Response对象的编码
:param response: Response对象
:param encoding: 指定的编码格式
:return: 设置编码后的Response对象
"""
...
def get_file_info(response: Response, save_path: Path, rename: str = None, suffix: str = None,
file_exists: str = None, encoding: Optional[str] = None, lock: Lock = None) -> dict:
"""获取文件信息,大小单位为byte
包括:size、path、skip
:param response: Response对象
:param save_path: 目标文件夹Path对象
:param rename: 重命名
:param suffix: 重命名后缀名
:param file_exists: 存在重名文件时的处理方式
:param encoding: 编码格式
:param lock: 线程锁
:return: 文件大小、完整路径、是否跳过、是否覆盖
"""
...
def get_file_name(response: Response, encoding: str) -> str:
"""从headers或url中获取文件名,如果获取不到,生成一个随机文件名
:param response: 返回的response
:param encoding: 在headers获取时指定编码格式
:return: 下载文件的文件名
"""
...
def set_session_cookies(session: Session, cookies: list) -> None:
"""设置Session对象的cookies
:param session: Session对象
:param cookies: cookies信息
:return: None
"""
...
@@ -0,0 +1,539 @@
# -*- coding:utf-8 -*-
from copy import copy
from pathlib import Path
from queue import Queue
from threading import Thread, Lock
from time import sleep, perf_counter
from requests.structures import CaseInsensitiveDict
from ._funcs import (FileExistsSetter, PathSetter, BlockSizeSetter, set_charset, get_file_info, get_file_exists_mode,
get_save_path)
from .mission import Task, Mission
from .setter import Setter
class DrissionGet(object):
file_exists = FileExistsSetter()
save_path = PathSetter()
block_size = BlockSizeSetter()
def __init__(self, save_path='.', roads=10, driver=None, file_exists='rename'):
self._roads = roads
self._missions = {}
self._threads = {i: None for i in range(self._roads)}
self._waiting_list = Queue()
self._missions_num = 0
self._running_count = 0 # 正在运行的任务数
self._stop_printing = False # 用于控制显示线程停止
self._lock = Lock()
self.page = None
self._retry = None
self._interval = None
self._timeout = None
self._encoding = None
self._setter = None
self._print_mode = 'all'
self._log_mode = None
self._logger = None
self.save_path = save_path
self.file_exists = file_exists
self.split = True
self.block_size = '50M'
self.set.driver(driver)
def __call__(self, url=None, save_path=None, rename=None, suffix=None,
file_exists=None, split=False, show_msg=True, file_url=None, **kwargs):
return self.download(url=url, save_path=save_path, rename=rename, suffix=suffix,
file_exists=file_exists, show_msg=show_msg, **kwargs)
@property
def set(self):
if self._setter is None:
self._setter = Setter(self)
return self._setter
@property
def roads(self):
return self._roads
@property
def retry(self):
if self._retry is not None:
return self._retry
elif self.page is not None:
return self.page.retry_times
else:
return 3
@property
def interval(self):
if self._interval is not None:
return self._interval
elif self.page is not None:
return self.page.retry_interval
else:
return 5
@property
def timeout(self):
if self._timeout is not None:
return self._timeout
elif self.page is not None:
return self.page.timeout
else:
return 20
@property
def waiting_list(self):
return self._waiting_list
@property
def session(self):
return self._session
@property
def is_running(self):
return self._running_count > 0
@property
def missions(self):
return self._missions
@property
def encoding(self):
"""返回指定的编码格式"""
return self._encoding
def add(self, url=None, save_path=None, rename=None, suffix=None, file_exists=None, split=None,
file_url=None, **kwargs):
if not url and file_url:
url = file_url
self._missions_num += 1
self._running_count += 1
if file_exists is not None:
file_exists = get_file_exists_mode(file_exists)
mission = Mission(self._missions_num, self, url, get_save_path(save_path) if save_path else self.save_path,
rename, suffix, file_exists or self.file_exists,
self.split if split is None else split, self._encoding, kwargs)
self._missions[self._missions_num] = mission
self._run_or_wait(mission)
return mission
def download(self, url=None, save_path=None, rename=None, suffix=None, file_exists=None, split=False,
show_msg=None, file_url=None, **kwargs):
if not url and file_url:
url = file_url
self._missions_num += 1
self._running_count += 1
if file_exists is not None:
file_exists = get_file_exists_mode(file_exists)
mission = Mission(self._missions_num, self, url, get_save_path(save_path) if save_path else self.save_path,
rename, suffix, file_exists or self.file_exists,
self.split if split is None else split, self._encoding, kwargs)
self._missions[self._missions_num] = mission
tmp, self._print_mode = self._print_mode, None
if show_msg is None:
show_msg = True if tmp else False
if split:
self._run_or_wait(mission)
r = mission.wait(show=show_msg)
self._print_mode = tmp
return r
ok, r, file_size = handle_mission(mission, url)
if show_msg:
print(f'url:{url}')
print(f'文件名:{mission.file_name}')
print(f'目标路径:{mission.path}')
if not mission.size:
print('下载中 ', end='')
if ok:
task1 = Task(mission, None, '1/1', file_size)
mission.tasks.append(task1)
do_download_(r, task1, show_msg, file_size)
if show_msg:
if mission.result is False:
print(f'下载失败 {mission.info}')
elif mission.result == 'success':
print('\r100% ', end='')
print(f'{"已覆盖" if mission._overwrote else "下载完成"} {mission.info}')
elif mission.result == 'skipped':
print(f'已跳过 {mission.info}')
print()
self._print_mode = tmp
return mission.result, mission.info
def by_browser(self, url, save_path=None, rename=None, suffix=None, timeout=None, file_exists=None):
if hasattr(self.page, '_download_by_browser'):
return self.page._download_by_browser(url=url, save_path=save_path, rename=rename, suffix=suffix,
timeout=timeout, file_exists=file_exists)
else:
raise RuntimeError('非浏览器调用无法使用浏览器下载。')
def _run_or_wait(self, mission):
thread_id = self._get_usable_thread()
if thread_id is None:
self._waiting_list.put(mission)
else:
thread = Thread(target=self._run, args=(thread_id, mission), daemon=False)
self._threads[thread_id] = {'thread': thread, 'mission': None}
thread.start()
def _run(self, ID, mission):
while True:
if not mission: # 如果没有任务,就从等候列表中取一个
if self._waiting_list.empty():
break
else:
try:
mission = self._waiting_list.get(True, .5)
except Exception:
self._waiting_list.task_done()
break
self._threads[ID]['mission'] = mission
self._download(mission, ID)
mission = None
self._threads[ID] = None
def get_mission(self, mission_or_id):
return self._missions[mission_or_id] if isinstance(mission_or_id, int) else mission_or_id
def get_failed_missions(self):
return [i for i in self._missions.values() if i.result is False]
def wait(self, mission=None, show=False, timeout=None):
timeout = 0 if timeout is None else timeout
if mission:
return self.get_mission(mission).wait(show, timeout)
else:
if show:
self.show(False)
else:
end_time = perf_counter() + timeout
while self.is_running and (perf_counter() < end_time or timeout == 0):
sleep(0.1)
def cancel(self):
for m in self._missions.values():
m.cancel()
def show(self, asyn=True, keep=False):
if asyn:
Thread(target=self._show, args=(2, keep)).start()
else:
self._show(0.1, keep)
def _show(self, wait, keep=False):
self._stop_printing = False
if keep:
Thread(target=self._stop_show).start()
end_time = perf_counter() + wait
while not self._stop_printing and (keep or self.is_running or perf_counter() < end_time):
print(f'\033[K', end='')
print(f'等待任务数:{self._waiting_list.qsize()}')
for k, v in self._threads.items():
m = v['mission'] if v else None
if m:
items = (m.mission.rate, m.mid) if isinstance(m, Task) else (m.rate, m.id)
path = f'M{items[1]} {items[0]}% {m}'
else:
path = '空闲'
print(f'\033[K', end='')
print(f'线程{k}:{path}')
print(f'\033[{self.roads + 1}A\r', end='')
sleep(0.4)
print(f'\033[1B', end='')
for i in range(self.roads):
print(f'\033[K', end='')
print(f'线程{i}:空闲')
print()
def _connect(self, url, session, _headers, method, encoding, **kwargs):
kwargs = CaseInsensitiveDict(kwargs)
if 'headers' in kwargs: # 不知道为什么添加这个才能正常使用多线程
kwargs['headers'] = CaseInsensitiveDict({**_headers, **kwargs['headers']})
else:
kwargs['headers'] = _headers
r = err = None
for i in range(self.retry + 1):
try:
if method == 'get':
r = session.get(url, **kwargs)
elif method == 'post':
r = session.post(url, **kwargs)
if r:
return set_charset(r, encoding), 'Success'
except Exception as e:
err = e
if r and r.status_code in (403, 404):
break
if i < self.retry:
sleep(self.interval)
# 返回失败结果
if r is None:
return None, '连接失败' if err is None else err
if not r.ok:
return r, f'状态码:{r.status_code}'
return None
def _get_usable_thread(self):
for k, v in self._threads.items():
if v is None:
return k
return None
def _stop_show(self):
input()
self._stop_printing = True
def _when_mission_done(self, mission):
self._running_count -= 1
if self._print_mode == 'all' or (self._print_mode == 'failed' and mission.result is False):
print(f'[{mission.RESULT_TEXTS[mission.result]}] {mission.info} {mission.data.url}')
if self._log_mode == 'all' or (self._log_mode == 'failed' and mission.result is False):
self._logger.add_data((mission.RESULT_TEXTS[mission.result], mission.data.url, mission.info,
mission.data.save_path, mission.data.rename, mission.data.kwargs))
mission.session.close()
def _download(self, mission_or_task, thread_id):
if mission_or_task.is_done:
return
if mission_or_task.state == 'cancel':
mission_or_task.state = 'done'
return
file_url = mission_or_task.data.url
if isinstance(mission_or_task, Task):
kwargs = copy(mission_or_task.data.kwargs)
kwargs['headers']['Range'] = f"bytes={mission_or_task.range[0]}-{mission_or_task.range[1]}"
r, inf = self._connect(file_url, mission_or_task.mission.session, mission_or_task.mission.headers,
mission_or_task.mission.method, mission_or_task.mission.encoding, **kwargs)
if r:
do_download(r, mission_or_task, False)
else:
mission_or_task._set_done(False, inf)
return
# ===================开始处理mission====================
mission = mission_or_task
ok, r, file_size = handle_mission(mission, file_url)
if not ok:
return
# -------------------设置分块任务-------------------
first = False
if mission.data.split and file_size and file_size > self.block_size and r.headers.get(
'Accept-Ranges') == 'bytes':
first = True
chunks = [[s, min(s + self.block_size, file_size) - 1] for s in range(0, file_size, self.block_size)]
chunks[-1][-1] = ''
chunks_len = len(chunks)
task1 = Task(mission, chunks[0], f'1/{chunks_len}', chunks[0][1] - chunks[0][0])
mission.tasks_count = chunks_len
mission.tasks = [task1]
for ind, chunk in enumerate(chunks[1:], 2):
s = file_size - chunk[0] if chunks_len == ind else chunk[1] - chunk[0]
task = Task(mission, chunk, f'{ind}/{chunks_len}', s)
mission.tasks.append(task)
self._run_or_wait(task)
else: # 不分块
task1 = Task(mission, None, '1/1', file_size)
mission.tasks.append(task1)
self._threads[thread_id]['mission'] = task1
do_download(r, task1, first)
def handle_mission(mission, url):
mission.info = '下载中'
mission.state = 'running'
kwargs = mission.data.kwargs
rename = mission.data.rename
suffix = mission.data.suffix
file_exists = mission.data.file_exists
mission.data.save_path.mkdir(parents=True, exist_ok=True)
if file_exists == 'skip' and rename:
tmp = mission.data.save_path / rename
if tmp.exists() and tmp.is_file():
mission.file_name = rename
mission._set_path(mission.data.save_path / rename)
mission._set_done('skipped', str(mission.path))
return False, None, None
r, inf = mission.owner._connect(url, mission.session, mission.headers, mission.method, mission.encoding, **kwargs)
if mission.is_done:
return False, None, None
if not r:
mission._break_mission(result=False, info=inf)
return False, None, None
# -------------------获取文件信息-------------------
file_info = get_file_info(r, mission.data.save_path, rename, suffix, file_exists,
mission.encoding, mission.owner._lock)
file_size = file_info['size']
full_path = file_info['path']
mission._set_path(full_path)
mission.size = file_size
mission._overwrote = file_info['overwrite']
if file_info['skip']:
mission._set_done('skipped', str(mission.path))
return False, None, None
full_Path = Path(full_path)
if file_exists == 'add' and full_Path.exists():
mission.data.offset = full_Path.stat().st_size
return True, r, file_size
def do_download_(r, task, show, size):
if task.is_done or task.mission.is_done:
return
task.set_states(result=None, info='下载中', state='running')
block_size = 524288 # 512k
result = None
try:
if show and size:
for chunk in r.iter_content(chunk_size=block_size):
if task.state in ('cancel', 'done'):
result = 'canceled'
task.clear_cache()
break
if chunk:
task.add_data(chunk, None)
print(f'\r{task.rate}% ', end='')
else:
for chunk in r.iter_content(chunk_size=block_size):
if task.state in ('cancel', 'done'):
result = 'canceled'
task.clear_cache()
break
if chunk:
task.add_data(chunk, None)
except Exception as e:
result, info = False, f'{r.status_code} {e}'
else:
result = 'success' if result is None else result
info = str(task.path)
finally:
r.close()
task._set_done(result=result, info=info)
def do_download(r, task, first=False):
if task.is_done or task.mission.is_done:
return
task.set_states(result=None, info='下载中', state='running')
block_size = 524288 # 512k
result = None
try:
if first: # 分块是第一块
if task.range[1] <= block_size or task.range[1] % block_size != 0:
r_content = r.iter_content(chunk_size=task.range[1] + 1)
task.add_data(next(r_content), seek=0 + task.mission.data.offset)
if task.state in ('cancel', 'done'):
result = 'canceled'
task.clear_cache()
else:
blocks = task.range[1] // block_size
remainder = task.range[1] % block_size
r_content = r.iter_content(chunk_size=block_size)
for b in range(blocks):
task.add_data(next(r_content), seek=b * block_size + task.mission.data.offset)
if task.state in ('cancel', 'done'):
result = 'canceled'
task.clear_cache()
break
if task.state in ('cancel', 'done'):
result = 'canceled'
task.clear_cache()
else:
task.add_data(next(r_content)[:remainder + 1], blocks * block_size + task.mission.data.offset)
else:
if task.range is None: # 不分块
for chunk in r.iter_content(chunk_size=block_size):
if task.state in ('cancel', 'done'):
result = 'canceled'
task.clear_cache()
break
if chunk:
task.add_data(chunk, None)
elif task.range[1] == '': # 结尾的数据块
begin = task.range[0]
for chunk in r.iter_content(chunk_size=block_size):
if task.state in ('cancel', 'done'):
result = 'canceled'
task.clear_cache()
break
if chunk:
task.add_data(chunk, seek=begin + task.mission.data.offset)
begin += len(chunk)
else: # 有始末数字的数据块
begin, end = task.range
num = (end - begin) // block_size
for ind, chunk in enumerate(r.iter_content(chunk_size=block_size), 1):
if task.state in ('cancel', 'done'):
result = 'canceled'
task.clear_cache()
break
if chunk:
task.add_data(chunk, seek=begin + task.mission.data.offset)
if ind <= num:
begin += block_size
except Exception as e:
result, info = False, f'{r.status_code} {e}'
else:
result = 'success' if result is None else result
info = str(task.path)
finally:
r.close()
task._set_done(result=result, info=info)
@@ -0,0 +1,353 @@
# -*- coding:utf-8 -*-
from pathlib import Path
from queue import Queue
from threading import Lock
from typing import Union, Tuple, Any, Literal, Optional
from DrissionPage import SessionOptions
from DrissionPage._base.base import BasePage
from DrissionPage._units.downloader import DownloadMission
from DrissionRecord import Recorder
from requests import Session, Response
from requests.structures import CaseInsensitiveDict
from ._funcs import FileExistsSetter, PathSetter, BlockSizeSetter
from .mission import Task, Mission, BaseTask
from .setter import Setter
FILE_EXISTS = Literal['add', 'skip', 'rename', 'overwrite', 'a', 's', 'r', 'o']
class DrissionGet(object):
file_exists: FileExistsSetter = ...
save_path: PathSetter = ...
block_size: BlockSizeSetter = ...
_roads: int = ...
_setter: Optional[Setter] = ...
_print_mode: Optional[str] = ...
_log_mode: Optional[str] = ...
_logger: Optional[Recorder] = ...
_retry: Optional[int] = ...
_interval: Optional[float] = ...
page: Optional[BasePage] = ...
_waiting_list: Queue = ...
_session: Session = ...
_running_count: int = ...
_missions_num: int = ...
_missions: dict = ...
_threads: dict = ...
_timeout: Union[int, float, None] = ...
_stop_printing: bool = ...
_lock: Lock = ...
split: bool = ...
_encoding: Optional[str] = ...
def __init__(self,
save_path: Union[str, Path] = None,
roads: int = 10,
driver: Union[Session, BasePage, SessionOptions] = None,
file_exists: FILE_EXISTS = 'rename'):
"""
:param save_path: 文件保存路径
:param roads: 可同时运行的线程数
:param driver: 使用的Session对象,或配置对象、页面对象等
:param file_exists: 有同名文件名时的处理方式,可选 'skip', 'overwrite', 'rename', 'add', 's', 'o', 'r', 'a'
"""
...
def __call__(self,
url: str,
save_path: Union[str, Path, None] = None,
rename: str = None,
suffix: Optional[str] = None,
file_exists: FILE_EXISTS = None,
show_msg: bool = True,
split: bool = False,
timeout: Optional[float] = None,
params: Optional[dict] = ...,
data: Any = ...,
json: Any = ...,
headers: Optional[dict] = ...,
cookies: Any = ...,
files: Any = ...,
auth: Any = ...,
allow_redirects: bool = ...,
proxies: Optional[dict] = ...,
hooks: Any = ...,
stream: Any = ...,
verify: Any = ...,
cert: Any = ...) -> tuple:
"""以阻塞的方式下载一个文件并返回结果
:param url: 文件网址
:param save_path: 保存路径
:param rename: 重命名的文件名
:param suffix: 指定后缀名
:param file_exists: 遇到同名文件时的处理方式,可选 'skip', 'overwrite', 'rename', 'add', 's', 'o', 'r', 'a',默认跟随实例属性
:param split: 是否分块下载,为None使用设置
:param show_msg: 是否打印进度
:param kwargs: 连接参数
:return: 任务结果和信息组成的tuple
"""
...
@property
def set(self) -> Setter:
"""用于设置打印和记录模式的对象"""
...
@property
def roads(self) -> int:
"""可同时运行的线程数"""
...
@property
def retry(self) -> int:
"""返回连接失败时重试次数"""
...
@property
def interval(self) -> float:
"""返回连接失败时重试间隔"""
...
@property
def timeout(self) -> float:
"""返回连接超时时间"""
...
@property
def waiting_list(self) -> Queue:
"""返回等待队列"""
...
@property
def session(self) -> Session:
"""返回用于保存默认连接设置的Session对象"""
...
@property
def is_running(self) -> bool:
"""返回是否有线程还在运行"""
...
@property
def missions(self) -> dict:
"""用list方式返回所有任务对象"""
...
@property
def encoding(self) -> Optional[str]:
...
def add(self,
url: str,
save_path: Union[str, Path, None] = None,
rename: str = None,
suffix: str = None,
file_exists: FILE_EXISTS = None,
split: bool = None,
timeout: Optional[float] = None,
params: Optional[dict] = ...,
data: Any = None,
json: Union[dict, str, None] = ...,
headers: Optional[dict] = ...,
cookies: Any = ...,
files: Any = ...,
auth: Any = ...,
allow_redirects: bool = ...,
proxies: Optional[dict] = ...,
hooks: Any = ...,
stream: Any = ...,
verify: Any = ...,
cert: Any = ...) -> Mission:
"""添加一个下载任务并将其返回
:param url: 文件网址
:param save_path: 保存路径
:param rename: 重命名的文件名
:param suffix: 重命名的文件后缀名
:param file_exists: 遇到同名文件时的处理方式,可选 'skip', 'overwrite', 'rename', 'add', 's', 'o', 'r', 'a',默认跟随实例属性
:param split: 是否允许多线程分块下载,为None则使用对象属性
:param kwargs: 连接参数
:return: 任务对象
"""
...
def download(self,
url: str,
save_path: Union[str, Path, None] = None,
rename: str = None,
suffix: str = None,
file_exists: FILE_EXISTS = None,
split: bool = False,
show_msg: bool = None,
timeout: Optional[float] = None,
params: Optional[dict] = ...,
data: Any = ...,
json: Any = ...,
headers: Optional[dict] = ...,
cookies: Any = ...,
files: Any = ...,
auth: Any = ...,
allow_redirects: bool = ...,
proxies: Optional[dict] = ...,
hooks: Any = ...,
stream: Any = ...,
verify: Any = ...,
cert: Any = ...) -> tuple:
"""以阻塞的方式下载一个文件并返回结果
:param url: 文件网址
:param save_path: 保存路径
:param rename: 重命名的文件名
:param suffix: 重命名的文件后缀名
:param file_exists: 遇到同名文件时的处理方式,可选 'skip', 'overwrite', 'rename', 'add', 's', 'o', 'r', 'a',默认跟随实例属性
:param split: 是否分块下载,为None使用设置
:param show_msg: 是否打印进度,为None使用设置
:param kwargs: 连接参数
:return: 任务结果和信息组成的tuple
"""
...
def by_browser(self,
url: str,
save_path: Union[str, Path] = None,
rename: str = None, suffix: str = None,
timeout: float = None,
file_exists: Literal['rename', 'overwrite', 'skip', 'r', 'o', 's'] = None) -> Union[
DownloadMission, False]:
"""触发浏览器下载
:param url: 下载地址
:param save_path: 保存路径,为None保存在原来设置的,如未设置保存到当前路径
:param rename: 重命名文件名
:param suffix: 指定文件后缀
:param timeout: 等待下载触发的超时时间,为None则使用页面对象设置
:param file_exists: 可在 'rename', 'overwrite', 'skip', 'r', 'o', 's'中选择
:return: DownloadMission对象,任务无触发时返回False
"""
...
def _run_or_wait(self, mission: BaseTask) -> None:
"""接收任务,有空线程则运行,没有则进入等待队列
:param mission: 任务对象
:return: None
"""
...
def _run(self, ID: int, mission: BaseTask) -> None:
"""线程函数
:param ID: 线程id
:param mission: 任务对象,Mission或Task
:return: None
"""
...
def get_mission(self, mission_or_id: Union[int, Mission]) -> Mission:
"""根据id值获取一个任务
:param mission_or_id: 任务或任务id
:return: 任务对象
"""
...
def get_failed_missions(self) -> list:
"""返回失败任务列表"""
...
def wait(self,
mission: Union[int, Mission] = None,
show: bool = False,
timeout: float = None) -> Optional[tuple]:
"""等待所有或指定任务完成
:param mission: 任务对象或任务id,为None时等待所有任务结束
:param show: 是否显示进度
:param timeout: 超时时间,None或0为无限
:return: 任务结果和信息组成的tuple
"""
...
def cancel(self) -> None:
"""取消所有等待中或执行中的任务"""
...
def show(self, asyn: bool = True, keep: bool = False) -> None:
"""实时显示所有线程进度
:param asyn: 是否以异步方式显示
:param keep: 任务列表为空时是否保持显示
:return: None
"""
...
def _show(self, wait: float, keep: bool = False) -> None:
"""实时显示所有线程进度
:param wait: 超时时间(秒)
:param keep: 任务列表为空时是否保持显示
:return: None
"""
...
def _connect(self, url: str, session: Session, _headers: CaseInsensitiveDict,
method: str, encoding: Optional[str], **kwargs) -> Tuple[Union[Response, None], str]:
"""生成response对象
:param url: 目标url
:param session: 用于连接的Session对象
:param _headers: 内置的headers参数
:param method: 请求方式
:param encoding: 编码格式
:param kwargs: 连接参数
:return: tuple,第一位为Response或None,第二位为出错信息或'Success'
"""
...
def _get_usable_thread(self) -> Optional[int]:
"""获取可用线程,没有则返回None"""
...
def _stop_show(self) -> None:
"""设置停止打印的变量"""
...
def _when_mission_done(self, mission: Mission) -> None:
"""当任务完成时执行的操作
:param mission: 完结的任务
:return: None
"""
...
def _download(self,
mission_or_task: Union[Mission, Task],
thread_id: int) -> None:
"""此方法是执行下载的线程方法,用于根据任务下载文件
:param mission_or_task: 下载任务对象
:param thread_id: 线程号
:return: None
"""
...
def handle_mission(mission: Mission, url: str) -> tuple:
"""对任务进行预处理,创建连接、获取信息
:param mission: 下载任务对象
:param url: 文件url
:return: (是否继续, Response对象, 文件大小)
"""
...
def do_download_(r, task: Task, show, size) -> None:
"""执行download()触发的下载任务
:param r: Response对象
:param task: 任务对象
:param show: 是否打印进度
:param size: 文件大小
:return: None
"""
...
def do_download(r: Response, task: Task, first: bool = False) -> None:
"""执行下载任务
:param r: Response对象
:param task: 任务对象
:param first: 是否第一个分块
:return: None
"""
...
@@ -0,0 +1,280 @@
# -*- coding:utf-8 -*-
from time import sleep, perf_counter
from urllib.parse import quote, urlparse
from DrissionRecord import Recorder
from requests.structures import CaseInsensitiveDict
from ._funcs import copy_session, set_session_cookies
class MissionData(object):
def __init__(self, url, save_path, rename, suffix, file_exists, split, kwargs, offset=0):
self.url = quote(url, safe='-_.~!*\'"();:@&=+$,/\\?#[]%')
self.save_path = save_path
self.rename = rename
self.suffix = suffix
self.file_exists = file_exists
self.split = split
self.kwargs = kwargs
self.offset = offset
class BaseTask(object):
_DONE = 'done'
RESULT_TEXTS = {'success': '成功', 'skipped': '跳过', 'canceled': '取消', False: '失败', None: '未知'}
def __init__(self, ID):
self._id = ID
self.state = 'waiting' # 'waiting', 'running', 'done'
self.result = None # 'success', 'skipped', 'canceled', False, None
self.info = '等待下载' # 信息
@property
def id(self):
return self._id
@property
def data(self):
return
@property
def is_done(self):
return self.state in ('done', 'cancel')
def set_states(self, result=None, info=None, state='done'):
self.result = result
self.info = info
self.state = state
class Mission(BaseTask):
def __init__(self, ID, owner, file_url, save_path, rename, suffix, file_exists, split, encoding, kwargs):
super().__init__(ID)
self.owner = owner
self.size = None
self.tasks = []
self.tasks_count = 1
self.done_tasks_count = 0
self.file_name = None
self._path = None # 文件完整路径,Path对象
self._recorder = None
self.encoding = encoding
self._set_session()
kwargs = self._handle_kwargs(file_url, kwargs)
self._data = MissionData(file_url, save_path, rename, suffix, file_exists, split, kwargs)
self._overwrote = False
self.method = 'post' if (self._data.kwargs.get('data', None) is not None or
self._data.kwargs.get('json', None) is not None) else 'get'
def __repr__(self):
return f'<Mission {self.id} {self.info} {self.file_name}>'
@property
def data(self):
return self._data
@property
def path(self):
return self._path
@property
def recorder(self):
if self._recorder is None:
self._recorder = Recorder(cache_size=100)
self._recorder.set.show_msg(False)
return self._recorder
@property
def rate(self):
if not self.size:
return None
c = 0
for t in self.tasks:
c += t._downloaded_size if t._downloaded_size else 0
return round((c / self.size) * 100, 2)
def cancel(self) -> None:
self._break_mission('canceled', '已取消')
def del_file(self):
if self.path and self.path.exists():
try:
self.path.unlink()
except Exception:
pass
def wait(self, show=True, timeout=0):
if show:
print(f'url:{self.data.url}')
t2 = perf_counter()
while self.file_name is None and perf_counter() - t2 < 4:
sleep(0.01)
print(f'文件名:{self.file_name}')
print(f'目标路径:{self.path}')
if not self.size:
print('未知大小 ', end='')
t1 = perf_counter()
while not self.is_done and (perf_counter() - t1 < timeout or timeout == 0):
try:
rate = round((self.path.stat().st_size / self.size) * 100, 2)
print(f'\r{rate}% ', end='')
except (FileNotFoundError, TypeError, ZeroDivisionError):
pass
sleep(0.1)
if self.result is False:
print(f'下载失败 {self.info}')
elif self.result == 'success':
print('\r100% ', end='')
print(f'{"已覆盖" if self._overwrote else "下载完成"} {self.info}')
elif self.result == 'skipped':
print(f'已跳过 {self.info}')
print()
else:
t1 = perf_counter()
while not self.is_done and (perf_counter() - t1 < timeout or timeout == 0):
sleep(0.1)
return self.result, self.info
def _set_session(self):
session = copy_session(self.owner.session)
headers = session.headers
session.headers = None
if self.owner.page:
set_session_cookies(session, self.owner.page.cookies())
if hasattr(self.owner.page, '_headers'):
headers = CaseInsensitiveDict({**self.owner.page._headers, **headers})
headers.update({"User-Agent": self.owner.page.user_agent})
self.session = session
self.headers = headers
def _handle_kwargs(self, url, kwargs):
if 'timeout' not in kwargs:
kwargs['timeout'] = self.owner.timeout
headers = CaseInsensitiveDict(kwargs['headers']) if 'headers' in kwargs else CaseInsensitiveDict()
parsed_url = urlparse(url)
hostname = parsed_url.hostname
scheme = parsed_url.scheme
if not ('Referer' in headers or 'Referer' in self.headers):
headers['Referer'] = self.owner.page.url if self.owner.page and self.owner.page.url \
else f'{scheme}://{hostname}'
if not ('Host' in headers or 'Host' in self.headers):
headers['Host'] = hostname
kwargs['headers'] = format_headers(headers)
if not kwargs['headers']['Host']:
kwargs['headers'].pop('Host')
if not kwargs['headers']['Referer']:
kwargs['headers'].pop('Referer')
return kwargs
def _set_path(self, path):
self.file_name = path.name
self._path = path
self.recorder.set.path(path, file_type='b')
def _set_done(self, result, info):
if result == 'skipped':
self.set_states(result=result, info=info, state=self._DONE)
elif result == 'canceled' or result is False:
self.recorder.clear()
self.set_states(result=result, info=info, state=self._DONE)
elif result == 'success':
self.recorder.record()
if self.size and self.path.stat().st_size < self.size:
self.del_file()
self.set_states(False, '下载失败', self._DONE)
else:
self.set_states('success', info, self._DONE)
self.owner._when_mission_done(self)
def _a_task_done(self, is_success, info):
if self.is_done:
return
if is_success is False:
self._break_mission(False, info)
return
self.done_tasks_count += 1
if self.done_tasks_count == self.tasks_count:
self._set_done('success', info)
def _break_mission(self, result, info):
if self.is_done:
return
for task in self.tasks:
if not task.is_done:
task.set_states(result=result, info=info, state='cancel')
while any((not i.is_done for i in self.tasks)):
sleep(.3)
self._set_done(result, info)
self.del_file()
class Task(BaseTask):
def __init__(self, mission, range_, ID, size):
super().__init__(ID)
self.mission = mission
self.range = range_
self.size = size
self._downloaded_size = 0
def __repr__(self):
return f'<Task M{self.mid} T{self._id} {self.rate}% {self.info} {self.file_name}>'
@property
def mid(self):
return self.mission.id
@property
def data(self):
return self.mission.data
@property
def path(self):
return self.mission.path
@property
def file_name(self):
return self.mission.file_name
@property
def rate(self):
return round((self._downloaded_size / self.size) * 100, 2) if self.size else None
def add_data(self, data, seek=None):
self._downloaded_size += len(data)
self.mission.recorder.add_data(data, coord=seek)
def clear_cache(self):
self.mission.recorder.clear()
def _set_done(self, result, info):
self.set_states(result=result, info=info, state=self._DONE)
self.mission._a_task_done(result, info)
def format_headers(headers):
for k, v in headers.items():
if k in (':method', ':scheme', ':authority', ':path'):
headers.pop(k)
elif v not in (None, False, True):
headers[k] = str(v)
return headers
@@ -0,0 +1,259 @@
# -*- coding:utf-8 -*-
from pathlib import Path
from typing import Union, List, Optional
from DrissionRecord import Recorder
from requests import Session
from requests.structures import CaseInsensitiveDict
from .drissionGet import DrissionGet
class MissionData(object):
"""保存任务数据的对象"""
url: str = ...
save_path: Path = ...
rename: Optional[str] = ...
suffix: Optional[str] = ...
file_exists: str = ...
split: bool = ...
kwargs: dict = ...
offset: int = ...
def __init__(self, url: str, save_path: Path, rename: Optional[str], suffix: Optional[str],
file_exists: str, split: bool, kwargs: dict, offset: int = 0):
"""保存任务数据的对象
:param url: 下载文件url
:param save_path: 保存文件夹
:param rename: 文件重命名
:param suffix: 文件重命名后缀名
:param file_exists: 存在重名文件时处理方式
:param split: 是否允许分块下载
:param kwargs: requests其它参数
:param offset: 文件存储偏移量
"""
...
class BaseTask(object):
"""任务类基类"""
_DONE: str = ...
RESULT_TEXTS: dict = ...
_id: str = ...
state: str = ...
result: Union[str, False, None] = ...
info: str = ...
def __init__(self, ID: Union[int, str]):
"""任务类基类
:param ID: 任务id
"""
...
@property
def id(self) -> Union[int, str]:
"""返回任务或子任务id"""
...
@property
def data(self):
"""返回任务数据"""
...
@property
def is_done(self) -> bool:
"""返回任务是否结束"""
...
def set_states(self,
result: Union[bool, str, None] = None,
info: str = None,
state: str = 'done') -> None:
"""设置任务结果值
:param result: 结果:'success', 'skipped', 'canceled', False, None
:param info: 任务信息
:param state: 任务状态:'waiting', 'running', 'done'
:return: None
"""
...
class Mission(BaseTask):
"""任务类"""
file_name: Optional[str] = ...
_data: MissionData = ...
_path: Optional[Path] = ...
_recorder: Optional[Recorder] = ...
size: Optional[float] = ...
done_tasks_count: int = ...
tasks_count: int = ...
tasks: List[Task] = ...
owner: DrissionGet = ...
session: Session = ...
headers: CaseInsensitiveDict = ...
method: str = ...
encoding: Optional[str] = ...
_overwrote: bool = ...
def __init__(self, ID: int, owner: DrissionGet, file_url: str, save_path: Path, rename: str,
suffix: str, file_exists: str, split: bool, encoding: Optional[str], kwargs: dict):
"""任务类
:param ID: 任务id
:param owner: 所属DrissionGet对象
:param file_url: 文件网址
:param save_path: 保存文件夹路径
:param rename: 重命名
:param suffix: 重命名后缀名
:param file_exists: 存在同名文件处理方式
:param split: 是否分块下载
:param encoding: 编码格式
:param kwargs: 连接参数
"""
...
def __repr__(self) -> str:
...
@property
def data(self) -> MissionData:
"""返回任务数据"""
...
@property
def path(self) -> Union[str, Path]:
"""返回文件保存路径"""
...
@property
def recorder(self) -> Recorder:
"""返回记录器对象"""
...
@property
def rate(self) -> Optional[float]:
"""返回下载进度百分比"""
...
def cancel(self) -> None:
"""取消该任务,停止未下载完的task"""
...
def del_file(self) -> None:
"""删除下载的文件"""
...
def wait(self, show: bool = True, timeout: float = 0) -> tuple:
"""等待当前任务完成
:param show: 是否显示下载进度
:param timeout: 超时时间
:return: 任务结果和信息组成的tuple
"""
...
def _set_session(self) -> Session:
"""复制Session对象,并设置cookies"""
...
def _handle_kwargs(self, url: str, kwargs: dict) -> dict:
"""处理接收到的参数
:param url: 要访问的url
:param kwargs: 传入的参数dict
:return: 处理后的参数dict
"""
...
def _set_path(self, path: Path) -> None:
"""设置文件保存路径"""
...
def _set_done(self, result: Union[bool, str, None], info: str) -> None:
"""设置一个任务为done状态
:param result: 结果:'success'、'skipped'、'canceled'、False、None
:param info: 任务信息
:return: None
"""
...
def _a_task_done(self, is_success: bool, info: str) -> None:
"""当一个task完成时调用
:param is_success: 该task是否成功
:param info: 该task传入的信息
:return: None
"""
...
def _break_mission(self, result: Union[bool, str, None], info: str) -> None:
"""中止该任务,停止未下载完的task
:param result: 结果:'success'、'skipped'、'canceled'、False、None
:param info: 任务信息
:return: None
"""
...
class Task(BaseTask):
"""子任务类"""
mission: Mission = ...
range: Optional[list] = ...
size: Optional[int] = ...
_downloaded_size: int = 0
def __init__(self, mission: Mission, range_: Optional[list], ID: str, size: Optional[int]):
"""子任务类
:param mission: 父任务对象
:param range_: 读取文件数据范围
:param ID: 任务id
"""
...
def __repr__(self) -> str:
...
@property
def mid(self) -> int:
"""返回父任务id"""
...
@property
def data(self) -> MissionData:
"""返回任务数据对象"""
...
@property
def path(self) -> str:
"""返回文件保存路径"""
...
@property
def file_name(self) -> str:
"""返回文件名"""
...
@property
def rate(self) -> Optional[float]:
"""返回下载进度百分比"""
...
def add_data(self, data: bytes, seek: int = None) -> None:
"""把数据输入到记录器
:param data: 文件字节数据
:param seek: 在文件中的位置,None表示最后
:return: None
"""
...
def clear_cache(self) -> None:
"""清除以接收但未写入硬盘的缓存"""
...
def _set_done(self, result: Union[bool, str, None], info: str) -> None:
"""设置一个子任务为done状态
:param result: 结果:'success'、'skipped'、'canceled'、False、None
:param info: 任务信息
:return: None
"""
...
def format_headers(headers: CaseInsensitiveDict): ...
@@ -0,0 +1,122 @@
# -*- coding:utf-8 -*-
from DrissionRecord import Recorder
from requests import Session
from ._funcs import get_file_exists_mode, get_save_path
class Setter(object):
def __init__(self, owner):
self._owner = owner
@property
def if_file_exists(self):
return FileExists(self)
def log(self, path='DrissionGet_log.csv', failed_only=False):
if not path:
self._owner._logger = self._owner._log_mode = None
return self
if self._owner._logger:
self._owner._logger.set.path(path)
else:
self._owner._logger = Recorder(path, cache_size=1)
self._owner._log_mode = 'failed' if failed_only else 'all'
return self
def show_msg(self, on_off=True, failed_only=False):
self._owner._print_mode = 'failed' if failed_only else 'all' if on_off else None
return self
def driver(self, driver):
if driver is None:
self._owner._session = Session()
return self
elif isinstance(driver, Session):
self._owner._session = driver
return self
_type = str(type(driver))
if _type.startswith('<SessionOptions '):
self._owner._session, headers = driver.make_session()
self._owner._session.headers = headers
return self
else:
self._owner._session = driver.session if hasattr(driver, 'session') else Session()
self._owner.page = driver
return self
def roads(self, num):
if self._owner.is_running:
print('有任务未完成时不能改变roads。')
return self
if num != self._owner.roads:
self._owner._roads = num
self._owner._threads = {i: None for i in range(num)}
return self
def retry(self, times):
if not isinstance(times, int) or times < 0:
raise TypeError('times参数只能接受int格式且不能小于0。')
self._owner._retry = times
return self
def interval(self, seconds):
if not isinstance(seconds, (int, float)) or seconds < 0:
raise TypeError('seconds参数只能接受int或float格式且不能小于0。')
self._owner._interval = seconds
return self
def timeout(self, seconds):
if not isinstance(seconds, (int, float)) or seconds < 0:
raise TypeError('seconds参数只能接受int或float格式且不能小于0。')
self._owner._timeout = seconds
return self
def save_path(self, path):
self._owner.save_path = get_save_path(path)
return self
def split(self, on_off=True):
self._owner.split = on_off
return self
def block_size(self, size):
self._owner.block_size = size
return self
def proxies(self, http=None, https=None):
self._owner._session.proxies = {'http': http, 'https': https}
return self
def encoding(self, encoding):
self._owner._encoding = encoding if encoding else None
return self
class FileExists(object):
def __init__(self, setter):
self._setter = setter
def __call__(self, mode):
self._setter._owner.file_exists = get_file_exists_mode(mode)
return self._setter
def skip(self):
self._setter._owner.file_exists = 'skip'
return self._setter
def rename(self):
self._setter._owner.file_exists = 'rename'
return self._setter
def overwrite(self):
self._setter._owner.file_exists = 'overwrite'
return self._setter
def add(self):
self._setter._owner.file_exists = 'add'
return self._setter
@@ -0,0 +1,147 @@
# -*- coding:utf-8 -*-
from pathlib import Path
from typing import Union, Literal, Optional
from DrissionPage import SessionOptions
from DrissionPage._base.base import BasePage
from requests import Session
from .drissionGet import DrissionGet
FILE_EXISTS = Literal['add', 'skip', 'rename', 'overwrite', 'a', 's', 'r', 'o']
class Setter(object):
_owner: DrissionGet = ...
def __init__(self, owner: DrissionGet):
"""
:param owner: DrissionGet对象
"""
...
@property
def if_file_exists(self) -> FileExists:
"""返回用于设置文件同名策略的对象"""
...
def log(self, path: Union[str, Path, None] = 'DrissionGet_log.csv', failed_only: bool = False) -> Setter:
"""设置下载完成日志
:param path: 日志文件路径,为None或False不记录
:param failed_only: 是否只记录失败任务
:return: Setter对象自己
"""
...
def show_msg(self, on_off: bool = True, failed_only: bool = False) -> Setter:
"""设置是否打印下载信息
:param on_off: bool代表开关
:param failed_only: 是否只显示失败任务,此设置仅多线程任务时生效
:return: Setter对象自己
"""
...
def driver(self, driver: Union[Session, BasePage, SessionOptions]) -> Setter:
"""设置驱动
:param driver: Session对象或DrissionPage的页面对象
:return: Setter对象自己
"""
...
def roads(self, num: int) -> Setter:
"""设置可同时运行的线程数
:param num: 线程数量
:return: Setter对象自己
"""
...
def retry(self, times: int) -> Setter:
"""设置连接失败时重试次数
:param times: 重试次数
:return: Setter对象自己
"""
...
def interval(self, seconds: float) -> Setter:
"""设置连接失败时重试间隔
:param seconds: 连接失败时重试间隔(秒)
:return: Setter对象自己
"""
...
def timeout(self, seconds: float) -> Setter:
"""设置连接超时时间
:param seconds: 超时时间(秒)
:return: Setter对象自己
"""
...
def save_path(self, path: Union[str, Path]) -> Setter:
"""设置文件保存路径
:param path: 文件路径,可以是str或Path
:return: Setter对象自己
"""
...
def split(self, on_off: bool = True) -> Setter:
"""设置大文件是否分块下载
:param on_off: bool代表开关
:return: Setter对象自己
"""
...
def block_size(self, size: Union[str, int]) -> Setter:
"""设置分块大小
:param size: 单位为字节,可用'K'、'M'、'G'为单位,如'50M'
:return: Setter对象自己
"""
...
def proxies(self, http: str = None, https: str = None) -> Setter:
"""设置代理地址及端口,例:'127.0.0.1:1080'
:param http: http代理地址及端口
:param https: https代理地址及端口
:return: Setter对象自己
"""
...
def encoding(self, encoding: Optional[str]) -> Setter:
"""设置编码
:param encoding: 编码名称,传入None取消之前的设置
:return: Setter对象自己
"""
...
class FileExists(object):
"""用于设置存在同名文件时处理方法"""
_setter: Setter = ...
def __init__(self, setter: Setter):
"""
:param setter: Setter对象
"""
...
def __call__(self, mode: FILE_EXISTS) -> Setter:
"""设置文件存在时的处理方式
:param mode: 'skip', 'rename', 'overwrite', 'add', 's', 'r', 'o', 'a'
:return: Setter 对象
"""
...
def skip(self) -> Setter:
"""设为跳过"""
...
def rename(self) -> Setter:
"""设为重命名,文件名后加序号"""
...
def overwrite(self) -> Setter:
"""设为覆盖"""
...
def add(self) -> Setter:
"""设为追加"""
...