Initial commit
This commit is contained in:
commit
57fc3525eb
7311 files changed
+1291398
No files matched your search
@@ -0,0 +1,8 @@
|
||||
# -*- coding:utf-8 -*-
|
||||
"""
|
||||
@Author : g1879
|
||||
@Contact : g1879@qq.com
|
||||
"""
|
||||
from .drissionGet import DrissionGet
|
||||
|
||||
__version__ = '1.2.1'
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -0,0 +1,218 @@
|
||||
# -*- coding:utf-8 -*-
|
||||
from copy import copy
|
||||
from os import path as os_PATH
|
||||
from os.path import sep
|
||||
from pathlib import Path
|
||||
from random import randint
|
||||
from re import search, sub
|
||||
from time import time
|
||||
from urllib.parse import unquote
|
||||
|
||||
from DrissionRecord.tools import get_usable_path, make_valid_name
|
||||
from requests import Session
|
||||
|
||||
FILE_EXISTS_MODE = {'rename': 'rename', 'overwrite': 'overwrite', 'skip': 'skip', 'add': 'add', 'r': 'rename',
|
||||
'o': 'overwrite', 's': 'skip', 'a': 'add'}
|
||||
|
||||
|
||||
def copy_session(session):
|
||||
new = Session()
|
||||
new.headers = session.headers.copy()
|
||||
new.cookies = session.cookies.copy()
|
||||
new.stream = True
|
||||
new.auth = session.auth
|
||||
new.proxies = dict(session.proxies).copy()
|
||||
new.params = copy(session.params) #
|
||||
new.cert = session.cert
|
||||
new.max_redirects = session.max_redirects
|
||||
new.trust_env = session.trust_env
|
||||
new.verify = session.verify
|
||||
|
||||
return new
|
||||
|
||||
|
||||
class BlockSizeSetter(object):
|
||||
def __set__(self, block_size, val):
|
||||
if isinstance(val, int) and val > 0:
|
||||
size = val
|
||||
elif isinstance(val, str):
|
||||
units = {'b': 1, 'k': 1024, 'm': 1048576, 'g': 21474836480}
|
||||
num = int(val[:-1])
|
||||
unit = units.get(val[-1].lower(), None)
|
||||
if unit and num > 0:
|
||||
size = num * unit
|
||||
else:
|
||||
raise ValueError('单位只支持B、K、M、G,数字必须为大于0的整数。')
|
||||
else:
|
||||
raise TypeError('split_size只能传入int或str,数字必须为大于0的整数。')
|
||||
|
||||
block_size._block_size = size
|
||||
|
||||
def __get__(self, block_size, objtype=None) -> int:
|
||||
return block_size._block_size
|
||||
|
||||
|
||||
class PathSetter(object):
|
||||
def __set__(self, save_path, val):
|
||||
if val is not None and not isinstance(val, (str, Path)):
|
||||
raise TypeError('路径只能是str或Path类型。')
|
||||
save_path._save_path = get_save_path(val)
|
||||
|
||||
def __get__(self, save_path, objtype=None):
|
||||
return save_path._save_path
|
||||
|
||||
|
||||
class FileExistsSetter(object):
|
||||
def __set__(self, file_exists, mode):
|
||||
file_exists._file_exists = get_file_exists_mode(mode)
|
||||
|
||||
def __get__(self, file_exists, objtype=None):
|
||||
return file_exists._file_exists
|
||||
|
||||
|
||||
def get_save_path(path):
|
||||
path = Path(path or '.').absolute()
|
||||
return Path(path.anchor + sub(r'[*:|<>?"]', '', sep.join(path.parts[1:])).strip())
|
||||
|
||||
|
||||
def get_file_exists_mode(mode):
|
||||
mode = FILE_EXISTS_MODE.get(mode, mode)
|
||||
if mode not in FILE_EXISTS_MODE:
|
||||
raise ValueError(f'''mode参数只能是 '{"', '".join(FILE_EXISTS_MODE.keys())}' 之一,现在是:{mode}''')
|
||||
return mode
|
||||
|
||||
|
||||
def set_charset(response, encoding):
|
||||
if encoding:
|
||||
response.encoding = encoding
|
||||
return response
|
||||
|
||||
# 在headers中获取编码
|
||||
content_type = response.headers.get('content-type', '').lower()
|
||||
if not content_type.endswith(';'):
|
||||
content_type += ';'
|
||||
charset = search(r'charset[=: ]*(.*)?;?', content_type)
|
||||
|
||||
if charset:
|
||||
response.encoding = charset.group(1)
|
||||
|
||||
# 在headers中获取不到编码,且如果是网页
|
||||
elif content_type.replace(' ', '').startswith('text/html'):
|
||||
re_result = search(b'<meta.*?charset=[ \\\'"]*([^"\\\' />]+).*?>', response.content)
|
||||
|
||||
if re_result:
|
||||
charset = re_result.group(1).decode()
|
||||
else:
|
||||
charset = response.apparent_encoding
|
||||
|
||||
response.encoding = charset
|
||||
|
||||
return response
|
||||
|
||||
|
||||
def get_file_info(response, save_path, rename=None, suffix=None, file_exists=None, encoding=None, lock=None):
|
||||
# ------------获取文件大小------------
|
||||
file_size = response.headers.get('Content-Length', None)
|
||||
file_size = None if file_size is None else int(file_size)
|
||||
|
||||
# ------------获取网络文件名------------
|
||||
file_name = get_file_name(response, encoding)
|
||||
|
||||
# ------------获取保存文件名------------
|
||||
# -------------------重命名-------------------
|
||||
if rename:
|
||||
if suffix is not None:
|
||||
full_name = f'{rename}.{suffix}' if suffix else rename
|
||||
|
||||
else:
|
||||
tmp = file_name.rsplit('.', 1)
|
||||
ext_name = f'.{tmp[-1]}' if len(tmp) > 1 else ''
|
||||
tmp = rename.rsplit('.', 1)
|
||||
ext_rename = f'.{tmp[-1]}' if len(tmp) > 1 else ''
|
||||
full_name = rename if ext_rename == ext_name else f'{rename}{ext_name}'
|
||||
|
||||
elif suffix is not None:
|
||||
full_name = file_name.rsplit(".", 1)[0]
|
||||
if suffix:
|
||||
full_name = f'{full_name}.{suffix}'
|
||||
|
||||
else:
|
||||
full_name = file_name
|
||||
|
||||
full_name = make_valid_name(full_name)
|
||||
|
||||
# -------------------生成路径-------------------
|
||||
skip = False
|
||||
overwrite = False
|
||||
full_path = save_path / full_name
|
||||
|
||||
with lock:
|
||||
if full_path.exists():
|
||||
if file_exists == 'rename':
|
||||
full_path = get_usable_path(full_path)
|
||||
|
||||
elif file_exists == 'skip':
|
||||
skip = True
|
||||
|
||||
elif file_exists == 'overwrite':
|
||||
overwrite = True
|
||||
full_path.unlink()
|
||||
|
||||
return {'size': file_size,
|
||||
'path': full_path,
|
||||
'skip': skip,
|
||||
'overwrite': overwrite}
|
||||
|
||||
|
||||
def get_file_name(response, encoding):
|
||||
file_name = ''
|
||||
charset = ''
|
||||
content_disposition = response.headers.get('content-disposition', '').replace(' ', '')
|
||||
|
||||
# 使用header里的文件名
|
||||
if content_disposition:
|
||||
txt = search(r'filename\*="?([^";]+)', content_disposition)
|
||||
if txt: # 文件名自带编码方式
|
||||
txt = txt.group(1).split("''", 1)
|
||||
if len(txt) == 2:
|
||||
charset, file_name = txt
|
||||
else:
|
||||
file_name = txt[0]
|
||||
|
||||
else: # 文件名没带编码方式
|
||||
txt = search(r'filename="?([^";]+)', content_disposition)
|
||||
if txt:
|
||||
file_name = txt.group(1)
|
||||
|
||||
# 获取编码(如有)
|
||||
charset = encoding or response.encoding
|
||||
|
||||
file_name = file_name.strip("'")
|
||||
|
||||
# 在url里获取文件名
|
||||
if not file_name and os_PATH.basename(response.url):
|
||||
file_name = os_PATH.basename(response.url).split("?")[0]
|
||||
|
||||
# 找不到则用时间和随机数生成文件名
|
||||
if not file_name:
|
||||
file_name = f'untitled_{time()}_{randint(0, 100)}'
|
||||
|
||||
# 去除非法字符
|
||||
charset = charset or 'utf-8'
|
||||
return unquote(file_name, charset)
|
||||
|
||||
|
||||
def set_session_cookies(session, cookies):
|
||||
# cookies = cookies_to_tuple(cookies)
|
||||
for cookie in cookies:
|
||||
if cookie['value'] is None:
|
||||
cookie['value'] = ''
|
||||
|
||||
kwargs = {x: cookie[x] for x in cookie
|
||||
if x.lower() in ('version', 'port', 'domain', 'path', 'secure',
|
||||
'expires', 'discard', 'comment', 'comment_url', 'rest')}
|
||||
|
||||
if 'expiry' in cookie:
|
||||
kwargs['expires'] = cookie['expiry']
|
||||
|
||||
session.cookies.set(cookie['name'], cookie['value'], **kwargs)
|
||||
@@ -0,0 +1,93 @@
|
||||
# -*- coding:utf-8 -*-
|
||||
from pathlib import Path
|
||||
from threading import Lock
|
||||
from typing import Union, Optional
|
||||
|
||||
from requests import Session, Response
|
||||
|
||||
FILE_EXISTS_MODE: dict = ...
|
||||
|
||||
|
||||
def copy_session(session: Session) -> Session:
|
||||
"""复制输入Session对象,返回一个新的
|
||||
:param session: 被复制的Session对象
|
||||
:return: 新Session对象
|
||||
"""
|
||||
...
|
||||
|
||||
|
||||
class BlockSizeSetter(object):
|
||||
def __set__(self, block_size, val: Union[str, int]): ...
|
||||
|
||||
def __get__(self, block_size, objtype=None) -> int: ...
|
||||
|
||||
|
||||
class PathSetter(object):
|
||||
def __set__(self, save_path, val: Union[str, Path]): ...
|
||||
|
||||
def __get__(self, save_path, objtype=None): ...
|
||||
|
||||
|
||||
class FileExistsSetter(object):
|
||||
def __set__(self, file_exists, mode: str): ...
|
||||
|
||||
def __get__(self, file_exists, objtype=None): ...
|
||||
|
||||
|
||||
def get_save_path(path: Union[str, Path]) -> Path:
|
||||
"""获取绝对路径,去除Windows系统非法字符
|
||||
:param path: 输入的路径
|
||||
:return: Path格式路径
|
||||
"""
|
||||
...
|
||||
|
||||
|
||||
def get_file_exists_mode(mode: str) -> str:
|
||||
"""获取文件重名时处理策略名称
|
||||
:param mode: 输入
|
||||
:return: 标准字符串
|
||||
"""
|
||||
...
|
||||
|
||||
|
||||
def set_charset(response: Response, encoding: Optional[str]) -> Response:
|
||||
"""设置Response对象的编码
|
||||
:param response: Response对象
|
||||
:param encoding: 指定的编码格式
|
||||
:return: 设置编码后的Response对象
|
||||
"""
|
||||
...
|
||||
|
||||
|
||||
def get_file_info(response: Response, save_path: Path, rename: str = None, suffix: str = None,
|
||||
file_exists: str = None, encoding: Optional[str] = None, lock: Lock = None) -> dict:
|
||||
"""获取文件信息,大小单位为byte
|
||||
包括:size、path、skip
|
||||
:param response: Response对象
|
||||
:param save_path: 目标文件夹Path对象
|
||||
:param rename: 重命名
|
||||
:param suffix: 重命名后缀名
|
||||
:param file_exists: 存在重名文件时的处理方式
|
||||
:param encoding: 编码格式
|
||||
:param lock: 线程锁
|
||||
:return: 文件大小、完整路径、是否跳过、是否覆盖
|
||||
"""
|
||||
...
|
||||
|
||||
|
||||
def get_file_name(response: Response, encoding: str) -> str:
|
||||
"""从headers或url中获取文件名,如果获取不到,生成一个随机文件名
|
||||
:param response: 返回的response
|
||||
:param encoding: 在headers获取时指定编码格式
|
||||
:return: 下载文件的文件名
|
||||
"""
|
||||
...
|
||||
|
||||
|
||||
def set_session_cookies(session: Session, cookies: list) -> None:
|
||||
"""设置Session对象的cookies
|
||||
:param session: Session对象
|
||||
:param cookies: cookies信息
|
||||
:return: None
|
||||
"""
|
||||
...
|
||||
@@ -0,0 +1,539 @@
|
||||
# -*- coding:utf-8 -*-
|
||||
from copy import copy
|
||||
from pathlib import Path
|
||||
from queue import Queue
|
||||
from threading import Thread, Lock
|
||||
from time import sleep, perf_counter
|
||||
|
||||
from requests.structures import CaseInsensitiveDict
|
||||
|
||||
from ._funcs import (FileExistsSetter, PathSetter, BlockSizeSetter, set_charset, get_file_info, get_file_exists_mode,
|
||||
get_save_path)
|
||||
from .mission import Task, Mission
|
||||
from .setter import Setter
|
||||
|
||||
|
||||
class DrissionGet(object):
|
||||
file_exists = FileExistsSetter()
|
||||
save_path = PathSetter()
|
||||
block_size = BlockSizeSetter()
|
||||
|
||||
def __init__(self, save_path='.', roads=10, driver=None, file_exists='rename'):
|
||||
self._roads = roads
|
||||
self._missions = {}
|
||||
self._threads = {i: None for i in range(self._roads)}
|
||||
self._waiting_list = Queue()
|
||||
self._missions_num = 0
|
||||
self._running_count = 0 # 正在运行的任务数
|
||||
self._stop_printing = False # 用于控制显示线程停止
|
||||
self._lock = Lock()
|
||||
self.page = None
|
||||
self._retry = None
|
||||
self._interval = None
|
||||
self._timeout = None
|
||||
self._encoding = None
|
||||
|
||||
self._setter = None
|
||||
self._print_mode = 'all'
|
||||
self._log_mode = None
|
||||
self._logger = None
|
||||
|
||||
self.save_path = save_path
|
||||
self.file_exists = file_exists
|
||||
self.split = True
|
||||
self.block_size = '50M'
|
||||
self.set.driver(driver)
|
||||
|
||||
def __call__(self, url=None, save_path=None, rename=None, suffix=None,
|
||||
file_exists=None, split=False, show_msg=True, file_url=None, **kwargs):
|
||||
return self.download(url=url, save_path=save_path, rename=rename, suffix=suffix,
|
||||
file_exists=file_exists, show_msg=show_msg, **kwargs)
|
||||
|
||||
@property
|
||||
def set(self):
|
||||
if self._setter is None:
|
||||
self._setter = Setter(self)
|
||||
return self._setter
|
||||
|
||||
@property
|
||||
def roads(self):
|
||||
return self._roads
|
||||
|
||||
@property
|
||||
def retry(self):
|
||||
if self._retry is not None:
|
||||
return self._retry
|
||||
elif self.page is not None:
|
||||
return self.page.retry_times
|
||||
else:
|
||||
return 3
|
||||
|
||||
@property
|
||||
def interval(self):
|
||||
if self._interval is not None:
|
||||
return self._interval
|
||||
elif self.page is not None:
|
||||
return self.page.retry_interval
|
||||
else:
|
||||
return 5
|
||||
|
||||
@property
|
||||
def timeout(self):
|
||||
if self._timeout is not None:
|
||||
return self._timeout
|
||||
elif self.page is not None:
|
||||
return self.page.timeout
|
||||
else:
|
||||
return 20
|
||||
|
||||
@property
|
||||
def waiting_list(self):
|
||||
return self._waiting_list
|
||||
|
||||
@property
|
||||
def session(self):
|
||||
return self._session
|
||||
|
||||
@property
|
||||
def is_running(self):
|
||||
return self._running_count > 0
|
||||
|
||||
@property
|
||||
def missions(self):
|
||||
return self._missions
|
||||
|
||||
@property
|
||||
def encoding(self):
|
||||
"""返回指定的编码格式"""
|
||||
return self._encoding
|
||||
|
||||
def add(self, url=None, save_path=None, rename=None, suffix=None, file_exists=None, split=None,
|
||||
file_url=None, **kwargs):
|
||||
if not url and file_url:
|
||||
url = file_url
|
||||
self._missions_num += 1
|
||||
self._running_count += 1
|
||||
if file_exists is not None:
|
||||
file_exists = get_file_exists_mode(file_exists)
|
||||
mission = Mission(self._missions_num, self, url, get_save_path(save_path) if save_path else self.save_path,
|
||||
rename, suffix, file_exists or self.file_exists,
|
||||
self.split if split is None else split, self._encoding, kwargs)
|
||||
self._missions[self._missions_num] = mission
|
||||
|
||||
self._run_or_wait(mission)
|
||||
return mission
|
||||
|
||||
def download(self, url=None, save_path=None, rename=None, suffix=None, file_exists=None, split=False,
|
||||
show_msg=None, file_url=None, **kwargs):
|
||||
if not url and file_url:
|
||||
url = file_url
|
||||
self._missions_num += 1
|
||||
self._running_count += 1
|
||||
if file_exists is not None:
|
||||
file_exists = get_file_exists_mode(file_exists)
|
||||
mission = Mission(self._missions_num, self, url, get_save_path(save_path) if save_path else self.save_path,
|
||||
rename, suffix, file_exists or self.file_exists,
|
||||
self.split if split is None else split, self._encoding, kwargs)
|
||||
self._missions[self._missions_num] = mission
|
||||
|
||||
tmp, self._print_mode = self._print_mode, None
|
||||
if show_msg is None:
|
||||
show_msg = True if tmp else False
|
||||
|
||||
if split:
|
||||
self._run_or_wait(mission)
|
||||
r = mission.wait(show=show_msg)
|
||||
self._print_mode = tmp
|
||||
return r
|
||||
|
||||
ok, r, file_size = handle_mission(mission, url)
|
||||
if show_msg:
|
||||
print(f'url:{url}')
|
||||
print(f'文件名:{mission.file_name}')
|
||||
print(f'目标路径:{mission.path}')
|
||||
if not mission.size:
|
||||
print('下载中 ', end='')
|
||||
|
||||
if ok:
|
||||
task1 = Task(mission, None, '1/1', file_size)
|
||||
mission.tasks.append(task1)
|
||||
do_download_(r, task1, show_msg, file_size)
|
||||
|
||||
if show_msg:
|
||||
if mission.result is False:
|
||||
print(f'下载失败 {mission.info}')
|
||||
elif mission.result == 'success':
|
||||
print('\r100% ', end='')
|
||||
print(f'{"已覆盖" if mission._overwrote else "下载完成"} {mission.info}')
|
||||
elif mission.result == 'skipped':
|
||||
print(f'已跳过 {mission.info}')
|
||||
print()
|
||||
|
||||
self._print_mode = tmp
|
||||
return mission.result, mission.info
|
||||
|
||||
def by_browser(self, url, save_path=None, rename=None, suffix=None, timeout=None, file_exists=None):
|
||||
if hasattr(self.page, '_download_by_browser'):
|
||||
return self.page._download_by_browser(url=url, save_path=save_path, rename=rename, suffix=suffix,
|
||||
timeout=timeout, file_exists=file_exists)
|
||||
else:
|
||||
raise RuntimeError('非浏览器调用无法使用浏览器下载。')
|
||||
|
||||
def _run_or_wait(self, mission):
|
||||
thread_id = self._get_usable_thread()
|
||||
if thread_id is None:
|
||||
self._waiting_list.put(mission)
|
||||
else:
|
||||
thread = Thread(target=self._run, args=(thread_id, mission), daemon=False)
|
||||
self._threads[thread_id] = {'thread': thread, 'mission': None}
|
||||
thread.start()
|
||||
|
||||
def _run(self, ID, mission):
|
||||
while True:
|
||||
if not mission: # 如果没有任务,就从等候列表中取一个
|
||||
if self._waiting_list.empty():
|
||||
break
|
||||
else:
|
||||
try:
|
||||
mission = self._waiting_list.get(True, .5)
|
||||
except Exception:
|
||||
self._waiting_list.task_done()
|
||||
break
|
||||
|
||||
self._threads[ID]['mission'] = mission
|
||||
self._download(mission, ID)
|
||||
mission = None
|
||||
|
||||
self._threads[ID] = None
|
||||
|
||||
def get_mission(self, mission_or_id):
|
||||
return self._missions[mission_or_id] if isinstance(mission_or_id, int) else mission_or_id
|
||||
|
||||
def get_failed_missions(self):
|
||||
return [i for i in self._missions.values() if i.result is False]
|
||||
|
||||
def wait(self, mission=None, show=False, timeout=None):
|
||||
timeout = 0 if timeout is None else timeout
|
||||
if mission:
|
||||
return self.get_mission(mission).wait(show, timeout)
|
||||
|
||||
else:
|
||||
if show:
|
||||
self.show(False)
|
||||
else:
|
||||
end_time = perf_counter() + timeout
|
||||
while self.is_running and (perf_counter() < end_time or timeout == 0):
|
||||
sleep(0.1)
|
||||
|
||||
def cancel(self):
|
||||
for m in self._missions.values():
|
||||
m.cancel()
|
||||
|
||||
def show(self, asyn=True, keep=False):
|
||||
if asyn:
|
||||
Thread(target=self._show, args=(2, keep)).start()
|
||||
else:
|
||||
self._show(0.1, keep)
|
||||
|
||||
def _show(self, wait, keep=False):
|
||||
self._stop_printing = False
|
||||
|
||||
if keep:
|
||||
Thread(target=self._stop_show).start()
|
||||
|
||||
end_time = perf_counter() + wait
|
||||
while not self._stop_printing and (keep or self.is_running or perf_counter() < end_time):
|
||||
print(f'\033[K', end='')
|
||||
print(f'等待任务数:{self._waiting_list.qsize()}')
|
||||
for k, v in self._threads.items():
|
||||
m = v['mission'] if v else None
|
||||
if m:
|
||||
items = (m.mission.rate, m.mid) if isinstance(m, Task) else (m.rate, m.id)
|
||||
path = f'M{items[1]} {items[0]}% {m}'
|
||||
else:
|
||||
path = '空闲'
|
||||
print(f'\033[K', end='')
|
||||
print(f'线程{k}:{path}')
|
||||
|
||||
print(f'\033[{self.roads + 1}A\r', end='')
|
||||
sleep(0.4)
|
||||
|
||||
print(f'\033[1B', end='')
|
||||
for i in range(self.roads):
|
||||
print(f'\033[K', end='')
|
||||
print(f'线程{i}:空闲')
|
||||
|
||||
print()
|
||||
|
||||
def _connect(self, url, session, _headers, method, encoding, **kwargs):
|
||||
kwargs = CaseInsensitiveDict(kwargs)
|
||||
if 'headers' in kwargs: # 不知道为什么添加这个才能正常使用多线程
|
||||
kwargs['headers'] = CaseInsensitiveDict({**_headers, **kwargs['headers']})
|
||||
else:
|
||||
kwargs['headers'] = _headers
|
||||
|
||||
r = err = None
|
||||
for i in range(self.retry + 1):
|
||||
try:
|
||||
if method == 'get':
|
||||
r = session.get(url, **kwargs)
|
||||
elif method == 'post':
|
||||
r = session.post(url, **kwargs)
|
||||
|
||||
if r:
|
||||
return set_charset(r, encoding), 'Success'
|
||||
|
||||
except Exception as e:
|
||||
err = e
|
||||
|
||||
if r and r.status_code in (403, 404):
|
||||
break
|
||||
if i < self.retry:
|
||||
sleep(self.interval)
|
||||
|
||||
# 返回失败结果
|
||||
if r is None:
|
||||
return None, '连接失败' if err is None else err
|
||||
if not r.ok:
|
||||
return r, f'状态码:{r.status_code}'
|
||||
return None
|
||||
|
||||
def _get_usable_thread(self):
|
||||
for k, v in self._threads.items():
|
||||
if v is None:
|
||||
return k
|
||||
return None
|
||||
|
||||
def _stop_show(self):
|
||||
input()
|
||||
self._stop_printing = True
|
||||
|
||||
def _when_mission_done(self, mission):
|
||||
self._running_count -= 1
|
||||
if self._print_mode == 'all' or (self._print_mode == 'failed' and mission.result is False):
|
||||
print(f'[{mission.RESULT_TEXTS[mission.result]}] {mission.info} {mission.data.url}')
|
||||
if self._log_mode == 'all' or (self._log_mode == 'failed' and mission.result is False):
|
||||
self._logger.add_data((mission.RESULT_TEXTS[mission.result], mission.data.url, mission.info,
|
||||
mission.data.save_path, mission.data.rename, mission.data.kwargs))
|
||||
mission.session.close()
|
||||
|
||||
def _download(self, mission_or_task, thread_id):
|
||||
if mission_or_task.is_done:
|
||||
return
|
||||
if mission_or_task.state == 'cancel':
|
||||
mission_or_task.state = 'done'
|
||||
return
|
||||
|
||||
file_url = mission_or_task.data.url
|
||||
if isinstance(mission_or_task, Task):
|
||||
kwargs = copy(mission_or_task.data.kwargs)
|
||||
kwargs['headers']['Range'] = f"bytes={mission_or_task.range[0]}-{mission_or_task.range[1]}"
|
||||
r, inf = self._connect(file_url, mission_or_task.mission.session, mission_or_task.mission.headers,
|
||||
mission_or_task.mission.method, mission_or_task.mission.encoding, **kwargs)
|
||||
|
||||
if r:
|
||||
do_download(r, mission_or_task, False)
|
||||
else:
|
||||
mission_or_task._set_done(False, inf)
|
||||
|
||||
return
|
||||
|
||||
# ===================开始处理mission====================
|
||||
mission = mission_or_task
|
||||
ok, r, file_size = handle_mission(mission, file_url)
|
||||
if not ok:
|
||||
return
|
||||
|
||||
# -------------------设置分块任务-------------------
|
||||
first = False
|
||||
if mission.data.split and file_size and file_size > self.block_size and r.headers.get(
|
||||
'Accept-Ranges') == 'bytes':
|
||||
first = True
|
||||
chunks = [[s, min(s + self.block_size, file_size) - 1] for s in range(0, file_size, self.block_size)]
|
||||
chunks[-1][-1] = ''
|
||||
chunks_len = len(chunks)
|
||||
|
||||
task1 = Task(mission, chunks[0], f'1/{chunks_len}', chunks[0][1] - chunks[0][0])
|
||||
mission.tasks_count = chunks_len
|
||||
mission.tasks = [task1]
|
||||
|
||||
for ind, chunk in enumerate(chunks[1:], 2):
|
||||
s = file_size - chunk[0] if chunks_len == ind else chunk[1] - chunk[0]
|
||||
task = Task(mission, chunk, f'{ind}/{chunks_len}', s)
|
||||
mission.tasks.append(task)
|
||||
self._run_or_wait(task)
|
||||
|
||||
else: # 不分块
|
||||
task1 = Task(mission, None, '1/1', file_size)
|
||||
mission.tasks.append(task1)
|
||||
|
||||
self._threads[thread_id]['mission'] = task1
|
||||
do_download(r, task1, first)
|
||||
|
||||
|
||||
def handle_mission(mission, url):
|
||||
mission.info = '下载中'
|
||||
mission.state = 'running'
|
||||
kwargs = mission.data.kwargs
|
||||
|
||||
rename = mission.data.rename
|
||||
suffix = mission.data.suffix
|
||||
file_exists = mission.data.file_exists
|
||||
mission.data.save_path.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
if file_exists == 'skip' and rename:
|
||||
tmp = mission.data.save_path / rename
|
||||
if tmp.exists() and tmp.is_file():
|
||||
mission.file_name = rename
|
||||
mission._set_path(mission.data.save_path / rename)
|
||||
mission._set_done('skipped', str(mission.path))
|
||||
return False, None, None
|
||||
|
||||
r, inf = mission.owner._connect(url, mission.session, mission.headers, mission.method, mission.encoding, **kwargs)
|
||||
|
||||
if mission.is_done:
|
||||
return False, None, None
|
||||
if not r:
|
||||
mission._break_mission(result=False, info=inf)
|
||||
return False, None, None
|
||||
|
||||
# -------------------获取文件信息-------------------
|
||||
file_info = get_file_info(r, mission.data.save_path, rename, suffix, file_exists,
|
||||
mission.encoding, mission.owner._lock)
|
||||
file_size = file_info['size']
|
||||
full_path = file_info['path']
|
||||
mission._set_path(full_path)
|
||||
mission.size = file_size
|
||||
mission._overwrote = file_info['overwrite']
|
||||
|
||||
if file_info['skip']:
|
||||
mission._set_done('skipped', str(mission.path))
|
||||
return False, None, None
|
||||
|
||||
full_Path = Path(full_path)
|
||||
if file_exists == 'add' and full_Path.exists():
|
||||
mission.data.offset = full_Path.stat().st_size
|
||||
|
||||
return True, r, file_size
|
||||
|
||||
|
||||
def do_download_(r, task, show, size):
|
||||
if task.is_done or task.mission.is_done:
|
||||
return
|
||||
|
||||
task.set_states(result=None, info='下载中', state='running')
|
||||
block_size = 524288 # 512k
|
||||
result = None
|
||||
|
||||
try:
|
||||
if show and size:
|
||||
for chunk in r.iter_content(chunk_size=block_size):
|
||||
if task.state in ('cancel', 'done'):
|
||||
result = 'canceled'
|
||||
task.clear_cache()
|
||||
break
|
||||
if chunk:
|
||||
task.add_data(chunk, None)
|
||||
print(f'\r{task.rate}% ', end='')
|
||||
|
||||
else:
|
||||
for chunk in r.iter_content(chunk_size=block_size):
|
||||
if task.state in ('cancel', 'done'):
|
||||
result = 'canceled'
|
||||
task.clear_cache()
|
||||
break
|
||||
if chunk:
|
||||
task.add_data(chunk, None)
|
||||
|
||||
except Exception as e:
|
||||
result, info = False, f'{r.status_code} {e}'
|
||||
|
||||
else:
|
||||
result = 'success' if result is None else result
|
||||
info = str(task.path)
|
||||
|
||||
finally:
|
||||
r.close()
|
||||
|
||||
task._set_done(result=result, info=info)
|
||||
|
||||
|
||||
def do_download(r, task, first=False):
|
||||
if task.is_done or task.mission.is_done:
|
||||
return
|
||||
|
||||
task.set_states(result=None, info='下载中', state='running')
|
||||
block_size = 524288 # 512k
|
||||
result = None
|
||||
|
||||
try:
|
||||
if first: # 分块是第一块
|
||||
if task.range[1] <= block_size or task.range[1] % block_size != 0:
|
||||
r_content = r.iter_content(chunk_size=task.range[1] + 1)
|
||||
task.add_data(next(r_content), seek=0 + task.mission.data.offset)
|
||||
if task.state in ('cancel', 'done'):
|
||||
result = 'canceled'
|
||||
task.clear_cache()
|
||||
|
||||
else:
|
||||
blocks = task.range[1] // block_size
|
||||
remainder = task.range[1] % block_size
|
||||
r_content = r.iter_content(chunk_size=block_size)
|
||||
for b in range(blocks):
|
||||
task.add_data(next(r_content), seek=b * block_size + task.mission.data.offset)
|
||||
if task.state in ('cancel', 'done'):
|
||||
result = 'canceled'
|
||||
task.clear_cache()
|
||||
break
|
||||
|
||||
if task.state in ('cancel', 'done'):
|
||||
result = 'canceled'
|
||||
task.clear_cache()
|
||||
else:
|
||||
task.add_data(next(r_content)[:remainder + 1], blocks * block_size + task.mission.data.offset)
|
||||
|
||||
else:
|
||||
if task.range is None: # 不分块
|
||||
for chunk in r.iter_content(chunk_size=block_size):
|
||||
if task.state in ('cancel', 'done'):
|
||||
result = 'canceled'
|
||||
task.clear_cache()
|
||||
break
|
||||
if chunk:
|
||||
task.add_data(chunk, None)
|
||||
|
||||
elif task.range[1] == '': # 结尾的数据块
|
||||
begin = task.range[0]
|
||||
for chunk in r.iter_content(chunk_size=block_size):
|
||||
if task.state in ('cancel', 'done'):
|
||||
result = 'canceled'
|
||||
task.clear_cache()
|
||||
break
|
||||
if chunk:
|
||||
task.add_data(chunk, seek=begin + task.mission.data.offset)
|
||||
begin += len(chunk)
|
||||
|
||||
else: # 有始末数字的数据块
|
||||
begin, end = task.range
|
||||
num = (end - begin) // block_size
|
||||
for ind, chunk in enumerate(r.iter_content(chunk_size=block_size), 1):
|
||||
if task.state in ('cancel', 'done'):
|
||||
result = 'canceled'
|
||||
task.clear_cache()
|
||||
break
|
||||
if chunk:
|
||||
task.add_data(chunk, seek=begin + task.mission.data.offset)
|
||||
if ind <= num:
|
||||
begin += block_size
|
||||
|
||||
except Exception as e:
|
||||
result, info = False, f'{r.status_code} {e}'
|
||||
|
||||
else:
|
||||
result = 'success' if result is None else result
|
||||
info = str(task.path)
|
||||
|
||||
finally:
|
||||
r.close()
|
||||
|
||||
task._set_done(result=result, info=info)
|
||||
@@ -0,0 +1,353 @@
|
||||
# -*- coding:utf-8 -*-
|
||||
from pathlib import Path
|
||||
from queue import Queue
|
||||
from threading import Lock
|
||||
from typing import Union, Tuple, Any, Literal, Optional
|
||||
|
||||
from DrissionPage import SessionOptions
|
||||
from DrissionPage._base.base import BasePage
|
||||
from DrissionPage._units.downloader import DownloadMission
|
||||
from DrissionRecord import Recorder
|
||||
from requests import Session, Response
|
||||
from requests.structures import CaseInsensitiveDict
|
||||
|
||||
from ._funcs import FileExistsSetter, PathSetter, BlockSizeSetter
|
||||
from .mission import Task, Mission, BaseTask
|
||||
from .setter import Setter
|
||||
|
||||
FILE_EXISTS = Literal['add', 'skip', 'rename', 'overwrite', 'a', 's', 'r', 'o']
|
||||
|
||||
|
||||
class DrissionGet(object):
|
||||
file_exists: FileExistsSetter = ...
|
||||
save_path: PathSetter = ...
|
||||
block_size: BlockSizeSetter = ...
|
||||
|
||||
_roads: int = ...
|
||||
_setter: Optional[Setter] = ...
|
||||
_print_mode: Optional[str] = ...
|
||||
_log_mode: Optional[str] = ...
|
||||
_logger: Optional[Recorder] = ...
|
||||
_retry: Optional[int] = ...
|
||||
_interval: Optional[float] = ...
|
||||
page: Optional[BasePage] = ...
|
||||
_waiting_list: Queue = ...
|
||||
_session: Session = ...
|
||||
_running_count: int = ...
|
||||
_missions_num: int = ...
|
||||
_missions: dict = ...
|
||||
_threads: dict = ...
|
||||
_timeout: Union[int, float, None] = ...
|
||||
_stop_printing: bool = ...
|
||||
_lock: Lock = ...
|
||||
split: bool = ...
|
||||
_encoding: Optional[str] = ...
|
||||
|
||||
def __init__(self,
|
||||
save_path: Union[str, Path] = None,
|
||||
roads: int = 10,
|
||||
driver: Union[Session, BasePage, SessionOptions] = None,
|
||||
file_exists: FILE_EXISTS = 'rename'):
|
||||
"""
|
||||
:param save_path: 文件保存路径
|
||||
:param roads: 可同时运行的线程数
|
||||
:param driver: 使用的Session对象,或配置对象、页面对象等
|
||||
:param file_exists: 有同名文件名时的处理方式,可选 'skip', 'overwrite', 'rename', 'add', 's', 'o', 'r', 'a'
|
||||
"""
|
||||
...
|
||||
|
||||
def __call__(self,
|
||||
url: str,
|
||||
save_path: Union[str, Path, None] = None,
|
||||
rename: str = None,
|
||||
suffix: Optional[str] = None,
|
||||
file_exists: FILE_EXISTS = None,
|
||||
show_msg: bool = True,
|
||||
split: bool = False,
|
||||
timeout: Optional[float] = None,
|
||||
params: Optional[dict] = ...,
|
||||
data: Any = ...,
|
||||
json: Any = ...,
|
||||
headers: Optional[dict] = ...,
|
||||
cookies: Any = ...,
|
||||
files: Any = ...,
|
||||
auth: Any = ...,
|
||||
allow_redirects: bool = ...,
|
||||
proxies: Optional[dict] = ...,
|
||||
hooks: Any = ...,
|
||||
stream: Any = ...,
|
||||
verify: Any = ...,
|
||||
cert: Any = ...) -> tuple:
|
||||
"""以阻塞的方式下载一个文件并返回结果
|
||||
:param url: 文件网址
|
||||
:param save_path: 保存路径
|
||||
:param rename: 重命名的文件名
|
||||
:param suffix: 指定后缀名
|
||||
:param file_exists: 遇到同名文件时的处理方式,可选 'skip', 'overwrite', 'rename', 'add', 's', 'o', 'r', 'a',默认跟随实例属性
|
||||
:param split: 是否分块下载,为None使用设置
|
||||
:param show_msg: 是否打印进度
|
||||
:param kwargs: 连接参数
|
||||
:return: 任务结果和信息组成的tuple
|
||||
"""
|
||||
...
|
||||
|
||||
@property
|
||||
def set(self) -> Setter:
|
||||
"""用于设置打印和记录模式的对象"""
|
||||
...
|
||||
|
||||
@property
|
||||
def roads(self) -> int:
|
||||
"""可同时运行的线程数"""
|
||||
...
|
||||
|
||||
@property
|
||||
def retry(self) -> int:
|
||||
"""返回连接失败时重试次数"""
|
||||
...
|
||||
|
||||
@property
|
||||
def interval(self) -> float:
|
||||
"""返回连接失败时重试间隔"""
|
||||
...
|
||||
|
||||
@property
|
||||
def timeout(self) -> float:
|
||||
"""返回连接超时时间"""
|
||||
...
|
||||
|
||||
@property
|
||||
def waiting_list(self) -> Queue:
|
||||
"""返回等待队列"""
|
||||
...
|
||||
|
||||
@property
|
||||
def session(self) -> Session:
|
||||
"""返回用于保存默认连接设置的Session对象"""
|
||||
...
|
||||
|
||||
@property
|
||||
def is_running(self) -> bool:
|
||||
"""返回是否有线程还在运行"""
|
||||
...
|
||||
|
||||
@property
|
||||
def missions(self) -> dict:
|
||||
"""用list方式返回所有任务对象"""
|
||||
...
|
||||
|
||||
@property
|
||||
def encoding(self) -> Optional[str]:
|
||||
...
|
||||
|
||||
def add(self,
|
||||
url: str,
|
||||
save_path: Union[str, Path, None] = None,
|
||||
rename: str = None,
|
||||
suffix: str = None,
|
||||
file_exists: FILE_EXISTS = None,
|
||||
split: bool = None,
|
||||
timeout: Optional[float] = None,
|
||||
params: Optional[dict] = ...,
|
||||
data: Any = None,
|
||||
json: Union[dict, str, None] = ...,
|
||||
headers: Optional[dict] = ...,
|
||||
cookies: Any = ...,
|
||||
files: Any = ...,
|
||||
auth: Any = ...,
|
||||
allow_redirects: bool = ...,
|
||||
proxies: Optional[dict] = ...,
|
||||
hooks: Any = ...,
|
||||
stream: Any = ...,
|
||||
verify: Any = ...,
|
||||
cert: Any = ...) -> Mission:
|
||||
"""添加一个下载任务并将其返回
|
||||
:param url: 文件网址
|
||||
:param save_path: 保存路径
|
||||
:param rename: 重命名的文件名
|
||||
:param suffix: 重命名的文件后缀名
|
||||
:param file_exists: 遇到同名文件时的处理方式,可选 'skip', 'overwrite', 'rename', 'add', 's', 'o', 'r', 'a',默认跟随实例属性
|
||||
:param split: 是否允许多线程分块下载,为None则使用对象属性
|
||||
:param kwargs: 连接参数
|
||||
:return: 任务对象
|
||||
"""
|
||||
...
|
||||
|
||||
def download(self,
|
||||
url: str,
|
||||
save_path: Union[str, Path, None] = None,
|
||||
rename: str = None,
|
||||
suffix: str = None,
|
||||
file_exists: FILE_EXISTS = None,
|
||||
split: bool = False,
|
||||
show_msg: bool = None,
|
||||
timeout: Optional[float] = None,
|
||||
params: Optional[dict] = ...,
|
||||
data: Any = ...,
|
||||
json: Any = ...,
|
||||
headers: Optional[dict] = ...,
|
||||
cookies: Any = ...,
|
||||
files: Any = ...,
|
||||
auth: Any = ...,
|
||||
allow_redirects: bool = ...,
|
||||
proxies: Optional[dict] = ...,
|
||||
hooks: Any = ...,
|
||||
stream: Any = ...,
|
||||
verify: Any = ...,
|
||||
cert: Any = ...) -> tuple:
|
||||
"""以阻塞的方式下载一个文件并返回结果
|
||||
:param url: 文件网址
|
||||
:param save_path: 保存路径
|
||||
:param rename: 重命名的文件名
|
||||
:param suffix: 重命名的文件后缀名
|
||||
:param file_exists: 遇到同名文件时的处理方式,可选 'skip', 'overwrite', 'rename', 'add', 's', 'o', 'r', 'a',默认跟随实例属性
|
||||
:param split: 是否分块下载,为None使用设置
|
||||
:param show_msg: 是否打印进度,为None使用设置
|
||||
:param kwargs: 连接参数
|
||||
:return: 任务结果和信息组成的tuple
|
||||
"""
|
||||
...
|
||||
|
||||
def by_browser(self,
|
||||
url: str,
|
||||
save_path: Union[str, Path] = None,
|
||||
rename: str = None, suffix: str = None,
|
||||
timeout: float = None,
|
||||
file_exists: Literal['rename', 'overwrite', 'skip', 'r', 'o', 's'] = None) -> Union[
|
||||
DownloadMission, False]:
|
||||
"""触发浏览器下载
|
||||
:param url: 下载地址
|
||||
:param save_path: 保存路径,为None保存在原来设置的,如未设置保存到当前路径
|
||||
:param rename: 重命名文件名
|
||||
:param suffix: 指定文件后缀
|
||||
:param timeout: 等待下载触发的超时时间,为None则使用页面对象设置
|
||||
:param file_exists: 可在 'rename', 'overwrite', 'skip', 'r', 'o', 's'中选择
|
||||
:return: DownloadMission对象,任务无触发时返回False
|
||||
"""
|
||||
...
|
||||
|
||||
def _run_or_wait(self, mission: BaseTask) -> None:
|
||||
"""接收任务,有空线程则运行,没有则进入等待队列
|
||||
:param mission: 任务对象
|
||||
:return: None
|
||||
"""
|
||||
...
|
||||
|
||||
def _run(self, ID: int, mission: BaseTask) -> None:
|
||||
"""线程函数
|
||||
:param ID: 线程id
|
||||
:param mission: 任务对象,Mission或Task
|
||||
:return: None
|
||||
"""
|
||||
...
|
||||
|
||||
def get_mission(self, mission_or_id: Union[int, Mission]) -> Mission:
|
||||
"""根据id值获取一个任务
|
||||
:param mission_or_id: 任务或任务id
|
||||
:return: 任务对象
|
||||
"""
|
||||
...
|
||||
|
||||
def get_failed_missions(self) -> list:
|
||||
"""返回失败任务列表"""
|
||||
...
|
||||
|
||||
def wait(self,
|
||||
mission: Union[int, Mission] = None,
|
||||
show: bool = False,
|
||||
timeout: float = None) -> Optional[tuple]:
|
||||
"""等待所有或指定任务完成
|
||||
:param mission: 任务对象或任务id,为None时等待所有任务结束
|
||||
:param show: 是否显示进度
|
||||
:param timeout: 超时时间,None或0为无限
|
||||
:return: 任务结果和信息组成的tuple
|
||||
"""
|
||||
...
|
||||
|
||||
def cancel(self) -> None:
|
||||
"""取消所有等待中或执行中的任务"""
|
||||
...
|
||||
|
||||
def show(self, asyn: bool = True, keep: bool = False) -> None:
|
||||
"""实时显示所有线程进度
|
||||
:param asyn: 是否以异步方式显示
|
||||
:param keep: 任务列表为空时是否保持显示
|
||||
:return: None
|
||||
"""
|
||||
...
|
||||
|
||||
def _show(self, wait: float, keep: bool = False) -> None:
|
||||
"""实时显示所有线程进度
|
||||
:param wait: 超时时间(秒)
|
||||
:param keep: 任务列表为空时是否保持显示
|
||||
:return: None
|
||||
"""
|
||||
...
|
||||
|
||||
def _connect(self, url: str, session: Session, _headers: CaseInsensitiveDict,
|
||||
method: str, encoding: Optional[str], **kwargs) -> Tuple[Union[Response, None], str]:
|
||||
"""生成response对象
|
||||
:param url: 目标url
|
||||
:param session: 用于连接的Session对象
|
||||
:param _headers: 内置的headers参数
|
||||
:param method: 请求方式
|
||||
:param encoding: 编码格式
|
||||
:param kwargs: 连接参数
|
||||
:return: tuple,第一位为Response或None,第二位为出错信息或'Success'
|
||||
"""
|
||||
...
|
||||
|
||||
def _get_usable_thread(self) -> Optional[int]:
|
||||
"""获取可用线程,没有则返回None"""
|
||||
...
|
||||
|
||||
def _stop_show(self) -> None:
|
||||
"""设置停止打印的变量"""
|
||||
...
|
||||
|
||||
def _when_mission_done(self, mission: Mission) -> None:
|
||||
"""当任务完成时执行的操作
|
||||
:param mission: 完结的任务
|
||||
:return: None
|
||||
"""
|
||||
...
|
||||
|
||||
def _download(self,
|
||||
mission_or_task: Union[Mission, Task],
|
||||
thread_id: int) -> None:
|
||||
"""此方法是执行下载的线程方法,用于根据任务下载文件
|
||||
:param mission_or_task: 下载任务对象
|
||||
:param thread_id: 线程号
|
||||
:return: None
|
||||
"""
|
||||
...
|
||||
|
||||
|
||||
def handle_mission(mission: Mission, url: str) -> tuple:
|
||||
"""对任务进行预处理,创建连接、获取信息
|
||||
:param mission: 下载任务对象
|
||||
:param url: 文件url
|
||||
:return: (是否继续, Response对象, 文件大小)
|
||||
"""
|
||||
...
|
||||
|
||||
|
||||
def do_download_(r, task: Task, show, size) -> None:
|
||||
"""执行download()触发的下载任务
|
||||
:param r: Response对象
|
||||
:param task: 任务对象
|
||||
:param show: 是否打印进度
|
||||
:param size: 文件大小
|
||||
:return: None
|
||||
"""
|
||||
...
|
||||
|
||||
|
||||
def do_download(r: Response, task: Task, first: bool = False) -> None:
|
||||
"""执行下载任务
|
||||
:param r: Response对象
|
||||
:param task: 任务对象
|
||||
:param first: 是否第一个分块
|
||||
:return: None
|
||||
"""
|
||||
...
|
||||
@@ -0,0 +1,280 @@
|
||||
# -*- coding:utf-8 -*-
|
||||
from time import sleep, perf_counter
|
||||
from urllib.parse import quote, urlparse
|
||||
|
||||
from DrissionRecord import Recorder
|
||||
from requests.structures import CaseInsensitiveDict
|
||||
|
||||
from ._funcs import copy_session, set_session_cookies
|
||||
|
||||
|
||||
class MissionData(object):
|
||||
def __init__(self, url, save_path, rename, suffix, file_exists, split, kwargs, offset=0):
|
||||
self.url = quote(url, safe='-_.~!*\'"();:@&=+$,/\\?#[]%')
|
||||
self.save_path = save_path
|
||||
self.rename = rename
|
||||
self.suffix = suffix
|
||||
self.file_exists = file_exists
|
||||
self.split = split
|
||||
self.kwargs = kwargs
|
||||
self.offset = offset
|
||||
|
||||
|
||||
class BaseTask(object):
|
||||
_DONE = 'done'
|
||||
RESULT_TEXTS = {'success': '成功', 'skipped': '跳过', 'canceled': '取消', False: '失败', None: '未知'}
|
||||
|
||||
def __init__(self, ID):
|
||||
self._id = ID
|
||||
self.state = 'waiting' # 'waiting', 'running', 'done'
|
||||
self.result = None # 'success', 'skipped', 'canceled', False, None
|
||||
self.info = '等待下载' # 信息
|
||||
|
||||
@property
|
||||
def id(self):
|
||||
return self._id
|
||||
|
||||
@property
|
||||
def data(self):
|
||||
return
|
||||
|
||||
@property
|
||||
def is_done(self):
|
||||
return self.state in ('done', 'cancel')
|
||||
|
||||
def set_states(self, result=None, info=None, state='done'):
|
||||
self.result = result
|
||||
self.info = info
|
||||
self.state = state
|
||||
|
||||
|
||||
class Mission(BaseTask):
|
||||
def __init__(self, ID, owner, file_url, save_path, rename, suffix, file_exists, split, encoding, kwargs):
|
||||
super().__init__(ID)
|
||||
self.owner = owner
|
||||
self.size = None
|
||||
|
||||
self.tasks = []
|
||||
self.tasks_count = 1
|
||||
self.done_tasks_count = 0
|
||||
|
||||
self.file_name = None
|
||||
self._path = None # 文件完整路径,Path对象
|
||||
self._recorder = None
|
||||
self.encoding = encoding
|
||||
|
||||
self._set_session()
|
||||
kwargs = self._handle_kwargs(file_url, kwargs)
|
||||
self._data = MissionData(file_url, save_path, rename, suffix, file_exists, split, kwargs)
|
||||
self._overwrote = False
|
||||
self.method = 'post' if (self._data.kwargs.get('data', None) is not None or
|
||||
self._data.kwargs.get('json', None) is not None) else 'get'
|
||||
|
||||
def __repr__(self):
|
||||
return f'<Mission {self.id} {self.info} {self.file_name}>'
|
||||
|
||||
@property
|
||||
def data(self):
|
||||
return self._data
|
||||
|
||||
@property
|
||||
def path(self):
|
||||
return self._path
|
||||
|
||||
@property
|
||||
def recorder(self):
|
||||
if self._recorder is None:
|
||||
self._recorder = Recorder(cache_size=100)
|
||||
self._recorder.set.show_msg(False)
|
||||
return self._recorder
|
||||
|
||||
@property
|
||||
def rate(self):
|
||||
if not self.size:
|
||||
return None
|
||||
c = 0
|
||||
for t in self.tasks:
|
||||
c += t._downloaded_size if t._downloaded_size else 0
|
||||
return round((c / self.size) * 100, 2)
|
||||
|
||||
def cancel(self) -> None:
|
||||
self._break_mission('canceled', '已取消')
|
||||
|
||||
def del_file(self):
|
||||
if self.path and self.path.exists():
|
||||
try:
|
||||
self.path.unlink()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
def wait(self, show=True, timeout=0):
|
||||
if show:
|
||||
print(f'url:{self.data.url}')
|
||||
t2 = perf_counter()
|
||||
while self.file_name is None and perf_counter() - t2 < 4:
|
||||
sleep(0.01)
|
||||
print(f'文件名:{self.file_name}')
|
||||
print(f'目标路径:{self.path}')
|
||||
if not self.size:
|
||||
print('未知大小 ', end='')
|
||||
|
||||
t1 = perf_counter()
|
||||
while not self.is_done and (perf_counter() - t1 < timeout or timeout == 0):
|
||||
try:
|
||||
rate = round((self.path.stat().st_size / self.size) * 100, 2)
|
||||
print(f'\r{rate}% ', end='')
|
||||
except (FileNotFoundError, TypeError, ZeroDivisionError):
|
||||
pass
|
||||
sleep(0.1)
|
||||
|
||||
if self.result is False:
|
||||
print(f'下载失败 {self.info}')
|
||||
elif self.result == 'success':
|
||||
print('\r100% ', end='')
|
||||
print(f'{"已覆盖" if self._overwrote else "下载完成"} {self.info}')
|
||||
elif self.result == 'skipped':
|
||||
print(f'已跳过 {self.info}')
|
||||
print()
|
||||
|
||||
else:
|
||||
t1 = perf_counter()
|
||||
while not self.is_done and (perf_counter() - t1 < timeout or timeout == 0):
|
||||
sleep(0.1)
|
||||
|
||||
return self.result, self.info
|
||||
|
||||
def _set_session(self):
|
||||
session = copy_session(self.owner.session)
|
||||
headers = session.headers
|
||||
session.headers = None
|
||||
if self.owner.page:
|
||||
set_session_cookies(session, self.owner.page.cookies())
|
||||
if hasattr(self.owner.page, '_headers'):
|
||||
headers = CaseInsensitiveDict({**self.owner.page._headers, **headers})
|
||||
headers.update({"User-Agent": self.owner.page.user_agent})
|
||||
self.session = session
|
||||
self.headers = headers
|
||||
|
||||
def _handle_kwargs(self, url, kwargs):
|
||||
if 'timeout' not in kwargs:
|
||||
kwargs['timeout'] = self.owner.timeout
|
||||
|
||||
headers = CaseInsensitiveDict(kwargs['headers']) if 'headers' in kwargs else CaseInsensitiveDict()
|
||||
|
||||
parsed_url = urlparse(url)
|
||||
hostname = parsed_url.hostname
|
||||
scheme = parsed_url.scheme
|
||||
|
||||
if not ('Referer' in headers or 'Referer' in self.headers):
|
||||
headers['Referer'] = self.owner.page.url if self.owner.page and self.owner.page.url \
|
||||
else f'{scheme}://{hostname}'
|
||||
if not ('Host' in headers or 'Host' in self.headers):
|
||||
headers['Host'] = hostname
|
||||
kwargs['headers'] = format_headers(headers)
|
||||
if not kwargs['headers']['Host']:
|
||||
kwargs['headers'].pop('Host')
|
||||
if not kwargs['headers']['Referer']:
|
||||
kwargs['headers'].pop('Referer')
|
||||
|
||||
return kwargs
|
||||
|
||||
def _set_path(self, path):
|
||||
self.file_name = path.name
|
||||
self._path = path
|
||||
self.recorder.set.path(path, file_type='b')
|
||||
|
||||
def _set_done(self, result, info):
|
||||
if result == 'skipped':
|
||||
self.set_states(result=result, info=info, state=self._DONE)
|
||||
|
||||
elif result == 'canceled' or result is False:
|
||||
self.recorder.clear()
|
||||
self.set_states(result=result, info=info, state=self._DONE)
|
||||
|
||||
elif result == 'success':
|
||||
self.recorder.record()
|
||||
if self.size and self.path.stat().st_size < self.size:
|
||||
self.del_file()
|
||||
self.set_states(False, '下载失败', self._DONE)
|
||||
else:
|
||||
self.set_states('success', info, self._DONE)
|
||||
|
||||
self.owner._when_mission_done(self)
|
||||
|
||||
def _a_task_done(self, is_success, info):
|
||||
if self.is_done:
|
||||
return
|
||||
|
||||
if is_success is False:
|
||||
self._break_mission(False, info)
|
||||
return
|
||||
|
||||
self.done_tasks_count += 1
|
||||
if self.done_tasks_count == self.tasks_count:
|
||||
self._set_done('success', info)
|
||||
|
||||
def _break_mission(self, result, info):
|
||||
if self.is_done:
|
||||
return
|
||||
|
||||
for task in self.tasks:
|
||||
if not task.is_done:
|
||||
task.set_states(result=result, info=info, state='cancel')
|
||||
|
||||
while any((not i.is_done for i in self.tasks)):
|
||||
sleep(.3)
|
||||
|
||||
self._set_done(result, info)
|
||||
self.del_file()
|
||||
|
||||
|
||||
class Task(BaseTask):
|
||||
def __init__(self, mission, range_, ID, size):
|
||||
super().__init__(ID)
|
||||
self.mission = mission
|
||||
self.range = range_
|
||||
self.size = size
|
||||
self._downloaded_size = 0
|
||||
|
||||
def __repr__(self):
|
||||
return f'<Task M{self.mid} T{self._id} {self.rate}% {self.info} {self.file_name}>'
|
||||
|
||||
@property
|
||||
def mid(self):
|
||||
return self.mission.id
|
||||
|
||||
@property
|
||||
def data(self):
|
||||
return self.mission.data
|
||||
|
||||
@property
|
||||
def path(self):
|
||||
return self.mission.path
|
||||
|
||||
@property
|
||||
def file_name(self):
|
||||
return self.mission.file_name
|
||||
|
||||
@property
|
||||
def rate(self):
|
||||
return round((self._downloaded_size / self.size) * 100, 2) if self.size else None
|
||||
|
||||
def add_data(self, data, seek=None):
|
||||
self._downloaded_size += len(data)
|
||||
self.mission.recorder.add_data(data, coord=seek)
|
||||
|
||||
def clear_cache(self):
|
||||
self.mission.recorder.clear()
|
||||
|
||||
def _set_done(self, result, info):
|
||||
self.set_states(result=result, info=info, state=self._DONE)
|
||||
self.mission._a_task_done(result, info)
|
||||
|
||||
|
||||
def format_headers(headers):
|
||||
for k, v in headers.items():
|
||||
if k in (':method', ':scheme', ':authority', ':path'):
|
||||
headers.pop(k)
|
||||
elif v not in (None, False, True):
|
||||
headers[k] = str(v)
|
||||
return headers
|
||||
@@ -0,0 +1,259 @@
|
||||
# -*- coding:utf-8 -*-
|
||||
from pathlib import Path
|
||||
from typing import Union, List, Optional
|
||||
|
||||
from DrissionRecord import Recorder
|
||||
from requests import Session
|
||||
from requests.structures import CaseInsensitiveDict
|
||||
|
||||
from .drissionGet import DrissionGet
|
||||
|
||||
|
||||
class MissionData(object):
|
||||
"""保存任务数据的对象"""
|
||||
url: str = ...
|
||||
save_path: Path = ...
|
||||
rename: Optional[str] = ...
|
||||
suffix: Optional[str] = ...
|
||||
file_exists: str = ...
|
||||
split: bool = ...
|
||||
kwargs: dict = ...
|
||||
offset: int = ...
|
||||
|
||||
def __init__(self, url: str, save_path: Path, rename: Optional[str], suffix: Optional[str],
|
||||
file_exists: str, split: bool, kwargs: dict, offset: int = 0):
|
||||
"""保存任务数据的对象
|
||||
:param url: 下载文件url
|
||||
:param save_path: 保存文件夹
|
||||
:param rename: 文件重命名
|
||||
:param suffix: 文件重命名后缀名
|
||||
:param file_exists: 存在重名文件时处理方式
|
||||
:param split: 是否允许分块下载
|
||||
:param kwargs: requests其它参数
|
||||
:param offset: 文件存储偏移量
|
||||
"""
|
||||
...
|
||||
|
||||
|
||||
class BaseTask(object):
|
||||
"""任务类基类"""
|
||||
_DONE: str = ...
|
||||
RESULT_TEXTS: dict = ...
|
||||
|
||||
_id: str = ...
|
||||
state: str = ...
|
||||
result: Union[str, False, None] = ...
|
||||
info: str = ...
|
||||
|
||||
def __init__(self, ID: Union[int, str]):
|
||||
"""任务类基类
|
||||
:param ID: 任务id
|
||||
"""
|
||||
...
|
||||
|
||||
@property
|
||||
def id(self) -> Union[int, str]:
|
||||
"""返回任务或子任务id"""
|
||||
...
|
||||
|
||||
@property
|
||||
def data(self):
|
||||
"""返回任务数据"""
|
||||
...
|
||||
|
||||
@property
|
||||
def is_done(self) -> bool:
|
||||
"""返回任务是否结束"""
|
||||
...
|
||||
|
||||
def set_states(self,
|
||||
result: Union[bool, str, None] = None,
|
||||
info: str = None,
|
||||
state: str = 'done') -> None:
|
||||
"""设置任务结果值
|
||||
:param result: 结果:'success', 'skipped', 'canceled', False, None
|
||||
:param info: 任务信息
|
||||
:param state: 任务状态:'waiting', 'running', 'done'
|
||||
:return: None
|
||||
"""
|
||||
...
|
||||
|
||||
|
||||
class Mission(BaseTask):
|
||||
"""任务类"""
|
||||
file_name: Optional[str] = ...
|
||||
_data: MissionData = ...
|
||||
_path: Optional[Path] = ...
|
||||
_recorder: Optional[Recorder] = ...
|
||||
size: Optional[float] = ...
|
||||
done_tasks_count: int = ...
|
||||
tasks_count: int = ...
|
||||
tasks: List[Task] = ...
|
||||
owner: DrissionGet = ...
|
||||
session: Session = ...
|
||||
headers: CaseInsensitiveDict = ...
|
||||
method: str = ...
|
||||
encoding: Optional[str] = ...
|
||||
_overwrote: bool = ...
|
||||
|
||||
def __init__(self, ID: int, owner: DrissionGet, file_url: str, save_path: Path, rename: str,
|
||||
suffix: str, file_exists: str, split: bool, encoding: Optional[str], kwargs: dict):
|
||||
"""任务类
|
||||
:param ID: 任务id
|
||||
:param owner: 所属DrissionGet对象
|
||||
:param file_url: 文件网址
|
||||
:param save_path: 保存文件夹路径
|
||||
:param rename: 重命名
|
||||
:param suffix: 重命名后缀名
|
||||
:param file_exists: 存在同名文件处理方式
|
||||
:param split: 是否分块下载
|
||||
:param encoding: 编码格式
|
||||
:param kwargs: 连接参数
|
||||
"""
|
||||
...
|
||||
|
||||
def __repr__(self) -> str:
|
||||
...
|
||||
|
||||
@property
|
||||
def data(self) -> MissionData:
|
||||
"""返回任务数据"""
|
||||
...
|
||||
|
||||
@property
|
||||
def path(self) -> Union[str, Path]:
|
||||
"""返回文件保存路径"""
|
||||
...
|
||||
|
||||
@property
|
||||
def recorder(self) -> Recorder:
|
||||
"""返回记录器对象"""
|
||||
...
|
||||
|
||||
@property
|
||||
def rate(self) -> Optional[float]:
|
||||
"""返回下载进度百分比"""
|
||||
...
|
||||
|
||||
def cancel(self) -> None:
|
||||
"""取消该任务,停止未下载完的task"""
|
||||
...
|
||||
|
||||
def del_file(self) -> None:
|
||||
"""删除下载的文件"""
|
||||
...
|
||||
|
||||
def wait(self, show: bool = True, timeout: float = 0) -> tuple:
|
||||
"""等待当前任务完成
|
||||
:param show: 是否显示下载进度
|
||||
:param timeout: 超时时间
|
||||
:return: 任务结果和信息组成的tuple
|
||||
"""
|
||||
...
|
||||
|
||||
def _set_session(self) -> Session:
|
||||
"""复制Session对象,并设置cookies"""
|
||||
...
|
||||
|
||||
def _handle_kwargs(self, url: str, kwargs: dict) -> dict:
|
||||
"""处理接收到的参数
|
||||
:param url: 要访问的url
|
||||
:param kwargs: 传入的参数dict
|
||||
:return: 处理后的参数dict
|
||||
"""
|
||||
...
|
||||
|
||||
def _set_path(self, path: Path) -> None:
|
||||
"""设置文件保存路径"""
|
||||
...
|
||||
|
||||
def _set_done(self, result: Union[bool, str, None], info: str) -> None:
|
||||
"""设置一个任务为done状态
|
||||
:param result: 结果:'success'、'skipped'、'canceled'、False、None
|
||||
:param info: 任务信息
|
||||
:return: None
|
||||
"""
|
||||
...
|
||||
|
||||
def _a_task_done(self, is_success: bool, info: str) -> None:
|
||||
"""当一个task完成时调用
|
||||
:param is_success: 该task是否成功
|
||||
:param info: 该task传入的信息
|
||||
:return: None
|
||||
"""
|
||||
...
|
||||
|
||||
def _break_mission(self, result: Union[bool, str, None], info: str) -> None:
|
||||
"""中止该任务,停止未下载完的task
|
||||
:param result: 结果:'success'、'skipped'、'canceled'、False、None
|
||||
:param info: 任务信息
|
||||
:return: None
|
||||
"""
|
||||
...
|
||||
|
||||
|
||||
class Task(BaseTask):
|
||||
"""子任务类"""
|
||||
mission: Mission = ...
|
||||
range: Optional[list] = ...
|
||||
size: Optional[int] = ...
|
||||
_downloaded_size: int = 0
|
||||
|
||||
def __init__(self, mission: Mission, range_: Optional[list], ID: str, size: Optional[int]):
|
||||
"""子任务类
|
||||
:param mission: 父任务对象
|
||||
:param range_: 读取文件数据范围
|
||||
:param ID: 任务id
|
||||
"""
|
||||
...
|
||||
|
||||
def __repr__(self) -> str:
|
||||
...
|
||||
|
||||
@property
|
||||
def mid(self) -> int:
|
||||
"""返回父任务id"""
|
||||
...
|
||||
|
||||
@property
|
||||
def data(self) -> MissionData:
|
||||
"""返回任务数据对象"""
|
||||
...
|
||||
|
||||
@property
|
||||
def path(self) -> str:
|
||||
"""返回文件保存路径"""
|
||||
...
|
||||
|
||||
@property
|
||||
def file_name(self) -> str:
|
||||
"""返回文件名"""
|
||||
...
|
||||
|
||||
@property
|
||||
def rate(self) -> Optional[float]:
|
||||
"""返回下载进度百分比"""
|
||||
...
|
||||
|
||||
def add_data(self, data: bytes, seek: int = None) -> None:
|
||||
"""把数据输入到记录器
|
||||
:param data: 文件字节数据
|
||||
:param seek: 在文件中的位置,None表示最后
|
||||
:return: None
|
||||
"""
|
||||
...
|
||||
|
||||
def clear_cache(self) -> None:
|
||||
"""清除以接收但未写入硬盘的缓存"""
|
||||
...
|
||||
|
||||
def _set_done(self, result: Union[bool, str, None], info: str) -> None:
|
||||
"""设置一个子任务为done状态
|
||||
:param result: 结果:'success'、'skipped'、'canceled'、False、None
|
||||
:param info: 任务信息
|
||||
:return: None
|
||||
"""
|
||||
...
|
||||
|
||||
|
||||
def format_headers(headers: CaseInsensitiveDict): ...
|
||||
@@ -0,0 +1,122 @@
|
||||
# -*- coding:utf-8 -*-
|
||||
from DrissionRecord import Recorder
|
||||
from requests import Session
|
||||
|
||||
from ._funcs import get_file_exists_mode, get_save_path
|
||||
|
||||
|
||||
class Setter(object):
|
||||
def __init__(self, owner):
|
||||
self._owner = owner
|
||||
|
||||
@property
|
||||
def if_file_exists(self):
|
||||
return FileExists(self)
|
||||
|
||||
def log(self, path='DrissionGet_log.csv', failed_only=False):
|
||||
if not path:
|
||||
self._owner._logger = self._owner._log_mode = None
|
||||
return self
|
||||
if self._owner._logger:
|
||||
self._owner._logger.set.path(path)
|
||||
else:
|
||||
self._owner._logger = Recorder(path, cache_size=1)
|
||||
self._owner._log_mode = 'failed' if failed_only else 'all'
|
||||
return self
|
||||
|
||||
def show_msg(self, on_off=True, failed_only=False):
|
||||
self._owner._print_mode = 'failed' if failed_only else 'all' if on_off else None
|
||||
return self
|
||||
|
||||
def driver(self, driver):
|
||||
if driver is None:
|
||||
self._owner._session = Session()
|
||||
return self
|
||||
|
||||
elif isinstance(driver, Session):
|
||||
self._owner._session = driver
|
||||
return self
|
||||
|
||||
_type = str(type(driver))
|
||||
if _type.startswith('<SessionOptions '):
|
||||
self._owner._session, headers = driver.make_session()
|
||||
self._owner._session.headers = headers
|
||||
return self
|
||||
|
||||
else:
|
||||
self._owner._session = driver.session if hasattr(driver, 'session') else Session()
|
||||
self._owner.page = driver
|
||||
return self
|
||||
|
||||
def roads(self, num):
|
||||
if self._owner.is_running:
|
||||
print('有任务未完成时不能改变roads。')
|
||||
return self
|
||||
if num != self._owner.roads:
|
||||
self._owner._roads = num
|
||||
self._owner._threads = {i: None for i in range(num)}
|
||||
return self
|
||||
|
||||
def retry(self, times):
|
||||
if not isinstance(times, int) or times < 0:
|
||||
raise TypeError('times参数只能接受int格式且不能小于0。')
|
||||
self._owner._retry = times
|
||||
return self
|
||||
|
||||
def interval(self, seconds):
|
||||
if not isinstance(seconds, (int, float)) or seconds < 0:
|
||||
raise TypeError('seconds参数只能接受int或float格式且不能小于0。')
|
||||
self._owner._interval = seconds
|
||||
return self
|
||||
|
||||
def timeout(self, seconds):
|
||||
if not isinstance(seconds, (int, float)) or seconds < 0:
|
||||
raise TypeError('seconds参数只能接受int或float格式且不能小于0。')
|
||||
self._owner._timeout = seconds
|
||||
return self
|
||||
|
||||
def save_path(self, path):
|
||||
self._owner.save_path = get_save_path(path)
|
||||
return self
|
||||
|
||||
def split(self, on_off=True):
|
||||
self._owner.split = on_off
|
||||
return self
|
||||
|
||||
def block_size(self, size):
|
||||
self._owner.block_size = size
|
||||
return self
|
||||
|
||||
def proxies(self, http=None, https=None):
|
||||
self._owner._session.proxies = {'http': http, 'https': https}
|
||||
return self
|
||||
|
||||
def encoding(self, encoding):
|
||||
self._owner._encoding = encoding if encoding else None
|
||||
return self
|
||||
|
||||
|
||||
class FileExists(object):
|
||||
|
||||
def __init__(self, setter):
|
||||
self._setter = setter
|
||||
|
||||
def __call__(self, mode):
|
||||
self._setter._owner.file_exists = get_file_exists_mode(mode)
|
||||
return self._setter
|
||||
|
||||
def skip(self):
|
||||
self._setter._owner.file_exists = 'skip'
|
||||
return self._setter
|
||||
|
||||
def rename(self):
|
||||
self._setter._owner.file_exists = 'rename'
|
||||
return self._setter
|
||||
|
||||
def overwrite(self):
|
||||
self._setter._owner.file_exists = 'overwrite'
|
||||
return self._setter
|
||||
|
||||
def add(self):
|
||||
self._setter._owner.file_exists = 'add'
|
||||
return self._setter
|
||||
@@ -0,0 +1,147 @@
|
||||
# -*- coding:utf-8 -*-
|
||||
from pathlib import Path
|
||||
from typing import Union, Literal, Optional
|
||||
|
||||
from DrissionPage import SessionOptions
|
||||
from DrissionPage._base.base import BasePage
|
||||
from requests import Session
|
||||
|
||||
from .drissionGet import DrissionGet
|
||||
|
||||
FILE_EXISTS = Literal['add', 'skip', 'rename', 'overwrite', 'a', 's', 'r', 'o']
|
||||
|
||||
|
||||
class Setter(object):
|
||||
_owner: DrissionGet = ...
|
||||
|
||||
def __init__(self, owner: DrissionGet):
|
||||
"""
|
||||
:param owner: DrissionGet对象
|
||||
"""
|
||||
...
|
||||
|
||||
@property
|
||||
def if_file_exists(self) -> FileExists:
|
||||
"""返回用于设置文件同名策略的对象"""
|
||||
...
|
||||
|
||||
def log(self, path: Union[str, Path, None] = 'DrissionGet_log.csv', failed_only: bool = False) -> Setter:
|
||||
"""设置下载完成日志
|
||||
:param path: 日志文件路径,为None或False不记录
|
||||
:param failed_only: 是否只记录失败任务
|
||||
:return: Setter对象自己
|
||||
"""
|
||||
...
|
||||
|
||||
def show_msg(self, on_off: bool = True, failed_only: bool = False) -> Setter:
|
||||
"""设置是否打印下载信息
|
||||
:param on_off: bool代表开关
|
||||
:param failed_only: 是否只显示失败任务,此设置仅多线程任务时生效
|
||||
:return: Setter对象自己
|
||||
"""
|
||||
...
|
||||
|
||||
def driver(self, driver: Union[Session, BasePage, SessionOptions]) -> Setter:
|
||||
"""设置驱动
|
||||
:param driver: Session对象或DrissionPage的页面对象
|
||||
:return: Setter对象自己
|
||||
"""
|
||||
...
|
||||
|
||||
def roads(self, num: int) -> Setter:
|
||||
"""设置可同时运行的线程数
|
||||
:param num: 线程数量
|
||||
:return: Setter对象自己
|
||||
"""
|
||||
...
|
||||
|
||||
def retry(self, times: int) -> Setter:
|
||||
"""设置连接失败时重试次数
|
||||
:param times: 重试次数
|
||||
:return: Setter对象自己
|
||||
"""
|
||||
...
|
||||
|
||||
def interval(self, seconds: float) -> Setter:
|
||||
"""设置连接失败时重试间隔
|
||||
:param seconds: 连接失败时重试间隔(秒)
|
||||
:return: Setter对象自己
|
||||
"""
|
||||
...
|
||||
|
||||
def timeout(self, seconds: float) -> Setter:
|
||||
"""设置连接超时时间
|
||||
:param seconds: 超时时间(秒)
|
||||
:return: Setter对象自己
|
||||
"""
|
||||
...
|
||||
|
||||
def save_path(self, path: Union[str, Path]) -> Setter:
|
||||
"""设置文件保存路径
|
||||
:param path: 文件路径,可以是str或Path
|
||||
:return: Setter对象自己
|
||||
"""
|
||||
...
|
||||
|
||||
def split(self, on_off: bool = True) -> Setter:
|
||||
"""设置大文件是否分块下载
|
||||
:param on_off: bool代表开关
|
||||
:return: Setter对象自己
|
||||
"""
|
||||
...
|
||||
|
||||
def block_size(self, size: Union[str, int]) -> Setter:
|
||||
"""设置分块大小
|
||||
:param size: 单位为字节,可用'K'、'M'、'G'为单位,如'50M'
|
||||
:return: Setter对象自己
|
||||
"""
|
||||
...
|
||||
|
||||
def proxies(self, http: str = None, https: str = None) -> Setter:
|
||||
"""设置代理地址及端口,例:'127.0.0.1:1080'
|
||||
:param http: http代理地址及端口
|
||||
:param https: https代理地址及端口
|
||||
:return: Setter对象自己
|
||||
"""
|
||||
...
|
||||
|
||||
def encoding(self, encoding: Optional[str]) -> Setter:
|
||||
"""设置编码
|
||||
:param encoding: 编码名称,传入None取消之前的设置
|
||||
:return: Setter对象自己
|
||||
"""
|
||||
...
|
||||
|
||||
|
||||
class FileExists(object):
|
||||
"""用于设置存在同名文件时处理方法"""
|
||||
_setter: Setter = ...
|
||||
|
||||
def __init__(self, setter: Setter):
|
||||
"""
|
||||
:param setter: Setter对象
|
||||
"""
|
||||
...
|
||||
|
||||
def __call__(self, mode: FILE_EXISTS) -> Setter:
|
||||
"""设置文件存在时的处理方式
|
||||
:param mode: 'skip', 'rename', 'overwrite', 'add', 's', 'r', 'o', 'a'
|
||||
:return: Setter 对象
|
||||
"""
|
||||
...
|
||||
|
||||
def skip(self) -> Setter:
|
||||
"""设为跳过"""
|
||||
...
|
||||
|
||||
def rename(self) -> Setter:
|
||||
"""设为重命名,文件名后加序号"""
|
||||
...
|
||||
|
||||
def overwrite(self) -> Setter:
|
||||
"""设为覆盖"""
|
||||
...
|
||||
|
||||
def add(self) -> Setter:
|
||||
"""设为追加"""
|
||||
...
|
||||
Reference in new issue
Block a user