Files
universal-web-api/venv/Lib/site-packages/DrissionGet/drissionGet.py
T
2026-11-02 22:54:14 +08:00

540 lines
19 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
# -*- coding:utf-8 -*-
from copy import copy
from pathlib import Path
from queue import Queue
from threading import Thread, Lock
from time import sleep, perf_counter
from requests.structures import CaseInsensitiveDict
from ._funcs import (FileExistsSetter, PathSetter, BlockSizeSetter, set_charset, get_file_info, get_file_exists_mode,
get_save_path)
from .mission import Task, Mission
from .setter import Setter
class DrissionGet(object):
file_exists = FileExistsSetter()
save_path = PathSetter()
block_size = BlockSizeSetter()
def __init__(self, save_path='.', roads=10, driver=None, file_exists='rename'):
self._roads = roads
self._missions = {}
self._threads = {i: None for i in range(self._roads)}
self._waiting_list = Queue()
self._missions_num = 0
self._running_count = 0 # 正在运行的任务数
self._stop_printing = False # 用于控制显示线程停止
self._lock = Lock()
self.page = None
self._retry = None
self._interval = None
self._timeout = None
self._encoding = None
self._setter = None
self._print_mode = 'all'
self._log_mode = None
self._logger = None
self.save_path = save_path
self.file_exists = file_exists
self.split = True
self.block_size = '50M'
self.set.driver(driver)
def __call__(self, url=None, save_path=None, rename=None, suffix=None,
file_exists=None, split=False, show_msg=True, file_url=None, **kwargs):
return self.download(url=url, save_path=save_path, rename=rename, suffix=suffix,
file_exists=file_exists, show_msg=show_msg, **kwargs)
@property
def set(self):
if self._setter is None:
self._setter = Setter(self)
return self._setter
@property
def roads(self):
return self._roads
@property
def retry(self):
if self._retry is not None:
return self._retry
elif self.page is not None:
return self.page.retry_times
else:
return 3
@property
def interval(self):
if self._interval is not None:
return self._interval
elif self.page is not None:
return self.page.retry_interval
else:
return 5
@property
def timeout(self):
if self._timeout is not None:
return self._timeout
elif self.page is not None:
return self.page.timeout
else:
return 20
@property
def waiting_list(self):
return self._waiting_list
@property
def session(self):
return self._session
@property
def is_running(self):
return self._running_count > 0
@property
def missions(self):
return self._missions
@property
def encoding(self):
"""返回指定的编码格式"""
return self._encoding
def add(self, url=None, save_path=None, rename=None, suffix=None, file_exists=None, split=None,
file_url=None, **kwargs):
if not url and file_url:
url = file_url
self._missions_num += 1
self._running_count += 1
if file_exists is not None:
file_exists = get_file_exists_mode(file_exists)
mission = Mission(self._missions_num, self, url, get_save_path(save_path) if save_path else self.save_path,
rename, suffix, file_exists or self.file_exists,
self.split if split is None else split, self._encoding, kwargs)
self._missions[self._missions_num] = mission
self._run_or_wait(mission)
return mission
def download(self, url=None, save_path=None, rename=None, suffix=None, file_exists=None, split=False,
show_msg=None, file_url=None, **kwargs):
if not url and file_url:
url = file_url
self._missions_num += 1
self._running_count += 1
if file_exists is not None:
file_exists = get_file_exists_mode(file_exists)
mission = Mission(self._missions_num, self, url, get_save_path(save_path) if save_path else self.save_path,
rename, suffix, file_exists or self.file_exists,
self.split if split is None else split, self._encoding, kwargs)
self._missions[self._missions_num] = mission
tmp, self._print_mode = self._print_mode, None
if show_msg is None:
show_msg = True if tmp else False
if split:
self._run_or_wait(mission)
r = mission.wait(show=show_msg)
self._print_mode = tmp
return r
ok, r, file_size = handle_mission(mission, url)
if show_msg:
print(f'url:{url}')
print(f'文件名:{mission.file_name}')
print(f'目标路径:{mission.path}')
if not mission.size:
print('下载中 ', end='')
if ok:
task1 = Task(mission, None, '1/1', file_size)
mission.tasks.append(task1)
do_download_(r, task1, show_msg, file_size)
if show_msg:
if mission.result is False:
print(f'下载失败 {mission.info}')
elif mission.result == 'success':
print('\r100% ', end='')
print(f'{"已覆盖" if mission._overwrote else "下载完成"} {mission.info}')
elif mission.result == 'skipped':
print(f'已跳过 {mission.info}')
print()
self._print_mode = tmp
return mission.result, mission.info
def by_browser(self, url, save_path=None, rename=None, suffix=None, timeout=None, file_exists=None):
if hasattr(self.page, '_download_by_browser'):
return self.page._download_by_browser(url=url, save_path=save_path, rename=rename, suffix=suffix,
timeout=timeout, file_exists=file_exists)
else:
raise RuntimeError('非浏览器调用无法使用浏览器下载。')
def _run_or_wait(self, mission):
thread_id = self._get_usable_thread()
if thread_id is None:
self._waiting_list.put(mission)
else:
thread = Thread(target=self._run, args=(thread_id, mission), daemon=False)
self._threads[thread_id] = {'thread': thread, 'mission': None}
thread.start()
def _run(self, ID, mission):
while True:
if not mission: # 如果没有任务,就从等候列表中取一个
if self._waiting_list.empty():
break
else:
try:
mission = self._waiting_list.get(True, .5)
except Exception:
self._waiting_list.task_done()
break
self._threads[ID]['mission'] = mission
self._download(mission, ID)
mission = None
self._threads[ID] = None
def get_mission(self, mission_or_id):
return self._missions[mission_or_id] if isinstance(mission_or_id, int) else mission_or_id
def get_failed_missions(self):
return [i for i in self._missions.values() if i.result is False]
def wait(self, mission=None, show=False, timeout=None):
timeout = 0 if timeout is None else timeout
if mission:
return self.get_mission(mission).wait(show, timeout)
else:
if show:
self.show(False)
else:
end_time = perf_counter() + timeout
while self.is_running and (perf_counter() < end_time or timeout == 0):
sleep(0.1)
def cancel(self):
for m in self._missions.values():
m.cancel()
def show(self, asyn=True, keep=False):
if asyn:
Thread(target=self._show, args=(2, keep)).start()
else:
self._show(0.1, keep)
def _show(self, wait, keep=False):
self._stop_printing = False
if keep:
Thread(target=self._stop_show).start()
end_time = perf_counter() + wait
while not self._stop_printing and (keep or self.is_running or perf_counter() < end_time):
print(f'\033[K', end='')
print(f'等待任务数:{self._waiting_list.qsize()}')
for k, v in self._threads.items():
m = v['mission'] if v else None
if m:
items = (m.mission.rate, m.mid) if isinstance(m, Task) else (m.rate, m.id)
path = f'M{items[1]} {items[0]}% {m}'
else:
path = '空闲'
print(f'\033[K', end='')
print(f'线程{k}:{path}')
print(f'\033[{self.roads + 1}A\r', end='')
sleep(0.4)
print(f'\033[1B', end='')
for i in range(self.roads):
print(f'\033[K', end='')
print(f'线程{i}:空闲')
print()
def _connect(self, url, session, _headers, method, encoding, **kwargs):
kwargs = CaseInsensitiveDict(kwargs)
if 'headers' in kwargs: # 不知道为什么添加这个才能正常使用多线程
kwargs['headers'] = CaseInsensitiveDict({**_headers, **kwargs['headers']})
else:
kwargs['headers'] = _headers
r = err = None
for i in range(self.retry + 1):
try:
if method == 'get':
r = session.get(url, **kwargs)
elif method == 'post':
r = session.post(url, **kwargs)
if r:
return set_charset(r, encoding), 'Success'
except Exception as e:
err = e
if r and r.status_code in (403, 404):
break
if i < self.retry:
sleep(self.interval)
# 返回失败结果
if r is None:
return None, '连接失败' if err is None else err
if not r.ok:
return r, f'状态码:{r.status_code}'
return None
def _get_usable_thread(self):
for k, v in self._threads.items():
if v is None:
return k
return None
def _stop_show(self):
input()
self._stop_printing = True
def _when_mission_done(self, mission):
self._running_count -= 1
if self._print_mode == 'all' or (self._print_mode == 'failed' and mission.result is False):
print(f'[{mission.RESULT_TEXTS[mission.result]}] {mission.info} {mission.data.url}')
if self._log_mode == 'all' or (self._log_mode == 'failed' and mission.result is False):
self._logger.add_data((mission.RESULT_TEXTS[mission.result], mission.data.url, mission.info,
mission.data.save_path, mission.data.rename, mission.data.kwargs))
mission.session.close()
def _download(self, mission_or_task, thread_id):
if mission_or_task.is_done:
return
if mission_or_task.state == 'cancel':
mission_or_task.state = 'done'
return
file_url = mission_or_task.data.url
if isinstance(mission_or_task, Task):
kwargs = copy(mission_or_task.data.kwargs)
kwargs['headers']['Range'] = f"bytes={mission_or_task.range[0]}-{mission_or_task.range[1]}"
r, inf = self._connect(file_url, mission_or_task.mission.session, mission_or_task.mission.headers,
mission_or_task.mission.method, mission_or_task.mission.encoding, **kwargs)
if r:
do_download(r, mission_or_task, False)
else:
mission_or_task._set_done(False, inf)
return
# ===================开始处理mission====================
mission = mission_or_task
ok, r, file_size = handle_mission(mission, file_url)
if not ok:
return
# -------------------设置分块任务-------------------
first = False
if mission.data.split and file_size and file_size > self.block_size and r.headers.get(
'Accept-Ranges') == 'bytes':
first = True
chunks = [[s, min(s + self.block_size, file_size) - 1] for s in range(0, file_size, self.block_size)]
chunks[-1][-1] = ''
chunks_len = len(chunks)
task1 = Task(mission, chunks[0], f'1/{chunks_len}', chunks[0][1] - chunks[0][0])
mission.tasks_count = chunks_len
mission.tasks = [task1]
for ind, chunk in enumerate(chunks[1:], 2):
s = file_size - chunk[0] if chunks_len == ind else chunk[1] - chunk[0]
task = Task(mission, chunk, f'{ind}/{chunks_len}', s)
mission.tasks.append(task)
self._run_or_wait(task)
else: # 不分块
task1 = Task(mission, None, '1/1', file_size)
mission.tasks.append(task1)
self._threads[thread_id]['mission'] = task1
do_download(r, task1, first)
def handle_mission(mission, url):
mission.info = '下载中'
mission.state = 'running'
kwargs = mission.data.kwargs
rename = mission.data.rename
suffix = mission.data.suffix
file_exists = mission.data.file_exists
mission.data.save_path.mkdir(parents=True, exist_ok=True)
if file_exists == 'skip' and rename:
tmp = mission.data.save_path / rename
if tmp.exists() and tmp.is_file():
mission.file_name = rename
mission._set_path(mission.data.save_path / rename)
mission._set_done('skipped', str(mission.path))
return False, None, None
r, inf = mission.owner._connect(url, mission.session, mission.headers, mission.method, mission.encoding, **kwargs)
if mission.is_done:
return False, None, None
if not r:
mission._break_mission(result=False, info=inf)
return False, None, None
# -------------------获取文件信息-------------------
file_info = get_file_info(r, mission.data.save_path, rename, suffix, file_exists,
mission.encoding, mission.owner._lock)
file_size = file_info['size']
full_path = file_info['path']
mission._set_path(full_path)
mission.size = file_size
mission._overwrote = file_info['overwrite']
if file_info['skip']:
mission._set_done('skipped', str(mission.path))
return False, None, None
full_Path = Path(full_path)
if file_exists == 'add' and full_Path.exists():
mission.data.offset = full_Path.stat().st_size
return True, r, file_size
def do_download_(r, task, show, size):
if task.is_done or task.mission.is_done:
return
task.set_states(result=None, info='下载中', state='running')
block_size = 524288 # 512k
result = None
try:
if show and size:
for chunk in r.iter_content(chunk_size=block_size):
if task.state in ('cancel', 'done'):
result = 'canceled'
task.clear_cache()
break
if chunk:
task.add_data(chunk, None)
print(f'\r{task.rate}% ', end='')
else:
for chunk in r.iter_content(chunk_size=block_size):
if task.state in ('cancel', 'done'):
result = 'canceled'
task.clear_cache()
break
if chunk:
task.add_data(chunk, None)
except Exception as e:
result, info = False, f'{r.status_code} {e}'
else:
result = 'success' if result is None else result
info = str(task.path)
finally:
r.close()
task._set_done(result=result, info=info)
def do_download(r, task, first=False):
if task.is_done or task.mission.is_done:
return
task.set_states(result=None, info='下载中', state='running')
block_size = 524288 # 512k
result = None
try:
if first: # 分块是第一块
if task.range[1] <= block_size or task.range[1] % block_size != 0:
r_content = r.iter_content(chunk_size=task.range[1] + 1)
task.add_data(next(r_content), seek=0 + task.mission.data.offset)
if task.state in ('cancel', 'done'):
result = 'canceled'
task.clear_cache()
else:
blocks = task.range[1] // block_size
remainder = task.range[1] % block_size
r_content = r.iter_content(chunk_size=block_size)
for b in range(blocks):
task.add_data(next(r_content), seek=b * block_size + task.mission.data.offset)
if task.state in ('cancel', 'done'):
result = 'canceled'
task.clear_cache()
break
if task.state in ('cancel', 'done'):
result = 'canceled'
task.clear_cache()
else:
task.add_data(next(r_content)[:remainder + 1], blocks * block_size + task.mission.data.offset)
else:
if task.range is None: # 不分块
for chunk in r.iter_content(chunk_size=block_size):
if task.state in ('cancel', 'done'):
result = 'canceled'
task.clear_cache()
break
if chunk:
task.add_data(chunk, None)
elif task.range[1] == '': # 结尾的数据块
begin = task.range[0]
for chunk in r.iter_content(chunk_size=block_size):
if task.state in ('cancel', 'done'):
result = 'canceled'
task.clear_cache()
break
if chunk:
task.add_data(chunk, seek=begin + task.mission.data.offset)
begin += len(chunk)
else: # 有始末数字的数据块
begin, end = task.range
num = (end - begin) // block_size
for ind, chunk in enumerate(r.iter_content(chunk_size=block_size), 1):
if task.state in ('cancel', 'done'):
result = 'canceled'
task.clear_cache()
break
if chunk:
task.add_data(chunk, seek=begin + task.mission.data.offset)
if ind <= num:
begin += block_size
except Exception as e:
result, info = False, f'{r.status_code} {e}'
else:
result = 'success' if result is None else result
info = str(task.path)
finally:
r.close()
task._set_done(result=result, info=info)