540 lines
19 KiB
Python
540 lines
19 KiB
Python
# -*- coding:utf-8 -*-
|
||
from copy import copy
|
||
from pathlib import Path
|
||
from queue import Queue
|
||
from threading import Thread, Lock
|
||
from time import sleep, perf_counter
|
||
|
||
from requests.structures import CaseInsensitiveDict
|
||
|
||
from ._funcs import (FileExistsSetter, PathSetter, BlockSizeSetter, set_charset, get_file_info, get_file_exists_mode,
|
||
get_save_path)
|
||
from .mission import Task, Mission
|
||
from .setter import Setter
|
||
|
||
|
||
class DrissionGet(object):
|
||
file_exists = FileExistsSetter()
|
||
save_path = PathSetter()
|
||
block_size = BlockSizeSetter()
|
||
|
||
def __init__(self, save_path='.', roads=10, driver=None, file_exists='rename'):
|
||
self._roads = roads
|
||
self._missions = {}
|
||
self._threads = {i: None for i in range(self._roads)}
|
||
self._waiting_list = Queue()
|
||
self._missions_num = 0
|
||
self._running_count = 0 # 正在运行的任务数
|
||
self._stop_printing = False # 用于控制显示线程停止
|
||
self._lock = Lock()
|
||
self.page = None
|
||
self._retry = None
|
||
self._interval = None
|
||
self._timeout = None
|
||
self._encoding = None
|
||
|
||
self._setter = None
|
||
self._print_mode = 'all'
|
||
self._log_mode = None
|
||
self._logger = None
|
||
|
||
self.save_path = save_path
|
||
self.file_exists = file_exists
|
||
self.split = True
|
||
self.block_size = '50M'
|
||
self.set.driver(driver)
|
||
|
||
def __call__(self, url=None, save_path=None, rename=None, suffix=None,
|
||
file_exists=None, split=False, show_msg=True, file_url=None, **kwargs):
|
||
return self.download(url=url, save_path=save_path, rename=rename, suffix=suffix,
|
||
file_exists=file_exists, show_msg=show_msg, **kwargs)
|
||
|
||
@property
|
||
def set(self):
|
||
if self._setter is None:
|
||
self._setter = Setter(self)
|
||
return self._setter
|
||
|
||
@property
|
||
def roads(self):
|
||
return self._roads
|
||
|
||
@property
|
||
def retry(self):
|
||
if self._retry is not None:
|
||
return self._retry
|
||
elif self.page is not None:
|
||
return self.page.retry_times
|
||
else:
|
||
return 3
|
||
|
||
@property
|
||
def interval(self):
|
||
if self._interval is not None:
|
||
return self._interval
|
||
elif self.page is not None:
|
||
return self.page.retry_interval
|
||
else:
|
||
return 5
|
||
|
||
@property
|
||
def timeout(self):
|
||
if self._timeout is not None:
|
||
return self._timeout
|
||
elif self.page is not None:
|
||
return self.page.timeout
|
||
else:
|
||
return 20
|
||
|
||
@property
|
||
def waiting_list(self):
|
||
return self._waiting_list
|
||
|
||
@property
|
||
def session(self):
|
||
return self._session
|
||
|
||
@property
|
||
def is_running(self):
|
||
return self._running_count > 0
|
||
|
||
@property
|
||
def missions(self):
|
||
return self._missions
|
||
|
||
@property
|
||
def encoding(self):
|
||
"""返回指定的编码格式"""
|
||
return self._encoding
|
||
|
||
def add(self, url=None, save_path=None, rename=None, suffix=None, file_exists=None, split=None,
|
||
file_url=None, **kwargs):
|
||
if not url and file_url:
|
||
url = file_url
|
||
self._missions_num += 1
|
||
self._running_count += 1
|
||
if file_exists is not None:
|
||
file_exists = get_file_exists_mode(file_exists)
|
||
mission = Mission(self._missions_num, self, url, get_save_path(save_path) if save_path else self.save_path,
|
||
rename, suffix, file_exists or self.file_exists,
|
||
self.split if split is None else split, self._encoding, kwargs)
|
||
self._missions[self._missions_num] = mission
|
||
|
||
self._run_or_wait(mission)
|
||
return mission
|
||
|
||
def download(self, url=None, save_path=None, rename=None, suffix=None, file_exists=None, split=False,
|
||
show_msg=None, file_url=None, **kwargs):
|
||
if not url and file_url:
|
||
url = file_url
|
||
self._missions_num += 1
|
||
self._running_count += 1
|
||
if file_exists is not None:
|
||
file_exists = get_file_exists_mode(file_exists)
|
||
mission = Mission(self._missions_num, self, url, get_save_path(save_path) if save_path else self.save_path,
|
||
rename, suffix, file_exists or self.file_exists,
|
||
self.split if split is None else split, self._encoding, kwargs)
|
||
self._missions[self._missions_num] = mission
|
||
|
||
tmp, self._print_mode = self._print_mode, None
|
||
if show_msg is None:
|
||
show_msg = True if tmp else False
|
||
|
||
if split:
|
||
self._run_or_wait(mission)
|
||
r = mission.wait(show=show_msg)
|
||
self._print_mode = tmp
|
||
return r
|
||
|
||
ok, r, file_size = handle_mission(mission, url)
|
||
if show_msg:
|
||
print(f'url:{url}')
|
||
print(f'文件名:{mission.file_name}')
|
||
print(f'目标路径:{mission.path}')
|
||
if not mission.size:
|
||
print('下载中 ', end='')
|
||
|
||
if ok:
|
||
task1 = Task(mission, None, '1/1', file_size)
|
||
mission.tasks.append(task1)
|
||
do_download_(r, task1, show_msg, file_size)
|
||
|
||
if show_msg:
|
||
if mission.result is False:
|
||
print(f'下载失败 {mission.info}')
|
||
elif mission.result == 'success':
|
||
print('\r100% ', end='')
|
||
print(f'{"已覆盖" if mission._overwrote else "下载完成"} {mission.info}')
|
||
elif mission.result == 'skipped':
|
||
print(f'已跳过 {mission.info}')
|
||
print()
|
||
|
||
self._print_mode = tmp
|
||
return mission.result, mission.info
|
||
|
||
def by_browser(self, url, save_path=None, rename=None, suffix=None, timeout=None, file_exists=None):
|
||
if hasattr(self.page, '_download_by_browser'):
|
||
return self.page._download_by_browser(url=url, save_path=save_path, rename=rename, suffix=suffix,
|
||
timeout=timeout, file_exists=file_exists)
|
||
else:
|
||
raise RuntimeError('非浏览器调用无法使用浏览器下载。')
|
||
|
||
def _run_or_wait(self, mission):
|
||
thread_id = self._get_usable_thread()
|
||
if thread_id is None:
|
||
self._waiting_list.put(mission)
|
||
else:
|
||
thread = Thread(target=self._run, args=(thread_id, mission), daemon=False)
|
||
self._threads[thread_id] = {'thread': thread, 'mission': None}
|
||
thread.start()
|
||
|
||
def _run(self, ID, mission):
|
||
while True:
|
||
if not mission: # 如果没有任务,就从等候列表中取一个
|
||
if self._waiting_list.empty():
|
||
break
|
||
else:
|
||
try:
|
||
mission = self._waiting_list.get(True, .5)
|
||
except Exception:
|
||
self._waiting_list.task_done()
|
||
break
|
||
|
||
self._threads[ID]['mission'] = mission
|
||
self._download(mission, ID)
|
||
mission = None
|
||
|
||
self._threads[ID] = None
|
||
|
||
def get_mission(self, mission_or_id):
|
||
return self._missions[mission_or_id] if isinstance(mission_or_id, int) else mission_or_id
|
||
|
||
def get_failed_missions(self):
|
||
return [i for i in self._missions.values() if i.result is False]
|
||
|
||
def wait(self, mission=None, show=False, timeout=None):
|
||
timeout = 0 if timeout is None else timeout
|
||
if mission:
|
||
return self.get_mission(mission).wait(show, timeout)
|
||
|
||
else:
|
||
if show:
|
||
self.show(False)
|
||
else:
|
||
end_time = perf_counter() + timeout
|
||
while self.is_running and (perf_counter() < end_time or timeout == 0):
|
||
sleep(0.1)
|
||
|
||
def cancel(self):
|
||
for m in self._missions.values():
|
||
m.cancel()
|
||
|
||
def show(self, asyn=True, keep=False):
|
||
if asyn:
|
||
Thread(target=self._show, args=(2, keep)).start()
|
||
else:
|
||
self._show(0.1, keep)
|
||
|
||
def _show(self, wait, keep=False):
|
||
self._stop_printing = False
|
||
|
||
if keep:
|
||
Thread(target=self._stop_show).start()
|
||
|
||
end_time = perf_counter() + wait
|
||
while not self._stop_printing and (keep or self.is_running or perf_counter() < end_time):
|
||
print(f'\033[K', end='')
|
||
print(f'等待任务数:{self._waiting_list.qsize()}')
|
||
for k, v in self._threads.items():
|
||
m = v['mission'] if v else None
|
||
if m:
|
||
items = (m.mission.rate, m.mid) if isinstance(m, Task) else (m.rate, m.id)
|
||
path = f'M{items[1]} {items[0]}% {m}'
|
||
else:
|
||
path = '空闲'
|
||
print(f'\033[K', end='')
|
||
print(f'线程{k}:{path}')
|
||
|
||
print(f'\033[{self.roads + 1}A\r', end='')
|
||
sleep(0.4)
|
||
|
||
print(f'\033[1B', end='')
|
||
for i in range(self.roads):
|
||
print(f'\033[K', end='')
|
||
print(f'线程{i}:空闲')
|
||
|
||
print()
|
||
|
||
def _connect(self, url, session, _headers, method, encoding, **kwargs):
|
||
kwargs = CaseInsensitiveDict(kwargs)
|
||
if 'headers' in kwargs: # 不知道为什么添加这个才能正常使用多线程
|
||
kwargs['headers'] = CaseInsensitiveDict({**_headers, **kwargs['headers']})
|
||
else:
|
||
kwargs['headers'] = _headers
|
||
|
||
r = err = None
|
||
for i in range(self.retry + 1):
|
||
try:
|
||
if method == 'get':
|
||
r = session.get(url, **kwargs)
|
||
elif method == 'post':
|
||
r = session.post(url, **kwargs)
|
||
|
||
if r:
|
||
return set_charset(r, encoding), 'Success'
|
||
|
||
except Exception as e:
|
||
err = e
|
||
|
||
if r and r.status_code in (403, 404):
|
||
break
|
||
if i < self.retry:
|
||
sleep(self.interval)
|
||
|
||
# 返回失败结果
|
||
if r is None:
|
||
return None, '连接失败' if err is None else err
|
||
if not r.ok:
|
||
return r, f'状态码:{r.status_code}'
|
||
return None
|
||
|
||
def _get_usable_thread(self):
|
||
for k, v in self._threads.items():
|
||
if v is None:
|
||
return k
|
||
return None
|
||
|
||
def _stop_show(self):
|
||
input()
|
||
self._stop_printing = True
|
||
|
||
def _when_mission_done(self, mission):
|
||
self._running_count -= 1
|
||
if self._print_mode == 'all' or (self._print_mode == 'failed' and mission.result is False):
|
||
print(f'[{mission.RESULT_TEXTS[mission.result]}] {mission.info} {mission.data.url}')
|
||
if self._log_mode == 'all' or (self._log_mode == 'failed' and mission.result is False):
|
||
self._logger.add_data((mission.RESULT_TEXTS[mission.result], mission.data.url, mission.info,
|
||
mission.data.save_path, mission.data.rename, mission.data.kwargs))
|
||
mission.session.close()
|
||
|
||
def _download(self, mission_or_task, thread_id):
|
||
if mission_or_task.is_done:
|
||
return
|
||
if mission_or_task.state == 'cancel':
|
||
mission_or_task.state = 'done'
|
||
return
|
||
|
||
file_url = mission_or_task.data.url
|
||
if isinstance(mission_or_task, Task):
|
||
kwargs = copy(mission_or_task.data.kwargs)
|
||
kwargs['headers']['Range'] = f"bytes={mission_or_task.range[0]}-{mission_or_task.range[1]}"
|
||
r, inf = self._connect(file_url, mission_or_task.mission.session, mission_or_task.mission.headers,
|
||
mission_or_task.mission.method, mission_or_task.mission.encoding, **kwargs)
|
||
|
||
if r:
|
||
do_download(r, mission_or_task, False)
|
||
else:
|
||
mission_or_task._set_done(False, inf)
|
||
|
||
return
|
||
|
||
# ===================开始处理mission====================
|
||
mission = mission_or_task
|
||
ok, r, file_size = handle_mission(mission, file_url)
|
||
if not ok:
|
||
return
|
||
|
||
# -------------------设置分块任务-------------------
|
||
first = False
|
||
if mission.data.split and file_size and file_size > self.block_size and r.headers.get(
|
||
'Accept-Ranges') == 'bytes':
|
||
first = True
|
||
chunks = [[s, min(s + self.block_size, file_size) - 1] for s in range(0, file_size, self.block_size)]
|
||
chunks[-1][-1] = ''
|
||
chunks_len = len(chunks)
|
||
|
||
task1 = Task(mission, chunks[0], f'1/{chunks_len}', chunks[0][1] - chunks[0][0])
|
||
mission.tasks_count = chunks_len
|
||
mission.tasks = [task1]
|
||
|
||
for ind, chunk in enumerate(chunks[1:], 2):
|
||
s = file_size - chunk[0] if chunks_len == ind else chunk[1] - chunk[0]
|
||
task = Task(mission, chunk, f'{ind}/{chunks_len}', s)
|
||
mission.tasks.append(task)
|
||
self._run_or_wait(task)
|
||
|
||
else: # 不分块
|
||
task1 = Task(mission, None, '1/1', file_size)
|
||
mission.tasks.append(task1)
|
||
|
||
self._threads[thread_id]['mission'] = task1
|
||
do_download(r, task1, first)
|
||
|
||
|
||
def handle_mission(mission, url):
|
||
mission.info = '下载中'
|
||
mission.state = 'running'
|
||
kwargs = mission.data.kwargs
|
||
|
||
rename = mission.data.rename
|
||
suffix = mission.data.suffix
|
||
file_exists = mission.data.file_exists
|
||
mission.data.save_path.mkdir(parents=True, exist_ok=True)
|
||
|
||
if file_exists == 'skip' and rename:
|
||
tmp = mission.data.save_path / rename
|
||
if tmp.exists() and tmp.is_file():
|
||
mission.file_name = rename
|
||
mission._set_path(mission.data.save_path / rename)
|
||
mission._set_done('skipped', str(mission.path))
|
||
return False, None, None
|
||
|
||
r, inf = mission.owner._connect(url, mission.session, mission.headers, mission.method, mission.encoding, **kwargs)
|
||
|
||
if mission.is_done:
|
||
return False, None, None
|
||
if not r:
|
||
mission._break_mission(result=False, info=inf)
|
||
return False, None, None
|
||
|
||
# -------------------获取文件信息-------------------
|
||
file_info = get_file_info(r, mission.data.save_path, rename, suffix, file_exists,
|
||
mission.encoding, mission.owner._lock)
|
||
file_size = file_info['size']
|
||
full_path = file_info['path']
|
||
mission._set_path(full_path)
|
||
mission.size = file_size
|
||
mission._overwrote = file_info['overwrite']
|
||
|
||
if file_info['skip']:
|
||
mission._set_done('skipped', str(mission.path))
|
||
return False, None, None
|
||
|
||
full_Path = Path(full_path)
|
||
if file_exists == 'add' and full_Path.exists():
|
||
mission.data.offset = full_Path.stat().st_size
|
||
|
||
return True, r, file_size
|
||
|
||
|
||
def do_download_(r, task, show, size):
|
||
if task.is_done or task.mission.is_done:
|
||
return
|
||
|
||
task.set_states(result=None, info='下载中', state='running')
|
||
block_size = 524288 # 512k
|
||
result = None
|
||
|
||
try:
|
||
if show and size:
|
||
for chunk in r.iter_content(chunk_size=block_size):
|
||
if task.state in ('cancel', 'done'):
|
||
result = 'canceled'
|
||
task.clear_cache()
|
||
break
|
||
if chunk:
|
||
task.add_data(chunk, None)
|
||
print(f'\r{task.rate}% ', end='')
|
||
|
||
else:
|
||
for chunk in r.iter_content(chunk_size=block_size):
|
||
if task.state in ('cancel', 'done'):
|
||
result = 'canceled'
|
||
task.clear_cache()
|
||
break
|
||
if chunk:
|
||
task.add_data(chunk, None)
|
||
|
||
except Exception as e:
|
||
result, info = False, f'{r.status_code} {e}'
|
||
|
||
else:
|
||
result = 'success' if result is None else result
|
||
info = str(task.path)
|
||
|
||
finally:
|
||
r.close()
|
||
|
||
task._set_done(result=result, info=info)
|
||
|
||
|
||
def do_download(r, task, first=False):
|
||
if task.is_done or task.mission.is_done:
|
||
return
|
||
|
||
task.set_states(result=None, info='下载中', state='running')
|
||
block_size = 524288 # 512k
|
||
result = None
|
||
|
||
try:
|
||
if first: # 分块是第一块
|
||
if task.range[1] <= block_size or task.range[1] % block_size != 0:
|
||
r_content = r.iter_content(chunk_size=task.range[1] + 1)
|
||
task.add_data(next(r_content), seek=0 + task.mission.data.offset)
|
||
if task.state in ('cancel', 'done'):
|
||
result = 'canceled'
|
||
task.clear_cache()
|
||
|
||
else:
|
||
blocks = task.range[1] // block_size
|
||
remainder = task.range[1] % block_size
|
||
r_content = r.iter_content(chunk_size=block_size)
|
||
for b in range(blocks):
|
||
task.add_data(next(r_content), seek=b * block_size + task.mission.data.offset)
|
||
if task.state in ('cancel', 'done'):
|
||
result = 'canceled'
|
||
task.clear_cache()
|
||
break
|
||
|
||
if task.state in ('cancel', 'done'):
|
||
result = 'canceled'
|
||
task.clear_cache()
|
||
else:
|
||
task.add_data(next(r_content)[:remainder + 1], blocks * block_size + task.mission.data.offset)
|
||
|
||
else:
|
||
if task.range is None: # 不分块
|
||
for chunk in r.iter_content(chunk_size=block_size):
|
||
if task.state in ('cancel', 'done'):
|
||
result = 'canceled'
|
||
task.clear_cache()
|
||
break
|
||
if chunk:
|
||
task.add_data(chunk, None)
|
||
|
||
elif task.range[1] == '': # 结尾的数据块
|
||
begin = task.range[0]
|
||
for chunk in r.iter_content(chunk_size=block_size):
|
||
if task.state in ('cancel', 'done'):
|
||
result = 'canceled'
|
||
task.clear_cache()
|
||
break
|
||
if chunk:
|
||
task.add_data(chunk, seek=begin + task.mission.data.offset)
|
||
begin += len(chunk)
|
||
|
||
else: # 有始末数字的数据块
|
||
begin, end = task.range
|
||
num = (end - begin) // block_size
|
||
for ind, chunk in enumerate(r.iter_content(chunk_size=block_size), 1):
|
||
if task.state in ('cancel', 'done'):
|
||
result = 'canceled'
|
||
task.clear_cache()
|
||
break
|
||
if chunk:
|
||
task.add_data(chunk, seek=begin + task.mission.data.offset)
|
||
if ind <= num:
|
||
begin += block_size
|
||
|
||
except Exception as e:
|
||
result, info = False, f'{r.status_code} {e}'
|
||
|
||
else:
|
||
result = 'success' if result is None else result
|
||
info = str(task.path)
|
||
|
||
finally:
|
||
r.close()
|
||
|
||
task._set_done(result=result, info=info)
|