Initial commit

This commit is contained in:
Alex committed 2026-11-02 22:54:14 +08:00
commit 57fc3525eb
7311 files changed
+1291398

No files matched your search

@@ -0,0 +1,539 @@
# -*- coding:utf-8 -*-
from copy import copy
from pathlib import Path
from queue import Queue
from threading import Thread, Lock
from time import sleep, perf_counter
from requests.structures import CaseInsensitiveDict
from ._funcs import (FileExistsSetter, PathSetter, BlockSizeSetter, set_charset, get_file_info, get_file_exists_mode,
get_save_path)
from .mission import Task, Mission
from .setter import Setter
class DrissionGet(object):
file_exists = FileExistsSetter()
save_path = PathSetter()
block_size = BlockSizeSetter()
def __init__(self, save_path='.', roads=10, driver=None, file_exists='rename'):
self._roads = roads
self._missions = {}
self._threads = {i: None for i in range(self._roads)}
self._waiting_list = Queue()
self._missions_num = 0
self._running_count = 0 # 正在运行的任务数
self._stop_printing = False # 用于控制显示线程停止
self._lock = Lock()
self.page = None
self._retry = None
self._interval = None
self._timeout = None
self._encoding = None
self._setter = None
self._print_mode = 'all'
self._log_mode = None
self._logger = None
self.save_path = save_path
self.file_exists = file_exists
self.split = True
self.block_size = '50M'
self.set.driver(driver)
def __call__(self, url=None, save_path=None, rename=None, suffix=None,
file_exists=None, split=False, show_msg=True, file_url=None, **kwargs):
return self.download(url=url, save_path=save_path, rename=rename, suffix=suffix,
file_exists=file_exists, show_msg=show_msg, **kwargs)
@property
def set(self):
if self._setter is None:
self._setter = Setter(self)
return self._setter
@property
def roads(self):
return self._roads
@property
def retry(self):
if self._retry is not None:
return self._retry
elif self.page is not None:
return self.page.retry_times
else:
return 3
@property
def interval(self):
if self._interval is not None:
return self._interval
elif self.page is not None:
return self.page.retry_interval
else:
return 5
@property
def timeout(self):
if self._timeout is not None:
return self._timeout
elif self.page is not None:
return self.page.timeout
else:
return 20
@property
def waiting_list(self):
return self._waiting_list
@property
def session(self):
return self._session
@property
def is_running(self):
return self._running_count > 0
@property
def missions(self):
return self._missions
@property
def encoding(self):
"""返回指定的编码格式"""
return self._encoding
def add(self, url=None, save_path=None, rename=None, suffix=None, file_exists=None, split=None,
file_url=None, **kwargs):
if not url and file_url:
url = file_url
self._missions_num += 1
self._running_count += 1
if file_exists is not None:
file_exists = get_file_exists_mode(file_exists)
mission = Mission(self._missions_num, self, url, get_save_path(save_path) if save_path else self.save_path,
rename, suffix, file_exists or self.file_exists,
self.split if split is None else split, self._encoding, kwargs)
self._missions[self._missions_num] = mission
self._run_or_wait(mission)
return mission
def download(self, url=None, save_path=None, rename=None, suffix=None, file_exists=None, split=False,
show_msg=None, file_url=None, **kwargs):
if not url and file_url:
url = file_url
self._missions_num += 1
self._running_count += 1
if file_exists is not None:
file_exists = get_file_exists_mode(file_exists)
mission = Mission(self._missions_num, self, url, get_save_path(save_path) if save_path else self.save_path,
rename, suffix, file_exists or self.file_exists,
self.split if split is None else split, self._encoding, kwargs)
self._missions[self._missions_num] = mission
tmp, self._print_mode = self._print_mode, None
if show_msg is None:
show_msg = True if tmp else False
if split:
self._run_or_wait(mission)
r = mission.wait(show=show_msg)
self._print_mode = tmp
return r
ok, r, file_size = handle_mission(mission, url)
if show_msg:
print(f'url:{url}')
print(f'文件名:{mission.file_name}')
print(f'目标路径:{mission.path}')
if not mission.size:
print('下载中 ', end='')
if ok:
task1 = Task(mission, None, '1/1', file_size)
mission.tasks.append(task1)
do_download_(r, task1, show_msg, file_size)
if show_msg:
if mission.result is False:
print(f'下载失败 {mission.info}')
elif mission.result == 'success':
print('\r100% ', end='')
print(f'{"已覆盖" if mission._overwrote else "下载完成"} {mission.info}')
elif mission.result == 'skipped':
print(f'已跳过 {mission.info}')
print()
self._print_mode = tmp
return mission.result, mission.info
def by_browser(self, url, save_path=None, rename=None, suffix=None, timeout=None, file_exists=None):
if hasattr(self.page, '_download_by_browser'):
return self.page._download_by_browser(url=url, save_path=save_path, rename=rename, suffix=suffix,
timeout=timeout, file_exists=file_exists)
else:
raise RuntimeError('非浏览器调用无法使用浏览器下载。')
def _run_or_wait(self, mission):
thread_id = self._get_usable_thread()
if thread_id is None:
self._waiting_list.put(mission)
else:
thread = Thread(target=self._run, args=(thread_id, mission), daemon=False)
self._threads[thread_id] = {'thread': thread, 'mission': None}
thread.start()
def _run(self, ID, mission):
while True:
if not mission: # 如果没有任务,就从等候列表中取一个
if self._waiting_list.empty():
break
else:
try:
mission = self._waiting_list.get(True, .5)
except Exception:
self._waiting_list.task_done()
break
self._threads[ID]['mission'] = mission
self._download(mission, ID)
mission = None
self._threads[ID] = None
def get_mission(self, mission_or_id):
return self._missions[mission_or_id] if isinstance(mission_or_id, int) else mission_or_id
def get_failed_missions(self):
return [i for i in self._missions.values() if i.result is False]
def wait(self, mission=None, show=False, timeout=None):
timeout = 0 if timeout is None else timeout
if mission:
return self.get_mission(mission).wait(show, timeout)
else:
if show:
self.show(False)
else:
end_time = perf_counter() + timeout
while self.is_running and (perf_counter() < end_time or timeout == 0):
sleep(0.1)
def cancel(self):
for m in self._missions.values():
m.cancel()
def show(self, asyn=True, keep=False):
if asyn:
Thread(target=self._show, args=(2, keep)).start()
else:
self._show(0.1, keep)
def _show(self, wait, keep=False):
self._stop_printing = False
if keep:
Thread(target=self._stop_show).start()
end_time = perf_counter() + wait
while not self._stop_printing and (keep or self.is_running or perf_counter() < end_time):
print(f'\033[K', end='')
print(f'等待任务数:{self._waiting_list.qsize()}')
for k, v in self._threads.items():
m = v['mission'] if v else None
if m:
items = (m.mission.rate, m.mid) if isinstance(m, Task) else (m.rate, m.id)
path = f'M{items[1]} {items[0]}% {m}'
else:
path = '空闲'
print(f'\033[K', end='')
print(f'线程{k}:{path}')
print(f'\033[{self.roads + 1}A\r', end='')
sleep(0.4)
print(f'\033[1B', end='')
for i in range(self.roads):
print(f'\033[K', end='')
print(f'线程{i}:空闲')
print()
def _connect(self, url, session, _headers, method, encoding, **kwargs):
kwargs = CaseInsensitiveDict(kwargs)
if 'headers' in kwargs: # 不知道为什么添加这个才能正常使用多线程
kwargs['headers'] = CaseInsensitiveDict({**_headers, **kwargs['headers']})
else:
kwargs['headers'] = _headers
r = err = None
for i in range(self.retry + 1):
try:
if method == 'get':
r = session.get(url, **kwargs)
elif method == 'post':
r = session.post(url, **kwargs)
if r:
return set_charset(r, encoding), 'Success'
except Exception as e:
err = e
if r and r.status_code in (403, 404):
break
if i < self.retry:
sleep(self.interval)
# 返回失败结果
if r is None:
return None, '连接失败' if err is None else err
if not r.ok:
return r, f'状态码:{r.status_code}'
return None
def _get_usable_thread(self):
for k, v in self._threads.items():
if v is None:
return k
return None
def _stop_show(self):
input()
self._stop_printing = True
def _when_mission_done(self, mission):
self._running_count -= 1
if self._print_mode == 'all' or (self._print_mode == 'failed' and mission.result is False):
print(f'[{mission.RESULT_TEXTS[mission.result]}] {mission.info} {mission.data.url}')
if self._log_mode == 'all' or (self._log_mode == 'failed' and mission.result is False):
self._logger.add_data((mission.RESULT_TEXTS[mission.result], mission.data.url, mission.info,
mission.data.save_path, mission.data.rename, mission.data.kwargs))
mission.session.close()
def _download(self, mission_or_task, thread_id):
if mission_or_task.is_done:
return
if mission_or_task.state == 'cancel':
mission_or_task.state = 'done'
return
file_url = mission_or_task.data.url
if isinstance(mission_or_task, Task):
kwargs = copy(mission_or_task.data.kwargs)
kwargs['headers']['Range'] = f"bytes={mission_or_task.range[0]}-{mission_or_task.range[1]}"
r, inf = self._connect(file_url, mission_or_task.mission.session, mission_or_task.mission.headers,
mission_or_task.mission.method, mission_or_task.mission.encoding, **kwargs)
if r:
do_download(r, mission_or_task, False)
else:
mission_or_task._set_done(False, inf)
return
# ===================开始处理mission====================
mission = mission_or_task
ok, r, file_size = handle_mission(mission, file_url)
if not ok:
return
# -------------------设置分块任务-------------------
first = False
if mission.data.split and file_size and file_size > self.block_size and r.headers.get(
'Accept-Ranges') == 'bytes':
first = True
chunks = [[s, min(s + self.block_size, file_size) - 1] for s in range(0, file_size, self.block_size)]
chunks[-1][-1] = ''
chunks_len = len(chunks)
task1 = Task(mission, chunks[0], f'1/{chunks_len}', chunks[0][1] - chunks[0][0])
mission.tasks_count = chunks_len
mission.tasks = [task1]
for ind, chunk in enumerate(chunks[1:], 2):
s = file_size - chunk[0] if chunks_len == ind else chunk[1] - chunk[0]
task = Task(mission, chunk, f'{ind}/{chunks_len}', s)
mission.tasks.append(task)
self._run_or_wait(task)
else: # 不分块
task1 = Task(mission, None, '1/1', file_size)
mission.tasks.append(task1)
self._threads[thread_id]['mission'] = task1
do_download(r, task1, first)
def handle_mission(mission, url):
mission.info = '下载中'
mission.state = 'running'
kwargs = mission.data.kwargs
rename = mission.data.rename
suffix = mission.data.suffix
file_exists = mission.data.file_exists
mission.data.save_path.mkdir(parents=True, exist_ok=True)
if file_exists == 'skip' and rename:
tmp = mission.data.save_path / rename
if tmp.exists() and tmp.is_file():
mission.file_name = rename
mission._set_path(mission.data.save_path / rename)
mission._set_done('skipped', str(mission.path))
return False, None, None
r, inf = mission.owner._connect(url, mission.session, mission.headers, mission.method, mission.encoding, **kwargs)
if mission.is_done:
return False, None, None
if not r:
mission._break_mission(result=False, info=inf)
return False, None, None
# -------------------获取文件信息-------------------
file_info = get_file_info(r, mission.data.save_path, rename, suffix, file_exists,
mission.encoding, mission.owner._lock)
file_size = file_info['size']
full_path = file_info['path']
mission._set_path(full_path)
mission.size = file_size
mission._overwrote = file_info['overwrite']
if file_info['skip']:
mission._set_done('skipped', str(mission.path))
return False, None, None
full_Path = Path(full_path)
if file_exists == 'add' and full_Path.exists():
mission.data.offset = full_Path.stat().st_size
return True, r, file_size
def do_download_(r, task, show, size):
if task.is_done or task.mission.is_done:
return
task.set_states(result=None, info='下载中', state='running')
block_size = 524288 # 512k
result = None
try:
if show and size:
for chunk in r.iter_content(chunk_size=block_size):
if task.state in ('cancel', 'done'):
result = 'canceled'
task.clear_cache()
break
if chunk:
task.add_data(chunk, None)
print(f'\r{task.rate}% ', end='')
else:
for chunk in r.iter_content(chunk_size=block_size):
if task.state in ('cancel', 'done'):
result = 'canceled'
task.clear_cache()
break
if chunk:
task.add_data(chunk, None)
except Exception as e:
result, info = False, f'{r.status_code} {e}'
else:
result = 'success' if result is None else result
info = str(task.path)
finally:
r.close()
task._set_done(result=result, info=info)
def do_download(r, task, first=False):
if task.is_done or task.mission.is_done:
return
task.set_states(result=None, info='下载中', state='running')
block_size = 524288 # 512k
result = None
try:
if first: # 分块是第一块
if task.range[1] <= block_size or task.range[1] % block_size != 0:
r_content = r.iter_content(chunk_size=task.range[1] + 1)
task.add_data(next(r_content), seek=0 + task.mission.data.offset)
if task.state in ('cancel', 'done'):
result = 'canceled'
task.clear_cache()
else:
blocks = task.range[1] // block_size
remainder = task.range[1] % block_size
r_content = r.iter_content(chunk_size=block_size)
for b in range(blocks):
task.add_data(next(r_content), seek=b * block_size + task.mission.data.offset)
if task.state in ('cancel', 'done'):
result = 'canceled'
task.clear_cache()
break
if task.state in ('cancel', 'done'):
result = 'canceled'
task.clear_cache()
else:
task.add_data(next(r_content)[:remainder + 1], blocks * block_size + task.mission.data.offset)
else:
if task.range is None: # 不分块
for chunk in r.iter_content(chunk_size=block_size):
if task.state in ('cancel', 'done'):
result = 'canceled'
task.clear_cache()
break
if chunk:
task.add_data(chunk, None)
elif task.range[1] == '': # 结尾的数据块
begin = task.range[0]
for chunk in r.iter_content(chunk_size=block_size):
if task.state in ('cancel', 'done'):
result = 'canceled'
task.clear_cache()
break
if chunk:
task.add_data(chunk, seek=begin + task.mission.data.offset)
begin += len(chunk)
else: # 有始末数字的数据块
begin, end = task.range
num = (end - begin) // block_size
for ind, chunk in enumerate(r.iter_content(chunk_size=block_size), 1):
if task.state in ('cancel', 'done'):
result = 'canceled'
task.clear_cache()
break
if chunk:
task.add_data(chunk, seek=begin + task.mission.data.offset)
if ind <= num:
begin += block_size
except Exception as e:
result, info = False, f'{r.status_code} {e}'
else:
result = 'success' if result is None else result
info = str(task.path)
finally:
r.close()
task._set_done(result=result, info=info)