# -*- coding:utf-8 -*- from copy import copy from os import path as os_PATH from os.path import sep from pathlib import Path from random import randint from re import search, sub from time import time from urllib.parse import unquote from DrissionRecord.tools import get_usable_path, make_valid_name from requests import Session FILE_EXISTS_MODE = {'rename': 'rename', 'overwrite': 'overwrite', 'skip': 'skip', 'add': 'add', 'r': 'rename', 'o': 'overwrite', 's': 'skip', 'a': 'add'} def copy_session(session): new = Session() new.headers = session.headers.copy() new.cookies = session.cookies.copy() new.stream = True new.auth = session.auth new.proxies = dict(session.proxies).copy() new.params = copy(session.params) # new.cert = session.cert new.max_redirects = session.max_redirects new.trust_env = session.trust_env new.verify = session.verify return new class BlockSizeSetter(object): def __set__(self, block_size, val): if isinstance(val, int) and val > 0: size = val elif isinstance(val, str): units = {'b': 1, 'k': 1024, 'm': 1048576, 'g': 21474836480} num = int(val[:-1]) unit = units.get(val[-1].lower(), None) if unit and num > 0: size = num * unit else: raise ValueError('单位只支持B、K、M、G,数字必须为大于0的整数。') else: raise TypeError('split_size只能传入int或str,数字必须为大于0的整数。') block_size._block_size = size def __get__(self, block_size, objtype=None) -> int: return block_size._block_size class PathSetter(object): def __set__(self, save_path, val): if val is not None and not isinstance(val, (str, Path)): raise TypeError('路径只能是str或Path类型。') save_path._save_path = get_save_path(val) def __get__(self, save_path, objtype=None): return save_path._save_path class FileExistsSetter(object): def __set__(self, file_exists, mode): file_exists._file_exists = get_file_exists_mode(mode) def __get__(self, file_exists, objtype=None): return file_exists._file_exists def get_save_path(path): path = Path(path or '.').absolute() return Path(path.anchor + sub(r'[*:|<>?"]', '', sep.join(path.parts[1:])).strip()) def get_file_exists_mode(mode): mode = FILE_EXISTS_MODE.get(mode, mode) if mode not in FILE_EXISTS_MODE: raise ValueError(f'''mode参数只能是 '{"', '".join(FILE_EXISTS_MODE.keys())}' 之一,现在是:{mode}''') return mode def set_charset(response, encoding): if encoding: response.encoding = encoding return response # 在headers中获取编码 content_type = response.headers.get('content-type', '').lower() if not content_type.endswith(';'): content_type += ';' charset = search(r'charset[=: ]*(.*)?;?', content_type) if charset: response.encoding = charset.group(1) # 在headers中获取不到编码,且如果是网页 elif content_type.replace(' ', '').startswith('text/html'): re_result = search(b']+).*?>', response.content) if re_result: charset = re_result.group(1).decode() else: charset = response.apparent_encoding response.encoding = charset return response def get_file_info(response, save_path, rename=None, suffix=None, file_exists=None, encoding=None, lock=None): # ------------获取文件大小------------ file_size = response.headers.get('Content-Length', None) file_size = None if file_size is None else int(file_size) # ------------获取网络文件名------------ file_name = get_file_name(response, encoding) # ------------获取保存文件名------------ # -------------------重命名------------------- if rename: if suffix is not None: full_name = f'{rename}.{suffix}' if suffix else rename else: tmp = file_name.rsplit('.', 1) ext_name = f'.{tmp[-1]}' if len(tmp) > 1 else '' tmp = rename.rsplit('.', 1) ext_rename = f'.{tmp[-1]}' if len(tmp) > 1 else '' full_name = rename if ext_rename == ext_name else f'{rename}{ext_name}' elif suffix is not None: full_name = file_name.rsplit(".", 1)[0] if suffix: full_name = f'{full_name}.{suffix}' else: full_name = file_name full_name = make_valid_name(full_name) # -------------------生成路径------------------- skip = False overwrite = False full_path = save_path / full_name with lock: if full_path.exists(): if file_exists == 'rename': full_path = get_usable_path(full_path) elif file_exists == 'skip': skip = True elif file_exists == 'overwrite': overwrite = True full_path.unlink() return {'size': file_size, 'path': full_path, 'skip': skip, 'overwrite': overwrite} def get_file_name(response, encoding): file_name = '' charset = '' content_disposition = response.headers.get('content-disposition', '').replace(' ', '') # 使用header里的文件名 if content_disposition: txt = search(r'filename\*="?([^";]+)', content_disposition) if txt: # 文件名自带编码方式 txt = txt.group(1).split("''", 1) if len(txt) == 2: charset, file_name = txt else: file_name = txt[0] else: # 文件名没带编码方式 txt = search(r'filename="?([^";]+)', content_disposition) if txt: file_name = txt.group(1) # 获取编码(如有) charset = encoding or response.encoding file_name = file_name.strip("'") # 在url里获取文件名 if not file_name and os_PATH.basename(response.url): file_name = os_PATH.basename(response.url).split("?")[0] # 找不到则用时间和随机数生成文件名 if not file_name: file_name = f'untitled_{time()}_{randint(0, 100)}' # 去除非法字符 charset = charset or 'utf-8' return unquote(file_name, charset) def set_session_cookies(session, cookies): # cookies = cookies_to_tuple(cookies) for cookie in cookies: if cookie['value'] is None: cookie['value'] = '' kwargs = {x: cookie[x] for x in cookie if x.lower() in ('version', 'port', 'domain', 'path', 'secure', 'expires', 'discard', 'comment', 'comment_url', 'rest')} if 'expiry' in cookie: kwargs['expires'] = cookie['expiry'] session.cookies.set(cookie['name'], cookie['value'], **kwargs)