# -*- coding:utf-8 -*- """ @Author : g1879 @Contact : g1879@qq.com @Website : https://DrissionPage.cn @Copyright: (c) 2020 by g1879, Inc. All Rights Reserved. """ from html import unescape from os.path import sep from pathlib import Path from re import sub from urllib.parse import urlparse, urljoin, urlunparse from DrissionRecord.tools import make_valid_name from requests.structures import CaseInsensitiveDict from .._functions.settings import Settings as _S def get_ele_txt(e): # 前面无须换行的元素 nowrap_list = ('br', 'sub', 'sup', 'em', 'strong', 'a', 'font', 'b', 'span', 's', 'i', 'del', 'ins', 'img', 'td', 'th', 'abbr', 'bdi', 'bdo', 'cite', 'code', 'data', 'dfn', 'kbd', 'mark', 'q', 'rp', 'rt', 'ruby', 'samp', 'small', 'time', 'u', 'var', 'wbr', 'button', 'slot', 'content') # 后面添加换行的元素 wrap_after_list = ('p', 'div', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'ol', 'li', 'blockquote', 'header', 'footer', 'address' 'article', 'aside', 'main', 'nav', 'section', 'figcaption', 'summary') # 不获取文本的元素 noText_list = ('script', 'style', 'video', 'audio', 'iframe', 'embed', 'noscript', 'canvas', 'template') # 用/t分隔的元素 tab_list = ('td', 'th') if e.tag in noText_list: return e.raw_text def get_node_txt(ele, pre=False) -> list: tag = ele.tag if tag == 'br': return [True] if not pre and tag == 'pre': pre = True str_list = [] if tag in noText_list and not pre: # 标签内的文本不返回 return str_list nodes = ele.eles('xpath:./text() | *') prev_ele = '' for el in nodes: if isinstance(el, str): # 字符节点 if pre: str_list.append(el) else: if sub('[ \n\t\r]', '', el) != '': # 字符除了回车和空格还有其它内容 txt = el if not pre: txt = txt.replace('\r\n', ' ').replace('\n', ' ') txt = sub(r' {2,}', ' ', txt) str_list.append(txt) else: # 元素节点 if el.tag not in nowrap_list and str_list and str_list[-1] != '\n': # 元素间换行的情况 str_list.append('\n') if el.tag in tab_list and prev_ele in tab_list: # 表格的行 str_list.append('\t') str_list.extend(get_node_txt(el, pre)) prev_ele = el.tag if tag in wrap_after_list and str_list and str_list[-1] not in ('\n', True): # 有些元素后面要添加回车 str_list.append('\n') return str_list re_str = get_node_txt(e) if re_str and re_str[-1] == '\n': re_str.pop() l = len(re_str) if l > 1: r = [] for i in range(l - 1): i1 = re_str[i] i2 = re_str[i + 1] if i1 is True: r.append('\n') continue elif i2 is True: r.append(i1) continue elif i1.endswith(' ') and i2.startswith(' '): i1 = i1[:-1] r.append(i1) r.append('\n' if re_str[-1] is True else re_str[-1]) re_str = ''.join(r) elif not l: re_str = '' else: re_str = re_str[0] if re_str[0] is not True else '\n' return format_html(re_str.strip()) def format_html(text): return unescape(text).replace('\xa0', ' ') if text else text def location_in_viewport(page, loc_x, loc_y): js = f'''function(){{let x = {loc_x}; let y = {loc_y}; const scrollLeft = document.documentElement.scrollLeft; const scrollTop = document.documentElement.scrollTop; const vWidth = document.documentElement.clientWidth; const vHeight = document.documentElement.clientHeight; if (x< scrollLeft || y < scrollTop || x > vWidth + scrollLeft || y > vHeight + scrollTop){{return false;}} return true;}}''' return page._run_js(js) def offset_scroll(ele, offset_x, offset_y): loc_x, loc_y = ele.rect.location cp_x, cp_y = ele.rect.click_point lx = loc_x + offset_x if offset_x else cp_x ly = loc_y + offset_y if offset_y else cp_y if not location_in_viewport(ele.owner, lx, ly): clientWidth = ele.owner._run_js('return document.body.clientWidth;') clientHeight = ele.owner._run_js('return document.body.clientHeight;') ele.owner.scroll.to_location(lx - clientWidth // 2, ly - clientHeight // 2) cl_x, cl_y = ele.rect.viewport_location ccp_x, ccp_y = ele.rect.viewport_click_point cx = cl_x + offset_x if offset_x else ccp_x cy = cl_y + offset_y if offset_y else ccp_y return cx, cy def make_absolute_link(link, baseURI=None): if not link: return link link = link.strip().replace('\\', '/') parsed = urlparse(link)._asdict() # if baseURI: # baseURI = baseURI.rstrip('/\\') # 是相对路径,与页面url拼接并返回 if not parsed['netloc']: return urljoin(baseURI, link) if baseURI else link # 是绝对路径但缺少协议,从页面url获取协议并修复 if not parsed['scheme'] and baseURI: parsed['scheme'] = urlparse(baseURI).scheme parsed = tuple(v for v in parsed.values()) return urlunparse(parsed) # 绝对路径且不缺协议,直接返回 return link def is_js_func(func): func = func.strip() if (func.startswith('function') or func.startswith('async ')) and func.endswith('}'): return True # elif '=>' in func: # return True return False def get_blob(page, url, as_bytes=True): if not url.startswith('blob'): raise ValueError(_S._lang.join(_S._lang.NOT_BLOB, url=url)) js = """ function fetchData(url) { return new Promise((resolve, reject) => { let xhr = new XMLHttpRequest(); xhr.responseType = 'blob'; xhr.onload = function() { let reader = new FileReader(); reader.onloadend = function(){resolve(reader.result);} reader.readAsDataURL(xhr.response); }; xhr.open('GET', url, true); xhr.send(); }); } """ try: result = page._run_js(js, url) except: raise RuntimeError(_S._lang.join(_S._lang.GET_BLOB_FAILED)) if as_bytes: from base64 import b64decode return b64decode(result.split(',', 1)[-1]) else: return result def save_page(tab, path=None, name=None, as_pdf=False, kwargs=None): if name: if name.endswith('.pdf'): name = name[:-4] as_pdf = True elif name.endswith('.mhtml'): name = name[:-6] as_pdf = False if path: path = Path(path) if path.suffix.lower() == '.mhtml': name = path.stem path = path.parent as_pdf = False elif path.suffix.lower() == '.pdf': name = path.stem path = path.parent as_pdf = True return get_pdf(tab, path, name, kwargs) if as_pdf else get_mhtml(tab, path, name) def get_mhtml(page, path=None, name=None): r = page._run_cdp('Page.captureSnapshot')['data'] if path is None and name is None: return r path = path or '.' Path(path).mkdir(parents=True, exist_ok=True) name = make_valid_name(name or page.title) with open(f'{path}{sep}{name}.mhtml', 'w', encoding='utf-8', newline='\n') as f: f.write(r.replace('\r\n', '\n')) return r def get_pdf(page, path=None, name=None, kwargs=None): if not kwargs: kwargs = {} kwargs['transferMode'] = 'ReturnAsBase64' if 'printBackground' not in kwargs: kwargs['printBackground'] = True try: r = page._run_cdp('Page.printToPDF', **kwargs)['data'] except: raise RuntimeError(_S._lang.join(_S._lang.GET_PDF_FAILED)) from base64 import b64decode r = b64decode(r) if path is None and name is None: return r path = path or '.' Path(path).mkdir(parents=True, exist_ok=True) name = make_valid_name(name or page.title) with open(f'{path}{sep}{name}.pdf', 'wb', newline='\n') as f: f.write(r) return r def tree(ele_or_page, text=False, show_js=False, show_css=False): def _tree(obj, last_one=True, body=''): list_ele = obj.children() length = len(list_ele) body_unit = ' ' if last_one else '│ ' tail = '├───' new_body = body + body_unit if length > 0: new_last_one = False for i in range(length): if i == length - 1: tail = '└───' new_last_one = True e = list_ele[i] attrs = ' '.join([f"{k}='{v}'" for k, v in e.attrs.items()]) show_text = f'{new_body}{tail}<{e.tag} {attrs}>'.replace('\n', ' ') if text: t = e('x:/text()') if t: t = t.replace('\n', ' ') if (e.tag not in ('script', 'style') or (e.tag == 'script' and show_js) or (e.tag == 'style' and show_css)): if text is not True: t = t[:text] show_text = f'{show_text} {t}' print(show_text) _tree(e, new_last_one, new_body) ele = ele_or_page.s_ele() attrs = ' '.join([f"{k}='{v}'" for k, v in ele.attrs.items()]) show_text = f'<{ele.tag} {attrs}>'.replace('\n', ' ') if text: t = ele('x:/text()') if t: t = t.replace('\n', ' ') if (ele.tag not in ('script', 'style') or (ele.tag == 'script' and show_js) or (ele.tag == 'style' and show_css)): if text is not True: t = t[:text] show_text = f'{show_text} {t}' print(show_text) _tree(ele) def format_headers(txt): if isinstance(txt, (dict, CaseInsensitiveDict)): for k, v in txt.items(): if v not in (None, False, True): txt[k] = str(v) for i in (':method', ':scheme', ':authority', ':path'): txt.pop(i, None) return txt headers = {} for header in txt.split('\n'): if header: name, value = header.split(': ', maxsplit=1) if name not in (':method', ':scheme', ':authority', ':path'): headers[name] = value return headers