319 lines
11 KiB
Python
319 lines
11 KiB
Python
# -*- coding:utf-8 -*-
|
|
"""
|
|
@Author : g1879
|
|
@Contact : g1879@qq.com
|
|
@Website : https://DrissionPage.cn
|
|
@Copyright: (c) 2020 by g1879, Inc. All Rights Reserved.
|
|
"""
|
|
from html import unescape
|
|
from os.path import sep
|
|
from pathlib import Path
|
|
from re import sub
|
|
from urllib.parse import urlparse, urljoin, urlunparse
|
|
|
|
from DrissionRecord.tools import make_valid_name
|
|
from requests.structures import CaseInsensitiveDict
|
|
|
|
from .._functions.settings import Settings as _S
|
|
|
|
|
|
def get_ele_txt(e):
|
|
# 前面无须换行的元素
|
|
nowrap_list = ('br', 'sub', 'sup', 'em', 'strong', 'a', 'font', 'b', 'span', 's', 'i', 'del', 'ins', 'img', 'td',
|
|
'th', 'abbr', 'bdi', 'bdo', 'cite', 'code', 'data', 'dfn', 'kbd', 'mark', 'q', 'rp', 'rt', 'ruby',
|
|
'samp', 'small', 'time', 'u', 'var', 'wbr', 'button', 'slot', 'content')
|
|
# 后面添加换行的元素
|
|
wrap_after_list = ('p', 'div', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'ol', 'li', 'blockquote', 'header',
|
|
'footer', 'address' 'article', 'aside', 'main', 'nav', 'section', 'figcaption', 'summary')
|
|
# 不获取文本的元素
|
|
noText_list = ('script', 'style', 'video', 'audio', 'iframe', 'embed', 'noscript', 'canvas', 'template')
|
|
# 用/t分隔的元素
|
|
tab_list = ('td', 'th')
|
|
|
|
if e.tag in noText_list:
|
|
return e.raw_text
|
|
|
|
def get_node_txt(ele, pre=False) -> list:
|
|
tag = ele.tag
|
|
if tag == 'br':
|
|
return [True]
|
|
if not pre and tag == 'pre':
|
|
pre = True
|
|
|
|
str_list = []
|
|
if tag in noText_list and not pre: # 标签内的文本不返回
|
|
return str_list
|
|
|
|
nodes = ele.eles('xpath:./text() | *')
|
|
prev_ele = ''
|
|
for el in nodes:
|
|
if isinstance(el, str): # 字符节点
|
|
if pre:
|
|
str_list.append(el)
|
|
|
|
else:
|
|
if sub('[ \n\t\r]', '', el) != '': # 字符除了回车和空格还有其它内容
|
|
txt = el
|
|
if not pre:
|
|
txt = txt.replace('\r\n', ' ').replace('\n', ' ')
|
|
txt = sub(r' {2,}', ' ', txt)
|
|
str_list.append(txt)
|
|
|
|
else: # 元素节点
|
|
if el.tag not in nowrap_list and str_list and str_list[-1] != '\n': # 元素间换行的情况
|
|
str_list.append('\n')
|
|
if el.tag in tab_list and prev_ele in tab_list: # 表格的行
|
|
str_list.append('\t')
|
|
|
|
str_list.extend(get_node_txt(el, pre))
|
|
prev_ele = el.tag
|
|
|
|
if tag in wrap_after_list and str_list and str_list[-1] not in ('\n', True): # 有些元素后面要添加回车
|
|
str_list.append('\n')
|
|
|
|
return str_list
|
|
|
|
re_str = get_node_txt(e)
|
|
if re_str and re_str[-1] == '\n':
|
|
re_str.pop()
|
|
|
|
l = len(re_str)
|
|
if l > 1:
|
|
r = []
|
|
for i in range(l - 1):
|
|
i1 = re_str[i]
|
|
i2 = re_str[i + 1]
|
|
if i1 is True:
|
|
r.append('\n')
|
|
continue
|
|
elif i2 is True:
|
|
r.append(i1)
|
|
continue
|
|
elif i1.endswith(' ') and i2.startswith(' '):
|
|
i1 = i1[:-1]
|
|
r.append(i1)
|
|
r.append('\n' if re_str[-1] is True else re_str[-1])
|
|
re_str = ''.join(r)
|
|
|
|
elif not l:
|
|
re_str = ''
|
|
else:
|
|
re_str = re_str[0] if re_str[0] is not True else '\n'
|
|
|
|
return format_html(re_str.strip())
|
|
|
|
|
|
def format_html(text):
|
|
return unescape(text).replace('\xa0', ' ') if text else text
|
|
|
|
|
|
def location_in_viewport(page, loc_x, loc_y):
|
|
js = f'''function(){{let x = {loc_x}; let y = {loc_y};
|
|
const scrollLeft = document.documentElement.scrollLeft;
|
|
const scrollTop = document.documentElement.scrollTop;
|
|
const vWidth = document.documentElement.clientWidth;
|
|
const vHeight = document.documentElement.clientHeight;
|
|
if (x< scrollLeft || y < scrollTop || x > vWidth + scrollLeft || y > vHeight + scrollTop){{return false;}}
|
|
return true;}}'''
|
|
return page._run_js(js)
|
|
|
|
|
|
def offset_scroll(ele, offset_x, offset_y):
|
|
loc_x, loc_y = ele.rect.location
|
|
cp_x, cp_y = ele.rect.click_point
|
|
lx = loc_x + offset_x if offset_x else cp_x
|
|
ly = loc_y + offset_y if offset_y else cp_y
|
|
if not location_in_viewport(ele.owner, lx, ly):
|
|
clientWidth = ele.owner._run_js('return document.body.clientWidth;')
|
|
clientHeight = ele.owner._run_js('return document.body.clientHeight;')
|
|
ele.owner.scroll.to_location(lx - clientWidth // 2, ly - clientHeight // 2)
|
|
cl_x, cl_y = ele.rect.viewport_location
|
|
ccp_x, ccp_y = ele.rect.viewport_click_point
|
|
cx = cl_x + offset_x if offset_x else ccp_x
|
|
cy = cl_y + offset_y if offset_y else ccp_y
|
|
return cx, cy
|
|
|
|
|
|
def make_absolute_link(link, baseURI=None):
|
|
if not link:
|
|
return link
|
|
|
|
link = link.strip().replace('\\', '/')
|
|
parsed = urlparse(link)._asdict()
|
|
# if baseURI:
|
|
# baseURI = baseURI.rstrip('/\\')
|
|
|
|
# 是相对路径,与页面url拼接并返回
|
|
if not parsed['netloc']:
|
|
return urljoin(baseURI, link) if baseURI else link
|
|
|
|
# 是绝对路径但缺少协议,从页面url获取协议并修复
|
|
if not parsed['scheme'] and baseURI:
|
|
parsed['scheme'] = urlparse(baseURI).scheme
|
|
parsed = tuple(v for v in parsed.values())
|
|
return urlunparse(parsed)
|
|
|
|
# 绝对路径且不缺协议,直接返回
|
|
return link
|
|
|
|
|
|
def is_js_func(func):
|
|
func = func.strip()
|
|
if (func.startswith('function') or func.startswith('async ')) and func.endswith('}'):
|
|
return True
|
|
# elif '=>' in func:
|
|
# return True
|
|
return False
|
|
|
|
|
|
def get_blob(page, url, as_bytes=True):
|
|
if not url.startswith('blob'):
|
|
raise ValueError(_S._lang.join(_S._lang.NOT_BLOB, url=url))
|
|
js = """
|
|
function fetchData(url) {
|
|
return new Promise((resolve, reject) => {
|
|
let xhr = new XMLHttpRequest();
|
|
xhr.responseType = 'blob';
|
|
xhr.onload = function() {
|
|
let reader = new FileReader();
|
|
reader.onloadend = function(){resolve(reader.result);}
|
|
reader.readAsDataURL(xhr.response);
|
|
};
|
|
xhr.open('GET', url, true);
|
|
xhr.send();
|
|
});
|
|
}
|
|
"""
|
|
try:
|
|
result = page._run_js(js, url)
|
|
except:
|
|
raise RuntimeError(_S._lang.join(_S._lang.GET_BLOB_FAILED))
|
|
if as_bytes:
|
|
from base64 import b64decode
|
|
return b64decode(result.split(',', 1)[-1])
|
|
else:
|
|
return result
|
|
|
|
|
|
def save_page(tab, path=None, name=None, as_pdf=False, kwargs=None):
|
|
if name:
|
|
if name.endswith('.pdf'):
|
|
name = name[:-4]
|
|
as_pdf = True
|
|
elif name.endswith('.mhtml'):
|
|
name = name[:-6]
|
|
as_pdf = False
|
|
|
|
if path:
|
|
path = Path(path)
|
|
if path.suffix.lower() == '.mhtml':
|
|
name = path.stem
|
|
path = path.parent
|
|
as_pdf = False
|
|
elif path.suffix.lower() == '.pdf':
|
|
name = path.stem
|
|
path = path.parent
|
|
as_pdf = True
|
|
|
|
return get_pdf(tab, path, name, kwargs) if as_pdf else get_mhtml(tab, path, name)
|
|
|
|
|
|
def get_mhtml(page, path=None, name=None):
|
|
r = page._run_cdp('Page.captureSnapshot')['data']
|
|
if path is None and name is None:
|
|
return r
|
|
|
|
path = path or '.'
|
|
Path(path).mkdir(parents=True, exist_ok=True)
|
|
name = make_valid_name(name or page.title)
|
|
with open(f'{path}{sep}{name}.mhtml', 'w', encoding='utf-8', newline='\n') as f:
|
|
f.write(r.replace('\r\n', '\n'))
|
|
return r
|
|
|
|
|
|
def get_pdf(page, path=None, name=None, kwargs=None):
|
|
if not kwargs:
|
|
kwargs = {}
|
|
kwargs['transferMode'] = 'ReturnAsBase64'
|
|
if 'printBackground' not in kwargs:
|
|
kwargs['printBackground'] = True
|
|
try:
|
|
r = page._run_cdp('Page.printToPDF', **kwargs)['data']
|
|
except:
|
|
raise RuntimeError(_S._lang.join(_S._lang.GET_PDF_FAILED))
|
|
from base64 import b64decode
|
|
r = b64decode(r)
|
|
if path is None and name is None:
|
|
return r
|
|
|
|
path = path or '.'
|
|
Path(path).mkdir(parents=True, exist_ok=True)
|
|
name = make_valid_name(name or page.title)
|
|
with open(f'{path}{sep}{name}.pdf', 'wb', newline='\n') as f:
|
|
f.write(r)
|
|
return r
|
|
|
|
|
|
def tree(ele_or_page, text=False, show_js=False, show_css=False):
|
|
def _tree(obj, last_one=True, body=''):
|
|
list_ele = obj.children()
|
|
length = len(list_ele)
|
|
body_unit = ' ' if last_one else '│ '
|
|
tail = '├───'
|
|
new_body = body + body_unit
|
|
|
|
if length > 0:
|
|
new_last_one = False
|
|
for i in range(length):
|
|
if i == length - 1:
|
|
tail = '└───'
|
|
new_last_one = True
|
|
e = list_ele[i]
|
|
|
|
attrs = ' '.join([f"{k}='{v}'" for k, v in e.attrs.items()])
|
|
show_text = f'{new_body}{tail}<{e.tag} {attrs}>'.replace('\n', ' ')
|
|
if text:
|
|
t = e('x:/text()')
|
|
if t:
|
|
t = t.replace('\n', ' ')
|
|
if (e.tag not in ('script', 'style') or (e.tag == 'script' and show_js)
|
|
or (e.tag == 'style' and show_css)):
|
|
if text is not True:
|
|
t = t[:text]
|
|
show_text = f'{show_text} {t}'
|
|
print(show_text)
|
|
|
|
_tree(e, new_last_one, new_body)
|
|
|
|
ele = ele_or_page.s_ele()
|
|
attrs = ' '.join([f"{k}='{v}'" for k, v in ele.attrs.items()])
|
|
show_text = f'<{ele.tag} {attrs}>'.replace('\n', ' ')
|
|
if text:
|
|
t = ele('x:/text()')
|
|
if t:
|
|
t = t.replace('\n', ' ')
|
|
if (ele.tag not in ('script', 'style') or (ele.tag == 'script' and show_js)
|
|
or (ele.tag == 'style' and show_css)):
|
|
if text is not True:
|
|
t = t[:text]
|
|
show_text = f'{show_text} {t}'
|
|
print(show_text)
|
|
_tree(ele)
|
|
|
|
|
|
def format_headers(txt):
|
|
if isinstance(txt, (dict, CaseInsensitiveDict)):
|
|
for k, v in txt.items():
|
|
if v not in (None, False, True):
|
|
txt[k] = str(v)
|
|
for i in (':method', ':scheme', ':authority', ':path'):
|
|
txt.pop(i, None)
|
|
return txt
|
|
headers = {}
|
|
for header in txt.split('\n'):
|
|
if header:
|
|
name, value = header.split(': ', maxsplit=1)
|
|
if name not in (':method', ':scheme', ':authority', ':path'):
|
|
headers[name] = value
|
|
return headers
|