Initial commit

This commit is contained in:
Alex committed 2026-11-02 22:54:14 +08:00
commit 57fc3525eb
7311 files changed
+1291398

No files matched your search

@@ -0,0 +1 @@
# -*- coding:utf-8 -*-
@@ -0,0 +1,215 @@
# -*- coding:utf-8 -*-
from re import match
from time import sleep
from ..tools import no_left_right, is_single_data, is_1D_data, add_left_right
from ..header import ZeroHeader, Header
class BaseHandler(object):
def __init__(self, recorder):
self._recorder = recorder
self.type = ''
self.clear()
@property
def path(self):
return self._recorder._path
def record(self):
raise RuntimeError('记录数据前需先指定文件路径。')
def clear(self):
self.data = [{'data': [], 'coord': 0}]
self.data_count = 0
def rows(self, cols=True, sign_col=True, signs=None, deny_sign=False, count=None, begin_row=None, end_row=None):
raise RuntimeError(f'{self.type}文件不支持rows()属性。')
def add(self, add_method, **data):
while self._recorder._pause_add: # 等待其它线程写入结束
sleep(.01)
self.__getattribute__(add_method)(**data)
self.data_count += 1
if 0 < self._recorder._cache_size <= self.data_count:
self._recorder.record()
def add_data(self, data, coord, table):
raise RuntimeError('请先指定文件路径。')
def add_link(self, link, coord, content=None, table=None):
raise TypeError('仅xlsx文件格式支持add_link()。')
def add_img(self, img_path, coord, width=None, height=None, table=None):
raise TypeError('仅xlsx文件格式支持add_img()。')
def add_styles(self, styles, coord=None, rows=None, cols=None, replace=True, table=None):
raise TypeError('仅xlsx文件格式支持add_styles()。')
def add_rows_height(self, height, rows=True, table=None):
raise TypeError('仅xlsx文件格式支持add_rows_height()。')
def add_cols_width(self, width, cols=True, table=None):
raise TypeError('仅xlsx文件格式支持add_cols_width()。')
def __getattr__(self, item):
try:
return self.__getattribute__(item)
except AttributeError:
if item.startswith('set_'):
raise TypeError(f'{self.type}文件不支持set.{item[4:]}()方法。')
else:
raise TypeError(f'{self.type}文件不支持{item}。')
class HasLeftRight(object):
@property
def right(self):
return self._recorder._settings.get('right', None)
@property
def left(self):
return self._recorder._settings.get('left', None)
@property
def _add_lr_method(self):
return self._recorder._settings.get('add_lr_method', no_left_right)
def set_left(self, data):
return self._set_left_right(True, data)
def set_right(self, data):
return self._set_left_right(False, data)
def _set_left_right(self, is_left, data):
if isinstance(data, (list, dict)):
data = data
elif isinstance(data, tuple):
data = list(data)
elif data is not None:
data = [data]
if is_left:
self._recorder._settings['left'] = data
else:
self._recorder._settings['right'] = data
if self.left or self.right:
self._recorder._settings['add_lr_method'] = add_left_right
else:
self._recorder._settings['add_lr_method'] = no_left_right
return self._recorder._setter
class SheetLikeHandler(BaseHandler, HasLeftRight):
# xlsx, db, csv
@property
def header(self):
return self._get_header()
@property
def auto_new_header(self):
return self._recorder._settings.get('auto_new_header', False)
def set_auto_new_header(self, on_off=True):
self._recorder.record()
self._recorder._settings['auto_new_header'] = on_off
return self._recorder._setter
def _parse_coord(self, coord):
return parse_coord(coord, self.data_col)
def _handle_data(self, data):
if is_single_data(data): # int, float, str, None
data = [self._add_lr_method(self, (data,))]
elif is_1D_data(data): # list, dict, tuple
data = [self._add_lr_method(self, data)]
else: # 二维数组
data = [self._add_lr_method(self, (d,)) if is_single_data(d)
else self._add_lr_method(self, d) for d in data]
return data
def _get_header(self) -> Header:
"""获取当前表格的表头"""
...
class TextLikeHandler(BaseHandler):
def __init__(self, recorder):
super().__init__(recorder)
self._fast = True
self._record_method = self._record_fast
@property
def encoding(self):
return self._recorder._settings.get('encoding', 'utf-8')
def set_encoding(self, encoding):
self._recorder._settings['encoding'] = encoding
return self._recorder._setter
def add_data(self, data, coord=None, **kwargs):
coord = self._parse_coord(coord)
data = self._handle_data(data)
if coord == self.data[-1]['coord'] == 0:
self.data[-1]['data'].extend(data)
else:
self.data.append({'data': data, 'coord': coord})
if self._fast and coord:
self._fast = False
self._record_method = self._record_slow
def _parse_coord(self, coord):
coord = coord or 0
if not isinstance(coord, int):
raise TypeError('txt文件的coord的参数必须是int。')
return coord
def record(self):
self._record_method()
self._record_method = self._record_fast
self._fast = True
def _record_fast(self):
pass
def _record_slow(self):
pass
def parse_coord(coord, data_col=1):
if not coord: # 新增一行,列为data_col
return_coord = 0, data_col
elif isinstance(coord, int):
return_coord = coord, data_col
elif isinstance(coord, str): # 'A3'格式
m = match(r'^[$]?([A-Za-z]{1,3})[$]?(-?\d+)$', coord)
if not m:
raise ValueError(f'{coord} 坐标格式不正确。')
y, x = m.groups()
return_coord = int(x), ZeroHeader()[y] or 1
elif isinstance(coord, (tuple, list)) and len(coord) == 2:
if isinstance(coord[0], int):
x = int(coord[0])
elif coord[0] is None:
x = 0
else:
raise TypeError('行格式不正确。')
if isinstance(coord[1], (str, int)):
y = coord[1]
elif coord[1] is None:
y = 0
else:
raise TypeError('列格式不正确。')
return_coord = x, y
else:
raise ValueError(f'{coord} 坐标格式不正确。')
return return_coord
@@ -0,0 +1,366 @@
# -*- coding:utf-8 -*-
from collections.abc import Callable
from pathlib import Path
from typing import Union, Optional, List, Any, Tuple
from ..cell_style import CellStyle
from ..data import *
from ..header import Header
from ..recorder import Recorder
class BaseHandler(object):
data: Union[list, dict] = ...
data_count: int = ...
type: str = ...
_recorder: Recorder = ...
def __init__(self, recorder: Recorder):
"""
:param recorder: Recorder对象
"""
...
def __getattr__(self, item: str): ...
@property
def path(self) -> Path:
"""返回文件路径"""
...
@property
def delimiter(self) -> str:
"""返回csv文件分隔符"""
...
@property
def quote_char(self) -> str:
"""返回csv文件引用符"""
...
@property
def header(self) -> Header:
"""返回表头,支持csv、xlsx和db格式文件"""
...
@property
def header_row(self) -> int:
"""返回表头行号"""
...
@property
def right(self) -> Union[list, dict, None]:
"""返回自动拼接到行右边的数据,支持csv、xlsx、db、jsonl格式文件"""
...
@property
def left(self) -> Union[list, dict, None]:
"""返回自动拼接到行左边的数据,支持csv、xlsx、db、jsonl格式文件"""
...
@property
def table(self) -> Optional[str]:
"""返回当前使用的表名,支持xlsx和db格式"""
...
@property
def tables(self) -> List[str]:
"""返回所有表名,支持xlsx和db格式"""
...
@property
def encoding(self) -> str:
"""返回文件编码格式,支持txt、csv、json、jsonl文件格式"""
...
@property
def data_col(self) -> Union[int, str]:
"""返回默认写入数据的列,支持xlsx和csv文件格式"""
...
def record(self) -> Path:
"""写入数据到文件,返回文件绝对路径"""
...
def clear(self) -> None:
"""清空已保存的数据"""
...
def run_sql(self, sql: str,
single: bool,
commit: bool) -> Union[None, list, tuple]:
"""执行sql语句并返回结果,只支持db文件
:param sql: sql语句
:param single: 是否只获取一个结果
:param commit: 是否提交到数据库
:return: 查找到的结果,没有结果时返回None
"""
...
def rows(self,
cols: Union[str, int, list, tuple, True] = True,
sign_col: Union[str, int, True] = True,
signs: Any = None,
deny_sign: bool = False,
count: int = None,
begin_row: Optional[int] = None,
end_row: Optional[int] = None) -> List[RowData, RowDict, RowList, RowStr]:
"""返回符合条件的行数据,可指定只要某些列。txt格式只有count、begin_row、end_row有效,不支持byte文件
:param cols: 要获取的列,可以是多列,传入表头值或列序号,要用列号用Col('a'),为True获取所有列
:param sign_col: 用于筛选数据的列,传入表头值或列序号,要用列号用Col('a'),为True获取所有行
:param signs: 按这个值筛选目标行,可用list, tuple, set设置多个
:param deny_sign: 是否反向匹配sign,即筛选值不是sign的行
:param count: 获取多少条数据,为None获取所有
:param begin_row: 数据开始的行,None表示header_row后面一行
:param end_row: 数据结束的行,None表示最后一行
:return: 数据对象组成的列表
"""
...
def add(self, add_method: str, **data) -> None:
"""调用添加数据到缓存的方法
:param add_method: 添加数据的方法,'add_data'、'add_link'等
:param data: 数据参数
:return: None
"""
...
def add_data(self,
data: Any,
coord: Union[list, Tuple[Union[None, int], Union[None, int, str]], str, int],
table: Union[str, bool]) -> None:
"""添加数据,可一次添加多条数据
:param data: 插入的数据,任意格式,可以为二维数据
:param coord: 要添加数据的坐标,'A3'、(3, Col('A'))或(3, '表头')格式坐标,行号,json为路径列表
:param table: 要写入的数据表,仅支持xlsx格式。为None表示用set.table()方法设置的值,为True表示活动的表格
:return: None
"""
...
def add_link(self,
link: Optional[str],
coord: Union[int, str, tuple],
content: Any,
table: Union[str, True, None]) -> None:
"""为单元格设置超链接,仅xlsx格式时有效
:param link: 超链接,为None时删除链接
:param coord: 单元格坐标,格式:'A3'、(3, Col('A'))或(3, '表头')格式坐标,行号
:param content: 单元格文本
:param table: 数据表名,仅支持xlsx格式。为None表示用set.table()方法设置的值,为Ture表示活动的表格
:return: None
"""
...
def add_img(self,
img_path: Union[None, str, Path, dict],
coord: Union[int, str, tuple],
width: float,
height: float,
table: Union[str, True, None]) -> None:
"""向单元格设置图片,仅xlsx格式时有效
:param img_path: 图片路径
:param coord: 单元格坐标,格式:'A3'、(3, Col('A'))或(3, '表头')格式坐标,行号
:param width: 图片宽
:param height: 图片高
:param table: 数据表名,仅支持xlsx格式。为None表示用set.table()方法设置的值,为Ture表示活动的表格
:return: None
"""
...
def add_styles(self,
styles: Union[CellStyle, dict, list, tuple, None],
coord: Union[str, tuple],
rows: Union[int, str, tuple, list],
cols: Union[int, str, tuple, list],
replace: bool,
table: Union[str, True, None]) -> None:
"""为单元格设置样式,可批量设置范围内的单元格,仅xlsx格式时有效
:param styles: CellStyle对象,可用列表传入多个;为None则清除单元格样式;可用dict设置指定多个单元格样式,此时coord、rows、cols参数无效
:param coord: 单元格坐标,str表示单个单元格'A1'或连续单元格'A1:C5',tuple为单个单元格坐标(1, '表头')
:param rows: 整行设置,int表示行号,str为'1:3'格式,可用列表传入多行
:param cols: 整列设置,int表示列序号,str表示表头值,长度为2的tuple传入连续多列的起止列,可用列表传入多列
:param replace: 是否直接覆盖所有已有样式,如为False只替换设置的属性
:param table: 数据表名,仅支持xlsx格式。为None表示用set.table()方法设置的值,为Ture表示活动的表格
:return: None
"""
...
def add_rows_height(self, height: Union[float, dict],
rows: Union[int, str, list, tuple, True],
table: Union[str, True, None]) -> None:
"""设置行高,可设置多行,仅xlsx格式时有效
:param height: 行高,为dict({1:30, 3:50})时可为每行指定行高,此时rows参数无效
:param rows: 行号,可指定多行(1、'1:4'、[1, 2, 3]),为Ture设置所有行
:param table: 数据表名,仅支持xlsx格式。为None表示用set.table()方法设置的值,为Ture表示活动的表格
:return: None
"""
...
def add_cols_width(self, width: Union[float, dict],
cols: Union[int, str, list, tuple, True],
table: Union[str, True, None]) -> None:
"""设置列宽,可设置多列,仅xlsx格式时有效
:param width: 列宽,为dict({1:30, '表头值':50})时可为每列指定行高,此时cols参数无效
:param cols: 用int表示列序号,str表示表头值,用Col('A')输入列号,用tuple设置连续起止列,用list指定离散列,为Ture设置所有列
:param table: 数据表名,仅支持xlsx格式。为None表示用set.table()方法设置的值,为Ture表示活动的表格
:return: None
"""
...
def set_table(self, name: str) -> None:
"""设置当前操作的表格,支持xlsx和db文件格式
:param name: 表名
:return: None
"""
...
def set_auto_new_header(self, on_off: bool) -> None:
"""设置写入数据时如表头不存在,是否自动添加,支持xlsx、csv、db文件格式
:param on_off: bool表示开或关
:return: None
"""
...
def set_left(self, data: Union[dict, list]) -> None:
"""设置左边数据
:param data: 数据值
:return: None
"""
...
def set_right(self, data: Union[dict, list]) -> None:
"""设置右边数据
:param data: 数据值
:return: None
"""
...
def set_encoding(self, encoding: str) -> None:
"""设置文件编码,支持csv、txt、json、jsonl文件格式
:param encoding: 编码
:return: None
"""
...
def set_header(self, header: Union[list, tuple],
table: Union[str, None, True],
to_file: bool,
row: int) -> None:
"""设置表头,支持xlsx和csv文件
:param header: 表头,列表或元组
:param table: 表名,只xlsx格式文件有效,为True表示活动数据表,为None表示不改变设置
:param to_file: 是否写入到文件
:param row: 指定写入文件的行号,不改变对象已设置的header_row属性,to_file为False时无效
:return: None
"""
...
def set_header_row(self, num: int,
table: Union[str, None, True]) -> None:
"""设置表头行号,支持xlsx和csv文件
:param num: 行号
:param table: 表名,为True表示活动数据表,为None表示不改变设置
:return: None
"""
...
def set_delimiter(self, delimiter: str) -> None:
"""设置csv文件分隔符
:param delimiter: 分隔符
:return: None
"""
...
def set_quote_char(self, quote_char: str) -> None:
"""设置csv文件引用符
:param quote_char: 引用符
:return: None
"""
...
def set_follow_styles(self, on_off: bool) -> None:
"""设置xlsx文件新行是否跟随上一行格式
:param on_off: bool表示开关
:return: None
"""
...
def set_new_row_height(self, height: float) -> None:
"""设置xlsx文件新行行高
:param height: 行高
:return: None
"""
...
def set_new_row_styles(self, styles: Union[CellStyle, List[CellStyle], None]) -> None:
"""设置xlsx文件新行样式,可传入多个,传入None则取消
:param styles: 传入CellStyle对象设置整个新行,传入CellStyle对象组成的列表设置多个,传入None清空设置
:return: None
"""
...
def set_data_col(self, col: Union[str, int]) -> None:
"""设置默认填充数据的列,支持xlsx和csv文件
:param col: 表头名或列序号,列序号从1开始,负数表示从后往前数,0表示新列(表头长度后一列),用Col('A')输入列号
:return: None
"""
...
def set_link_style(self, style: Union[CellStyle, True]) -> None:
"""设置单元格的链接样式
:param style: CellStyle对象,为True时使用内置的默认样式
:return: None
"""
...
class HasLeftRight(object):
_recorder: Recorder = ...
@property
def _add_lr_method(self) -> Callable:
"""返回执行添加左右数据的方法"""
...
class SheetLikeHandler(BaseHandler):
def __init__(self, recorder: Recorder):
"""
:param recorder: Recorder对象
"""
...
@property
def auto_new_header(self) -> bool:
"""返回写入数据时如表头不存在,是否自动添加的设置,支持xlsx、csv、db文件格式"""
...
def _parse_coord(self, coord) -> Tuple[int, Union[int, str]]:
"""格式化坐标,将坐标以(1, 1)格式返回(csv、xlsx)
:param coord: 初始坐标
:return: 坐标tuple
"""
...
def _handle_data(self, data) -> list:
"""格式化数据,以列表或二维列表格式返回(csv、xlsx、db)
:param data: 初始数据
:return: 数据组成的列表
"""
...
class TextLikeHandler(BaseHandler):
_fast: bool = ...
_record_method: Callable = ...
def parse_coord(coord: Union[int, str, list, tuple, None],
data_col: int = 1) -> Tuple[Optional[int], Optional[int]]:
"""处理坐标格式
:param coord: 'A3'格式坐标、(3, 1)或(3, '列名')格式坐标、行号
:param data_col: 列号,用于只传入行号的情况
:return: 坐标tuple:(行, 列)坐标中的None表示新行或列
"""
...
@@ -0,0 +1,36 @@
# -*- coding:utf-8 -*-
from .base_handler import BaseHandler
class ByteHandler(BaseHandler):
__END = (0, 2)
def __init__(self, recorder):
super().__init__(recorder)
self.type = 'byte'
def add_data(self, data, **kwargs):
if not isinstance(data, bytes):
raise TypeError('只能接受bytes类型数据。')
coord = kwargs['coord']
if coord is not None and not (isinstance(coord, int) and coord >= 0):
raise ValueError('seek参数只能接受None或大于等于0的整数。')
self.data.append((data, coord))
def record(self):
if not self._recorder._file_exists and not self._recorder._path.exists():
with open(self.path, 'wb'):
pass
with open(self.path, 'rb+') as f:
previous = None
for i in self.data:
loc = ByteHandler.__END if i[1] is None else (i[1], 0)
if not (previous == loc == ByteHandler.__END):
f.seek(loc[0], loc[1])
previous = loc
f.write(i[0])
def clear(self):
self.data = []
self.data_count = 0
@@ -0,0 +1,311 @@
# -*- coding:utf-8 -*-
from csv import reader as csv_reader, writer as csv_writer
from .base_handler import SheetLikeHandler, TextLikeHandler
from ..header import Header, ZeroHeader
from ..tools import data2str, get_first_dict_csv, get_key_cols, get_real_row
class CSVHandler(SheetLikeHandler, TextLikeHandler):
def __init__(self, recorder):
super().__init__(recorder)
self.type = 'csv'
self._header = None
self._header_row = 1
self._data_col = 1
@property
def delimiter(self):
return self._recorder._settings.get('delimiter', ',')
@property
def quote_char(self):
return self._recorder._settings.get('quote_char', '"')
@property
def header(self):
return self._get_header()
@property
def header_row(self):
return self._header_row
@property
def data_col(self):
return self._data_col
def add_data(self, data, coord, **kwargs):
coord = self._parse_coord(coord)
data = self._handle_data(data)
if coord == self.data[-1]['coord'] and coord[0] == 0:
self.data[-1]['data'].extend(data)
else:
self.data.append({'data': data, 'coord': coord})
if self._fast and coord[0]:
self._fast = False
self._record_method = self._record_slow
def clear(self):
self.data = [{'data': [], 'coord': (0, 1)}]
self.data_count = 0
def rows(self, cols=True, sign_col=True, signs=None, deny_sign=False, count=None, begin_row=None, end_row=None):
header = self.header
if sign_col is not True:
sign_col = header.get_num(sign_col) or 1
cols = get_key_cols(cols, header)
if not begin_row:
begin_row = self._header_row + 1
return get_csv_rows(self, header=header, cols=cols, begin_row=begin_row, end_row=end_row or 0,
sign_col=sign_col, sign=signs, deny_sign=deny_sign, count=count)
def set_header(self, header, to_file=True, row=None, **kwargs):
if not isinstance(header, (list, tuple)):
raise ValueError('header必须为list或tuple格式。')
self._recorder.record()
row = row or self._header_row
with self._recorder._lock:
header = Header(header)
self._header = header
if to_file:
self._header2file(header, row)
return self._recorder._setter
def set_header_row(self, num, **kwargs):
if num < 0:
raise ValueError('num不能小于0。')
self._recorder.record()
with self._recorder._lock:
self._header_row = num
self._header = ZeroHeader() if num == 0 else None
return self._recorder._setter
def set_data_col(self, col):
if col is None:
self._data_col = 0
elif not isinstance(col, (int, str)) or col == '':
raise TypeError('col值只能是int、None或非空str。')
else:
self._data_col = col
return self._recorder._setter
def set_delimiter(self, delimiter):
self._recorder._settings['delimiter'] = delimiter
return self._recorder._setter
def set_quote_char(self, quote_char):
self._recorder._settings['quote_char'] = quote_char
return self._recorder._setter
def _record_fast(self):
file, new_csv = get_csv(self._recorder)
writer = csv_writer(file, delimiter=self.delimiter, quotechar=self.quote_char)
get_and_set_csv_header(self, new_csv, file, writer)
rewrite_method = 'make_insert_list_rewrite' if self.auto_new_header else 'make_insert_list'
rewrite = False
for d in self.data:
col = self._header._get_num(d['coord'][1])
for data in d['data']:
data, rewrite = self._header.__getattribute__(rewrite_method)(data, 'csv', rewrite)
data = [None] * (col - 1) + data
writer.writerow(data)
file.close()
if rewrite:
self._header2file(self._header, self._header_row)
def _record_slow(self):
file, new_csv = get_csv(self._recorder)
writer = csv_writer(file, delimiter=self.delimiter, quotechar=self.quote_char)
get_and_set_csv_header(self, new_csv, file, writer)
file.seek(0)
reader = csv_reader(file, delimiter=self.delimiter, quotechar=self.quote_char)
lines = list(reader)
lines_count = len(lines)
header = self._header
rewrite = False
method = 'make_change_list_rewrite' if self.auto_new_header else 'make_change_list'
for i in self.data:
data = i['data']
row = get_real_row(i['coord'][0], lines_count)
col = header._get_num(i['coord'][1])
for r, da in enumerate(data, row):
add_rows = r - lines_count
if add_rows > 0: # 若行数不够,填充行数
[lines.append([]) for _ in range(add_rows)]
lines_count += add_rows
row_num = r - 1
lines[row_num], rewrite = self._header.__getattribute__(method)(lines[row_num], da, col,
'csv', rewrite)
if rewrite:
[lines.append([]) for _ in range(self._header_row - lines_count)] # 若行数不够,填充行数
lines[self._header_row - 1] = list(header.num_key.values())
file.close()
writer = csv_writer(open(self.path, 'w', encoding=self.encoding, newline=''),
delimiter=self.delimiter, quotechar=self.quote_char)
writer.writerows(lines)
def _header2file(self, header, row):
if not self.path:
raise FileNotFoundError('未指定文件。')
from csv import writer
if self._recorder._file_exists or self._recorder._path.exists():
with open(self.path, 'r', newline='', encoding=self.encoding) as f:
lines = f.readlines()
content1 = lines[:row - 1]
content2 = lines[row:]
with open(self.path, 'w', newline='', encoding=self.encoding) as f:
f.write("".join(content1))
csv_write = writer(f, delimiter=self.delimiter, quotechar=self.quote_char)
con_len = len(content1)
if con_len < row - 1:
for _ in range(row - con_len - 1):
csv_write.writerow([])
csv_write.writerow([data2str(i) for i in header.values()])
with open(self.path, 'a+', newline='', encoding=self.encoding) as f:
f.write("".join(content2))
else:
self._recorder._path.parent.mkdir(parents=True, exist_ok=True)
with open(self.path, 'w', newline='', encoding=self.encoding) as f:
csv_write = writer(f, delimiter=self.delimiter, quotechar=self.quote_char)
for _ in range(row - 1):
csv_write.writerow([])
csv_write.writerow([data2str(i) for i in header.values()])
self._recorder._file_exists = True
def _get_header(self):
if self._header is not None:
return self._header
if not self.path or not self._recorder._path.exists():
return None
from csv import reader
with open(self.path, 'r', newline='', encoding=self.encoding) as f:
u = reader(f, delimiter=self.delimiter, quotechar=self.quote_char)
try:
for _ in range(self.header_row):
header = next(u)
except StopIteration: # 文件是空的
header = []
self._header = Header(header)
return self._header
def get_csv(recorder):
new_csv = not recorder._file_exists and not recorder._path.exists()
return open(recorder.path, 'a+', newline='', encoding=recorder._handler.encoding), new_csv
def get_and_set_csv_header(handler, new_csv, file, writer):
if not handler._header_row:
return
if new_csv:
if handler._header:
for _ in range(handler._header_row - 1):
writer.writerow([])
writer.writerow([data2str(i) for i in handler._header.values()])
elif handler._header is None and handler.data_count:
data = get_first_dict_csv(handler.data)
if data:
handler._header = Header([h for h in data.keys() if isinstance(h, str)])
else:
handler._header = Header()
if handler._header:
writer.writerow([data2str(i) for i in handler._header.values()])
else:
handler._header = Header()
elif handler._header is None: # 从文件读取表头
file.seek(0)
reader = csv_reader(file, delimiter=handler.delimiter, quotechar=handler.quote_char)
header = []
try:
for _ in range(handler._header_row):
header = next(reader)
except StopIteration:
pass
handler._header = Header(header)
file.seek(2)
def get_csv_rows(handler, header, cols, begin_row, end_row, sign_col, sign, deny_sign, count):
sign = ['' if i is None else str(i) for i in sign]
if begin_row>1:
begin_row -= 1
res = []
with open(handler.path, 'r', encoding=handler.encoding) as f:
try:
for i in range(begin_row):
next(f)
except StopIteration:
return res
reader = csv_reader(f, delimiter=handler.delimiter, quotechar=handler.quote_char)
if sign_col is True: # 获取所有行
header_len = len(header)
if count or end_row:
end = min(count + begin_row, end_row) if count and end_row else (end_row or count + begin_row)
else:
end = False
method = get_csv_rows_key_is_True if cols is True else get_csv_rows_key_not_True
for ind, line in enumerate(reader, begin_row + 1):
if end and ind > end:
break
method(line, res, header, ind, cols, header_len)
else: # 获取符合条件的行
if sign_col > 1:
sign_col -= 1
get_csv_rows_with_count(reader, begin_row, end_row, sign_col, sign, deny_sign,
cols, res, header, count)
return res
def get_csv_rows_key_is_True(line, res, header, ind, cols, header_len):
if not line:
res.append(header.make_row_data(ind, {col: '' for col in range(1, header_len + 1)}))
else:
line_len = len(line)
x = max(header_len, line_len)
res.append(header.make_row_data(ind, {col: line[col - 1] if col <= line_len else ''
for col in range(1, x + 1)}))
def get_csv_rows_key_not_True(line, res, header, ind, cols, header_len):
x = len(line) + 1
res.append(header.make_row_data(ind, {col: line[col - 1] if col < x else '' for col in cols}))
def get_csv_rows_with_count(lines, begin_row, end_row, sign_col, sign, deny_sign, cols, res, header, count):
got = 0
header_len = len(header)
for ind, line in enumerate(lines, begin_row + 1):
if (end_row and ind > end_row) or (count and got == count):
break
row_sign = '' if sign_col > len(line) - 1 else line[sign_col]
if (row_sign not in sign) if deny_sign else (row_sign in sign):
if cols is True: # 获取整行
if not line:
res.append(header.make_row_data(ind, {col: '' for col in range(1, header_len + 1)}))
else:
line_len = len(line)
x = max(header_len, line_len)
res.append(header.make_row_data(ind, {col: line[col - 1] if col <= line_len else ''
for col in range(1, x + 1)}))
else: # 只获取对应的列
x = len(line) + 1
res.append(header.make_row_data(ind, {col: line[col - 1] if col < x else '' for col in cols}))
got += 1
@@ -0,0 +1,128 @@
# -*- coding:utf-8 -*-
from _csv import Reader, Writer
from typing import TextIO, Union, Optional, Any, List, Tuple
from .base_handler import SheetLikeHandler, TextLikeHandler
from ..data import RowData
from ..header import Header
class CSVHandler(SheetLikeHandler, TextLikeHandler):
_header: Optional[Header] = ...
_header_row: int = ...
_data_col: int = ...
def _header2file(self, header: Header, row: int) -> None:
"""将header数据写入文件
:param header: Header对象
:param row: 表头所在行号
:return: None
"""
...
def get_csv(recorder) -> Tuple[TextIO, bool]:
"""获取csv文件对象及是否新文件
:param recorder: Recorder对象
:return: (文件对象, 是否新文件)
"""
...
def get_and_set_csv_header(handler: CSVHandler,
new_csv: bool,
file: TextIO,
writer: Writer) -> None:
"""从csv获取表头或把已获取的表头设置到新csv
:param handler: CSVHandler对象
:param new_csv: 是否新csv文件
:param file: 文件对象
:param writer: csv Writer对象
:return: None
"""
...
def get_csv_rows(handler: CSVHandler,
header: Header,
cols: Union[list, True],
begin_row: Optional[int],
end_row: Optional[int],
sign_col: Union[str, int, bool],
sign: Any,
deny_sign: bool,
count: int) -> List[RowData]:
"""获取csv文件指定行数据
:param handler: CSVHandler对象
:param header: Header对象
:param cols: 要获取的列,为True获取所有,可指定多列
:param begin_row: 开始行号
:param end_row: 结束行号,None为最后一行
:param sign_col: 作为条件的列
:param sign: 按这个值筛选目标行,可设置多个
:param deny_sign: 是否反向匹配sign,即筛选指不是sign的行
:param count: 获取多少条数据,为None获取所有
:return: 获取到的数据列表
"""
...
def get_csv_rows_key_is_True(line: Union[list, dict],
res: list,
header: Header,
ind: int,
cols: list,
header_len: int) -> None:
"""把csv文件一行所有列数据加入到结果集合中
:param line: 行数据
:param res: 保存最终结果的列表
:param header: Header对象
:param ind: 行号
:param cols: 没有作用
:param header_len: 表头长度
:return: None
"""
...
def get_csv_rows_key_not_True(line: Union[list, dict],
res: list,
header: Header,
ind: int,
cols: list,
header_len: int) -> None:
"""把csv文件一行指定列数据加入到结果集合中
:param line: 行数据
:param res: 保存最终结果的列表
:param header: Header对象
:param ind: 行号
:param cols: 要获取的列,True为所有
:param header_len: 没有作用
:return: None
"""
...
def get_csv_rows_with_count(lines: Reader,
begin_row: Optional[int],
end_row: Optional[int],
sign_col: Union[str, int, bool],
sign: Any,
deny_sign: bool,
cols: Union[list, True],
res: list,
header: Header, count: int) -> List[RowData]:
"""执行从csv中获取数据,有指定数量
:param lines: csv Reader对象
:param begin_row: 开始行号
:param end_row: 结束行号,None为最后一行
:param sign_col: 用于筛选数据的列
:param sign: 用于筛选数据的值
:param deny_sign: 是否反向匹配sign,即筛选指不是sign的行
:param cols: 要获取的列,True为所有
:param res: 结果列表
:param header: Header对象
:param count: 数据总条数
:return: 数据对象列表
"""
...
@@ -0,0 +1,222 @@
# -*- coding:utf-8 -*-
from sqlite3 import connect
from .base_handler import SheetLikeHandler
from ..data import RowData
from ..header import Header
from ..tools import process_content_json
class DBHandler(SheetLikeHandler):
def __init__(self, recorder):
super().__init__(recorder)
self.type = 'db'
self._conn = None
self._cur = None
self._table = None
@property
def tables(self):
self._connect()
self._cur.execute("select name from sqlite_master where type='table'")
tables = self._cur.fetchall()
self._close_connection()
return [i[0] for i in tables]
@property
def table(self):
return self._table
def add_data(self, data, **kwargs):
table = kwargs['table'] or self.table
if not isinstance(table, str):
raise RuntimeError('未指定数据库表名。')
data = self._handle_data(data)
self.data.setdefault(table, []).extend(data)
def record(self):
if not self._recorder._file_exists:
self._recorder._path.parent.mkdir(parents=True, exist_ok=True)
self._connect()
self._cur.execute("select name from sqlite_master where type='table'")
tables = {t[0]: [] for t in self._cur.fetchall()}
for table, data in self.data.items():
if table in tables:
self._cur.execute(f"PRAGMA table_info(`{table}`)")
tables[table] = [i[1] for i in self._cur.fetchall()]
else:
if isinstance(data[0], list):
self._close_connection()
raise TypeError('新建表格首次须接收数据需为dict格式。')
self._cur.execute(f"CREATE TABLE `{table}` (`{'`,`'.join(data[0].keys())}`)")
tables[table] = list(data[0])
if isinstance(data[0], dict):
curr_keys = set(data[0].keys())
else:
curr_keys = len(data[0])
data_list = []
table_cols_set = set(tables[table])
table_cols_long = len(tables[table])
for d in data:
if isinstance(d, dict):
tmp_keys = set(d.keys())
diff = tmp_keys - table_cols_set
if diff:
if self.auto_new_header:
for key in tmp_keys - table_cols_set:
add_db_col(self._cur, table, key)
table_cols_set.add(key)
tables[table].append(key)
table_cols_long = len(tables[table])
else:
for key in diff:
d.pop(key)
tmp_keys.remove(key)
d = {k: process_content_json(d[k]) for k in tables[table] if k in d}
else:
tmp_keys = len(d)
if table_cols_long > tmp_keys:
d = ok_list_db(d)
d.extend([None] * (table_cols_long - tmp_keys))
else:
self._close_connection()
raise RuntimeError('数据个数大于列数(注意left和right属性)。')
if tmp_keys != curr_keys:
self._to_database(data_list, table, tables)
curr_keys = tmp_keys
data_list = []
data_list.append(d)
if data_list:
self._to_database(data_list, table, tables)
self._conn.commit()
self._close_connection()
def rows(self, cols=True, sign_col=True, signs=None, deny_sign=False, count=None, begin_row=None, end_row=None):
self._connect()
table = self._table
if not table:
raise RuntimeError('使用rows()前必须先用set.table()指定表名。')
self._cur.execute(f"PRAGMA table_info(`{table}`)")
all_cols = [i[1] for i in self._cur.fetchall()]
header = Header(all_cols)
if cols is True:
cols_txt = '*'
cols = list(header)
else:
if isinstance(cols, (str, int)):
cols = (cols,)
elif not isinstance(cols, (list, tuple)):
raise TypeError('cols必须为list、tuple、str或int。')
header_len = len(header)
cols = [header.get_key(i) for i in cols if isinstance(i, str) or i <= header_len]
cols_txt = '`' + '`,`'.join(cols) + '`'
sql = f'select {cols_txt} from `{table}`'
if sign_col is not True:
if not isinstance(signs, (list, tuple)):
signs = (signs,)
sign_col = header.get_key(sign_col)
if isinstance(sign_col, int):
raise RuntimeError('cols指定的列数超出表头。')
is_not = 'not ' if deny_sign else ''
all_signs = []
for s in signs:
if isinstance(s, str):
all_signs.append(f"'{s}'")
elif s is None:
all_signs.append('null')
else:
all_signs.append(str(s))
all_signs = ', '.join(all_signs)
sql = f'{sql} where {sign_col} {is_not}in ({all_signs})'
begin_row = begin_row or 1
if end_row and end_row > begin_row:
sql = f"{sql} limit {end_row - 1}"
if begin_row > 1:
sql = f"{sql} offset {begin_row - 1}"
self._cur.execute(sql)
res = self._cur.fetchall()
if count and count < len(res):
res = res[:count]
r = [RowData(0, header, None, dict(zip(cols, i))) for i in res]
self._close_connection()
return r
def run_sql(self, sql, single=True, commit=False):
self._connect()
self._cur.execute(sql)
r = self._cur.fetchone() if single else self._cur.fetchall()
if commit:
self._conn.commit()
self._close_connection()
return r
def clear(self):
self.data = {}
self.data_count = 0
def set_table(self, name):
if '`' in name:
raise ValueError('table名称不能包含字符"`"。')
self._table = name
return self._recorder._setter
def _get_header(self):
if not self._table:
return None
self._connect()
try:
self._cur.execute(f"PRAGMA table_info(`{self._table}`)")
columns = self._cur.fetchall()
return Header([col[1] for col in columns])
finally:
self._close_connection()
def _connect(self):
self._conn = connect(self.path)
self._cur = self._conn.cursor()
self._recorder._file_exists = True
def _close_connection(self):
if self._conn is not None:
try:
self._cur.close()
self._conn.close()
except:
pass
def _to_database(self, data_list, table, tables):
if isinstance(data_list[0], dict):
question_masks = ','.join('?' * len(data_list[0]))
keys_txt = '`' + '`,`'.join(data_list[0]) + '`'
values = [list(i.values()) for i in data_list]
sql = f'INSERT INTO `{table}` ({keys_txt}) values ({question_masks})'
else:
question_masks = ','.join('?' * len(tables[table]))
values = data_list
sql = f'INSERT INTO `{table}` values ({question_masks})'
self._cur.executemany(sql, values)
def ok_list_db(data_list):
if isinstance(data_list, (dict, Header)):
data_list = data_list.values()
return [process_content_json(i) for i in data_list]
def add_db_col(cur, table, key):
sql = f'ALTER TABLE `{table}` ADD COLUMN `{key}`'
cur.execute(sql)
@@ -0,0 +1,49 @@
# -*- coding:utf-8 -*-
from sqlite3 import Connection, Cursor
from typing import Optional
from .base_handler import SheetLikeHandler
class DBHandler(SheetLikeHandler):
_conn: Optional[Connection] = ...
_cur: Optional[Cursor] = ...
_table: Optional[str] = ...
def _connect(self) -> None:
"""连接数据库"""
...
def _close_connection(self) -> None:
"""关闭数据库 """
...
def _to_database(self,
data_list: list,
table: str,
tables: dict) -> None:
"""把数据批量写入指定数据表
:param data_list: 要写入的数据组成的列表
:param table: 要写入数据的数据表名称
:param tables: 数据库中数据表和列信息
:return: None
"""
...
def ok_list_db(data_list: list) -> list:
"""格式化要写入db的数据
:param data_list: 数据组成的列表
:return: 格式化后的列表
"""
...
def add_db_col(cur: Cursor, table: str, key: str) -> None:
"""为数据库添加一列
:param cur: 连接对象
:param table: 表名
:param key: 列名
:return: None
"""
...
@@ -0,0 +1,122 @@
# -*- coding:utf-8 -*-
from json import load, dump, JSONDecodeError
from .base_handler import TextLikeHandler, HasLeftRight
from ..tools import is_single_data, process_content_json
class JSONHandler(TextLikeHandler, HasLeftRight):
def __init__(self, recorder):
super().__init__(recorder)
self.type = 'json'
def add_data(self, data, coord=None, **kwargs):
coord = _parse_coord(coord)
data = self._handle_data(data)
self.data.append({'data': data, 'coord': coord})
def record(self):
if not self._recorder._file_exists and not self._recorder._path.exists():
with open(self.path, 'w', encoding=self.encoding):
pass
if isinstance(self.data[0]['coord'][0], int):
data = []
elif isinstance(self.data[0]['coord'][0], str):
data = {}
else:
raise TypeError('coord内元素只能是int或str。')
else:
with open(self.path, 'r', encoding=self.encoding) as f:
try:
data = load(f)
except JSONDecodeError:
if isinstance(self.data[0]['coord'][0], int):
data = []
elif isinstance(self.data[0]['coord'][0], str):
data = {}
else:
raise TypeError('coord内元素只能是int或str。')
for i in self.data:
r, k = nav(data, i['coord'])
if isinstance(r, dict):
if isinstance(k, str):
r[k] = i['data']
else:
raise TypeError(f"字典中的索引需使用str,而非 {k}")
elif isinstance(r, list):
if k == 0:
r.append(i['data'])
else:
while len(r) < k:
r.append(None)
r[k - 1] = i['data']
with open(self.path, 'w', encoding=self.encoding) as f:
dump(data, f, ensure_ascii=False, indent=2, default=process_content_json)
def rows(self, **kwargs):
with open(self.path, 'r', encoding=self.encoding) as f:
return load(f)
def clear(self):
self.data = []
self.data_count = 0
def _handle_data(self, data):
return data if is_single_data(data) else self._add_lr_method(self, data)
def _parse_coord(coord):
if isinstance(coord, (list, tuple)):
return coord
elif isinstance(coord, (int, str)):
return [coord]
elif coord is None:
return [0]
else:
raise TypeError(f'coord参数只能是int、str、list或tuple,现在是{coord}')
def nav(data, parts):
current = data
parent = None
last_key = None
for i, part in enumerate(parts):
parent = current
last_key = part
is_0 = False
if isinstance(part, int):
if part > 0:
part -= 1
elif part == 0:
is_0 = True
if i == len(parts) - 1:
break
if isinstance(current, dict):
if not isinstance(part, str):
raise TypeError(f"字典中的索引需使用str,而非 {part}")
if part not in current:
current[part] = {} if isinstance(parts[i + 1], str) else []
current = current[part]
elif isinstance(current, list):
if isinstance(part, int):
if is_0:
current.append({} if isinstance(parts[i + 1], str) else [])
current = current[-1]
elif len(current) < part:
while len(current) < part+1:
current.append(None)
current[-1] = {} if isinstance(parts[i + 1], str) else []
current = current[-1]
else:
current = current[part]
else:
raise TypeError(f"列表索引必须是int,而非 {part}")
else:
raise TypeError(f"无法导航至 {current}")
return parent, last_key
@@ -0,0 +1,31 @@
# -*- coding:utf-8 -*-
from typing import Any, Union
from .base_handler import TextLikeHandler
class JSONHandler(TextLikeHandler):
def _handle_data(self, data: Any) -> Any:
"""如果是独立数据直接返回,否则为其添加左右数据再返回
:param data: 初始数据
:return: 处理后的数据
"""
...
def _parse_coord(coord: Any) -> Union[list, tuple]:
"""解析坐标
:param coord: 初始坐标
:return: 处理后的坐标
"""
...
def nav(data: Any, parts: list) -> tuple:
"""在json数据中获取指定位置数据的坐标
:param data: 在这个数据中定位
:param parts: 位置信息组成的列表
:return: 返回父元素和最后一个索引组成的tuple
"""
...
@@ -0,0 +1,196 @@
# -*- coding:utf-8 -*-
from json import dumps, loads, JSONDecodeError
from .base_handler import TextLikeHandler, HasLeftRight
from ..data import *
from ..tools import process_content_json, get_real_row, is_single_data
class JSONLHandler(TextLikeHandler, HasLeftRight):
def __init__(self, recorder):
super().__init__(recorder)
self.type = 'jsonl'
def rows(self, cols=True, sign_col=True, signs=None, deny_sign=False, count=None, begin_row=None, end_row=None):
if cols is not True and isinstance(cols, (int, str)):
cols = (cols,)
if not begin_row:
begin_row = 1
return get_jsonl_rows(self, cols=cols, begin_row=begin_row, end_row=end_row or 0,
sign_col=sign_col, sign=signs, deny_sign=deny_sign, count=count)
def _handle_data(self, data):
return [data] if is_single_data(data) else [self._add_lr_method(self, data)]
def _record_fast(self):
with open(self.path, 'a+', encoding=self.encoding) as f:
for data in self.data:
for d in data['data']:
f.write(handle_jsonl_data(d))
def _record_slow(self):
if not self._recorder._file_exists and not self._recorder._path.exists():
with open(self.path, 'w', encoding=self.encoding):
pass
with open(self.path, 'r', encoding=self.encoding) as f:
lines = f.readlines()
handle_txt_lines(self.data, lines)
with open(self.path, 'w', encoding=self.encoding) as f:
f.writelines(lines)
def handle_txt_lines(data_lst, lines):
lines_len = len(lines)
for data in data_lst:
num = get_real_row(data['coord'], lines_len)
data_end = num + len(data['data'])
if lines_len < data_end:
diff = data_end - lines_len - 1
[lines.append('\n') for _ in range(diff)]
lines_len += diff
for num, i in enumerate(data['data'], num - 1):
lines[num] = f'{handle_jsonl_data(i)}\n'
def handle_jsonl_data(data):
if data is None:
return ''
elif isinstance(data, str):
return data
else:
return dumps(data, ensure_ascii=False, default=process_content_json)
def get_jsonl_rows(handler, cols, begin_row, end_row, sign_col, sign, deny_sign, count):
begin_row -= 1
res = []
with open(handler.path, 'r', encoding=handler.encoding) as f:
try:
for i in range(begin_row):
next(f)
except StopIteration:
return res
if sign_col is True: # 获取所有行
if count or end_row:
end = min(count + begin_row, end_row) if count and end_row else (end_row or count + begin_row)
else:
end = False
method = get_jsonl_row_key_is_True if cols is True else get_jsonl_row_key_not_True
for ind, line in enumerate(f, begin_row + 1):
if end and ind > end:
break
method(handle_line_jsonl(line), res, ind, cols)
else: # 获取符合条件的行
get_jsonl_rows_with_count(f, begin_row, end_row, sign_col, sign, deny_sign,
cols, res, count, handle_line_jsonl)
return res
def get_jsonl_row_key_is_True(line, res, ind, cols):
res.append(data2DataWithRow(line, ind))
def get_jsonl_row_key_not_True(line, res, ind, cols):
if isinstance(line, dict):
new_line = {}
keys = list(line.keys())
for col in cols:
if isinstance(col, str):
new_line[col] = line.get(col, None)
elif isinstance(col, int):
n_col = col - 1 if col > 0 else col
try:
new_line[col] = line.get(keys[n_col], None)
except (IndexError, TypeError):
new_line[col] = None
res.append(RowDict(ind, new_line))
elif isinstance(line, list):
new_line = []
for col in cols:
try:
if col > 0:
col -= 1
new_line.append(line[col])
except (IndexError, TypeError):
pass
res.append(RowList(ind, new_line))
else:
res.append(data2DataWithRow(line, ind) if 1 in cols else RowList(ind, []))
def get_jsonl_rows_with_count(lines, begin_row, end_row, sign_col, sign, deny_sign, cols, res, count, method):
got = 0
for ind, line in enumerate(lines, begin_row + 1):
if (end_row and ind > end_row) or (count and got == count):
break
line = method(line)
if isinstance(line, list) and isinstance(sign_col, int):
if sign_col > 0:
sign_col -= 1
try:
row_val = line[sign_col]
except IndexError:
continue
elif isinstance(line, dict):
if isinstance(sign_col, str):
row_val = line.get(sign_col, None)
elif isinstance(sign_col, int):
if sign_col > 0:
sign_col -= 1
try:
row_val = list(line.values())[sign_col]
except IndexError:
continue
else:
continue
elif sign_col == 1: # 只有一个单独数据
row_val = line
else:
continue
if (row_val not in sign) if deny_sign else (row_val in sign):
if cols is True: # 获取整行
get_jsonl_row_key_is_True(line, res, ind, cols)
else: # 只获取对应的列
get_jsonl_row_key_not_True(line, res, ind, cols)
got += 1
def handle_line_jsonl(line):
line = line.strip()
if not line:
return None
try:
return loads(line)
except JSONDecodeError:
return line
def data2DataWithRow(data, row):
if isinstance(data, dict):
data = RowDict(row, data)
elif isinstance(data, list):
data = RowList(row, data)
elif isinstance(data, str):
data = RowStr(data)
data.row = row
elif isinstance(data, int):
data = RowStr(data)
data.row = row
elif isinstance(data, float):
data = RowFloat(data)
data.row = row
elif data is None:
data = RowNone(row)
return data
@@ -0,0 +1,122 @@
# -*- coding:utf-8 -*-
from typing import Any, Callable, Union, Optional, List, TextIO
from .base_handler import TextLikeHandler, BaseHandler
from ..data import *
class JSONLHandler(TextLikeHandler):
@property
def _add_lr_method(self) -> Callable:
"""向数据添加left和right数据的方法"""
...
def _handle_data(self, data: Any) -> list:
"""格式化数据,以列表格式返回
:param data: 初始数据
:return: 数据组成的列表
"""
...
def _record_fast(self) -> None:
"""执行快速记录到文件"""
...
def _record_slow(self) -> None:
"""执行慢速记录到文件"""
...
def handle_txt_lines(data_lst: list, lines: list) -> None:
"""txt、json、jsonl格式处理修改中间行时的逻辑
:param data_lst: 数据总列表
:param lines: readlines()从文件读取的原数据列表
:return: None
"""
...
def handle_jsonl_data(data: Union[dict, list]) -> None:
"""处理jsonl格式单个数据的方法,对应handle_txt_lines()的method
:param data: 要写入的数据
:return: None
"""
...
def get_jsonl_rows(handler: BaseHandler, cols: Union[list, True],
begin_row: Optional[int], end_row: Optional[int],
sign_col: Union[str, int, bool], sign: Any,
deny_sign: bool, count: int) -> List[RowData]:
"""获取csv文件指定行数据
:param handler: BaseHandler对象
:param cols: 要获取的列,为True获取所有,可指定多列
:param begin_row: 开始行号
:param end_row: 结束行号,None为最后一行
:param sign_col: 作为条件的列
:param sign: 按这个值筛选目标行,可设置多个
:param deny_sign: 是否反向匹配sign,即筛选指不是sign的行
:param count: 获取多少条数据,为None获取所有
:return: 获取到的数据列表
"""
...
def get_jsonl_row_key_is_True(line: Union[list, dict], res: list, ind: int, cols: list) -> None:
"""把jsonl文件一行所有列数据加入到结果集合中
:param line: dict或list格式的行数据
:param res: 获取到的所有行数据
:param ind: 行号
:param cols: 仅用于与另一个方法对应
:return: None
"""
...
def get_jsonl_row_key_not_True(line: Union[list, dict], res: list, ind: int, cols: list) -> None:
"""把jsonl文件一行指定列数据加入到结果集合中
:param line: dict或list格式的行数据
:param res: 获取到的所有行数据
:param ind: 行号
:param cols: 要获取的列,为str时用于dict数据,为int时可用于list和dict(序号从1开始)数据
:return: None
"""
...
def get_jsonl_rows_with_count(lines: Union[TextIO, list], begin_row: Optional[int], end_row: Optional[int],
sign_col: Union[str, int, bool], sign: Any, deny_sign: bool,
cols: Union[list, True], res: list, count: int, method: Callable) -> None:
"""执行从jsonl或json文件中获取数据,有指定数量。两种文件区别在于行数据传入格式
:param lines: jsonl文件传入文件读取对象,json文件传入读取到的list
:param begin_row: 开始行号
:param end_row: 结束行号,None为最后一行
:param sign_col: 作为条件的列
:param sign: 按这个值筛选目标行,可设置多个
:param deny_sign: 是否反向匹配sign,即筛选指不是sign的行
:param cols: 要获取的列,为True获取所有,可指定多列
:param res: 保存最终数据的list
:param count: 获取多少条数据,为None获取所有
:param method: 读取每行数据的方法,jsonl和json不一样
:return: None
"""
...
def handle_line_jsonl(line: str) -> Union[dict, list]:
"""读取数据时从文本解析json
:param line: 一行数据
:return: dict或list格式的数据
"""
...
def data2DataWithRow(data: Union[str, int, float, dict, list, None],
row: int) -> Union[RowInt, RowDict, RowList, RowStr, RowFloat, RowNone]:
"""把普通数据包装成行数据类型
:param data: 原数据
:param row: 行号
:return: 行数据类型
"""
...
@@ -0,0 +1,75 @@
# -*- coding:utf-8 -*-
from .base_handler import TextLikeHandler
from ..data import RowStr
from ..tools import get_real_row, data2str
class TXTHandler(TextLikeHandler):
def __init__(self, recorder):
super().__init__(recorder)
self.type = 'txt'
def rows(self, **kwargs):
begin_row, end_row, count = kwargs['begin_row'] or 1, kwargs['end_row'], kwargs['count']
begin_row -= 1
res = []
with open(self._recorder.path, 'r', encoding=self.encoding) as f:
try:
for i in range(begin_row):
next(f)
except StopIteration:
return res
got = 0
for ind, line in enumerate(f, begin_row + 1):
if (end_row and ind > end_row) or (count and got == count):
break
t = RowStr(line.strip('\n'))
t.row = ind
res.append(t)
got += 1
return res
def _handle_data(self, data):
if not isinstance(data, (list, tuple)):
data = [data]
elif not data:
data = ['']
return data
def _record_fast(self):
with open(self.path, 'a+', encoding=self.encoding) as f:
for data in self.data:
for d in data['data']:
f.write(f'{data2str(d)}\n')
def _record_slow(self):
if not self._recorder._file_exists and not self._recorder._path.exists():
with open(self.path, 'w', encoding=self.encoding):
pass
with open(self.path, 'r', encoding=self.encoding) as f:
lines = f.readlines()
slowData2list(self.data, lines)
with open(self.path, 'w', encoding=self.encoding) as f:
f.writelines(lines)
def slowData2list(data_lst, lines):
for data in data_lst:
if data['coord'] == 0:
[lines.extend([f'{dd}\n' for dd in data2str(d).split('\n')]) for d in data['data']]
continue
lines_len = len(lines)
num = get_real_row(data['coord'], lines_len)
cur_data = []
[cur_data.extend(data2str(d).split('\n')) for d in data['data']]
data_end = num + len(cur_data)
if lines_len < data_end:
diff = data_end - lines_len - 1
[lines.append('\n') for _ in range(diff)]
lines_len += diff
for num, i in enumerate(cur_data, num - 1):
lines[num] = f'{i}\n'
@@ -0,0 +1,44 @@
# -*- coding:utf-8 -*-
from typing import List, Any
from .base_handler import TextLikeHandler
from ..data import *
from ..recorder import Recorder
class TXTHandler(TextLikeHandler):
def __init__(self, recorder: Recorder):
"""
:param recorder: Recorder对象
"""
...
def rows(self, **kwargs) -> List[RowData, RowDict, RowList, RowStr]:
"""返回符合条件的行数据,可指定只要某些列
:param count: 获取多少条数据,为None获取所有
:param begin_row: 数据开始的行,None表示header_row后面一行
:param end_row: 数据结束的行,None表示最后一行
:return: 数据对象组成的列表
"""
...
def _handle_data(self, data:Any) -> list:
"""格式化数据,以列表格式返回"""
...
def _record_fast(self) -> None:
"""执行快速记录到文件"""
...
def _record_slow(self) -> None:
"""执行慢速记录到文件"""
...
def slowData2list(data_lst, lines) -> None:
"""处理要写入文件的数据
:param data_lst: 数据组成的列表
:param lines: 初始文件行
:return: None
"""
...
@@ -0,0 +1,786 @@
# -*- coding:utf-8 -*-
from openpyxl import load_workbook, Workbook
from .base_handler import SheetLikeHandler, parse_coord
from ..cell_style import CellStyle, NoneStyle, CellStyleCopier
from ..header import Header, ZeroHeader
from ..tools import process_content_xlsx, get_real_row, get_first_dict_xlsx, get_key_cols
class XLSXHandler(SheetLikeHandler):
def __init__(self, recorder):
super().__init__(recorder)
self.type = 'xlsx'
self._header = {None: None}
self._header_row = {None: 1}
self._data_col = 1
self._table = None
self._None_header_is_newest = None
self._None_header_row_is_newest = None
self._methods = {'img': img2ws,
'link': link2ws,
'style': styles2ws,
'height': height2ws,
'width': width2ws,
'data': data2ws}
@property
def table(self):
return self._table
@property
def tables(self):
if not self.path:
raise RuntimeError('未指定文件路径。')
return get_tables(self.path)
@property
def header_row(self):
return self._header_row.get(self._table, 1)
@property
def data_col(self):
return self._data_col
@property
def follow_styles(self):
return self._recorder._settings.get('new_row_styles', False)
@property
def new_row_height(self):
return self._recorder._settings.get('new_row_height', None)
@property
def new_row_styles(self):
return self._recorder._settings.get('new_row_styles', None)
@property
def link_style(self):
return self._recorder._settings.get('link_style', None)
def add_data(self, data, coord=None, table=None):
coord = self._parse_coord(coord)
data = self._handle_data(data)
if (self.data.get(table, None) and coord == self.data[table][-1]['coord']
and self.data[table][-1]['type'] == 'data'):
self.data[table][-1]['data'].extend(data)
else:
self.data.setdefault(table, []).append({'type': 'data', 'data': data, 'coord': coord})
def add_link(self, link, coord, content=None, table=None):
self.data.setdefault(table, []).append({'type': 'link', 'link': link, 'content': content,
'coord': self._parse_coord(coord)})
def add_img(self, img_path, coord, width=None, height=None, table=None):
self.data.setdefault(table, []).append({'type': 'img', 'imgPath': img_path, 'width': width, 'height': height,
'coord': self._parse_coord(coord)})
def add_styles(self, styles, coord=None, rows=None, cols=None, replace=True, table=None):
self.data.setdefault(table, []).append({'type': 'style', 'mode': 'replace' if replace else 'cover',
'styles': styles, 'coord': (1, 1), 'real_coord': coord, 'rows': rows,
'cols': cols})
def add_rows_height(self, height, rows=True, table=None):
self.data.setdefault(table, []).append({'type': 'height', 'rows': rows, 'height': height})
def add_cols_width(self, width, cols=True, table=None):
self.data.setdefault(table, []).append({'type': 'width', 'cols': cols, 'width': width})
def record(self):
wb, new_file = get_wb(self._recorder)
tables = wb.sheetnames
rewrite_method = 'make_num_dict_rewrite' if self.auto_new_header else 'make_num_dict'
for table, data in self.data.items():
ws, new_sheet = get_ws(wb, table, tables, new_file)
new_file = False
if table is None:
if self._None_header_is_newest or ws.title not in self._header:
self._header[ws.title] = self._header[None]
self._None_header_is_newest = None
if self._None_header_row_is_newest or ws.title not in self._header_row:
self._header_row[ws.title] = self._header_row[None]
begin_row = True
if new_sheet:
begin_row = handle_new_sheet(self, ws, data)
elif self._header.get(ws.title, None) is None:
self._header[ws.title] = (Header([c.value for c in ws[self._header_row[ws.title]]])
if ws.title in self._header_row else Header())
header = self._header[ws.title]
rewrite = False
if not begin_row and not data[0]['coord'][0]: # 首行为空,将数据填入首行
cur = data[0]
rewrite = self._methods[cur['type']](
**{'handler': self,
'ws': ws,
'data': cur,
'coord': (1, header._get_num(cur.get('coord', (1, 1))[1])),
'new_row': not cur.get('coord', (1, 1))[0],
'header': header,
'rewrite': rewrite,
'rewrite_method': rewrite_method})
data = data[1:]
for cur in data:
rewrite = self._methods[cur['type']](
**{'handler': self,
'ws': ws,
'data': cur,
'coord': get_ws_real_coord(cur.get('coord', (1, 1)), ws, header),
'new_row': not cur.get('coord', (1, 1))[0],
'header': header,
'rewrite': rewrite,
'rewrite_method': rewrite_method})
if rewrite:
for c in range(1, ws.max_column + 1):
ws.cell(self._header_row[ws.title], c, value=header[c])
wb.save(self.path)
wb.close()
def rows(self, cols=True, sign_col=True, signs=None, deny_sign=False, count=None, begin_row=None, end_row=None):
wb = load_workbook(self.path, data_only=True, read_only=True)
if self._table and self._table not in [i.title for i in wb.worksheets]:
raise RuntimeError(f'xlsx文件未包含指定工作表:{self._table}')
ws = wb[self._table] if self._table else wb.active
if ws.max_column is None: # 遇到过read_only时无法获取列数的文件
wb.close()
wb = load_workbook(self.path, data_only=True)
ws = wb[self._table] if self._table else wb.active
header = self._get_header(ws)
if sign_col is not True:
sign_col = header.get_num(sign_col) or 1
cols = get_key_cols(cols, header)
if not begin_row:
begin_row = self._header_row.get(self._table, self._header_row[None]) + 1
return get_xlsx_rows(header=header, cols=cols, begin_row=begin_row, end_row=end_row or 0,
sign_col=sign_col, sign=signs, deny_sign=deny_sign, count=count, ws=ws)
def clear(self):
self.data = {}
self.data_count = 0
def set_header(self, header, to_file=True, row=None, table=None):
if not isinstance(header, (list, tuple)):
raise ValueError('header必须为list或tuple格式。')
self._recorder.record()
row = row or self._header_row.get(table, 1)
with self._recorder._lock:
header = Header(header)
if table is None:
table = self._recorder.table
elif table is True:
table = None
elif not isinstance(table, str):
raise ValueError('table只能是None、True或str。')
self._None_header_is_newest = table is None
self._header[table] = header
if to_file:
set_xlsx_header(self._recorder, header, table, row)
return self._recorder._setter
def set_header_row(self, num, table=None):
if num < 0:
raise ValueError('num不能小于0。')
self._recorder.record()
with self._recorder._lock:
if table is None:
table = self._table
elif table is True:
table = None
elif not isinstance(table, str):
raise ValueError('table只能是None、True或str。')
self._header_row[table] = num
self._header[table] = ZeroHeader() if num == 0 else None
self._None_header_is_newest = table is None
self._None_header_row_is_newest = table is None
return self._recorder._setter
def set_data_col(self, col):
if col is None:
self._data_col = 0
elif not isinstance(col, (int, str)) or col == '':
raise TypeError('col值只能是int、None或非空str。')
else:
self._data_col = col
return self._recorder._setter
def set_table(self, name):
self._table = name if name is not True else None
return self._recorder._setter
def set_new_row_height(self, height):
self._recorder._settings['new_row_height'] = height
if height is not None:
self._recorder._settings['follow_styles'] = False
self._methods['data'] = data2ws_style
else:
self._methods['data'] = data2ws
return self._recorder._setter
def set_new_row_styles(self, styles):
self._recorder.record()
self._recorder._settings['new_row_styles'] = styles
if styles is not None:
self._recorder._settings['follow_styles'] = False
self._methods['data'] = data2ws_style
else:
self._methods['data'] = data2ws
return self._recorder._setter
def set_follow_styles(self, on_off=True):
self._recorder._settings['follow_styles'] = on_off
if on_off:
self._recorder._settings['styles'] = None
self._recorder._settings['new_row_height'] = None
self._methods['data'] = data2ws_follow
else:
self._methods['data'] = data2ws
return self._recorder._setter
def set_link_style(self, style=True):
if style is True:
style = CellStyle()
style.font.set_color("0000FF")
style.font.set_underline('single')
self._recorder._settings['link_style'] = style
return self._recorder._setter
def _get_header(self, ws=None):
header = self._header.get(self._table, None)
if header is not None:
return header
if not self.path or not self._recorder._path.exists():
return None
if not ws:
wb = load_workbook(self.path)
if not self.table:
ws = wb.active
elif self.table not in wb.sheetnames:
wb.close()
return Header()
else:
ws = wb[self.table]
else:
wb = None
header_row = self._header_row.get(self.table, self._header_row[None])
if header_row > ws.max_row:
self._header[self.table] = Header()
else:
self._header[self.table] = Header([i.value for i in ws[self._header_row.get(self.table,
self._header_row[None])]])
if wb:
wb.close()
return self._header[self.table]
def set_xlsx_header(recorder, header, table, row):
if not recorder.path:
raise FileNotFoundError('未指定文件。')
if recorder._file_exists or recorder._path.exists():
wb = load_workbook(recorder.path)
if table:
ws = wb[table] if table in [i.title for i in wb.worksheets] else wb.create_sheet(title=table)
else:
ws = wb.active
else:
recorder._path.parent.mkdir(parents=True, exist_ok=True)
wb = Workbook()
ws = wb.active
if table:
ws.title = table
for c, i in header.items():
ws.cell(row, c, value=process_content_xlsx(i))
len_row = len(ws[row])
len_header = len(header)
if len_row > len_header:
for c in range(len_header + 1, len_row + 1):
ws.cell(row, c, value=None)
wb.save(recorder.path)
wb.close()
recorder._file_exists = True
def line2ws(ws, header, row, col, data, rewrite_method, rewrite):
if isinstance(data, dict):
data, rewrite, header_len = header.__getattribute__(rewrite_method)(data, 'xlsx', rewrite)
for c, val in data.items():
ws.cell(row, c, value=process_content_xlsx(val))
else:
for key, val in enumerate(data):
ws.cell(row, col + key, value=process_content_xlsx(val))
return rewrite
def line2ws_follow(ws, header, row, col, data, rewrite_method, rewrite, styles, height, new_row):
if new_row:
styles2new_row(ws, styles.values(), height, row)
if isinstance(data, dict):
data, rewrite, header_len = header.__getattribute__(rewrite_method)(data, 'xlsx', rewrite)
if new_row:
for c, val in data.items():
ws.cell(row, c, value=process_content_xlsx(val))
else:
for c, val in data.items():
styles.get(c, NoneStyle()).to_cell(ws.cell(row, c, value=process_content_xlsx(val)))
else:
if new_row:
for key, val in enumerate(data):
ws.cell(row, col + key, value=process_content_xlsx(val))
else:
for key, val in enumerate(data):
col1 = col + key
styles.get(col1, NoneStyle()).to_cell(ws.cell(row, col1, value=process_content_xlsx(val)))
return rewrite
def data2ws(handler, ws, data, coord, header, rewrite, rewrite_method, new_row):
row, col = coord
for r, d in enumerate(data['data'], row):
rewrite = line2ws(ws, header, r, col, d, rewrite_method, rewrite)
return rewrite
def data2ws_follow(handler, ws, data, coord, header, rewrite, rewrite_method, new_row):
row, col = coord
if row > 1:
styles = {ind: CellStyleCopier(cell) for ind, cell in enumerate(ws[row - 1], 1)}
height = ws.row_dimensions[row - 1].height
for r, d in enumerate(data['data'], row):
rewrite = line2ws_follow(ws, header, r, col, d, rewrite_method, rewrite, styles, height, new_row)
else:
for r, d in enumerate(data['data'], row):
rewrite = line2ws(ws, header, r, col, d, rewrite_method, rewrite)
return rewrite
def data2ws_style(handler, ws, data, coord, header, rewrite, rewrite_method, new_row):
row, col = coord
if new_row:
styles = handler.new_row_styles
if isinstance(styles, dict):
styles = header.make_num_dict(styles, None)[0]
styles = [styles.get(c, None) for c in range(1, ws.max_column + 1)]
elif isinstance(styles, CellStyle):
styles = [styles] * ws.max_column
height = ws.row_dimensions[row].height
for r, d in enumerate(data['data'], row):
rewrite = line2ws(ws, header, r, col, d, rewrite_method, rewrite)
styles2new_row(ws, styles, height, r)
else:
for r, d in enumerate(data['data'], row):
rewrite = line2ws(ws, header, r, col, d, rewrite_method, rewrite)
return rewrite
def styles2new_row(ws, styles, height, row):
if height is not None:
ws.row_dimensions[row].height = height
if styles:
for c, s in enumerate(styles, start=1):
if s:
s.to_cell(ws.cell(row=row, column=c))
def styles2ws(**kwargs):
ws = kwargs['ws']
header = kwargs['header']
data = kwargs['data']
styles = data['styles']
coord = data['real_coord'] # 'A3'、'A1:C3'、(1, 3)、['A1', 'B2', 'C3']
rows = data['rows'] # 3、'1:3'、[1, 2, 3]
cols = data['cols']
mode = data['mode'] == 'replace'
if not styles:
styles = [NoneStyle()]
elif isinstance(styles, CellStyle):
styles = [styles]
if isinstance(styles, dict):
for coord, val in styles.items():
styles2ws(ws=ws, header=header,
data={'styles': val, 'real_coord': coord, 'rows': None, 'cols': None, 'mode': mode})
return
if rows:
if isinstance(rows, int):
styles_len = len(styles)
for col, cell in enumerate(ws[header.get_num(rows)]):
styles[col % styles_len].to_cell(cell, replace=mode)
elif isinstance(rows, str) and ':' in rows:
begin, end = rows.split(':', 1)
try:
begin = header.get_num(int(begin))
end = header.get_num(int(end))
except ValueError:
raise ValueError('行号必须是数字,现在是:', rows)
if begin > end:
begin, end = end, begin
for i in range(begin, end + 1):
styles2ws(ws=ws, header=header,
data={'styles': styles, 'real_coord': None, 'rows': i, 'cols': None, 'mode': mode})
elif isinstance(rows, (tuple, list)):
for i in rows:
styles2ws(ws=ws, header=header,
data={'styles': styles, 'real_coord': None, 'rows': i, 'cols': None, 'mode': mode})
if cols:
if isinstance(cols, int): # 列序号
styles_len = len(styles)
for col, cell in enumerate(ws[header.get_col(cols)]):
styles[col % styles_len].to_cell(cell, replace=mode)
elif isinstance(cols, str): # 表头值
cols = header.get_num(cols)
if cols:
styles2ws(ws=ws, header=header,
data={'styles': styles, 'real_coord': None, 'rows': None, 'cols': cols, 'mode': mode})
elif isinstance(cols, tuple) and len(cols) == 2:
begin, end = cols
begin = header.get_num(begin)
end = header.get_num(end)
if begin > end:
begin, end = end, begin
for i in range(begin, end + 1):
styles2ws(ws=ws, header=header,
data={'styles': styles, 'real_coord': None, 'rows': None, 'cols': i, 'mode': mode})
elif isinstance(cols, (tuple, list)):
for i in cols:
styles2ws(ws=ws, header=header,
data={'styles': styles, 'real_coord': None, 'rows': None, 'cols': i, 'mode': mode})
if coord:
if isinstance(coord, str):
if ':' in coord:
begin, end = coord.split(':', 1)
begin = parse_coord(begin)
end = parse_coord(end)
begin = f'{header.get_col(begin[1])}{begin[0]}'
end = f'{header.get_col(end[1])}{end[0]}'
styles_len = len(styles)
for row in ws[f'{begin}:{end}']:
for col, cell in enumerate(row):
styles[col % styles_len].to_cell(cell, replace=mode)
else:
coord = parse_coord(coord)
coord = f'{header.get_col(coord[1])}{coord[0]}'
styles[0].to_cell(ws[coord], replace=mode)
elif isinstance(coord, tuple) and len(coord) == 2:
coord = parse_coord(coord)
coord = f'{header.get_col(coord[1])}{coord[0]}'
styles[0].to_cell(ws[coord], replace=mode)
elif isinstance(coord, (tuple, list)):
for i in coord:
styles2ws(ws=ws, header=header,
data={'styles': styles, 'real_coord': i, 'rows': None, 'cols': None, 'mode': mode})
def link2ws(**kwargs):
handler = kwargs['handler']
data = kwargs['data']
cell = kwargs['ws'].cell(*kwargs['coord'])
has_link = bool(cell.hyperlink)
cell.hyperlink = data['link']
if data['content'] is not None:
cell.value = process_content_xlsx(data['content'])
if data['link']:
if handler.link_style:
handler.link_style.to_cell(cell, replace=False)
elif has_link:
NoneStyle().to_cell(cell, replace=False)
def img2ws(**kwargs):
row, col = kwargs['coord']
data = kwargs['data']
ws = kwargs['ws']
from openpyxl.drawing.image import Image
img = Image(data['imgPath'])
width, height = data['width'], data['height']
if width and height:
img.width = width
img.height = height
elif width:
img.height = int(img.height * (width / img.width))
img.width = width
elif height:
img.width = int(img.width * (height / img.height))
img.height = height
# ws.add_image(img, (row, Header._NUM_KEY[col]))
ws.add_image(img, f'{Header._NUM_KEY[col]}{row}')
def width2ws(**kwargs):
# 用int表示列序号,str表示表头值,用tuple设置某列到某列,用list指定每一列,为Ture设置所有列
cols = kwargs['data']['cols']
width = kwargs['data']['width']
ws = kwargs['ws']
header = kwargs['header']
if isinstance(width, dict):
for col, val in width.items():
width2ws(ws=ws, header=header, data={'cols': col, 'width': val})
elif isinstance(cols, (int, str)): # 表头值或列序号
cols = header.get_col(cols)
if cols:
ws.column_dimensions[cols].width = width
elif isinstance(cols, tuple) and len(cols) == 2: # 连续多列
beg, end = cols
if not beg:
beg = 1
if not end:
end = -1
beg = header.get_num(beg)
end = header.get_num(end)
if beg and end:
if beg > end:
beg, end = end, beg
for c in range(beg, end + 1):
ws.column_dimensions[header.get_col(c)].width = width
elif isinstance(cols, list):
for col in cols:
width2ws(ws=ws, header=header, data={'cols': col, 'width': width})
elif cols is True:
for col in range(1, ws.max_column + 1):
ws.column_dimensions[ZeroHeader()[col]].width = width
def height2ws(**kwargs):
# int表示行号,str为'1:3'格式,dict可指定行独立设置高度,list或tuple指定多行设置同一个高度。True设置所有行
rows = kwargs['data']['rows']
height = kwargs['data']['height']
ws = kwargs['ws']
if isinstance(height, dict):
for row, val in height.items():
height2ws(ws=ws, data={'rows': row, 'height': val})
elif isinstance(rows, int):
if rows < 1:
rows = get_real_row(rows, ws.max_row)
ws.row_dimensions[rows].height = height
elif isinstance(rows, str) and ':' in rows:
beg, end = rows.split(':', 1)
if beg == '':
beg = 1
if end == '':
end = -1
beg = int(beg)
end = int(end)
if beg < 1 or end < 1:
max_row = ws.max_row
beg = get_real_row(beg, max_row)
end = get_real_row(end, max_row)
if beg > end:
beg, end = end, beg
for c in range(beg, end + 1):
ws.row_dimensions[c].height = height
elif isinstance(rows, (list, tuple)):
for row in rows:
height2ws(ws=ws, data={'rows': row, 'height': height})
elif rows is True:
for i in range(1, ws.max_row + 1):
ws.row_dimensions[i].height = height
def get_wb(recorder):
if recorder._file_exists or recorder._path.exists():
wb = load_workbook(recorder.path)
new_file = False
else:
wb = Workbook()
new_file = True
return wb, new_file
def get_ws(wb, table, tables, new_file):
new_sheet = new_file
if table is None:
ws = wb.active
if ws.max_row == 1 and ws.max_column == 1 and ws.cell(row=1, column=1).value is None:
new_sheet = True
elif table in tables:
ws = wb[table]
if ws.max_row == 1 and ws.max_column == 1 and not ws.cell(row=1, column=1).value:
new_sheet = True
elif new_file is True:
ws = wb.active
tables.remove(ws.title)
ws.title = table
tables.append(table)
new_sheet = True
else:
ws = wb.create_sheet(title=table)
tables.append(table)
new_sheet = True
return ws, new_sheet
def handle_new_sheet(handler, ws, data):
if not handler._header_row:
return 0
if handler._header.get(ws.title, None) is not None:
for c, h in handler._header[ws.title].items():
ws.cell(row=handler._header_row[ws.title], column=c, value=h)
begin_row = handler._header_row
else:
data = get_first_dict_xlsx(data)
if data:
header = Header([h for h in data.keys() if isinstance(h, str)])
handler._header[ws.title] = header
for c, h in header.items():
ws.cell(row=handler._header_row[ws.title], column=c, value=h)
begin_row = handler._header_row
else:
handler._header[ws.title] = Header()
begin_row = 0
return begin_row
def get_ws_real_coord(coord, ws, header):
row, col = coord
if row <= 0:
row = ws.max_row + row + 1
return 1 if row < 1 else row, header._get_num(col)
def get_xlsx_rows(header, cols, begin_row, end_row, sign_col, sign, deny_sign, count, ws):
rows = ws.rows
try:
for _ in range(begin_row - 1):
next(rows)
except StopIteration:
return []
if sign_col is True or sign_col > ws.max_column: # 获取所有行
if count or end_row:
rows = list(rows)[:(min(count, end_row - begin_row + 1)
if count and end_row else (count or end_row - begin_row + 1))]
if cols is True: # 获取整行
res = [header.make_row_data(ind, {col: cell.value for col, cell in enumerate(row, 1)})
for ind, row in enumerate(rows, begin_row)]
else: # 只获取对应的列
res = [header.make_row_data(ind, {col: row[col - 1].value for col in cols})
for ind, row in enumerate(rows, begin_row)]
else: # 获取符合条件的行
if count:
res = get_xlsx_rows_with_count(cols, deny_sign, header, rows,
begin_row, end_row, sign_col, sign, count)
else:
res = get_xlsx_rows_without_count(cols, deny_sign, header, rows, begin_row, end_row,
sign_col, sign)
ws.parent.close()
return res
def get_xlsx_rows_with_count(cols, deny_sign, header, rows, begin_row, end_row, sign_col, sign, count):
got = 0
res = []
if cols is True: # 获取整行
if deny_sign:
for ind, row in enumerate(rows, begin_row):
if got == count or (end_row and ind > end_row):
break
if row[sign_col - 1].value not in sign:
res.append(header.make_row_data(ind, {col: cell.value for col, cell in enumerate(row, 1)}))
got += 1
else:
for ind, row in enumerate(rows, begin_row):
if got == count or (end_row and ind > end_row):
break
if row[sign_col - 1].value in sign:
res.append(header.make_row_data(ind, {col: cell.value for col, cell in enumerate(row, 1)}))
got += 1
else: # 只获取对应的列
if deny_sign:
for ind, row in enumerate(rows, begin_row):
if got == count or (end_row and ind > end_row):
break
if row[sign_col - 1].value not in sign:
res.append(header.make_row_data(ind, {col: row[col - 1].value for col in cols}))
got += 1
else:
for ind, row in enumerate(rows, begin_row):
if got == count or (end_row and ind > end_row):
break
if row[sign_col - 1].value in sign:
res.append(header.make_row_data(ind, {col: row[col - 1].value for col in cols}))
got += 1
return res
def get_xlsx_rows_without_count(cols, deny_sign, header, rows, begin_row, end_row, sign_col, sign):
if end_row:
if end_row < begin_row:
return []
rows = list(rows)[:end_row - begin_row + 1]
if cols is True: # 获取整行
if deny_sign:
return [header.make_row_data(ind, {col: cell.value for col, cell in enumerate(row, 1)})
for ind, row in enumerate(rows, begin_row)
if row[sign_col - 1].value not in sign]
else:
return [header.make_row_data(ind, {col: cell.value for col, cell in enumerate(row, 1)})
for ind, row in enumerate(rows, begin_row)
if row[sign_col - 1].value in sign]
else: # 只获取对应的列
if deny_sign:
return [header.make_row_data(ind, {col: row[col - 1].value for col in cols})
for ind, row in enumerate(rows, begin_row)
if row[sign_col - 1].value not in sign]
else:
return [header.make_row_data(ind, {col: row[col - 1].value for col in cols})
for ind, row in enumerate(rows, begin_row)
if row[sign_col - 1].value in sign]
def get_tables(path):
wb = load_workbook(path)
tables = wb.sheetnames
wb.close()
return tables
@@ -0,0 +1,281 @@
# -*- coding:utf-8 -*-
from pathlib import Path
from typing import Dict, Optional, Callable, Tuple, Literal, Union, Any, List, Iterable
from openpyxl import Workbook
from openpyxl.worksheet.worksheet import Worksheet
from .base_handler import SheetLikeHandler, BaseHandler
from ..cell_style import CellStyleCopier, CellStyle
from ..data import RowData
from ..header import Header
from ..recorder import Recorder
REWRITE_METHOD = Literal['make_num_dict_rewrite', 'make_num_dict']
class XLSXHandler(SheetLikeHandler):
type: str = ...
_header: Dict[Optional[str], Optional[Header]] = ...
_header_row: Dict[Optional[str], int] = ...
_data_col: int = ...
_table: Optional[str] = ...
_None_header_is_newest: Optional[bool] = ...
_None_header_row_is_newest: Optional[bool] = ...
_methods: Dict[str, Callable] = ...
def __init__(self, recorder: Recorder):
""""""
...
@property
def follow_styles(self) -> bool:
""""""
...
@property
def new_row_height(self) -> Optional[float]:
""""""
...
@property
def new_row_styles(self) -> Union[CellStyle, List[CellStyle], None]:
""""""
...
@property
def link_style(self) -> Optional[CellStyle]:
""""""
...
@property
def header_row(self) -> Dict[Optional[str], Optional[Header]]:
...
def _get_header(self, ws: Worksheet = None) -> Header:
"""获取当前指定的table的header
:param ws: Worksheet对象
:return: Header对象
"""
...
def set_xlsx_header(recorder: Recorder,
header: Header,
table: str,
row: int) -> None:
"""设置xlsx文件的表头
:param recorder: Recorder对象
:param header: 表头列表或元组
:param table: 工作表名称
:param row: 行号
:return: None
"""
...
def line2ws(ws: Worksheet, header: Header, row: int, col: int, data: Union[dict, list], rewrite_method: REWRITE_METHOD,
rewrite: bool) -> bool:
"""把一行数据写入数据表,不设置样式
:param ws: Worksheet对象
:param header: Header对象
:param row: 行号
:param col: 列序号
:param data: 行数据
:param rewrite_method: 'make_num_dict_rewrite'或'make_num_dict'
:param rewrite: 是否重写表头
:return: 是否重写表头
"""
...
def line2ws_follow(ws: Worksheet, header: Header, row: int, col: int, data: Union[dict, list],
rewrite_method: REWRITE_METHOD, rewrite: bool, styles: Dict[int, CellStyleCopier],
height: Optional[float], new_row: bool) -> bool:
"""把一行数据写入数据表,并设置样式
:param ws: Worksheet对象
:param header: Header对象
:param row: 行号
:param col: 列序号
:param data: 行数据
:param rewrite_method: 'make_num_dict_rewrite'或'make_num_dict'
:param rewrite: 是否重写表头
:param styles: 样式对象了列表
:param height: 行高,仅新行时有效
:param new_row: 是否新行
:return: 是否重写表头
"""
...
def data2ws(recorder: Recorder, ws: Worksheet, data: dict, coord: Tuple[int, int],
header: Header, rewrite: bool, rewrite_method: REWRITE_METHOD) -> bool:
"""数据写入数据表
:param recorder: Recorder对象
:param ws: Worksheet对象
:param data: 标准数据 {'type': 'data', 'data': [(1, 2, 3, 4)], 'coord': (0, 1)}
:param coord: 要写入的坐标
:param header: Header对象
:param rewrite: 是否重写表头
:param rewrite_method: 'make_num_dict_rewrite'或'make_num_dict'
:return: 是否重写表头
"""
...
def data2ws_follow(handler: BaseHandler, ws: Worksheet, data: dict, coord: Tuple[int, int],
header: Header, rewrite: bool, rewrite_method: REWRITE_METHOD) -> None:
"""数据写入数据表,跟随上一行样式
:param handler: BaseHandler对象
:param ws: Worksheet对象
:param data: 标准数据 {'type': 'data', 'data': [(1, 2, 3, 4)], 'coord': (0, 1)}
:param coord: 要写入的坐标
:param header: Header对象
:param rewrite: 是否重写表头
:param rewrite_method: 'make_num_dict_rewrite'或'make_num_dict'
:return: 是否重写表头
"""
...
def data2ws_style(handler: BaseHandler, ws: Worksheet, data: dict, coord: Tuple[int, int],
header: Header, rewrite: bool, rewrite_method: REWRITE_METHOD) -> None:
"""数据写入数据表,并设置指定样式
:param handler: BaseHandler对象
:param ws: Worksheet对象
:param data: 标准数据 {'type': 'data', 'data': [(1, 2, 3, 4)], 'coord': (0, 1)}
:param coord: 要写入的坐标
:param header: Header对象
:param rewrite: 是否重写表头
:param rewrite_method: 'make_num_dict_rewrite'或'make_num_dict'
:return: 是否重写表头
"""
...
def styles2new_row(ws, styles, height, row):
""""""
...
def styles2ws(**kwargs) -> None:
"""把样式写入数据表"""
...
def link2ws(**kwargs) -> None:
"""把擦后入到单元格"""
...
def img2ws(**kwargs) -> None:
"""把图片到单元格"""
...
def width2ws(**kwargs) -> None:
"""把列宽设置到数据表"""
...
def height2ws(**kwargs) -> None:
"""把行高设置到数据表"""
...
def get_wb(recorder: Recorder) -> Tuple[Workbook, bool]:
"""获取Workbook对象
:param recorder: Recorder对象
:return: (Workbook对象, 是否新文件)
"""
...
def get_ws(wb: Workbook, table: Optional[str], tables: List[str], new_file: bool) -> Tuple[Worksheet, bool]:
"""获取Worksheet对象
:param wb: Workbook对象
:param table: 表名,None代表活动表格
:param tables: 工作簿所有表名组成的列表
:param new_file: 是否新文件
:return: (Worksheet对象, 是否新文件)
"""
...
def handle_new_sheet(handler: BaseHandler, ws: Worksheet, data: list) -> int:
"""从设置或第一条dict数据获取表头并向新表写入
:param handler: BaseHandler对象
:param ws: 数据表对象
:param data: 对应数据表的数据列表
:return: 开始写数据的行的前一行
"""
...
def get_ws_real_coord(coord: tuple, ws: Worksheet, header: Header) -> Tuple[int, int]:
"""返回真正写入xlsx文件的坐标
:param coord: 已初步格式化的坐标,如(1, 2)、(0, 3)、(-3, -2)
:param ws: Worksheet对象
:param header: Header对象
:return: 真正写入文件的坐标,tuple格式
"""
...
def get_xlsx_rows(header: Header, cols: Union[list, True],
begin_row: Optional[int], end_row: Optional[int],
sign_col: Union[str, int, bool], sign: Any,
deny_sign: bool, count: int, ws: Worksheet) -> List[RowData]:
"""获取xlsx文件指定行数据
:param header: Header对象
:param cols: 要获取的列,为True获取所有,可指定多列
:param begin_row: 开始行号
:param end_row: 结束行号,None为最后一行
:param sign_col: 作为条件的列
:param sign: 按这个值筛选目标行,可设置多个
:param deny_sign: 是否反向匹配sign,即筛选指不是sign的行
:param count: 获取多少条数据,为None获取所有
:param ws: Worksheet对象
:return: 获取到的数据列表
"""
...
def get_xlsx_rows_with_count(cols: Union[list, True], deny_sign: bool, header: Header, rows: Iterable,
begin_row: Optional[int], end_row: Optional[int],
sign_col: Union[str, int, bool], sign: Any, count: int) -> List[RowData]:
"""执行从xlsx中获取数据,有指定数量
:param cols: 要获取的列,True为所有
:param deny_sign: 是否反向匹配sign,即筛选指不是sign的行
:param header: Header对象
:param rows: 行组成的列表
:param begin_row: 开始行号
:param end_row: 结束行号,None为最后一行
:param sign_col: 用于筛选数据的列
:param sign: 用于筛选数据的值
:param count: 数据总条数
:return: 数据对象列表
"""
...
def get_xlsx_rows_without_count(cols: Union[list, True], deny_sign: bool, header: Header, rows: Iterable,
begin_row: Optional[int], end_row: Optional[int],
sign_col: Union[str, int, bool], sign: Any) -> List[RowData]:
"""执行从xlsx中获取全部数据
:param cols: 要获取的列,True为所有
:param deny_sign: 是否反向匹配sign,即筛选指不是sign的行
:param header: Header对象
:param rows: 行组成的列表
:param begin_row: 开始行号
:param end_row: 结束行号,None为最后一行
:param sign_col: 用于筛选数据的列
:param sign: 用于筛选数据的值
:return: 数据对象列表
"""
...
def get_tables(path: Path) -> List[str]:
"""获取所有表格名称"""
...