Initial commit

This commit is contained in:
Alex committed 2026-11-02 22:54:14 +08:00
commit 57fc3525eb
7311 files changed
+1291398

No files matched your search

+143
View File
@@ -0,0 +1,143 @@
"""
app/utils/similarity.py - 文本相似度计算工具
用于验证提取器提取结果的准确性
"""
import re
from difflib import SequenceMatcher
from typing import Tuple
def normalize_text(text: str) -> str:
"""
标准化文本(用于比对)
处理:
- 统一换行符
- 压缩连续空白
- 去除首尾空白
"""
if not text:
return ""
# 统一换行符
text = text.replace('\r\n', '\n').replace('\r', '\n')
# 压缩连续空白(保留单个换行)
text = re.sub(r'[ \t]+', ' ', text)
text = re.sub(r'\n{3,}', '\n\n', text)
# 去除首尾空白
return text.strip()
def calculate_similarity(text1: str, text2: str, normalize: bool = True) -> float:
"""
计算两段文本的相似度
Args:
text1: 第一段文本
text2: 第二段文本
normalize: 是否先标准化文本
Returns:
相似度 (0.0 - 1.0)
"""
if normalize:
text1 = normalize_text(text1)
text2 = normalize_text(text2)
if not text1 and not text2:
return 1.0
if not text1 or not text2:
return 0.0
# 使用 SequenceMatcher 计算相似度
matcher = SequenceMatcher(None, text1, text2)
return matcher.ratio()
def verify_extraction(extracted: str, expected: str, threshold: float = 0.95) -> Tuple[bool, float, str]:
"""
验证提取结果
Args:
extracted: 提取的文本
expected: 预期的文本
threshold: 通过阈值(默认 0.95)
Returns:
(是否通过, 相似度, 消息)
"""
similarity = calculate_similarity(extracted, expected)
passed = similarity >= threshold
if passed:
message = f"验证通过 (相似度: {similarity:.1%})"
else:
# 计算差异信息
extracted_len = len(normalize_text(extracted))
expected_len = len(normalize_text(expected))
len_diff = abs(extracted_len - expected_len)
message = (
f"验证未通过 (相似度: {similarity:.1%}, "
f"提取长度: {extracted_len}, 预期长度: {expected_len}, "
f"差异: {len_diff} 字符)"
)
return passed, similarity, message
def get_diff_summary(text1: str, text2: str, context_chars: int = 50) -> str:
"""
获取差异摘要
Args:
text1: 第一段文本
text2: 第二段文本
context_chars: 上下文字符数
Returns:
差异摘要字符串
"""
t1 = normalize_text(text1)
t2 = normalize_text(text2)
if t1 == t2:
return "完全相同"
# 找到第一个不同的位置
min_len = min(len(t1), len(t2))
first_diff = -1
for i in range(min_len):
if t1[i] != t2[i]:
first_diff = i
break
if first_diff == -1:
first_diff = min_len
# 获取差异上下文
start = max(0, first_diff - context_chars)
context1 = t1[start:first_diff + context_chars] if first_diff < len(t1) else t1[start:]
context2 = t2[start:first_diff + context_chars] if first_diff < len(t2) else t2[start:]
return (
f"首个差异位置: {first_diff}\n"
f"文本1 ({len(t1)} 字符): ...{repr(context1)}...\n"
f"文本2 ({len(t2)} 字符): ...{repr(context2)}..."
)
__all__ = [
'normalize_text',
'calculate_similarity',
'verify_extraction',
'get_diff_summary'
]