Initial commit
This commit is contained in:
commit
57fc3525eb
7311 files changed
+1291398
No files matched your search
@@ -0,0 +1,693 @@
|
||||
"""
|
||||
doubao_parser.py - Doubao SSE response parser.
|
||||
|
||||
Observed stream traits:
|
||||
- event: STREAM_MSG_NOTIFY carries the first assistant block
|
||||
- event: STREAM_CHUNK may carry extra content_block patches
|
||||
- event: CHUNK_DELTA carries plain text deltas
|
||||
- event: SSE_REPLY_END marks the answer as finished
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
from typing import Any, Dict, List, Tuple
|
||||
|
||||
from app.core.config import logger
|
||||
from .base import ResponseParser
|
||||
|
||||
|
||||
class DoubaoParser(ResponseParser):
|
||||
"""Parse Doubao's SSE response stream."""
|
||||
|
||||
# 生成图片以 image_ori / image_thumb 等字段挂在 SSE 事件里,
|
||||
# 同一张图的各尺寸共享 tos 路径(差异仅在 ~tplv 模板与签名参数)。
|
||||
_IMAGE_FIELD_RE = re.compile(
|
||||
r'"(image_ori|image_ori_raw|image_preview_resize|image_preview|image_thumb)"\s*:\s*'
|
||||
r'\{[^{}]*?"url"\s*:\s*"(https?:\\?/\\?/[^"\s]+)"'
|
||||
)
|
||||
_IMAGE_KEY_PRIORITY = {
|
||||
"image_ori": 0,
|
||||
"image_ori_raw": 1,
|
||||
"image_preview": 2,
|
||||
"image_preview_resize": 3,
|
||||
"image_thumb": 4,
|
||||
}
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._last_raw = ""
|
||||
self._pending = ""
|
||||
self._block_texts: Dict[str, str] = {}
|
||||
self._last_full_message = ""
|
||||
self._assembled_text = ""
|
||||
self._has_seen_visible_text = False
|
||||
self._seen_image_keys: set = set()
|
||||
|
||||
def _collect_image_urls(self, new_data: str) -> List[str]:
|
||||
"""从新的 SSE 数据中提取生成图片 URL,按图片去重、尺寸优先级取最优。"""
|
||||
found: Dict[str, Tuple[int, str]] = {}
|
||||
for match in self._IMAGE_FIELD_RE.finditer(new_data):
|
||||
key, raw_url = match.group(1), match.group(2)
|
||||
url = raw_url.replace(r"\/", "/")
|
||||
# 去掉主机与签名参数/裁剪模板,同一张图的不同 CDN 副本视为一条
|
||||
base_key = re.sub(r"^https?://[^/]+/", "", url).split("?", 1)[0].split("~tplv", 1)[0]
|
||||
priority = self._IMAGE_KEY_PRIORITY.get(key, 99)
|
||||
current = found.get(base_key)
|
||||
if current is None or priority < current[0]:
|
||||
found[base_key] = (priority, url)
|
||||
fresh: List[str] = []
|
||||
for base_key, (_, url) in found.items():
|
||||
if base_key in self._seen_image_keys:
|
||||
continue
|
||||
self._seen_image_keys.add(base_key)
|
||||
fresh.append(url)
|
||||
return fresh
|
||||
|
||||
def parse_chunk(self, raw_response: str) -> Dict[str, Any]:
|
||||
result: Dict[str, Any] = {
|
||||
"content": "",
|
||||
"images": [],
|
||||
"done": False,
|
||||
"error": None,
|
||||
}
|
||||
|
||||
try:
|
||||
if isinstance(raw_response, (bytes, bytearray)):
|
||||
raw_response = raw_response.decode("utf-8", errors="ignore")
|
||||
elif not isinstance(raw_response, str):
|
||||
raw_response = str(raw_response)
|
||||
|
||||
if not raw_response:
|
||||
return result
|
||||
|
||||
if self._last_raw and raw_response == self._last_raw:
|
||||
return result
|
||||
|
||||
if self._last_raw and raw_response.startswith(self._last_raw):
|
||||
new_data = raw_response[len(self._last_raw):]
|
||||
else:
|
||||
new_data = raw_response
|
||||
|
||||
self._last_raw = raw_response
|
||||
|
||||
result["images"] = self._collect_image_urls(new_data)
|
||||
|
||||
direct_content, direct_done = self._parse_direct_payload(new_data)
|
||||
if direct_content or direct_done:
|
||||
repaired = self._repair_text(direct_content)
|
||||
result["content"] = repaired
|
||||
if repaired:
|
||||
self._assembled_text += repaired
|
||||
self._has_seen_visible_text = True
|
||||
result["done"] = bool(direct_done and self._can_accept_done_signal(repaired))
|
||||
return result
|
||||
|
||||
delta_content, done = self._consume_new_data(new_data)
|
||||
if delta_content:
|
||||
repaired = self._repair_text(delta_content)
|
||||
if repaired != delta_content and self._assembled_text.endswith(delta_content):
|
||||
self._assembled_text = self._assembled_text[:-len(delta_content)] + repaired
|
||||
result["content"] = repaired
|
||||
result["done"] = done
|
||||
|
||||
except Exception as e:
|
||||
logger.debug(f"[DoubaoParser] parse exception: {e}")
|
||||
result["error"] = str(e)
|
||||
|
||||
return result
|
||||
|
||||
def reset(self) -> None:
|
||||
self._last_raw = ""
|
||||
self._pending = ""
|
||||
self._block_texts.clear()
|
||||
self._last_full_message = ""
|
||||
self._assembled_text = ""
|
||||
self._has_seen_visible_text = False
|
||||
self._seen_image_keys.clear()
|
||||
|
||||
def _consume_new_data(self, new_data: str) -> Tuple[str, bool]:
|
||||
normalized = (self._pending + new_data).replace("\r\n", "\n")
|
||||
if not normalized:
|
||||
return "", False
|
||||
|
||||
blocks = normalized.split("\n\n")
|
||||
if normalized.endswith("\n\n"):
|
||||
self._pending = ""
|
||||
complete_blocks = [block for block in blocks if block.strip()]
|
||||
else:
|
||||
self._pending = blocks.pop() if blocks else normalized
|
||||
complete_blocks = [block for block in blocks if block.strip()]
|
||||
|
||||
content_parts: List[str] = []
|
||||
done = False
|
||||
|
||||
for block in complete_blocks:
|
||||
block_content, block_done = self._parse_event_block(block)
|
||||
if block_content:
|
||||
content_parts.append(block_content)
|
||||
self._assembled_text += block_content
|
||||
self._has_seen_visible_text = True
|
||||
if block_done:
|
||||
done = True
|
||||
|
||||
return "".join(content_parts), done
|
||||
|
||||
def _parse_direct_payload(self, payload: str) -> Tuple[str, bool]:
|
||||
stripped = (payload or "").strip()
|
||||
if not stripped or not stripped.startswith("{"):
|
||||
return "", False
|
||||
|
||||
try:
|
||||
data = json.loads(stripped)
|
||||
except json.JSONDecodeError:
|
||||
return "", False
|
||||
|
||||
logger.debug(f"[DoubaoParser][DirectJSON] {self._describe_direct_json(data)}")
|
||||
|
||||
embedded_stream = self._extract_embedded_stream_payload(data)
|
||||
if embedded_stream:
|
||||
logger.debug(
|
||||
"[DoubaoParser][DirectJSON] branch=embedded_stream "
|
||||
f"payload_len={len(embedded_stream)}"
|
||||
)
|
||||
return self._consume_new_data(embedded_stream)
|
||||
|
||||
logger.debug("[DoubaoParser][DirectJSON] branch=direct_content")
|
||||
return self._extract_direct_json_content(data)
|
||||
|
||||
@staticmethod
|
||||
def _extract_embedded_stream_payload(data: Any) -> str:
|
||||
if not isinstance(data, dict):
|
||||
return ""
|
||||
|
||||
containers: List[Dict[str, Any]] = [data]
|
||||
body = data.get("body")
|
||||
if isinstance(body, dict):
|
||||
containers.append(body)
|
||||
|
||||
for container in containers:
|
||||
for stream_key in ("_stream", "stream"):
|
||||
stream_data = container.get(stream_key)
|
||||
if not isinstance(stream_data, dict):
|
||||
continue
|
||||
|
||||
full_text = stream_data.get("fullText")
|
||||
if isinstance(full_text, str) and full_text:
|
||||
return full_text
|
||||
|
||||
chunks = stream_data.get("chunks")
|
||||
if not isinstance(chunks, list):
|
||||
continue
|
||||
|
||||
parts: List[str] = []
|
||||
for chunk in chunks:
|
||||
if not isinstance(chunk, dict):
|
||||
continue
|
||||
chunk_data = chunk.get("data")
|
||||
if isinstance(chunk_data, str) and chunk_data:
|
||||
parts.append(chunk_data)
|
||||
if parts:
|
||||
return "".join(parts)
|
||||
|
||||
return ""
|
||||
|
||||
def _parse_event_block(self, block: str) -> Tuple[str, bool]:
|
||||
event_name = ""
|
||||
data_lines: List[str] = []
|
||||
|
||||
for raw_line in block.split("\n"):
|
||||
line = raw_line.strip()
|
||||
if not line:
|
||||
continue
|
||||
if line.startswith("event:"):
|
||||
event_name = line[6:].strip()
|
||||
elif line.startswith("data:"):
|
||||
data_lines.append(line[5:].strip())
|
||||
|
||||
payload = "\n".join(data_lines).strip()
|
||||
if not event_name:
|
||||
return "", False
|
||||
|
||||
if not payload:
|
||||
return "", False
|
||||
|
||||
if event_name in {"STREAM_FINISH", "STREAM_END", "DONE", "COMPLETION"}:
|
||||
logger.debug(f"[DoubaoParser][SSE] event={event_name} done_only=1")
|
||||
return "", self._can_accept_done_signal()
|
||||
|
||||
if event_name == "CHUNK_DELTA":
|
||||
text = self._extract_chunk_delta(payload)
|
||||
text = self._repair_text(text)
|
||||
if text:
|
||||
logger.debug(
|
||||
f"[DoubaoParser][SSE] event=CHUNK_DELTA text_len={len(text)} "
|
||||
f"preview={text[:60]!r}"
|
||||
)
|
||||
return text, False
|
||||
|
||||
if event_name == "STREAM_MSG_NOTIFY":
|
||||
text = self._extract_stream_msg_notify(payload)
|
||||
text = self._repair_text(text)
|
||||
if text:
|
||||
logger.debug(
|
||||
f"[DoubaoParser][SSE] event=STREAM_MSG_NOTIFY text_len={len(text)} "
|
||||
f"preview={text[:60]!r}"
|
||||
)
|
||||
return text, False
|
||||
|
||||
if event_name == "STREAM_CHUNK":
|
||||
text, done = self._extract_stream_chunk(payload)
|
||||
text = self._repair_text(text)
|
||||
if text or done:
|
||||
logger.debug(
|
||||
f"[DoubaoParser][SSE] event=STREAM_CHUNK text_len={len(text)} "
|
||||
f"done={done} preview={text[:60]!r}"
|
||||
)
|
||||
return text, bool(done and self._can_accept_done_signal(text))
|
||||
|
||||
if event_name == "FULL_MSG_NOTIFY":
|
||||
text = self._extract_full_msg_notify(payload)
|
||||
text = self._repair_text(text)
|
||||
if text:
|
||||
logger.debug(
|
||||
f"[DoubaoParser][SSE] event=FULL_MSG_NOTIFY text_len={len(text)} "
|
||||
f"preview={text[:60]!r}"
|
||||
)
|
||||
return text, False
|
||||
|
||||
if event_name == "SSE_REPLY_END":
|
||||
text, done = self._extract_reply_end(payload)
|
||||
text = self._repair_text(text)
|
||||
logger.debug(
|
||||
f"[DoubaoParser][SSE] event=SSE_REPLY_END text_len={len(text)} "
|
||||
f"done={done} preview={text[:60]!r}"
|
||||
)
|
||||
return text, bool(done and self._can_accept_done_signal(text))
|
||||
|
||||
return "", False
|
||||
|
||||
def _extract_direct_json_content(self, data: Any) -> Tuple[str, bool]:
|
||||
if not isinstance(data, dict):
|
||||
return "", False
|
||||
|
||||
role = data.get("role")
|
||||
content = data.get("content")
|
||||
if role == "assistant" and isinstance(content, str) and content:
|
||||
logger.debug(
|
||||
"[DoubaoParser][DirectJSON] content_source=role.content "
|
||||
f"len={len(content)}"
|
||||
)
|
||||
return self._extract_full_message_delta(content), True
|
||||
|
||||
message = data.get("message")
|
||||
if isinstance(message, dict):
|
||||
role = message.get("role")
|
||||
content = message.get("content")
|
||||
if role == "assistant" and isinstance(content, str) and content:
|
||||
logger.debug(
|
||||
"[DoubaoParser][DirectJSON] content_source=message.content "
|
||||
f"len={len(content)}"
|
||||
)
|
||||
return self._extract_full_message_delta(content), True
|
||||
|
||||
choices = data.get("choices")
|
||||
if isinstance(choices, list):
|
||||
parts: List[str] = []
|
||||
for choice in choices:
|
||||
if not isinstance(choice, dict):
|
||||
continue
|
||||
delta = choice.get("delta")
|
||||
if isinstance(delta, dict):
|
||||
text = delta.get("content")
|
||||
if isinstance(text, str) and text:
|
||||
parts.append(text)
|
||||
message_obj = choice.get("message")
|
||||
if isinstance(message_obj, dict):
|
||||
text = message_obj.get("content")
|
||||
if isinstance(text, str) and text:
|
||||
parts.append(text)
|
||||
if parts:
|
||||
logger.debug(
|
||||
"[DoubaoParser][DirectJSON] content_source=choices "
|
||||
f"parts={len(parts)} total_len={len(''.join(parts))}"
|
||||
)
|
||||
return self._extract_full_message_delta("".join(parts)), bool(data.get("done", True))
|
||||
|
||||
return "", False
|
||||
|
||||
@staticmethod
|
||||
def _describe_direct_json(data: Any) -> str:
|
||||
if not isinstance(data, dict):
|
||||
return f"type={type(data).__name__}"
|
||||
|
||||
def _keys_of(obj: Any) -> list[str]:
|
||||
if isinstance(obj, dict):
|
||||
return [str(k) for k in list(obj.keys())[:8]]
|
||||
return []
|
||||
|
||||
message = data.get("message")
|
||||
body = data.get("body")
|
||||
message_content = message.get("content") if isinstance(message, dict) else None
|
||||
message_content_len = len(message_content) if isinstance(message_content, str) else 0
|
||||
choices = data.get("choices")
|
||||
choices_len = len(choices) if isinstance(choices, list) else 0
|
||||
|
||||
return (
|
||||
f"keys={_keys_of(data)}, "
|
||||
f"stream_keys={_keys_of(data.get('stream'))}, "
|
||||
f"_stream_keys={_keys_of(data.get('_stream'))}, "
|
||||
f"body_keys={_keys_of(body)}, "
|
||||
f"message_keys={_keys_of(message)}, "
|
||||
f"message_content_len={message_content_len}, "
|
||||
f"choices_len={choices_len}"
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _extract_chunk_delta(payload: str) -> str:
|
||||
try:
|
||||
data = json.loads(payload)
|
||||
except json.JSONDecodeError:
|
||||
return ""
|
||||
|
||||
text = data.get("text", "")
|
||||
return text if isinstance(text, str) else ""
|
||||
|
||||
def _extract_stream_msg_notify(self, payload: str) -> str:
|
||||
try:
|
||||
data = json.loads(payload)
|
||||
except json.JSONDecodeError:
|
||||
return ""
|
||||
|
||||
content = data.get("content", {})
|
||||
return self._extract_content_blocks(content.get("content_block"))
|
||||
|
||||
def _extract_full_msg_notify(self, payload: str) -> str:
|
||||
try:
|
||||
data = json.loads(payload)
|
||||
except json.JSONDecodeError:
|
||||
return ""
|
||||
|
||||
message = data.get("message")
|
||||
if not isinstance(message, dict):
|
||||
return ""
|
||||
|
||||
if message.get("user_type") != 2:
|
||||
return ""
|
||||
|
||||
content_block_text = self._extract_content_blocks(message.get("content_block"))
|
||||
if content_block_text:
|
||||
return content_block_text
|
||||
|
||||
raw_content = message.get("content")
|
||||
if not isinstance(raw_content, str) or not raw_content.strip():
|
||||
return ""
|
||||
|
||||
try:
|
||||
parsed_content = json.loads(raw_content)
|
||||
except json.JSONDecodeError:
|
||||
return ""
|
||||
|
||||
return self._extract_content_blocks(parsed_content)
|
||||
|
||||
def _extract_stream_chunk(self, payload: str) -> Tuple[str, bool]:
|
||||
try:
|
||||
data = json.loads(payload)
|
||||
except json.JSONDecodeError:
|
||||
return "", False
|
||||
|
||||
content_parts: List[str] = []
|
||||
done = False
|
||||
|
||||
for patch in data.get("patch_op", []) or []:
|
||||
if not isinstance(patch, dict):
|
||||
continue
|
||||
|
||||
patch_value = patch.get("patch_value")
|
||||
if not isinstance(patch_value, dict):
|
||||
continue
|
||||
|
||||
text = self._extract_content_blocks(patch_value.get("content_block"))
|
||||
if text:
|
||||
content_parts.append(text)
|
||||
|
||||
ext = patch_value.get("ext")
|
||||
if isinstance(ext, dict) and str(ext.get("is_finish", "")) == "1":
|
||||
done = True
|
||||
|
||||
return "".join(content_parts), done
|
||||
|
||||
def _extract_reply_end(self, payload: str) -> Tuple[str, bool]:
|
||||
try:
|
||||
data = json.loads(payload)
|
||||
except json.JSONDecodeError:
|
||||
return "", True
|
||||
|
||||
if self._assembled_text:
|
||||
return "", True
|
||||
|
||||
finish_attr = data.get("msg_finish_attr")
|
||||
if not isinstance(finish_attr, dict):
|
||||
return "", True
|
||||
|
||||
brief = finish_attr.get("brief")
|
||||
if not isinstance(brief, str) or not brief:
|
||||
return "", True
|
||||
|
||||
return self._extract_full_message_delta(brief), True
|
||||
|
||||
def _can_accept_done_signal(self, pending_text: str = "") -> bool:
|
||||
return bool(
|
||||
self._has_seen_visible_text
|
||||
or self._assembled_text
|
||||
or str(pending_text or "")
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _extract_text_from_block(block: Any) -> Tuple[str, str]:
|
||||
if not isinstance(block, dict):
|
||||
return "", ""
|
||||
|
||||
block_id = block.get("block_id") or block.get("id") or ""
|
||||
if not isinstance(block_id, str):
|
||||
block_id = str(block_id)
|
||||
|
||||
content = block.get("content")
|
||||
if not isinstance(content, dict):
|
||||
return block_id, ""
|
||||
|
||||
text_block = content.get("text_block")
|
||||
if not isinstance(text_block, dict):
|
||||
return block_id, ""
|
||||
|
||||
text = text_block.get("text", "")
|
||||
return block_id, text if isinstance(text, str) else ""
|
||||
|
||||
def _extract_content_blocks(self, blocks: Any) -> str:
|
||||
if not isinstance(blocks, list):
|
||||
return ""
|
||||
|
||||
parts: List[str] = []
|
||||
for block in blocks:
|
||||
block_id, text = self._extract_text_from_block(block)
|
||||
if not text:
|
||||
continue
|
||||
|
||||
previous = self._block_texts.get(block_id, "")
|
||||
if previous and text.startswith(previous):
|
||||
delta = text[len(previous):]
|
||||
elif previous == text:
|
||||
delta = ""
|
||||
else:
|
||||
delta = text
|
||||
|
||||
self._block_texts[block_id] = text
|
||||
|
||||
if delta:
|
||||
parts.append(delta)
|
||||
|
||||
return "".join(parts)
|
||||
|
||||
def _extract_full_message_delta(self, text: str) -> str:
|
||||
if not text:
|
||||
return ""
|
||||
|
||||
previous = self._last_full_message
|
||||
self._last_full_message = text
|
||||
|
||||
if self._assembled_text and text.startswith(self._assembled_text):
|
||||
return text[len(self._assembled_text):]
|
||||
if previous and text.startswith(previous):
|
||||
return text[len(previous):]
|
||||
if previous == text:
|
||||
return ""
|
||||
return text
|
||||
|
||||
@staticmethod
|
||||
def _repair_text(text: str) -> str:
|
||||
if not text:
|
||||
return text
|
||||
|
||||
repaired_whole = DoubaoParser._repair_text_candidate(text)
|
||||
if repaired_whole != text:
|
||||
return repaired_whole
|
||||
|
||||
repaired_segments: List[str] = []
|
||||
changed = False
|
||||
buffer = ""
|
||||
buffer_is_suspicious: bool | None = None
|
||||
|
||||
def flush() -> None:
|
||||
nonlocal buffer, buffer_is_suspicious, changed
|
||||
if not buffer:
|
||||
return
|
||||
if buffer_is_suspicious:
|
||||
repaired = DoubaoParser._repair_text_candidate(buffer)
|
||||
if repaired != buffer:
|
||||
changed = True
|
||||
repaired_segments.append(repaired)
|
||||
else:
|
||||
repaired_segments.append(buffer)
|
||||
buffer = ""
|
||||
buffer_is_suspicious = None
|
||||
|
||||
for ch in text:
|
||||
is_suspicious = DoubaoParser._is_suspicious_char(ch)
|
||||
if buffer_is_suspicious is None:
|
||||
buffer = ch
|
||||
buffer_is_suspicious = is_suspicious
|
||||
continue
|
||||
if is_suspicious == buffer_is_suspicious:
|
||||
buffer += ch
|
||||
continue
|
||||
flush()
|
||||
buffer = ch
|
||||
buffer_is_suspicious = is_suspicious
|
||||
|
||||
flush()
|
||||
if changed:
|
||||
repaired_text = "".join(repaired_segments)
|
||||
if repaired_text != text:
|
||||
logger.debug(
|
||||
"[DoubaoParser][Repair] repaired mixed mojibake "
|
||||
f"(before_len={len(text)}, after_len={len(repaired_text)})"
|
||||
)
|
||||
return repaired_text
|
||||
return text
|
||||
|
||||
@staticmethod
|
||||
def _repair_text_candidate(text: str) -> str:
|
||||
if not text or not DoubaoParser._should_try_repair_text(text):
|
||||
return text
|
||||
|
||||
for source_encoding in ("latin1", "cp1252"):
|
||||
repaired = DoubaoParser._try_redecode_text(text, source_encoding)
|
||||
if not repaired:
|
||||
continue
|
||||
|
||||
if DoubaoParser._looks_more_readable(repaired, text):
|
||||
logger.debug(
|
||||
"[DoubaoParser][Repair] repaired mojibake "
|
||||
f"(enc={source_encoding}, before_len={len(text)}, after_len={len(repaired)})"
|
||||
)
|
||||
return repaired
|
||||
return text
|
||||
|
||||
@staticmethod
|
||||
def _should_try_repair_text(text: str) -> bool:
|
||||
if not text:
|
||||
return False
|
||||
|
||||
if any(0x4E00 <= ord(ch) <= 0x9FFF or ord(ch) > 0xFFFF for ch in text):
|
||||
return False
|
||||
|
||||
cp1252_extra = {
|
||||
0x0152, 0x0153, 0x0160, 0x0161, 0x0178, 0x017D, 0x017E,
|
||||
0x02C6, 0x02DC, 0x2013, 0x2014, 0x2018, 0x2019, 0x201A,
|
||||
0x201C, 0x201D, 0x201E, 0x2020, 0x2021, 0x2022, 0x2026,
|
||||
0x2030, 0x2039, 0x203A, 0x20AC, 0x2122,
|
||||
}
|
||||
suspicious_count = 0
|
||||
for ch in text:
|
||||
if DoubaoParser._is_suspicious_char(ch, cp1252_extra):
|
||||
suspicious_count += 1
|
||||
|
||||
return suspicious_count >= max(2, len(text) // 4)
|
||||
|
||||
@staticmethod
|
||||
def _is_suspicious_char(ch: str, cp1252_extra: set[int] | None = None) -> bool:
|
||||
if not ch:
|
||||
return False
|
||||
if cp1252_extra is None:
|
||||
cp1252_extra = {
|
||||
0x0152, 0x0153, 0x0160, 0x0161, 0x0178, 0x017D, 0x017E,
|
||||
0x02C6, 0x02DC, 0x2013, 0x2014, 0x2018, 0x2019, 0x201A,
|
||||
0x201C, 0x201D, 0x201E, 0x2020, 0x2021, 0x2022, 0x2026,
|
||||
0x2030, 0x2039, 0x203A, 0x20AC, 0x2122,
|
||||
}
|
||||
code = ord(ch)
|
||||
return 0x80 <= code <= 0x00FF or code in cp1252_extra
|
||||
|
||||
@staticmethod
|
||||
def _try_redecode_text(text: str, source_encoding: str) -> str:
|
||||
data = bytearray()
|
||||
|
||||
for ch in text:
|
||||
code = ord(ch)
|
||||
if code <= 0xFF:
|
||||
data.append(code)
|
||||
continue
|
||||
|
||||
try:
|
||||
encoded = ch.encode(source_encoding)
|
||||
except Exception:
|
||||
return ""
|
||||
|
||||
if len(encoded) != 1:
|
||||
return ""
|
||||
|
||||
data.extend(encoded)
|
||||
|
||||
try:
|
||||
return bytes(data).decode("utf-8")
|
||||
except UnicodeDecodeError:
|
||||
return bytes(data).decode("utf-8", errors="ignore")
|
||||
|
||||
@staticmethod
|
||||
def _looks_more_readable(candidate: str, original: str) -> bool:
|
||||
readable_punct = set(".,!?;:'\"()[]{}-_/~`@#$%^&*+=<>,。!?:;()【】《》、~“”‘’…")
|
||||
|
||||
def score(value: str) -> int:
|
||||
total = 0
|
||||
for ch in value:
|
||||
code = ord(ch)
|
||||
if 0x4E00 <= code <= 0x9FFF:
|
||||
total += 2
|
||||
elif code > 0xFFFF:
|
||||
total += 2
|
||||
elif ch.isascii() and (ch.isalnum() or ch.isspace() or ch in readable_punct):
|
||||
total += 1
|
||||
elif ch in readable_punct or ch.isspace():
|
||||
total += 1
|
||||
return total
|
||||
|
||||
return score(candidate) > score(original)
|
||||
|
||||
@classmethod
|
||||
def get_id(cls) -> str:
|
||||
return "doubao"
|
||||
|
||||
@classmethod
|
||||
def get_name(cls) -> str:
|
||||
return "Doubao"
|
||||
|
||||
@classmethod
|
||||
def get_description(cls) -> str:
|
||||
return "Parse Doubao SSE streams with STREAM_MSG_NOTIFY / CHUNK_DELTA events"
|
||||
|
||||
@classmethod
|
||||
def get_supported_patterns(cls) -> List[str]:
|
||||
return ["**/chat/completion**"]
|
||||
|
||||
|
||||
__all__ = ["DoubaoParser"]
|
||||
Reference in new issue
Block a user