694 lines
23 KiB
Python
694 lines
23 KiB
Python
"""
|
|
doubao_parser.py - Doubao SSE response parser.
|
|
|
|
Observed stream traits:
|
|
- event: STREAM_MSG_NOTIFY carries the first assistant block
|
|
- event: STREAM_CHUNK may carry extra content_block patches
|
|
- event: CHUNK_DELTA carries plain text deltas
|
|
- event: SSE_REPLY_END marks the answer as finished
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import re
|
|
from typing import Any, Dict, List, Tuple
|
|
|
|
from app.core.config import logger
|
|
from .base import ResponseParser
|
|
|
|
|
|
class DoubaoParser(ResponseParser):
|
|
"""Parse Doubao's SSE response stream."""
|
|
|
|
# 生成图片以 image_ori / image_thumb 等字段挂在 SSE 事件里,
|
|
# 同一张图的各尺寸共享 tos 路径(差异仅在 ~tplv 模板与签名参数)。
|
|
_IMAGE_FIELD_RE = re.compile(
|
|
r'"(image_ori|image_ori_raw|image_preview_resize|image_preview|image_thumb)"\s*:\s*'
|
|
r'\{[^{}]*?"url"\s*:\s*"(https?:\\?/\\?/[^"\s]+)"'
|
|
)
|
|
_IMAGE_KEY_PRIORITY = {
|
|
"image_ori": 0,
|
|
"image_ori_raw": 1,
|
|
"image_preview": 2,
|
|
"image_preview_resize": 3,
|
|
"image_thumb": 4,
|
|
}
|
|
|
|
def __init__(self) -> None:
|
|
self._last_raw = ""
|
|
self._pending = ""
|
|
self._block_texts: Dict[str, str] = {}
|
|
self._last_full_message = ""
|
|
self._assembled_text = ""
|
|
self._has_seen_visible_text = False
|
|
self._seen_image_keys: set = set()
|
|
|
|
def _collect_image_urls(self, new_data: str) -> List[str]:
|
|
"""从新的 SSE 数据中提取生成图片 URL,按图片去重、尺寸优先级取最优。"""
|
|
found: Dict[str, Tuple[int, str]] = {}
|
|
for match in self._IMAGE_FIELD_RE.finditer(new_data):
|
|
key, raw_url = match.group(1), match.group(2)
|
|
url = raw_url.replace(r"\/", "/")
|
|
# 去掉主机与签名参数/裁剪模板,同一张图的不同 CDN 副本视为一条
|
|
base_key = re.sub(r"^https?://[^/]+/", "", url).split("?", 1)[0].split("~tplv", 1)[0]
|
|
priority = self._IMAGE_KEY_PRIORITY.get(key, 99)
|
|
current = found.get(base_key)
|
|
if current is None or priority < current[0]:
|
|
found[base_key] = (priority, url)
|
|
fresh: List[str] = []
|
|
for base_key, (_, url) in found.items():
|
|
if base_key in self._seen_image_keys:
|
|
continue
|
|
self._seen_image_keys.add(base_key)
|
|
fresh.append(url)
|
|
return fresh
|
|
|
|
def parse_chunk(self, raw_response: str) -> Dict[str, Any]:
|
|
result: Dict[str, Any] = {
|
|
"content": "",
|
|
"images": [],
|
|
"done": False,
|
|
"error": None,
|
|
}
|
|
|
|
try:
|
|
if isinstance(raw_response, (bytes, bytearray)):
|
|
raw_response = raw_response.decode("utf-8", errors="ignore")
|
|
elif not isinstance(raw_response, str):
|
|
raw_response = str(raw_response)
|
|
|
|
if not raw_response:
|
|
return result
|
|
|
|
if self._last_raw and raw_response == self._last_raw:
|
|
return result
|
|
|
|
if self._last_raw and raw_response.startswith(self._last_raw):
|
|
new_data = raw_response[len(self._last_raw):]
|
|
else:
|
|
new_data = raw_response
|
|
|
|
self._last_raw = raw_response
|
|
|
|
result["images"] = self._collect_image_urls(new_data)
|
|
|
|
direct_content, direct_done = self._parse_direct_payload(new_data)
|
|
if direct_content or direct_done:
|
|
repaired = self._repair_text(direct_content)
|
|
result["content"] = repaired
|
|
if repaired:
|
|
self._assembled_text += repaired
|
|
self._has_seen_visible_text = True
|
|
result["done"] = bool(direct_done and self._can_accept_done_signal(repaired))
|
|
return result
|
|
|
|
delta_content, done = self._consume_new_data(new_data)
|
|
if delta_content:
|
|
repaired = self._repair_text(delta_content)
|
|
if repaired != delta_content and self._assembled_text.endswith(delta_content):
|
|
self._assembled_text = self._assembled_text[:-len(delta_content)] + repaired
|
|
result["content"] = repaired
|
|
result["done"] = done
|
|
|
|
except Exception as e:
|
|
logger.debug(f"[DoubaoParser] parse exception: {e}")
|
|
result["error"] = str(e)
|
|
|
|
return result
|
|
|
|
def reset(self) -> None:
|
|
self._last_raw = ""
|
|
self._pending = ""
|
|
self._block_texts.clear()
|
|
self._last_full_message = ""
|
|
self._assembled_text = ""
|
|
self._has_seen_visible_text = False
|
|
self._seen_image_keys.clear()
|
|
|
|
def _consume_new_data(self, new_data: str) -> Tuple[str, bool]:
|
|
normalized = (self._pending + new_data).replace("\r\n", "\n")
|
|
if not normalized:
|
|
return "", False
|
|
|
|
blocks = normalized.split("\n\n")
|
|
if normalized.endswith("\n\n"):
|
|
self._pending = ""
|
|
complete_blocks = [block for block in blocks if block.strip()]
|
|
else:
|
|
self._pending = blocks.pop() if blocks else normalized
|
|
complete_blocks = [block for block in blocks if block.strip()]
|
|
|
|
content_parts: List[str] = []
|
|
done = False
|
|
|
|
for block in complete_blocks:
|
|
block_content, block_done = self._parse_event_block(block)
|
|
if block_content:
|
|
content_parts.append(block_content)
|
|
self._assembled_text += block_content
|
|
self._has_seen_visible_text = True
|
|
if block_done:
|
|
done = True
|
|
|
|
return "".join(content_parts), done
|
|
|
|
def _parse_direct_payload(self, payload: str) -> Tuple[str, bool]:
|
|
stripped = (payload or "").strip()
|
|
if not stripped or not stripped.startswith("{"):
|
|
return "", False
|
|
|
|
try:
|
|
data = json.loads(stripped)
|
|
except json.JSONDecodeError:
|
|
return "", False
|
|
|
|
logger.debug(f"[DoubaoParser][DirectJSON] {self._describe_direct_json(data)}")
|
|
|
|
embedded_stream = self._extract_embedded_stream_payload(data)
|
|
if embedded_stream:
|
|
logger.debug(
|
|
"[DoubaoParser][DirectJSON] branch=embedded_stream "
|
|
f"payload_len={len(embedded_stream)}"
|
|
)
|
|
return self._consume_new_data(embedded_stream)
|
|
|
|
logger.debug("[DoubaoParser][DirectJSON] branch=direct_content")
|
|
return self._extract_direct_json_content(data)
|
|
|
|
@staticmethod
|
|
def _extract_embedded_stream_payload(data: Any) -> str:
|
|
if not isinstance(data, dict):
|
|
return ""
|
|
|
|
containers: List[Dict[str, Any]] = [data]
|
|
body = data.get("body")
|
|
if isinstance(body, dict):
|
|
containers.append(body)
|
|
|
|
for container in containers:
|
|
for stream_key in ("_stream", "stream"):
|
|
stream_data = container.get(stream_key)
|
|
if not isinstance(stream_data, dict):
|
|
continue
|
|
|
|
full_text = stream_data.get("fullText")
|
|
if isinstance(full_text, str) and full_text:
|
|
return full_text
|
|
|
|
chunks = stream_data.get("chunks")
|
|
if not isinstance(chunks, list):
|
|
continue
|
|
|
|
parts: List[str] = []
|
|
for chunk in chunks:
|
|
if not isinstance(chunk, dict):
|
|
continue
|
|
chunk_data = chunk.get("data")
|
|
if isinstance(chunk_data, str) and chunk_data:
|
|
parts.append(chunk_data)
|
|
if parts:
|
|
return "".join(parts)
|
|
|
|
return ""
|
|
|
|
def _parse_event_block(self, block: str) -> Tuple[str, bool]:
|
|
event_name = ""
|
|
data_lines: List[str] = []
|
|
|
|
for raw_line in block.split("\n"):
|
|
line = raw_line.strip()
|
|
if not line:
|
|
continue
|
|
if line.startswith("event:"):
|
|
event_name = line[6:].strip()
|
|
elif line.startswith("data:"):
|
|
data_lines.append(line[5:].strip())
|
|
|
|
payload = "\n".join(data_lines).strip()
|
|
if not event_name:
|
|
return "", False
|
|
|
|
if not payload:
|
|
return "", False
|
|
|
|
if event_name in {"STREAM_FINISH", "STREAM_END", "DONE", "COMPLETION"}:
|
|
logger.debug(f"[DoubaoParser][SSE] event={event_name} done_only=1")
|
|
return "", self._can_accept_done_signal()
|
|
|
|
if event_name == "CHUNK_DELTA":
|
|
text = self._extract_chunk_delta(payload)
|
|
text = self._repair_text(text)
|
|
if text:
|
|
logger.debug(
|
|
f"[DoubaoParser][SSE] event=CHUNK_DELTA text_len={len(text)} "
|
|
f"preview={text[:60]!r}"
|
|
)
|
|
return text, False
|
|
|
|
if event_name == "STREAM_MSG_NOTIFY":
|
|
text = self._extract_stream_msg_notify(payload)
|
|
text = self._repair_text(text)
|
|
if text:
|
|
logger.debug(
|
|
f"[DoubaoParser][SSE] event=STREAM_MSG_NOTIFY text_len={len(text)} "
|
|
f"preview={text[:60]!r}"
|
|
)
|
|
return text, False
|
|
|
|
if event_name == "STREAM_CHUNK":
|
|
text, done = self._extract_stream_chunk(payload)
|
|
text = self._repair_text(text)
|
|
if text or done:
|
|
logger.debug(
|
|
f"[DoubaoParser][SSE] event=STREAM_CHUNK text_len={len(text)} "
|
|
f"done={done} preview={text[:60]!r}"
|
|
)
|
|
return text, bool(done and self._can_accept_done_signal(text))
|
|
|
|
if event_name == "FULL_MSG_NOTIFY":
|
|
text = self._extract_full_msg_notify(payload)
|
|
text = self._repair_text(text)
|
|
if text:
|
|
logger.debug(
|
|
f"[DoubaoParser][SSE] event=FULL_MSG_NOTIFY text_len={len(text)} "
|
|
f"preview={text[:60]!r}"
|
|
)
|
|
return text, False
|
|
|
|
if event_name == "SSE_REPLY_END":
|
|
text, done = self._extract_reply_end(payload)
|
|
text = self._repair_text(text)
|
|
logger.debug(
|
|
f"[DoubaoParser][SSE] event=SSE_REPLY_END text_len={len(text)} "
|
|
f"done={done} preview={text[:60]!r}"
|
|
)
|
|
return text, bool(done and self._can_accept_done_signal(text))
|
|
|
|
return "", False
|
|
|
|
def _extract_direct_json_content(self, data: Any) -> Tuple[str, bool]:
|
|
if not isinstance(data, dict):
|
|
return "", False
|
|
|
|
role = data.get("role")
|
|
content = data.get("content")
|
|
if role == "assistant" and isinstance(content, str) and content:
|
|
logger.debug(
|
|
"[DoubaoParser][DirectJSON] content_source=role.content "
|
|
f"len={len(content)}"
|
|
)
|
|
return self._extract_full_message_delta(content), True
|
|
|
|
message = data.get("message")
|
|
if isinstance(message, dict):
|
|
role = message.get("role")
|
|
content = message.get("content")
|
|
if role == "assistant" and isinstance(content, str) and content:
|
|
logger.debug(
|
|
"[DoubaoParser][DirectJSON] content_source=message.content "
|
|
f"len={len(content)}"
|
|
)
|
|
return self._extract_full_message_delta(content), True
|
|
|
|
choices = data.get("choices")
|
|
if isinstance(choices, list):
|
|
parts: List[str] = []
|
|
for choice in choices:
|
|
if not isinstance(choice, dict):
|
|
continue
|
|
delta = choice.get("delta")
|
|
if isinstance(delta, dict):
|
|
text = delta.get("content")
|
|
if isinstance(text, str) and text:
|
|
parts.append(text)
|
|
message_obj = choice.get("message")
|
|
if isinstance(message_obj, dict):
|
|
text = message_obj.get("content")
|
|
if isinstance(text, str) and text:
|
|
parts.append(text)
|
|
if parts:
|
|
logger.debug(
|
|
"[DoubaoParser][DirectJSON] content_source=choices "
|
|
f"parts={len(parts)} total_len={len(''.join(parts))}"
|
|
)
|
|
return self._extract_full_message_delta("".join(parts)), bool(data.get("done", True))
|
|
|
|
return "", False
|
|
|
|
@staticmethod
|
|
def _describe_direct_json(data: Any) -> str:
|
|
if not isinstance(data, dict):
|
|
return f"type={type(data).__name__}"
|
|
|
|
def _keys_of(obj: Any) -> list[str]:
|
|
if isinstance(obj, dict):
|
|
return [str(k) for k in list(obj.keys())[:8]]
|
|
return []
|
|
|
|
message = data.get("message")
|
|
body = data.get("body")
|
|
message_content = message.get("content") if isinstance(message, dict) else None
|
|
message_content_len = len(message_content) if isinstance(message_content, str) else 0
|
|
choices = data.get("choices")
|
|
choices_len = len(choices) if isinstance(choices, list) else 0
|
|
|
|
return (
|
|
f"keys={_keys_of(data)}, "
|
|
f"stream_keys={_keys_of(data.get('stream'))}, "
|
|
f"_stream_keys={_keys_of(data.get('_stream'))}, "
|
|
f"body_keys={_keys_of(body)}, "
|
|
f"message_keys={_keys_of(message)}, "
|
|
f"message_content_len={message_content_len}, "
|
|
f"choices_len={choices_len}"
|
|
)
|
|
|
|
@staticmethod
|
|
def _extract_chunk_delta(payload: str) -> str:
|
|
try:
|
|
data = json.loads(payload)
|
|
except json.JSONDecodeError:
|
|
return ""
|
|
|
|
text = data.get("text", "")
|
|
return text if isinstance(text, str) else ""
|
|
|
|
def _extract_stream_msg_notify(self, payload: str) -> str:
|
|
try:
|
|
data = json.loads(payload)
|
|
except json.JSONDecodeError:
|
|
return ""
|
|
|
|
content = data.get("content", {})
|
|
return self._extract_content_blocks(content.get("content_block"))
|
|
|
|
def _extract_full_msg_notify(self, payload: str) -> str:
|
|
try:
|
|
data = json.loads(payload)
|
|
except json.JSONDecodeError:
|
|
return ""
|
|
|
|
message = data.get("message")
|
|
if not isinstance(message, dict):
|
|
return ""
|
|
|
|
if message.get("user_type") != 2:
|
|
return ""
|
|
|
|
content_block_text = self._extract_content_blocks(message.get("content_block"))
|
|
if content_block_text:
|
|
return content_block_text
|
|
|
|
raw_content = message.get("content")
|
|
if not isinstance(raw_content, str) or not raw_content.strip():
|
|
return ""
|
|
|
|
try:
|
|
parsed_content = json.loads(raw_content)
|
|
except json.JSONDecodeError:
|
|
return ""
|
|
|
|
return self._extract_content_blocks(parsed_content)
|
|
|
|
def _extract_stream_chunk(self, payload: str) -> Tuple[str, bool]:
|
|
try:
|
|
data = json.loads(payload)
|
|
except json.JSONDecodeError:
|
|
return "", False
|
|
|
|
content_parts: List[str] = []
|
|
done = False
|
|
|
|
for patch in data.get("patch_op", []) or []:
|
|
if not isinstance(patch, dict):
|
|
continue
|
|
|
|
patch_value = patch.get("patch_value")
|
|
if not isinstance(patch_value, dict):
|
|
continue
|
|
|
|
text = self._extract_content_blocks(patch_value.get("content_block"))
|
|
if text:
|
|
content_parts.append(text)
|
|
|
|
ext = patch_value.get("ext")
|
|
if isinstance(ext, dict) and str(ext.get("is_finish", "")) == "1":
|
|
done = True
|
|
|
|
return "".join(content_parts), done
|
|
|
|
def _extract_reply_end(self, payload: str) -> Tuple[str, bool]:
|
|
try:
|
|
data = json.loads(payload)
|
|
except json.JSONDecodeError:
|
|
return "", True
|
|
|
|
if self._assembled_text:
|
|
return "", True
|
|
|
|
finish_attr = data.get("msg_finish_attr")
|
|
if not isinstance(finish_attr, dict):
|
|
return "", True
|
|
|
|
brief = finish_attr.get("brief")
|
|
if not isinstance(brief, str) or not brief:
|
|
return "", True
|
|
|
|
return self._extract_full_message_delta(brief), True
|
|
|
|
def _can_accept_done_signal(self, pending_text: str = "") -> bool:
|
|
return bool(
|
|
self._has_seen_visible_text
|
|
or self._assembled_text
|
|
or str(pending_text or "")
|
|
)
|
|
|
|
@staticmethod
|
|
def _extract_text_from_block(block: Any) -> Tuple[str, str]:
|
|
if not isinstance(block, dict):
|
|
return "", ""
|
|
|
|
block_id = block.get("block_id") or block.get("id") or ""
|
|
if not isinstance(block_id, str):
|
|
block_id = str(block_id)
|
|
|
|
content = block.get("content")
|
|
if not isinstance(content, dict):
|
|
return block_id, ""
|
|
|
|
text_block = content.get("text_block")
|
|
if not isinstance(text_block, dict):
|
|
return block_id, ""
|
|
|
|
text = text_block.get("text", "")
|
|
return block_id, text if isinstance(text, str) else ""
|
|
|
|
def _extract_content_blocks(self, blocks: Any) -> str:
|
|
if not isinstance(blocks, list):
|
|
return ""
|
|
|
|
parts: List[str] = []
|
|
for block in blocks:
|
|
block_id, text = self._extract_text_from_block(block)
|
|
if not text:
|
|
continue
|
|
|
|
previous = self._block_texts.get(block_id, "")
|
|
if previous and text.startswith(previous):
|
|
delta = text[len(previous):]
|
|
elif previous == text:
|
|
delta = ""
|
|
else:
|
|
delta = text
|
|
|
|
self._block_texts[block_id] = text
|
|
|
|
if delta:
|
|
parts.append(delta)
|
|
|
|
return "".join(parts)
|
|
|
|
def _extract_full_message_delta(self, text: str) -> str:
|
|
if not text:
|
|
return ""
|
|
|
|
previous = self._last_full_message
|
|
self._last_full_message = text
|
|
|
|
if self._assembled_text and text.startswith(self._assembled_text):
|
|
return text[len(self._assembled_text):]
|
|
if previous and text.startswith(previous):
|
|
return text[len(previous):]
|
|
if previous == text:
|
|
return ""
|
|
return text
|
|
|
|
@staticmethod
|
|
def _repair_text(text: str) -> str:
|
|
if not text:
|
|
return text
|
|
|
|
repaired_whole = DoubaoParser._repair_text_candidate(text)
|
|
if repaired_whole != text:
|
|
return repaired_whole
|
|
|
|
repaired_segments: List[str] = []
|
|
changed = False
|
|
buffer = ""
|
|
buffer_is_suspicious: bool | None = None
|
|
|
|
def flush() -> None:
|
|
nonlocal buffer, buffer_is_suspicious, changed
|
|
if not buffer:
|
|
return
|
|
if buffer_is_suspicious:
|
|
repaired = DoubaoParser._repair_text_candidate(buffer)
|
|
if repaired != buffer:
|
|
changed = True
|
|
repaired_segments.append(repaired)
|
|
else:
|
|
repaired_segments.append(buffer)
|
|
buffer = ""
|
|
buffer_is_suspicious = None
|
|
|
|
for ch in text:
|
|
is_suspicious = DoubaoParser._is_suspicious_char(ch)
|
|
if buffer_is_suspicious is None:
|
|
buffer = ch
|
|
buffer_is_suspicious = is_suspicious
|
|
continue
|
|
if is_suspicious == buffer_is_suspicious:
|
|
buffer += ch
|
|
continue
|
|
flush()
|
|
buffer = ch
|
|
buffer_is_suspicious = is_suspicious
|
|
|
|
flush()
|
|
if changed:
|
|
repaired_text = "".join(repaired_segments)
|
|
if repaired_text != text:
|
|
logger.debug(
|
|
"[DoubaoParser][Repair] repaired mixed mojibake "
|
|
f"(before_len={len(text)}, after_len={len(repaired_text)})"
|
|
)
|
|
return repaired_text
|
|
return text
|
|
|
|
@staticmethod
|
|
def _repair_text_candidate(text: str) -> str:
|
|
if not text or not DoubaoParser._should_try_repair_text(text):
|
|
return text
|
|
|
|
for source_encoding in ("latin1", "cp1252"):
|
|
repaired = DoubaoParser._try_redecode_text(text, source_encoding)
|
|
if not repaired:
|
|
continue
|
|
|
|
if DoubaoParser._looks_more_readable(repaired, text):
|
|
logger.debug(
|
|
"[DoubaoParser][Repair] repaired mojibake "
|
|
f"(enc={source_encoding}, before_len={len(text)}, after_len={len(repaired)})"
|
|
)
|
|
return repaired
|
|
return text
|
|
|
|
@staticmethod
|
|
def _should_try_repair_text(text: str) -> bool:
|
|
if not text:
|
|
return False
|
|
|
|
if any(0x4E00 <= ord(ch) <= 0x9FFF or ord(ch) > 0xFFFF for ch in text):
|
|
return False
|
|
|
|
cp1252_extra = {
|
|
0x0152, 0x0153, 0x0160, 0x0161, 0x0178, 0x017D, 0x017E,
|
|
0x02C6, 0x02DC, 0x2013, 0x2014, 0x2018, 0x2019, 0x201A,
|
|
0x201C, 0x201D, 0x201E, 0x2020, 0x2021, 0x2022, 0x2026,
|
|
0x2030, 0x2039, 0x203A, 0x20AC, 0x2122,
|
|
}
|
|
suspicious_count = 0
|
|
for ch in text:
|
|
if DoubaoParser._is_suspicious_char(ch, cp1252_extra):
|
|
suspicious_count += 1
|
|
|
|
return suspicious_count >= max(2, len(text) // 4)
|
|
|
|
@staticmethod
|
|
def _is_suspicious_char(ch: str, cp1252_extra: set[int] | None = None) -> bool:
|
|
if not ch:
|
|
return False
|
|
if cp1252_extra is None:
|
|
cp1252_extra = {
|
|
0x0152, 0x0153, 0x0160, 0x0161, 0x0178, 0x017D, 0x017E,
|
|
0x02C6, 0x02DC, 0x2013, 0x2014, 0x2018, 0x2019, 0x201A,
|
|
0x201C, 0x201D, 0x201E, 0x2020, 0x2021, 0x2022, 0x2026,
|
|
0x2030, 0x2039, 0x203A, 0x20AC, 0x2122,
|
|
}
|
|
code = ord(ch)
|
|
return 0x80 <= code <= 0x00FF or code in cp1252_extra
|
|
|
|
@staticmethod
|
|
def _try_redecode_text(text: str, source_encoding: str) -> str:
|
|
data = bytearray()
|
|
|
|
for ch in text:
|
|
code = ord(ch)
|
|
if code <= 0xFF:
|
|
data.append(code)
|
|
continue
|
|
|
|
try:
|
|
encoded = ch.encode(source_encoding)
|
|
except Exception:
|
|
return ""
|
|
|
|
if len(encoded) != 1:
|
|
return ""
|
|
|
|
data.extend(encoded)
|
|
|
|
try:
|
|
return bytes(data).decode("utf-8")
|
|
except UnicodeDecodeError:
|
|
return bytes(data).decode("utf-8", errors="ignore")
|
|
|
|
@staticmethod
|
|
def _looks_more_readable(candidate: str, original: str) -> bool:
|
|
readable_punct = set(".,!?;:'\"()[]{}-_/~`@#$%^&*+=<>,。!?:;()【】《》、~“”‘’…")
|
|
|
|
def score(value: str) -> int:
|
|
total = 0
|
|
for ch in value:
|
|
code = ord(ch)
|
|
if 0x4E00 <= code <= 0x9FFF:
|
|
total += 2
|
|
elif code > 0xFFFF:
|
|
total += 2
|
|
elif ch.isascii() and (ch.isalnum() or ch.isspace() or ch in readable_punct):
|
|
total += 1
|
|
elif ch in readable_punct or ch.isspace():
|
|
total += 1
|
|
return total
|
|
|
|
return score(candidate) > score(original)
|
|
|
|
@classmethod
|
|
def get_id(cls) -> str:
|
|
return "doubao"
|
|
|
|
@classmethod
|
|
def get_name(cls) -> str:
|
|
return "Doubao"
|
|
|
|
@classmethod
|
|
def get_description(cls) -> str:
|
|
return "Parse Doubao SSE streams with STREAM_MSG_NOTIFY / CHUNK_DELTA events"
|
|
|
|
@classmethod
|
|
def get_supported_patterns(cls) -> List[str]:
|
|
return ["**/chat/completion**"]
|
|
|
|
|
|
__all__ = ["DoubaoParser"]
|