Files
universal-web-api/app/core/parsers/mimo_runtime_parser.py
T
2026-11-02 22:54:14 +08:00

339 lines
9.9 KiB
Python

"""
mimo_parser.py - Xiaomi MiMo SSE response parser.
Observed stream traits:
- POST /open-apis/bot/chat?... returns text/event-stream
- answer chunks arrive as event:message with JSON data {"type":"text","content":"..."}
- event:finish with {"content":"[DONE]"} ends the stream
- reasoning text is wrapped inside <think>...</think> and should be ignored
"""
from __future__ import annotations
import json
from typing import Any, Dict, List, Tuple
from app.core.config import logger
from .base import ResponseParser
class MimoParser(ResponseParser):
"""Parse Xiaomi MiMo SSE streams and ignore think-phase content."""
_SUSPICIOUS_CHARS = set(
"ÃÂâæåçèéêëìíîïðñòóôõöùúûüýþÿ"
"€‚ƒ„…†‡ˆ‰Š‹ŒŽ‘’“”•–—˜™š›œžŸ"
)
_REVERSE_CP1252_BYTES = {
0x20AC: 0x80,
0x201A: 0x82,
0x0192: 0x83,
0x201E: 0x84,
0x2026: 0x85,
0x2020: 0x86,
0x2021: 0x87,
0x02C6: 0x88,
0x2030: 0x89,
0x0160: 0x8A,
0x2039: 0x8B,
0x0152: 0x8C,
0x017D: 0x8E,
0x2018: 0x91,
0x2019: 0x92,
0x201C: 0x93,
0x201D: 0x94,
0x2022: 0x95,
0x2013: 0x96,
0x2014: 0x97,
0x02DC: 0x98,
0x2122: 0x99,
0x0161: 0x9A,
0x203A: 0x9B,
0x0153: 0x9C,
0x017E: 0x9E,
0x0178: 0x9F,
}
def __init__(self) -> None:
self._last_raw_length = 0
self._pending = ""
self._inside_think = False
self._has_seen_visible_text = False
def reset(self) -> None:
self._last_raw_length = 0
self._last_raw_response = ""
self._pending = ""
self._inside_think = False
self._has_seen_visible_text = False
def parse_chunk(self, raw_response: str) -> Dict[str, Any]:
result: Dict[str, Any] = {
"content": "",
"images": [],
"done": False,
"error": None,
}
try:
if isinstance(raw_response, (bytes, bytearray)):
raw_response = raw_response.decode("utf-8", errors="ignore")
elif not isinstance(raw_response, str):
raw_response = str(raw_response)
new_data = self._prepare_incremental_raw_response(raw_response)
if not new_data:
return result
delta_content, done = self._consume_new_data(new_data)
if delta_content:
result["content"] = delta_content
result["done"] = done
except Exception as e:
logger.debug(f"[MimoParser] parse exception: {e}")
result["error"] = str(e)
return result
def _consume_new_data(self, new_data: str) -> Tuple[str, bool]:
normalized = (self._pending + new_data).replace("\r\n", "\n")
if not normalized:
return "", False
blocks = normalized.split("\n\n")
if normalized.endswith("\n\n"):
self._pending = ""
complete_blocks = [block for block in blocks if block.strip()]
else:
self._pending = blocks.pop() if blocks else normalized
complete_blocks = [block for block in blocks if block.strip()]
content_parts: List[str] = []
done = False
for block in complete_blocks:
block_content, block_done = self._parse_event_block(block)
if block_content:
content_parts.append(block_content)
if block_done:
done = True
return "".join(content_parts), done
def _parse_event_block(self, block: str) -> Tuple[str, bool]:
event_name = ""
data_lines: List[str] = []
for raw_line in block.split("\n"):
line = raw_line.strip()
if not line:
continue
if line.startswith("event:"):
event_name = line[6:].strip()
elif line.startswith("data:"):
data_lines.append(line[5:].strip())
if event_name == "finish":
return "", self._has_seen_visible_text
if event_name != "message":
return "", False
payload = "\n".join(data_lines).strip()
if not payload:
return "", False
try:
data = json.loads(payload)
except json.JSONDecodeError:
return "", False
if not isinstance(data, dict):
return "", False
if str(data.get("type") or "").strip().lower() != "text":
return "", False
text = data.get("content", "")
if not isinstance(text, str) or not text:
return "", False
visible_text = self._strip_think_content(text)
visible_text = self._repair_mojibake_text(visible_text)
if visible_text:
self._has_seen_visible_text = True
return visible_text, False
def _strip_think_content(self, text: str) -> str:
if not text:
return ""
normalized = text.replace("\x00", "")
visible_parts: List[str] = []
cursor = 0
while cursor < len(normalized):
if self._inside_think:
end_idx = normalized.find("</think>", cursor)
if end_idx == -1:
return "".join(visible_parts)
self._inside_think = False
cursor = end_idx + len("</think>")
continue
start_idx = normalized.find("<think>", cursor)
if start_idx == -1:
visible_parts.append(normalized[cursor:])
break
if start_idx > cursor:
visible_parts.append(normalized[cursor:start_idx])
self._inside_think = True
cursor = start_idx + len("<think>")
return "".join(visible_parts)
@classmethod
def _count_cjk(cls, text: str) -> int:
total = 0
for ch in text or "":
code = ord(ch)
if (
0x4E00 <= code <= 0x9FFF
or 0x3400 <= code <= 0x4DBF
or 0xF900 <= code <= 0xFAFF
):
total += 1
return total
@classmethod
def _count_suspicious(cls, text: str) -> int:
return sum(1 for ch in (text or "") if ch in cls._SUSPICIOUS_CHARS)
@classmethod
def _is_byte_like_char(cls, ch: str) -> bool:
code = ord(ch)
return code in cls._REVERSE_CP1252_BYTES or 0 <= code <= 0xFF
@classmethod
def _has_high_byte_signal(cls, text: str) -> bool:
for ch in text or "":
code = ord(ch)
if code in cls._REVERSE_CP1252_BYTES or 0x80 <= code <= 0xFF:
return True
return False
@classmethod
def _looks_like_utf8_mojibake(cls, text: str) -> bool:
if not text or len(text) < 2:
return False
cjk = cls._count_cjk(text)
if cjk == 0:
return cls._has_high_byte_signal(text)
suspicious = cls._count_suspicious(text)
return suspicious >= 1 and cls._has_high_byte_signal(text)
@classmethod
def _quality_score(cls, text: str) -> tuple[int, int, int]:
return (
cls._count_cjk(text),
-cls._count_suspicious(text),
-(text.count("\ufffd") if text else 0),
)
@classmethod
def _try_redecode(cls, text: str) -> str:
try:
payload = bytearray()
for ch in text:
code = ord(ch)
if code in cls._REVERSE_CP1252_BYTES:
payload.append(cls._REVERSE_CP1252_BYTES[code])
continue
if 0 <= code <= 0xFF:
payload.append(code)
continue
return text
return bytes(payload).decode("utf-8")
except Exception:
return text
@classmethod
def _repair_segmentwise(cls, text: str) -> str:
if not text:
return text
pieces: List[str] = []
buffer: List[str] = []
def _flush_buffer() -> None:
if not buffer:
return
segment = "".join(buffer)
buffer.clear()
if (
len(segment) >= 2
and cls._count_cjk(segment) == 0
and cls._has_high_byte_signal(segment)
):
candidate = cls._try_redecode(segment)
if cls._quality_score(candidate) > cls._quality_score(segment):
pieces.append(candidate)
return
pieces.append(segment)
for ch in text:
if cls._is_byte_like_char(ch):
buffer.append(ch)
else:
_flush_buffer()
pieces.append(ch)
_flush_buffer()
return "".join(pieces)
@classmethod
def _repair_mojibake_text(cls, text: str) -> str:
if not text or not cls._looks_like_utf8_mojibake(text):
return text
best = text
best_score = cls._quality_score(text)
candidate = cls._try_redecode(text)
if candidate != best:
candidate_score = cls._quality_score(candidate)
if candidate_score > best_score:
best = candidate
best_score = candidate_score
candidate = cls._repair_segmentwise(best)
if candidate != best:
candidate_score = cls._quality_score(candidate)
if candidate_score > best_score:
best = candidate
best_score = candidate_score
return best
@classmethod
def get_id(cls) -> str:
return "mimo"
@classmethod
def get_name(cls) -> str:
return "Xiaomi MiMo"
@classmethod
def get_description(cls) -> str:
return "Parse Xiaomi MiMo SSE streams and ignore think-phase text"
@classmethod
def get_supported_patterns(cls) -> List[str]:
return ["**/open-apis/bot/chat**"]
__all__ = ["MimoParser"]