""" chatgpt_parser.py - ChatGPT 响应解析器 响应格式特征: - SSE (Server-Sent Events) 流式响应 - 使用 v1 delta 编码 - 增量文本通过 event: delta 事件传递 - 结束标志: data: [DONE] """ import json import re from typing import Dict, Any, List, Optional from app.core.config import logger from .base import ResponseParser class ChatGPTParser(ResponseParser): """ ChatGPT API 响应解析器 URL 特征: /backend-api/f/conversation 响应格式: SSE (text/event-stream) with v1 delta encoding """ def __init__(self): self._accumulated_content = "" self._last_raw_length = 0 self._last_raw_response = "" self._rendered_content = "" self._message_parts: Dict[int, str] = {} self._message_parts_joined = "" self._message_parts_dirty = False self._content_references: List[Dict[str, Any]] = [] self._content_references_sorted: List[Dict[str, Any]] = [] self._content_references_dirty = False self._active_part_index: Optional[int] = None self._pending_sse_chunk = "" self._ref_start = "\ue200" self._ref_end = "\ue201" self._ref_sep = "\ue202" self._media_generation_state: Dict[str, Any] = {} self._inline_ref_pattern = re.compile( rf"{re.escape(self._ref_start)}([^{re.escape(self._ref_sep)}{re.escape(self._ref_end)}]+)" rf"{re.escape(self._ref_sep)}(.*?){re.escape(self._ref_end)}", re.DOTALL, ) self._dangling_ref_pattern = re.compile( rf"{re.escape(self._ref_start)}[^{re.escape(self._ref_end)}]*$", re.DOTALL, ) def parse_chunk(self, raw_response: str) -> Dict[str, Any]: """ 解析SSE响应流(返回增量) """ result = { "content": "", "images": [], "done": False, "error": None } try: # 确保是字符串 if isinstance(raw_response, bytes): raw_response = raw_response.decode('utf-8', errors='ignore') if not isinstance(raw_response, str): raw_response = str(raw_response) # 只处理新增部分;非前缀新快照会在 helper 内重置解析状态。 new_data = self._prepare_incremental_raw_response(raw_response) if not new_data: return result # 解析新增的SSE事件 delta_content, done = self._parse_sse_chunk(new_data) if delta_content: result["content"] = delta_content self._accumulated_content += delta_content # 检测结束标志 if done: result["done"] = True except Exception as e: logger.debug(f"[ChatGPTParser] 解析异常: {e}") result["error"] = str(e) return result def _parse_sse_chunk(self, chunk: str) -> tuple[str, bool]: """解析SSE数据块,提取文本增量""" chunk = self._extract_complete_sse_chunk(chunk) if not chunk: return "", False previous_content = self._rendered_content done = False for current_event, data in self._iter_sse_json_events(chunk): if not isinstance(data, dict): continue self._consume_event(current_event, data) for raw_line in str(chunk or "").split('\n'): line = raw_line.strip() if line.startswith("data:") and line[5:].strip() == "[DONE]": done = True break current_content = self._render_visible_text() self._rendered_content = current_content return self._compute_delta(previous_content, current_content), done def _extract_complete_sse_chunk(self, chunk: str) -> str: """Buffer partial SSE frames until a blank line terminates the event.""" combined = f"{self._pending_sse_chunk}{str(chunk or '')}" if not combined: return "" normalized = combined.replace("\r\n", "\n").replace("\r", "\n") parts = normalized.split("\n\n") if normalized.endswith("\n\n"): self._pending_sse_chunk = "" complete_parts = parts[:-1] elif normalized.rstrip("\n").endswith("data: [DONE]"): self._pending_sse_chunk = "" complete_parts = [normalized.rstrip("\n")] else: self._pending_sse_chunk = parts.pop() if parts else normalized complete_parts = parts if not complete_parts: return "" return "\n\n".join(complete_parts) + "\n\n" def _iter_sse_json_events(self, chunk: str): """遍历 SSE 中可解析为 JSON 的事件。""" current_event = None for raw_line in str(chunk or "").split('\n'): line = raw_line.strip() if not line: current_event = None continue if line.startswith('event:'): current_event = line[6:].strip() continue if not line.startswith('data:'): continue data_str = line[5:].strip() if not data_str or data_str == "[DONE]": continue try: payload = json.loads(data_str) except json.JSONDecodeError: continue yield current_event, payload def _consume_event(self, event_name: Optional[str], data: Dict[str, Any]) -> None: self._update_media_generation_state(event_name, data) if event_name != "delta": return operations: List[Dict[str, Any]] = [] if data.get("o") == "patch" and isinstance(data.get("v"), list): operations = [item for item in data.get("v", []) if isinstance(item, dict)] elif isinstance(data.get("v"), list): operations = [item for item in data.get("v", []) if isinstance(item, dict)] elif isinstance(data.get("p"), str) and data.get("o") is not None: operations = [data] if operations: for operation in operations: self._apply_operation(operation) return value = data.get("v") if isinstance(value, str) and self._active_part_index is not None: self._append_message_part(self._active_part_index, value) return self._sync_message_snapshot_from_payload(data) def _sync_message_snapshot_from_payload(self, payload: Dict[str, Any]) -> None: message = None value = payload.get("v") if isinstance(value, dict) and isinstance(value.get("message"), dict): message = value.get("message") or {} elif isinstance(payload.get("message"), dict): message = payload.get("message") or {} if not isinstance(message, dict): return author = message.get("author") or {} if str(author.get("role") or "").strip().lower() != "assistant": return content = message.get("content") or {} if not isinstance(content, dict): return if str(content.get("content_type") or "").strip().lower() != "text": return parts = content.get("parts") if isinstance(parts, list): self._set_message_parts(parts) self._active_part_index = max(self._message_parts.keys(), default=0) metadata = message.get("metadata") or {} refs = metadata.get("content_references") if isinstance(refs, list): self._content_references = [ dict(item) for item in refs if isinstance(item, dict) ] self._mark_content_references_dirty() def _apply_operation(self, operation: Dict[str, Any]) -> None: path = str(operation.get("p") or "") action = str(operation.get("o") or "").strip().lower() value = operation.get("v") if self._apply_content_part_operation(path, action, value): return if self._apply_content_reference_operation(path, action, value): return def _apply_content_part_operation(self, path: str, action: str, value: Any) -> bool: if not path.startswith("/message/content/parts"): return False if path == "/message/content/parts": if action in {"add", "replace"} and isinstance(value, list): self._set_message_parts(value) self._active_part_index = max(self._message_parts.keys(), default=0) elif action == "append" and isinstance(value, list): start_index = max(self._message_parts.keys(), default=-1) + 1 for offset, part in enumerate(value): if isinstance(part, str): self._set_message_part(start_index + offset, part) self._active_part_index = max(self._message_parts.keys(), default=0) elif action == "remove": self._clear_message_parts() self._active_part_index = None return True match = re.match(r"^/message/content/parts/(\d+)$", path) if not match: return True index = int(match.group(1)) if action == "append" and isinstance(value, str): self._append_message_part(index, value) self._active_part_index = index elif action in {"add", "replace"}: if isinstance(value, str): self._set_message_part(index, value) self._active_part_index = index elif value is None: self._set_message_part(index, "") self._active_part_index = index elif action == "remove": self._remove_message_part(index) if self._active_part_index == index: self._active_part_index = max(self._message_parts.keys(), default=None) return True def _apply_content_reference_operation(self, path: str, action: str, value: Any) -> bool: if not path.startswith("/message/metadata/content_references"): return False match = re.match(r"^/message/metadata/content_references(?:/(\d+)(?:/(.+))?)?$", path) if not match: return True index_text, subpath = match.groups() if index_text is None: if action in {"add", "replace"} and isinstance(value, list): self._content_references = [ dict(item) for item in value if isinstance(item, dict) ] self._mark_content_references_dirty() elif action == "append": changed = False items = value if isinstance(value, list) else [value] for item in items: if isinstance(item, dict): self._content_references.append(dict(item)) changed = True if changed: self._mark_content_references_dirty() elif action == "remove": self._clear_content_references() return True index = int(index_text) if self._ensure_reference_index(index): self._mark_content_references_dirty() if not subpath: if action == "remove": if 0 <= index < len(self._content_references): self._content_references.pop(index) self._mark_content_references_dirty() elif action in {"add", "replace"} and isinstance(value, dict): self._content_references[index] = dict(value) self._mark_content_references_dirty() elif action == "append" and isinstance(value, dict): self._content_references[index].update(value) self._mark_content_references_dirty() return True target = self._content_references[index] self._apply_nested_patch(target, subpath.split("/"), action, value) self._mark_content_references_dirty() return True def _ensure_reference_index(self, index: int) -> bool: changed = False while len(self._content_references) <= index: self._content_references.append({}) changed = True return changed def _apply_nested_patch( self, container: Dict[str, Any], path_segments: List[str], action: str, value: Any, ) -> None: current = container for segment in path_segments[:-1]: next_value = current.get(segment) if not isinstance(next_value, dict): next_value = {} current[segment] = next_value current = next_value key = path_segments[-1] existing = current.get(key) if action == "append": if isinstance(existing, str) and isinstance(value, str): current[key] = existing + value elif isinstance(existing, list) and isinstance(value, list): current[key] = existing + value elif isinstance(existing, dict) and isinstance(value, dict): merged = dict(existing) merged.update(value) current[key] = merged elif existing is None: current[key] = value else: current[key] = value elif action in {"add", "replace"}: current[key] = value elif action == "remove": current.pop(key, None) def _render_visible_text(self) -> str: if not self._message_parts: return self._rendered_content text = self._render_message_parts() text = self._replace_content_references(text) text = self._strip_private_markup(text) return text def _set_message_parts(self, parts: List[Any]) -> None: self._message_parts = { index: str(part) for index, part in enumerate(parts) if isinstance(part, str) } self._mark_message_parts_dirty() def _set_message_part(self, index: int, value: str) -> None: self._message_parts[index] = str(value) self._mark_message_parts_dirty() def _append_message_part(self, index: int, value: str) -> None: if not value: return self._message_parts[index] = self._message_parts.get(index, "") + str(value) self._mark_message_parts_dirty() def _remove_message_part(self, index: int) -> None: if index in self._message_parts: self._message_parts.pop(index, None) self._mark_message_parts_dirty() def _clear_message_parts(self) -> None: if self._message_parts or self._message_parts_joined: self._message_parts = {} self._message_parts_joined = "" self._message_parts_dirty = False def _mark_message_parts_dirty(self) -> None: self._message_parts_dirty = True def _render_message_parts(self) -> str: if self._message_parts_dirty: self._message_parts_joined = "".join( self._message_parts[index] for index in sorted(self._message_parts) ) self._message_parts_dirty = False return self._message_parts_joined def _mark_content_references_dirty(self) -> None: self._content_references_dirty = True def _clear_content_references(self) -> None: if self._content_references or self._content_references_sorted: self._content_references = [] self._content_references_sorted = [] self._content_references_dirty = False def _sorted_content_references(self) -> List[Dict[str, Any]]: if self._content_references_dirty: self._content_references_sorted = sorted( ( ref for ref in self._content_references if isinstance(ref, dict) and str(ref.get("matched_text") or "") ), key=lambda ref: len(str(ref.get("matched_text") or "")), reverse=True, ) self._content_references_dirty = False return self._content_references_sorted def _replace_content_references(self, text: str) -> str: rendered = text for ref in self._sorted_content_references(): matched_text = str(ref.get("matched_text") or "") if not matched_text or matched_text not in rendered: continue rendered = rendered.replace( matched_text, self._build_reference_replacement(ref), ) rendered = self._inline_ref_pattern.sub( lambda match: self._decode_inline_reference(match.group(1), match.group(2)), rendered, ) return rendered def _build_reference_replacement(self, ref: Dict[str, Any]) -> str: ref_type = str(ref.get("type") or "").strip().lower() if ref_type == "hidden": return "" label = "" for key in ("alt", "prompt_text", "name"): value = str(ref.get(key) or "").strip() if value: label = value break if not label: matched_text = str(ref.get("matched_text") or "") if matched_text: label = self._decode_inline_reference_from_text(matched_text) safe_urls = ref.get("safe_urls") first_url = "" if isinstance(safe_urls, list): for item in safe_urls: candidate = str(item or "").strip() if candidate: first_url = candidate break if first_url and label: if first_url in label or "](" in label or label.startswith("http://") or label.startswith("https://"): return label return f"{label} ({first_url})" if label: return label if first_url: return first_url return "" def _decode_inline_reference_from_text(self, text: str) -> str: match = self._inline_ref_pattern.search(str(text or "")) if not match: return "" return self._decode_inline_reference(match.group(1), match.group(2)) def _decode_inline_reference(self, ref_kind: str, payload: str) -> str: kind = str(ref_kind or "").strip().lower() payload_text = str(payload or "") if kind == "entity": try: decoded = json.loads(payload_text) if isinstance(decoded, list): for index in (1, 0, 2): if index < len(decoded): value = str(decoded[index] or "").strip() if value: return value except Exception: return "" return "" def _strip_private_markup(self, text: str) -> str: cleaned = self._dangling_ref_pattern.sub("", text) cleaned = cleaned.replace(self._ref_start, "") cleaned = cleaned.replace(self._ref_sep, "") cleaned = cleaned.replace(self._ref_end, "") return cleaned def _update_media_generation_state(self, event_name: Optional[str], payload: Dict[str, Any]) -> None: pending = False media_type = str(self._media_generation_state.get("media_type") or "") hint_parts = [ item for item in str(self._media_generation_state.get("hint_text") or "").split("\n\n") if item ] wait_timeout_seconds = self._media_generation_state.get("wait_timeout_seconds") payload_type = str(payload.get("type") or "").strip().lower() if payload_type == "server_ste_metadata": metadata = payload.get("metadata") or {} if str(metadata.get("turn_use_case") or "").strip().lower() == "image gen": pending = True media_type = media_type or "image" wait_timeout_seconds = max(float(wait_timeout_seconds or 0), 90.0) message = None if event_name == "delta": value = payload.get("v") if isinstance(value, dict) and isinstance(value.get("message"), dict): message = value.get("message") or {} elif isinstance(payload.get("message"), dict): message = payload.get("message") or {} if isinstance(message, dict): metadata = message.get("metadata") or {} content = message.get("content") or {} if str(metadata.get("image_gen_task_id") or "").strip(): pending = True media_type = media_type or "image" wait_timeout_seconds = max(float(wait_timeout_seconds or 0), 90.0) if bool(metadata.get("image_gen_multi_stream")): pending = True media_type = media_type or "image" wait_timeout_seconds = max(float(wait_timeout_seconds or 0), 90.0) for field in ("ui_card_title", "ui_card_description"): value = str(metadata.get(field) or "").strip() if value and value not in hint_parts: hint_parts.append(value) if isinstance(content, dict): parts = content.get("parts") if isinstance(parts, list): for item in parts: value = str(item or "").strip() if isinstance(item, str) else "" if value and value not in hint_parts: hint_parts.append(value) if pending and media_type: self._media_generation_state = { "pending": True, "media_type": media_type, "hint_text": "\n\n".join(hint_parts[:3]), "wait_timeout_seconds": wait_timeout_seconds, } @staticmethod def _compute_delta(previous: str, current: str) -> str: if not current or current == previous: return "" if not previous: return current if current.startswith(previous): return current[len(previous):] prefix_len = 0 max_prefix = min(len(previous), len(current)) while prefix_len < max_prefix and previous[prefix_len] == current[prefix_len]: prefix_len += 1 return current[prefix_len:] def reset(self): """重置状态""" self._accumulated_content = "" self._last_raw_length = 0 self._last_raw_response = "" self._rendered_content = "" self._message_parts = {} self._message_parts_joined = "" self._message_parts_dirty = False self._content_references = [] self._content_references_sorted = [] self._content_references_dirty = False self._active_part_index = None self._pending_sse_chunk = "" self._media_generation_state = {} def get_media_generation_state( self, raw_response: str = "", parse_result: Optional[Dict[str, Any]] = None, ) -> Dict[str, Any]: return dict(self._media_generation_state or {}) @classmethod def get_id(cls) -> str: return "chatgpt" @classmethod def get_name(cls) -> str: return "ChatGPT API" @classmethod def get_description(cls) -> str: return "解析 ChatGPT 的 API 响应(SSE流式,v1 delta编码)" @classmethod def get_supported_patterns(cls) -> List[str]: return ["**/backend-api/f/conversation**"] __all__ = ['ChatGPTParser']