Initial commit

This commit is contained in:
Alex committed 2026-11-02 22:54:14 +08:00
commit 57fc3525eb
7311 files changed
+1291398

No files matched your search

@@ -0,0 +1 @@
"""Helpers used by string parsing."""
@@ -0,0 +1,115 @@
from collections.abc import Callable
from typing import TYPE_CHECKING, Literal
from ..utils.constants import STRING_DELIMITERS # noqa: TID252
if TYPE_CHECKING:
from ..json_parser import JSONParser # noqa: TID252
ObjectValueCommaClassification = Literal["container", "member", "string", "string_no_future_delimiter"]
def _bare_member_has_recoverable_value(
parser: "JSONParser",
value_idx: int,
skip_to_character: Callable[[str | list[str], int], int],
) -> bool:
value_start_idx = parser.scroll_whitespaces(idx=value_idx)
value_start = parser.get_char_at(value_start_idx)
if value_start in [*STRING_DELIMITERS, "{", "[", "-"]:
return True
if value_start and value_start.isdigit():
return True
for literal in ["true", "false", "null"]:
if all(parser.get_char_at(value_start_idx + offset) == char for offset, char in enumerate(literal)):
value_end = parser.get_char_at(value_start_idx + len(literal))
if value_end is None or value_end.isspace() or value_end in [",", "}", "]"]:
return True
# An unquoted value is only a safe member boundary when its object closes
# before the current string can close. Otherwise prose such as
# `, floof: explanation` is more likely to belong to the string.
value_end_idx = skip_to_character([*STRING_DELIMITERS, "}"], value_start_idx)
return parser.get_char_at(value_end_idx) == "}"
def classify_object_value_comma(
parser: "JSONParser",
cached_skip_to_character: Callable[[str | list[str], int], int] | None = None,
) -> ObjectValueCommaClassification:
skip_to_character = cached_skip_to_character or parser.skip_to_character
next_idx = parser.scroll_whitespaces(idx=1)
next_c = parser.get_char_at(next_idx)
if next_c in ["}", None]:
return "member"
if next_c in STRING_DELIMITERS:
key_end_idx = parser.skip_to_character(character=next_c, idx=next_idx + 1)
if not parser.get_char_at(key_end_idx):
return "string"
key_end_idx = parser.scroll_whitespaces(idx=key_end_idx + 1)
return "member" if parser.get_char_at(key_end_idx) == ":" else "string"
if next_c == "`":
bare_key_idx = next_idx + 1
while True:
key_char = parser.get_char_at(bare_key_idx)
if not key_char or not (key_char.isalnum() or key_char in ["_", "-"]):
break
bare_key_idx += 1
bare_key_idx = parser.scroll_whitespaces(idx=bare_key_idx)
return "member" if parser.get_char_at(bare_key_idx) == ":" else "string"
if next_c and (next_c.isalnum() or next_c == "_"):
bare_key_idx = next_idx
while True:
key_char = parser.get_char_at(bare_key_idx)
if not key_char or not (key_char.isalnum() or key_char in ["_", "-"]):
break
bare_key_idx += 1
bare_key_idx = parser.scroll_whitespaces(idx=bare_key_idx)
if parser.get_char_at(bare_key_idx) == ":" and _bare_member_has_recoverable_value(
parser,
bare_key_idx + 1,
skip_to_character,
):
return "member"
if next_c in ["{", "["]:
return "container"
next_special_idx = skip_to_character([*STRING_DELIMITERS, "{", "["], next_idx)
next_special = parser.get_char_at(next_special_idx)
if not next_special:
return "string_no_future_delimiter"
if next_special in ["{", "["]:
return "string"
key_end_idx = skip_to_character(next_special, next_special_idx + 1)
if not parser.get_char_at(key_end_idx):
return "string"
key_end_idx = parser.scroll_whitespaces(idx=key_end_idx + 1)
return "member" if parser.get_char_at(key_end_idx) == ":" else "string"
def update_inline_container_stack(
char: str,
pending_inline_container: bool,
inline_container_stack: list[str],
) -> tuple[bool, bool]:
if char in ["{", "["]:
if pending_inline_container:
inline_container_stack.append(char)
return False, False
if inline_container_stack:
inline_container_stack.append(char)
if inline_container_stack and (
(char == "}" and inline_container_stack[-1] == "{") or (char == "]" and inline_container_stack[-1] == "[")
):
inline_container_stack.pop()
return pending_inline_container, True
return pending_inline_container, False
@@ -0,0 +1,35 @@
from typing import TYPE_CHECKING
if TYPE_CHECKING:
from ..json_parser import JSONParser # noqa: TID252
LITERAL_VALUES: dict[str, bool | None] = {
"true": True,
"false": False,
"null": None,
"none": None,
}
def parse_boolean_or_null(parser: "JSONParser") -> tuple[bool, bool | None]:
"""Return whether an unquoted complete literal was found and its value."""
char = (parser.get_char_at() or "").lower()
starting_index = parser.index
for literal, value in LITERAL_VALUES.items():
if char != literal[0]:
continue
parser.index = starting_index
for expected_char in literal:
if (parser.get_char_at() or "").lower() != expected_char:
break
parser.index += 1
else:
next_char = parser.get_char_at()
if next_char is None or next_char.isspace() or next_char in {",", ")", "]", "}"}:
if literal == "none":
parser.log("Converted unquoted Python None literal to JSON null")
return True, value
parser.index = starting_index
return False, None
@@ -0,0 +1,19 @@
from typing import TYPE_CHECKING
from ..utils.constants import JSONReturnType # noqa: TID252
if TYPE_CHECKING:
from ..json_parser import JSONParser # noqa: TID252
def parse_json_llm_block(parser: "JSONParser") -> JSONReturnType:
"""
Extracts and normalizes JSON enclosed in ```json ... ``` blocks.
"""
# Try to find a ```json ... ``` block
if parser.json_str[parser.index : parser.index + 7] == "```json":
i = parser.skip_to_character("`", idx=7)
if parser.json_str[parser.index + i : parser.index + i + 3] == "```":
parser.index += 7 # Move past ```json
return parser.parse_json()
return False