Initial commit
This commit is contained in:
commit
57fc3525eb
7311 files changed
+1291398
No files matched your search
@@ -0,0 +1 @@
|
||||
"""Helpers used by string parsing."""
|
||||
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
@@ -0,0 +1,115 @@
|
||||
from collections.abc import Callable
|
||||
from typing import TYPE_CHECKING, Literal
|
||||
|
||||
from ..utils.constants import STRING_DELIMITERS # noqa: TID252
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from ..json_parser import JSONParser # noqa: TID252
|
||||
|
||||
|
||||
ObjectValueCommaClassification = Literal["container", "member", "string", "string_no_future_delimiter"]
|
||||
|
||||
|
||||
def _bare_member_has_recoverable_value(
|
||||
parser: "JSONParser",
|
||||
value_idx: int,
|
||||
skip_to_character: Callable[[str | list[str], int], int],
|
||||
) -> bool:
|
||||
value_start_idx = parser.scroll_whitespaces(idx=value_idx)
|
||||
value_start = parser.get_char_at(value_start_idx)
|
||||
if value_start in [*STRING_DELIMITERS, "{", "[", "-"]:
|
||||
return True
|
||||
if value_start and value_start.isdigit():
|
||||
return True
|
||||
|
||||
for literal in ["true", "false", "null"]:
|
||||
if all(parser.get_char_at(value_start_idx + offset) == char for offset, char in enumerate(literal)):
|
||||
value_end = parser.get_char_at(value_start_idx + len(literal))
|
||||
if value_end is None or value_end.isspace() or value_end in [",", "}", "]"]:
|
||||
return True
|
||||
|
||||
# An unquoted value is only a safe member boundary when its object closes
|
||||
# before the current string can close. Otherwise prose such as
|
||||
# `, floof: explanation` is more likely to belong to the string.
|
||||
value_end_idx = skip_to_character([*STRING_DELIMITERS, "}"], value_start_idx)
|
||||
return parser.get_char_at(value_end_idx) == "}"
|
||||
|
||||
|
||||
def classify_object_value_comma(
|
||||
parser: "JSONParser",
|
||||
cached_skip_to_character: Callable[[str | list[str], int], int] | None = None,
|
||||
) -> ObjectValueCommaClassification:
|
||||
skip_to_character = cached_skip_to_character or parser.skip_to_character
|
||||
next_idx = parser.scroll_whitespaces(idx=1)
|
||||
next_c = parser.get_char_at(next_idx)
|
||||
if next_c in ["}", None]:
|
||||
return "member"
|
||||
|
||||
if next_c in STRING_DELIMITERS:
|
||||
key_end_idx = parser.skip_to_character(character=next_c, idx=next_idx + 1)
|
||||
if not parser.get_char_at(key_end_idx):
|
||||
return "string"
|
||||
key_end_idx = parser.scroll_whitespaces(idx=key_end_idx + 1)
|
||||
return "member" if parser.get_char_at(key_end_idx) == ":" else "string"
|
||||
|
||||
if next_c == "`":
|
||||
bare_key_idx = next_idx + 1
|
||||
while True:
|
||||
key_char = parser.get_char_at(bare_key_idx)
|
||||
if not key_char or not (key_char.isalnum() or key_char in ["_", "-"]):
|
||||
break
|
||||
bare_key_idx += 1
|
||||
bare_key_idx = parser.scroll_whitespaces(idx=bare_key_idx)
|
||||
return "member" if parser.get_char_at(bare_key_idx) == ":" else "string"
|
||||
|
||||
if next_c and (next_c.isalnum() or next_c == "_"):
|
||||
bare_key_idx = next_idx
|
||||
while True:
|
||||
key_char = parser.get_char_at(bare_key_idx)
|
||||
if not key_char or not (key_char.isalnum() or key_char in ["_", "-"]):
|
||||
break
|
||||
bare_key_idx += 1
|
||||
bare_key_idx = parser.scroll_whitespaces(idx=bare_key_idx)
|
||||
if parser.get_char_at(bare_key_idx) == ":" and _bare_member_has_recoverable_value(
|
||||
parser,
|
||||
bare_key_idx + 1,
|
||||
skip_to_character,
|
||||
):
|
||||
return "member"
|
||||
|
||||
if next_c in ["{", "["]:
|
||||
return "container"
|
||||
|
||||
next_special_idx = skip_to_character([*STRING_DELIMITERS, "{", "["], next_idx)
|
||||
next_special = parser.get_char_at(next_special_idx)
|
||||
if not next_special:
|
||||
return "string_no_future_delimiter"
|
||||
if next_special in ["{", "["]:
|
||||
return "string"
|
||||
|
||||
key_end_idx = skip_to_character(next_special, next_special_idx + 1)
|
||||
if not parser.get_char_at(key_end_idx):
|
||||
return "string"
|
||||
key_end_idx = parser.scroll_whitespaces(idx=key_end_idx + 1)
|
||||
return "member" if parser.get_char_at(key_end_idx) == ":" else "string"
|
||||
|
||||
|
||||
def update_inline_container_stack(
|
||||
char: str,
|
||||
pending_inline_container: bool,
|
||||
inline_container_stack: list[str],
|
||||
) -> tuple[bool, bool]:
|
||||
if char in ["{", "["]:
|
||||
if pending_inline_container:
|
||||
inline_container_stack.append(char)
|
||||
return False, False
|
||||
if inline_container_stack:
|
||||
inline_container_stack.append(char)
|
||||
|
||||
if inline_container_stack and (
|
||||
(char == "}" and inline_container_stack[-1] == "{") or (char == "]" and inline_container_stack[-1] == "[")
|
||||
):
|
||||
inline_container_stack.pop()
|
||||
return pending_inline_container, True
|
||||
|
||||
return pending_inline_container, False
|
||||
@@ -0,0 +1,35 @@
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from ..json_parser import JSONParser # noqa: TID252
|
||||
|
||||
|
||||
LITERAL_VALUES: dict[str, bool | None] = {
|
||||
"true": True,
|
||||
"false": False,
|
||||
"null": None,
|
||||
"none": None,
|
||||
}
|
||||
|
||||
|
||||
def parse_boolean_or_null(parser: "JSONParser") -> tuple[bool, bool | None]:
|
||||
"""Return whether an unquoted complete literal was found and its value."""
|
||||
char = (parser.get_char_at() or "").lower()
|
||||
starting_index = parser.index
|
||||
for literal, value in LITERAL_VALUES.items():
|
||||
if char != literal[0]:
|
||||
continue
|
||||
parser.index = starting_index
|
||||
for expected_char in literal:
|
||||
if (parser.get_char_at() or "").lower() != expected_char:
|
||||
break
|
||||
parser.index += 1
|
||||
else:
|
||||
next_char = parser.get_char_at()
|
||||
if next_char is None or next_char.isspace() or next_char in {",", ")", "]", "}"}:
|
||||
if literal == "none":
|
||||
parser.log("Converted unquoted Python None literal to JSON null")
|
||||
return True, value
|
||||
|
||||
parser.index = starting_index
|
||||
return False, None
|
||||
@@ -0,0 +1,19 @@
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from ..utils.constants import JSONReturnType # noqa: TID252
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from ..json_parser import JSONParser # noqa: TID252
|
||||
|
||||
|
||||
def parse_json_llm_block(parser: "JSONParser") -> JSONReturnType:
|
||||
"""
|
||||
Extracts and normalizes JSON enclosed in ```json ... ``` blocks.
|
||||
"""
|
||||
# Try to find a ```json ... ``` block
|
||||
if parser.json_str[parser.index : parser.index + 7] == "```json":
|
||||
i = parser.skip_to_character("`", idx=7)
|
||||
if parser.json_str[parser.index + i : parser.index + i + 3] == "```":
|
||||
parser.index += 7 # Move past ```json
|
||||
return parser.parse_json()
|
||||
return False
|
||||
Reference in new issue
Block a user