Initial commit
This commit is contained in:
commit
57fc3525eb
7311 files changed
+1291398
No files matched your search
@@ -0,0 +1,481 @@
|
||||
import json
|
||||
from collections.abc import Callable
|
||||
from contextlib import ExitStack
|
||||
from typing import TYPE_CHECKING, Any, TextIO
|
||||
|
||||
from .parse_array import parse_array as _parse_array
|
||||
from .parse_comment import parse_comment as _parse_comment
|
||||
from .parse_number import parse_number as _parse_number
|
||||
from .parse_object import parse_object as _parse_object
|
||||
from .parse_string import parse_string as _parse_string
|
||||
from .parser_parenthesized import parenthesized_is_explicit_tuple, top_level_parenthesized_can_start_value
|
||||
from .utils.constants import STRING_DELIMITERS, JSONReturnType
|
||||
from .utils.json_context import ContextValues, JsonContext
|
||||
from .utils.object_comparer import ObjectComparer
|
||||
from .utils.string_file_wrapper import StringFileWrapper
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from .schema_repair import SchemaRepairer
|
||||
|
||||
|
||||
class JSONParser:
|
||||
# Split the parse methods into separate files because this one was like 3000 lines
|
||||
def parse_array(
|
||||
self,
|
||||
schema: dict[str, Any] | bool | None = None,
|
||||
path: str = "$",
|
||||
closing_delimiter: str = "]",
|
||||
) -> list[JSONReturnType]:
|
||||
return _parse_array(self, schema, path, closing_delimiter)
|
||||
|
||||
def parse_comment(self, record_top_level_value: bool = False) -> JSONReturnType:
|
||||
return _parse_comment(self, record_top_level_value=record_top_level_value)
|
||||
|
||||
def parse_number(self) -> JSONReturnType:
|
||||
return _parse_number(self)
|
||||
|
||||
def parse_object(
|
||||
self,
|
||||
schema: dict[str, Any] | bool | None = None,
|
||||
path: str = "$",
|
||||
) -> JSONReturnType:
|
||||
return _parse_object(self, schema, path)
|
||||
|
||||
def parse_string(self) -> JSONReturnType:
|
||||
return _parse_string(self)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
json_str: str | StringFileWrapper,
|
||||
json_fd: TextIO | None,
|
||||
logging: bool | None,
|
||||
json_fd_chunk_length: int = 0,
|
||||
stream_stable: bool = False,
|
||||
strict: bool = False,
|
||||
try_valid_json_suffix: bool = False,
|
||||
) -> None:
|
||||
# The string to parse
|
||||
self.json_str: str | StringFileWrapper = json_str
|
||||
# Alternatively, the file description with a json file in it
|
||||
if json_fd:
|
||||
# This is a trick we do to treat the file wrapper as an array
|
||||
self.json_str = StringFileWrapper(json_fd, json_fd_chunk_length)
|
||||
# Index is our iterator that will keep track of which character we are looking at right now
|
||||
self.index: int = 0
|
||||
# This is used in the object member parsing to manage the special cases of missing quotes in key or value
|
||||
self.context = JsonContext()
|
||||
self.deferred_contexts: list[ContextValues] = []
|
||||
# Use this to log the activity, but only if logging is active
|
||||
|
||||
# This is a trick but a beautiful one. We call self.log in the code over and over even if it's not needed.
|
||||
# We could add a guard in the code for each call but that would make this code unreadable, so here's this neat trick
|
||||
# Replace self.log with a noop
|
||||
self.logging = logging
|
||||
self.logger: list[dict[str, str]] = []
|
||||
if logging:
|
||||
self.log = self._log
|
||||
else:
|
||||
# No-op
|
||||
self.log = lambda *args, **kwargs: None # noqa: ARG005
|
||||
# When the json to be repaired is the accumulation of streaming json at a certain moment.
|
||||
# e.g. json obtained from llm response.
|
||||
# If this parameter to True will keep the repair results stable. For example:
|
||||
# case 1: '{"key": "val\\' => '{"key": "val"}'
|
||||
# case 2: '{"key": "val\\n' => '{"key": "val\\n"}'
|
||||
# case 3: '{"key": "val\\n123,`key2:value2' => '{"key": "val\\n123,`key2:value2"}'
|
||||
# case 4: '{"key": "val\\n123,`key2:value2`"}' => '{"key": "val\\n123,`key2:value2`"}'
|
||||
self.stream_stable = stream_stable
|
||||
# Over time the library got more and more complex heuristics to repair JSON. Some of these heuristics
|
||||
# may not be desirable in some use cases and the user would prefer json_repair to return an exception.
|
||||
# So strict mode was added to disable some of those heuristics.
|
||||
self.strict = strict
|
||||
self.try_valid_json_suffix = try_valid_json_suffix
|
||||
self.has_tried_valid_json_suffix = False
|
||||
self._last_parse_found_value = False
|
||||
self.schema_repairer: SchemaRepairer | None = None
|
||||
|
||||
def parse(
|
||||
self,
|
||||
) -> JSONReturnType:
|
||||
return self._parse_top_level(self.parse_json)
|
||||
|
||||
def parse_with_schema(
|
||||
self,
|
||||
repairer: "SchemaRepairer",
|
||||
schema: dict[str, Any] | bool,
|
||||
) -> JSONReturnType:
|
||||
"""Parse with schema guidance enabled for all nested values."""
|
||||
self.schema_repairer = repairer
|
||||
if repairer.schema_repair_mode == "salvage":
|
||||
return self._parse_top_level_salvage_with_schema(repairer, schema)
|
||||
return self._parse_top_level(lambda: self.parse_json(schema, "$"))
|
||||
|
||||
def _parse_top_level_salvage_with_schema(
|
||||
self,
|
||||
repairer: "SchemaRepairer",
|
||||
schema: dict[str, Any] | bool,
|
||||
) -> JSONReturnType:
|
||||
"""Return the first schema-valid top-level fragment while salvaging."""
|
||||
last_error: ValueError | None = None
|
||||
while self.index < len(self.json_str):
|
||||
self.context.clear()
|
||||
self.deferred_contexts.clear()
|
||||
value = self.parse_json(schema, "$", finalize_schema=False, record_top_level_value=True)
|
||||
if not self._last_parse_found_value:
|
||||
break
|
||||
try:
|
||||
repaired = repairer.repair_value(value, schema, "$")
|
||||
repairer.validate(repaired, schema)
|
||||
except ValueError as exc:
|
||||
last_error = exc
|
||||
if self.index >= len(self.json_str):
|
||||
break
|
||||
self.log("Skipped top-level fragment that did not match schema while salvaging")
|
||||
continue
|
||||
else:
|
||||
return repaired
|
||||
if last_error is not None:
|
||||
raise last_error
|
||||
return ""
|
||||
|
||||
# Consolidate top-level parsing so we handle multiple sequential JSON values consistently
|
||||
# (including update semantics and strict-mode validation).
|
||||
def _parse_top_level(self, parse_element: Callable[[], JSONReturnType]) -> JSONReturnType:
|
||||
json = parse_element()
|
||||
if self.index < len(self.json_str):
|
||||
self.log(
|
||||
"The parser returned early, checking if there's more json elements",
|
||||
)
|
||||
json = [json]
|
||||
while self.index < len(self.json_str):
|
||||
self.context.clear()
|
||||
self.deferred_contexts.clear()
|
||||
is_comma_separated = self._next_top_level_value_is_comma_separated()
|
||||
element_start_index = self.index
|
||||
j = parse_element()
|
||||
if self.strict and self.index > element_start_index:
|
||||
self.log(
|
||||
"Multiple top-level JSON elements found in strict mode, raising an error",
|
||||
)
|
||||
raise ValueError("Multiple top-level JSON elements found in strict mode.")
|
||||
if j:
|
||||
if not is_comma_separated and ObjectComparer.is_same_object(json[-1], j):
|
||||
# Treat repeated objects as updates: keep the newest value.
|
||||
json.pop()
|
||||
else:
|
||||
if not json[-1]:
|
||||
json.pop()
|
||||
json.append(j)
|
||||
else:
|
||||
self.index += 1
|
||||
if len(json) == 1:
|
||||
self.log(
|
||||
"There were no more elements, returning the element without the array",
|
||||
)
|
||||
json = json[0]
|
||||
return json
|
||||
|
||||
def _next_top_level_value_is_comma_separated(self) -> bool:
|
||||
idx = self.scroll_whitespaces()
|
||||
if self.get_char_at(idx) == ",":
|
||||
return True
|
||||
|
||||
idx = self.index - 1
|
||||
while idx >= 0 and self.json_str[idx].isspace():
|
||||
idx -= 1
|
||||
return idx >= 0 and self.json_str[idx] == ","
|
||||
|
||||
def _initial_container_has_non_comma_trailing_content(self) -> bool:
|
||||
if self.index != 0 or not isinstance(self.json_str, str):
|
||||
return True
|
||||
|
||||
containers: list[str] = []
|
||||
in_string = False
|
||||
escaped = False
|
||||
closing_delimiters = {"{": "}", "[": "]"}
|
||||
for index, char in enumerate(self.json_str):
|
||||
if in_string:
|
||||
if escaped:
|
||||
escaped = False
|
||||
elif char == "\\":
|
||||
escaped = True
|
||||
elif char == '"':
|
||||
in_string = False
|
||||
continue
|
||||
|
||||
if char == '"':
|
||||
in_string = True
|
||||
elif char in closing_delimiters:
|
||||
containers.append(char)
|
||||
elif char in ["}", "]"]:
|
||||
if not containers or char != closing_delimiters[containers.pop()]:
|
||||
return False
|
||||
if not containers:
|
||||
trailing_index = index + 1
|
||||
while trailing_index < len(self.json_str) and self.json_str[trailing_index].isspace():
|
||||
trailing_index += 1
|
||||
return trailing_index < len(self.json_str) and self.json_str[trailing_index] != ","
|
||||
|
||||
return False
|
||||
|
||||
def _try_parse_valid_json_value(self) -> tuple[bool, JSONReturnType]:
|
||||
if (
|
||||
not self.try_valid_json_suffix
|
||||
or self.has_tried_valid_json_suffix
|
||||
or not self.context.empty
|
||||
or not isinstance(self.json_str, str)
|
||||
):
|
||||
return False, ""
|
||||
|
||||
self.has_tried_valid_json_suffix = True
|
||||
try:
|
||||
value, end_idx = json.JSONDecoder().raw_decode(self.json_str[self.index :])
|
||||
except json.JSONDecodeError:
|
||||
return False, ""
|
||||
|
||||
self.index += end_idx
|
||||
return True, value
|
||||
|
||||
def _try_parse_missing_opening_object(self, delimiter: str) -> tuple[bool, JSONReturnType]:
|
||||
if self.scroll_whitespaces(-self.index) != 0:
|
||||
return False, ""
|
||||
|
||||
offset = self.skip_to_character(delimiter, idx=1)
|
||||
if self.get_char_at(offset) is None:
|
||||
return False, ""
|
||||
|
||||
offset = self.scroll_whitespaces(offset + 1)
|
||||
if self.get_char_at(offset) != ":":
|
||||
return False, ""
|
||||
|
||||
end_offset = self.scroll_whitespaces(len(self.json_str) - self.index - 1, backwards=True)
|
||||
if self.get_char_at(end_offset) != "}":
|
||||
return False, ""
|
||||
|
||||
# Require a single insertion to explain the whole input, rather than
|
||||
# treating a quoted prose prefix or an inner container as an object.
|
||||
candidate = "{" + self.json_str[:]
|
||||
try:
|
||||
value, end_idx = json.JSONDecoder().raw_decode(candidate)
|
||||
except ValueError:
|
||||
# Decoder limits (such as oversized integers) also mean the
|
||||
# insertion is insufficient; leave these inputs to normal repair.
|
||||
return False, ""
|
||||
if candidate[end_idx:].strip(" \t\r\n"):
|
||||
return False, ""
|
||||
|
||||
if self.strict:
|
||||
raise ValueError("Missing opening object brace in strict mode.")
|
||||
self.log("Inserted missing opening object brace")
|
||||
self.index = len(self.json_str)
|
||||
return True, value
|
||||
|
||||
def parse_json(
|
||||
self,
|
||||
schema: dict[str, Any] | bool | None = None,
|
||||
path: str = "$",
|
||||
finalize_schema: bool = True,
|
||||
record_top_level_value: bool = False,
|
||||
) -> JSONReturnType:
|
||||
"""Parse the next JSON value and, when configured, enforce schema constraints."""
|
||||
if record_top_level_value:
|
||||
self._last_parse_found_value = False
|
||||
if self.deferred_contexts:
|
||||
deferred_contexts, self.deferred_contexts = self.deferred_contexts, []
|
||||
with ExitStack() as stack:
|
||||
for context_value in deferred_contexts:
|
||||
stack.enter_context(self.context.enter(context_value))
|
||||
return self.parse_json(schema, path, finalize_schema, record_top_level_value)
|
||||
|
||||
repairer, schema = self._resolve_schema_for_parse(schema)
|
||||
check_initial_quote = self.index == 0 and self.context.empty
|
||||
|
||||
while True:
|
||||
char = self.get_char_at()
|
||||
# None means that we are at the end of the string provided
|
||||
if char is None:
|
||||
return ""
|
||||
if (
|
||||
self.try_valid_json_suffix
|
||||
and char in ["{", "["]
|
||||
and self._initial_container_has_non_comma_trailing_content()
|
||||
):
|
||||
parsed_suffix, value = self._try_parse_valid_json_value()
|
||||
if parsed_suffix:
|
||||
return self._finalize_parsed_value(
|
||||
value, repairer, schema, path, finalize_schema, record_top_level_value
|
||||
)
|
||||
# <object> starts with '{'
|
||||
if char == "{":
|
||||
self.index += 1
|
||||
value = self.parse_object(schema, path) if repairer else self.parse_object()
|
||||
return self._finalize_parsed_value(
|
||||
value, repairer, schema, path, finalize_schema, record_top_level_value
|
||||
)
|
||||
# <array> starts with '['
|
||||
if char == "[":
|
||||
self.index += 1
|
||||
value = self.parse_array(schema, path) if repairer else self.parse_array()
|
||||
return self._finalize_parsed_value(
|
||||
value, repairer, schema, path, finalize_schema, record_top_level_value
|
||||
)
|
||||
# Python tuple literals and grouped values start with '('
|
||||
if char == "(":
|
||||
# Keep top-level tuple detection conservative so inline prose like
|
||||
# "note (clarification):" does not hijack later JSON blocks.
|
||||
if not self.context.empty or self.top_level_parenthesized_can_start_value():
|
||||
value = self.parse_parenthesized(schema, path) if repairer else self.parse_parenthesized()
|
||||
return self._finalize_parsed_value(
|
||||
value, repairer, schema, path, finalize_schema, record_top_level_value
|
||||
)
|
||||
self.index += 1
|
||||
continue
|
||||
# <string> starts with a quote
|
||||
if not self.context.empty and (char in STRING_DELIMITERS or char.isalpha()):
|
||||
value = self.parse_string()
|
||||
return self._finalize_parsed_value(
|
||||
value, repairer, schema, path, finalize_schema, record_top_level_value
|
||||
)
|
||||
# <number> starts with [0-9] or minus
|
||||
if not self.context.empty and (char.isdigit() or char == "-" or char == "."):
|
||||
value = self.parse_number()
|
||||
return self._finalize_parsed_value(
|
||||
value, repairer, schema, path, finalize_schema, record_top_level_value
|
||||
)
|
||||
if char in ["#", "/"]:
|
||||
value = self.parse_comment(record_top_level_value)
|
||||
return self._finalize_parsed_value(value, repairer, schema, path, finalize_schema)
|
||||
# Only probe a missing brace when normal dispatch would discard the
|
||||
# initial quote; later quoted prose must not start this recovery.
|
||||
if char in STRING_DELIMITERS and check_initial_quote:
|
||||
check_initial_quote = False
|
||||
parsed_object, value = self._try_parse_missing_opening_object(char)
|
||||
if parsed_object:
|
||||
return self._finalize_parsed_value(
|
||||
value, repairer, schema, path, finalize_schema, record_top_level_value
|
||||
)
|
||||
# If everything else fails, we just ignore and move on
|
||||
self.index += 1
|
||||
|
||||
def _resolve_schema_for_parse(
|
||||
self,
|
||||
schema: dict[str, Any] | bool | None,
|
||||
) -> tuple["SchemaRepairer | None", dict[str, Any] | bool | None]:
|
||||
repairer = self.schema_repairer if self.schema_repairer is not None and schema not in (None, True) else None
|
||||
if repairer is None:
|
||||
return None, schema
|
||||
|
||||
schema = repairer.resolve_schema(schema)
|
||||
if schema is True:
|
||||
return None, schema
|
||||
if schema is False:
|
||||
raise ValueError("Schema does not allow any values.")
|
||||
return repairer, schema
|
||||
|
||||
def _finalize_parsed_value(
|
||||
self,
|
||||
value: JSONReturnType,
|
||||
repairer: "SchemaRepairer | None",
|
||||
schema: dict[str, Any] | bool | None,
|
||||
path: str,
|
||||
finalize_schema: bool = True,
|
||||
record_top_level_value: bool = False,
|
||||
) -> JSONReturnType:
|
||||
if record_top_level_value:
|
||||
self._last_parse_found_value = True
|
||||
if repairer is None or not finalize_schema:
|
||||
return value
|
||||
return repairer.repair_value(value, schema, path)
|
||||
|
||||
def get_char_at(self, count: int = 0) -> str | None:
|
||||
# Why not use something simpler? Because try/except in python is a faster alternative to an "if" statement that is often True
|
||||
try:
|
||||
return self.json_str[self.index + count]
|
||||
except IndexError:
|
||||
return None
|
||||
|
||||
def skip_whitespaces(self) -> None:
|
||||
"""
|
||||
This function quickly iterates on whitespaces, moving the self.index forward
|
||||
"""
|
||||
try:
|
||||
char = self.json_str[self.index]
|
||||
while char.isspace():
|
||||
self.index += 1
|
||||
char = self.json_str[self.index]
|
||||
except IndexError:
|
||||
pass
|
||||
|
||||
def scroll_whitespaces(self, idx: int = 0, *, backwards: bool = False) -> int:
|
||||
"""
|
||||
Skip whitespace at idx without moving self.index, returning the relative offset.
|
||||
Set backwards to scan toward the start.
|
||||
"""
|
||||
step = -1 if backwards else 1
|
||||
try:
|
||||
char = self.json_str[self.index + idx]
|
||||
while self.index + idx >= 0 and char.isspace():
|
||||
idx += step
|
||||
char = self.json_str[self.index + idx]
|
||||
except IndexError:
|
||||
pass
|
||||
return idx
|
||||
|
||||
def skip_to_character(self, character: str | list[str], idx: int = 0) -> int:
|
||||
"""
|
||||
Advance from (self.index + idx) until we hit an *unescaped* target character.
|
||||
Returns the offset (idx) from self.index to that position, or the distance to the end if not found.
|
||||
"""
|
||||
targets = set(character) if isinstance(character, list) else {character}
|
||||
i = self.index + idx
|
||||
n = len(self.json_str)
|
||||
backslashes = 0 # count of consecutive '\' immediately before current char
|
||||
|
||||
while i < n:
|
||||
ch = self.json_str[i]
|
||||
|
||||
if ch == "\\":
|
||||
backslashes += 1
|
||||
i += 1
|
||||
continue
|
||||
|
||||
# ch is not a backslash; if it's a target and not escaped (even backslashes), we're done
|
||||
if ch in targets and (backslashes % 2 == 0):
|
||||
return i - self.index
|
||||
|
||||
# reset backslash run when we see a non-backslash
|
||||
backslashes = 0
|
||||
i += 1
|
||||
|
||||
# not found; return distance to end
|
||||
return n - self.index
|
||||
|
||||
def parenthesized_is_explicit_tuple(self) -> bool:
|
||||
return parenthesized_is_explicit_tuple(self)
|
||||
|
||||
def top_level_parenthesized_can_start_value(self) -> bool:
|
||||
return top_level_parenthesized_can_start_value(self)
|
||||
|
||||
def parse_parenthesized(
|
||||
self,
|
||||
schema: dict[str, Any] | bool | None = None,
|
||||
path: str = "$",
|
||||
) -> JSONReturnType:
|
||||
explicit_tuple = self.parenthesized_is_explicit_tuple()
|
||||
self.index += 1
|
||||
values = self.parse_array(schema, path, closing_delimiter=")")
|
||||
if explicit_tuple or len(values) != 1:
|
||||
return values
|
||||
return values[0]
|
||||
|
||||
def _log(self, text: str) -> None:
|
||||
window: int = 10
|
||||
start: int = max(self.index - window, 0)
|
||||
end: int = min(self.index + window, len(self.json_str))
|
||||
context: str = self.json_str[start:end]
|
||||
self.logger.append(
|
||||
{
|
||||
"text": text,
|
||||
"context": context,
|
||||
}
|
||||
)
|
||||
Reference in new issue
Block a user