Initial commit
This commit is contained in:
commit
57fc3525eb
7311 files changed
+1291398
No files matched your search
@@ -0,0 +1,4 @@
|
||||
from .json_repair import from_file, load, loads, repair_json
|
||||
from .utils.constants import JSONReturnType
|
||||
|
||||
__all__ = ["JSONReturnType", "from_file", "load", "loads", "repair_json"]
|
||||
@@ -0,0 +1,6 @@
|
||||
import sys
|
||||
|
||||
from .json_repair import cli
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(cli())
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -0,0 +1,481 @@
|
||||
import json
|
||||
from collections.abc import Callable
|
||||
from contextlib import ExitStack
|
||||
from typing import TYPE_CHECKING, Any, TextIO
|
||||
|
||||
from .parse_array import parse_array as _parse_array
|
||||
from .parse_comment import parse_comment as _parse_comment
|
||||
from .parse_number import parse_number as _parse_number
|
||||
from .parse_object import parse_object as _parse_object
|
||||
from .parse_string import parse_string as _parse_string
|
||||
from .parser_parenthesized import parenthesized_is_explicit_tuple, top_level_parenthesized_can_start_value
|
||||
from .utils.constants import STRING_DELIMITERS, JSONReturnType
|
||||
from .utils.json_context import ContextValues, JsonContext
|
||||
from .utils.object_comparer import ObjectComparer
|
||||
from .utils.string_file_wrapper import StringFileWrapper
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from .schema_repair import SchemaRepairer
|
||||
|
||||
|
||||
class JSONParser:
|
||||
# Split the parse methods into separate files because this one was like 3000 lines
|
||||
def parse_array(
|
||||
self,
|
||||
schema: dict[str, Any] | bool | None = None,
|
||||
path: str = "$",
|
||||
closing_delimiter: str = "]",
|
||||
) -> list[JSONReturnType]:
|
||||
return _parse_array(self, schema, path, closing_delimiter)
|
||||
|
||||
def parse_comment(self, record_top_level_value: bool = False) -> JSONReturnType:
|
||||
return _parse_comment(self, record_top_level_value=record_top_level_value)
|
||||
|
||||
def parse_number(self) -> JSONReturnType:
|
||||
return _parse_number(self)
|
||||
|
||||
def parse_object(
|
||||
self,
|
||||
schema: dict[str, Any] | bool | None = None,
|
||||
path: str = "$",
|
||||
) -> JSONReturnType:
|
||||
return _parse_object(self, schema, path)
|
||||
|
||||
def parse_string(self) -> JSONReturnType:
|
||||
return _parse_string(self)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
json_str: str | StringFileWrapper,
|
||||
json_fd: TextIO | None,
|
||||
logging: bool | None,
|
||||
json_fd_chunk_length: int = 0,
|
||||
stream_stable: bool = False,
|
||||
strict: bool = False,
|
||||
try_valid_json_suffix: bool = False,
|
||||
) -> None:
|
||||
# The string to parse
|
||||
self.json_str: str | StringFileWrapper = json_str
|
||||
# Alternatively, the file description with a json file in it
|
||||
if json_fd:
|
||||
# This is a trick we do to treat the file wrapper as an array
|
||||
self.json_str = StringFileWrapper(json_fd, json_fd_chunk_length)
|
||||
# Index is our iterator that will keep track of which character we are looking at right now
|
||||
self.index: int = 0
|
||||
# This is used in the object member parsing to manage the special cases of missing quotes in key or value
|
||||
self.context = JsonContext()
|
||||
self.deferred_contexts: list[ContextValues] = []
|
||||
# Use this to log the activity, but only if logging is active
|
||||
|
||||
# This is a trick but a beautiful one. We call self.log in the code over and over even if it's not needed.
|
||||
# We could add a guard in the code for each call but that would make this code unreadable, so here's this neat trick
|
||||
# Replace self.log with a noop
|
||||
self.logging = logging
|
||||
self.logger: list[dict[str, str]] = []
|
||||
if logging:
|
||||
self.log = self._log
|
||||
else:
|
||||
# No-op
|
||||
self.log = lambda *args, **kwargs: None # noqa: ARG005
|
||||
# When the json to be repaired is the accumulation of streaming json at a certain moment.
|
||||
# e.g. json obtained from llm response.
|
||||
# If this parameter to True will keep the repair results stable. For example:
|
||||
# case 1: '{"key": "val\\' => '{"key": "val"}'
|
||||
# case 2: '{"key": "val\\n' => '{"key": "val\\n"}'
|
||||
# case 3: '{"key": "val\\n123,`key2:value2' => '{"key": "val\\n123,`key2:value2"}'
|
||||
# case 4: '{"key": "val\\n123,`key2:value2`"}' => '{"key": "val\\n123,`key2:value2`"}'
|
||||
self.stream_stable = stream_stable
|
||||
# Over time the library got more and more complex heuristics to repair JSON. Some of these heuristics
|
||||
# may not be desirable in some use cases and the user would prefer json_repair to return an exception.
|
||||
# So strict mode was added to disable some of those heuristics.
|
||||
self.strict = strict
|
||||
self.try_valid_json_suffix = try_valid_json_suffix
|
||||
self.has_tried_valid_json_suffix = False
|
||||
self._last_parse_found_value = False
|
||||
self.schema_repairer: SchemaRepairer | None = None
|
||||
|
||||
def parse(
|
||||
self,
|
||||
) -> JSONReturnType:
|
||||
return self._parse_top_level(self.parse_json)
|
||||
|
||||
def parse_with_schema(
|
||||
self,
|
||||
repairer: "SchemaRepairer",
|
||||
schema: dict[str, Any] | bool,
|
||||
) -> JSONReturnType:
|
||||
"""Parse with schema guidance enabled for all nested values."""
|
||||
self.schema_repairer = repairer
|
||||
if repairer.schema_repair_mode == "salvage":
|
||||
return self._parse_top_level_salvage_with_schema(repairer, schema)
|
||||
return self._parse_top_level(lambda: self.parse_json(schema, "$"))
|
||||
|
||||
def _parse_top_level_salvage_with_schema(
|
||||
self,
|
||||
repairer: "SchemaRepairer",
|
||||
schema: dict[str, Any] | bool,
|
||||
) -> JSONReturnType:
|
||||
"""Return the first schema-valid top-level fragment while salvaging."""
|
||||
last_error: ValueError | None = None
|
||||
while self.index < len(self.json_str):
|
||||
self.context.clear()
|
||||
self.deferred_contexts.clear()
|
||||
value = self.parse_json(schema, "$", finalize_schema=False, record_top_level_value=True)
|
||||
if not self._last_parse_found_value:
|
||||
break
|
||||
try:
|
||||
repaired = repairer.repair_value(value, schema, "$")
|
||||
repairer.validate(repaired, schema)
|
||||
except ValueError as exc:
|
||||
last_error = exc
|
||||
if self.index >= len(self.json_str):
|
||||
break
|
||||
self.log("Skipped top-level fragment that did not match schema while salvaging")
|
||||
continue
|
||||
else:
|
||||
return repaired
|
||||
if last_error is not None:
|
||||
raise last_error
|
||||
return ""
|
||||
|
||||
# Consolidate top-level parsing so we handle multiple sequential JSON values consistently
|
||||
# (including update semantics and strict-mode validation).
|
||||
def _parse_top_level(self, parse_element: Callable[[], JSONReturnType]) -> JSONReturnType:
|
||||
json = parse_element()
|
||||
if self.index < len(self.json_str):
|
||||
self.log(
|
||||
"The parser returned early, checking if there's more json elements",
|
||||
)
|
||||
json = [json]
|
||||
while self.index < len(self.json_str):
|
||||
self.context.clear()
|
||||
self.deferred_contexts.clear()
|
||||
is_comma_separated = self._next_top_level_value_is_comma_separated()
|
||||
element_start_index = self.index
|
||||
j = parse_element()
|
||||
if self.strict and self.index > element_start_index:
|
||||
self.log(
|
||||
"Multiple top-level JSON elements found in strict mode, raising an error",
|
||||
)
|
||||
raise ValueError("Multiple top-level JSON elements found in strict mode.")
|
||||
if j:
|
||||
if not is_comma_separated and ObjectComparer.is_same_object(json[-1], j):
|
||||
# Treat repeated objects as updates: keep the newest value.
|
||||
json.pop()
|
||||
else:
|
||||
if not json[-1]:
|
||||
json.pop()
|
||||
json.append(j)
|
||||
else:
|
||||
self.index += 1
|
||||
if len(json) == 1:
|
||||
self.log(
|
||||
"There were no more elements, returning the element without the array",
|
||||
)
|
||||
json = json[0]
|
||||
return json
|
||||
|
||||
def _next_top_level_value_is_comma_separated(self) -> bool:
|
||||
idx = self.scroll_whitespaces()
|
||||
if self.get_char_at(idx) == ",":
|
||||
return True
|
||||
|
||||
idx = self.index - 1
|
||||
while idx >= 0 and self.json_str[idx].isspace():
|
||||
idx -= 1
|
||||
return idx >= 0 and self.json_str[idx] == ","
|
||||
|
||||
def _initial_container_has_non_comma_trailing_content(self) -> bool:
|
||||
if self.index != 0 or not isinstance(self.json_str, str):
|
||||
return True
|
||||
|
||||
containers: list[str] = []
|
||||
in_string = False
|
||||
escaped = False
|
||||
closing_delimiters = {"{": "}", "[": "]"}
|
||||
for index, char in enumerate(self.json_str):
|
||||
if in_string:
|
||||
if escaped:
|
||||
escaped = False
|
||||
elif char == "\\":
|
||||
escaped = True
|
||||
elif char == '"':
|
||||
in_string = False
|
||||
continue
|
||||
|
||||
if char == '"':
|
||||
in_string = True
|
||||
elif char in closing_delimiters:
|
||||
containers.append(char)
|
||||
elif char in ["}", "]"]:
|
||||
if not containers or char != closing_delimiters[containers.pop()]:
|
||||
return False
|
||||
if not containers:
|
||||
trailing_index = index + 1
|
||||
while trailing_index < len(self.json_str) and self.json_str[trailing_index].isspace():
|
||||
trailing_index += 1
|
||||
return trailing_index < len(self.json_str) and self.json_str[trailing_index] != ","
|
||||
|
||||
return False
|
||||
|
||||
def _try_parse_valid_json_value(self) -> tuple[bool, JSONReturnType]:
|
||||
if (
|
||||
not self.try_valid_json_suffix
|
||||
or self.has_tried_valid_json_suffix
|
||||
or not self.context.empty
|
||||
or not isinstance(self.json_str, str)
|
||||
):
|
||||
return False, ""
|
||||
|
||||
self.has_tried_valid_json_suffix = True
|
||||
try:
|
||||
value, end_idx = json.JSONDecoder().raw_decode(self.json_str[self.index :])
|
||||
except json.JSONDecodeError:
|
||||
return False, ""
|
||||
|
||||
self.index += end_idx
|
||||
return True, value
|
||||
|
||||
def _try_parse_missing_opening_object(self, delimiter: str) -> tuple[bool, JSONReturnType]:
|
||||
if self.scroll_whitespaces(-self.index) != 0:
|
||||
return False, ""
|
||||
|
||||
offset = self.skip_to_character(delimiter, idx=1)
|
||||
if self.get_char_at(offset) is None:
|
||||
return False, ""
|
||||
|
||||
offset = self.scroll_whitespaces(offset + 1)
|
||||
if self.get_char_at(offset) != ":":
|
||||
return False, ""
|
||||
|
||||
end_offset = self.scroll_whitespaces(len(self.json_str) - self.index - 1, backwards=True)
|
||||
if self.get_char_at(end_offset) != "}":
|
||||
return False, ""
|
||||
|
||||
# Require a single insertion to explain the whole input, rather than
|
||||
# treating a quoted prose prefix or an inner container as an object.
|
||||
candidate = "{" + self.json_str[:]
|
||||
try:
|
||||
value, end_idx = json.JSONDecoder().raw_decode(candidate)
|
||||
except ValueError:
|
||||
# Decoder limits (such as oversized integers) also mean the
|
||||
# insertion is insufficient; leave these inputs to normal repair.
|
||||
return False, ""
|
||||
if candidate[end_idx:].strip(" \t\r\n"):
|
||||
return False, ""
|
||||
|
||||
if self.strict:
|
||||
raise ValueError("Missing opening object brace in strict mode.")
|
||||
self.log("Inserted missing opening object brace")
|
||||
self.index = len(self.json_str)
|
||||
return True, value
|
||||
|
||||
def parse_json(
|
||||
self,
|
||||
schema: dict[str, Any] | bool | None = None,
|
||||
path: str = "$",
|
||||
finalize_schema: bool = True,
|
||||
record_top_level_value: bool = False,
|
||||
) -> JSONReturnType:
|
||||
"""Parse the next JSON value and, when configured, enforce schema constraints."""
|
||||
if record_top_level_value:
|
||||
self._last_parse_found_value = False
|
||||
if self.deferred_contexts:
|
||||
deferred_contexts, self.deferred_contexts = self.deferred_contexts, []
|
||||
with ExitStack() as stack:
|
||||
for context_value in deferred_contexts:
|
||||
stack.enter_context(self.context.enter(context_value))
|
||||
return self.parse_json(schema, path, finalize_schema, record_top_level_value)
|
||||
|
||||
repairer, schema = self._resolve_schema_for_parse(schema)
|
||||
check_initial_quote = self.index == 0 and self.context.empty
|
||||
|
||||
while True:
|
||||
char = self.get_char_at()
|
||||
# None means that we are at the end of the string provided
|
||||
if char is None:
|
||||
return ""
|
||||
if (
|
||||
self.try_valid_json_suffix
|
||||
and char in ["{", "["]
|
||||
and self._initial_container_has_non_comma_trailing_content()
|
||||
):
|
||||
parsed_suffix, value = self._try_parse_valid_json_value()
|
||||
if parsed_suffix:
|
||||
return self._finalize_parsed_value(
|
||||
value, repairer, schema, path, finalize_schema, record_top_level_value
|
||||
)
|
||||
# <object> starts with '{'
|
||||
if char == "{":
|
||||
self.index += 1
|
||||
value = self.parse_object(schema, path) if repairer else self.parse_object()
|
||||
return self._finalize_parsed_value(
|
||||
value, repairer, schema, path, finalize_schema, record_top_level_value
|
||||
)
|
||||
# <array> starts with '['
|
||||
if char == "[":
|
||||
self.index += 1
|
||||
value = self.parse_array(schema, path) if repairer else self.parse_array()
|
||||
return self._finalize_parsed_value(
|
||||
value, repairer, schema, path, finalize_schema, record_top_level_value
|
||||
)
|
||||
# Python tuple literals and grouped values start with '('
|
||||
if char == "(":
|
||||
# Keep top-level tuple detection conservative so inline prose like
|
||||
# "note (clarification):" does not hijack later JSON blocks.
|
||||
if not self.context.empty or self.top_level_parenthesized_can_start_value():
|
||||
value = self.parse_parenthesized(schema, path) if repairer else self.parse_parenthesized()
|
||||
return self._finalize_parsed_value(
|
||||
value, repairer, schema, path, finalize_schema, record_top_level_value
|
||||
)
|
||||
self.index += 1
|
||||
continue
|
||||
# <string> starts with a quote
|
||||
if not self.context.empty and (char in STRING_DELIMITERS or char.isalpha()):
|
||||
value = self.parse_string()
|
||||
return self._finalize_parsed_value(
|
||||
value, repairer, schema, path, finalize_schema, record_top_level_value
|
||||
)
|
||||
# <number> starts with [0-9] or minus
|
||||
if not self.context.empty and (char.isdigit() or char == "-" or char == "."):
|
||||
value = self.parse_number()
|
||||
return self._finalize_parsed_value(
|
||||
value, repairer, schema, path, finalize_schema, record_top_level_value
|
||||
)
|
||||
if char in ["#", "/"]:
|
||||
value = self.parse_comment(record_top_level_value)
|
||||
return self._finalize_parsed_value(value, repairer, schema, path, finalize_schema)
|
||||
# Only probe a missing brace when normal dispatch would discard the
|
||||
# initial quote; later quoted prose must not start this recovery.
|
||||
if char in STRING_DELIMITERS and check_initial_quote:
|
||||
check_initial_quote = False
|
||||
parsed_object, value = self._try_parse_missing_opening_object(char)
|
||||
if parsed_object:
|
||||
return self._finalize_parsed_value(
|
||||
value, repairer, schema, path, finalize_schema, record_top_level_value
|
||||
)
|
||||
# If everything else fails, we just ignore and move on
|
||||
self.index += 1
|
||||
|
||||
def _resolve_schema_for_parse(
|
||||
self,
|
||||
schema: dict[str, Any] | bool | None,
|
||||
) -> tuple["SchemaRepairer | None", dict[str, Any] | bool | None]:
|
||||
repairer = self.schema_repairer if self.schema_repairer is not None and schema not in (None, True) else None
|
||||
if repairer is None:
|
||||
return None, schema
|
||||
|
||||
schema = repairer.resolve_schema(schema)
|
||||
if schema is True:
|
||||
return None, schema
|
||||
if schema is False:
|
||||
raise ValueError("Schema does not allow any values.")
|
||||
return repairer, schema
|
||||
|
||||
def _finalize_parsed_value(
|
||||
self,
|
||||
value: JSONReturnType,
|
||||
repairer: "SchemaRepairer | None",
|
||||
schema: dict[str, Any] | bool | None,
|
||||
path: str,
|
||||
finalize_schema: bool = True,
|
||||
record_top_level_value: bool = False,
|
||||
) -> JSONReturnType:
|
||||
if record_top_level_value:
|
||||
self._last_parse_found_value = True
|
||||
if repairer is None or not finalize_schema:
|
||||
return value
|
||||
return repairer.repair_value(value, schema, path)
|
||||
|
||||
def get_char_at(self, count: int = 0) -> str | None:
|
||||
# Why not use something simpler? Because try/except in python is a faster alternative to an "if" statement that is often True
|
||||
try:
|
||||
return self.json_str[self.index + count]
|
||||
except IndexError:
|
||||
return None
|
||||
|
||||
def skip_whitespaces(self) -> None:
|
||||
"""
|
||||
This function quickly iterates on whitespaces, moving the self.index forward
|
||||
"""
|
||||
try:
|
||||
char = self.json_str[self.index]
|
||||
while char.isspace():
|
||||
self.index += 1
|
||||
char = self.json_str[self.index]
|
||||
except IndexError:
|
||||
pass
|
||||
|
||||
def scroll_whitespaces(self, idx: int = 0, *, backwards: bool = False) -> int:
|
||||
"""
|
||||
Skip whitespace at idx without moving self.index, returning the relative offset.
|
||||
Set backwards to scan toward the start.
|
||||
"""
|
||||
step = -1 if backwards else 1
|
||||
try:
|
||||
char = self.json_str[self.index + idx]
|
||||
while self.index + idx >= 0 and char.isspace():
|
||||
idx += step
|
||||
char = self.json_str[self.index + idx]
|
||||
except IndexError:
|
||||
pass
|
||||
return idx
|
||||
|
||||
def skip_to_character(self, character: str | list[str], idx: int = 0) -> int:
|
||||
"""
|
||||
Advance from (self.index + idx) until we hit an *unescaped* target character.
|
||||
Returns the offset (idx) from self.index to that position, or the distance to the end if not found.
|
||||
"""
|
||||
targets = set(character) if isinstance(character, list) else {character}
|
||||
i = self.index + idx
|
||||
n = len(self.json_str)
|
||||
backslashes = 0 # count of consecutive '\' immediately before current char
|
||||
|
||||
while i < n:
|
||||
ch = self.json_str[i]
|
||||
|
||||
if ch == "\\":
|
||||
backslashes += 1
|
||||
i += 1
|
||||
continue
|
||||
|
||||
# ch is not a backslash; if it's a target and not escaped (even backslashes), we're done
|
||||
if ch in targets and (backslashes % 2 == 0):
|
||||
return i - self.index
|
||||
|
||||
# reset backslash run when we see a non-backslash
|
||||
backslashes = 0
|
||||
i += 1
|
||||
|
||||
# not found; return distance to end
|
||||
return n - self.index
|
||||
|
||||
def parenthesized_is_explicit_tuple(self) -> bool:
|
||||
return parenthesized_is_explicit_tuple(self)
|
||||
|
||||
def top_level_parenthesized_can_start_value(self) -> bool:
|
||||
return top_level_parenthesized_can_start_value(self)
|
||||
|
||||
def parse_parenthesized(
|
||||
self,
|
||||
schema: dict[str, Any] | bool | None = None,
|
||||
path: str = "$",
|
||||
) -> JSONReturnType:
|
||||
explicit_tuple = self.parenthesized_is_explicit_tuple()
|
||||
self.index += 1
|
||||
values = self.parse_array(schema, path, closing_delimiter=")")
|
||||
if explicit_tuple or len(values) != 1:
|
||||
return values
|
||||
return values[0]
|
||||
|
||||
def _log(self, text: str) -> None:
|
||||
window: int = 10
|
||||
start: int = max(self.index - window, 0)
|
||||
end: int = min(self.index + window, len(self.json_str))
|
||||
context: str = self.json_str[start:end]
|
||||
self.logger.append(
|
||||
{
|
||||
"text": text,
|
||||
"context": context,
|
||||
}
|
||||
)
|
||||
@@ -0,0 +1,484 @@
|
||||
"""
|
||||
This module will parse the JSON file following the BNF definition:
|
||||
|
||||
<json> ::= <container>
|
||||
|
||||
<primitive> ::= <number> | <string> | <boolean>
|
||||
; Where:
|
||||
; <number> is a valid real number expressed in one of a number of given formats
|
||||
; <string> is a string of valid characters enclosed in quotes
|
||||
; <boolean> is one of the literal strings 'true', 'false', or 'null' (unquoted)
|
||||
|
||||
<container> ::= <object> | <array>
|
||||
<array> ::= '[' [ <json> *(', ' <json>) ] ']' ; A sequence of JSON values separated by commas
|
||||
<object> ::= '{' [ <member> *(', ' <member>) ] '}' ; A sequence of 'members'
|
||||
<member> ::= <string> ': ' <json> ; A pair consisting of a name, and a JSON value
|
||||
|
||||
If something is wrong (a missing parentheses or quotes for example) it will use a few simple heuristics to fix the JSON string:
|
||||
- Add the missing parentheses if the parser believes that the array or object should be closed
|
||||
- Quote strings or add missing single quotes
|
||||
- Adjust whitespaces and remove line breaks
|
||||
|
||||
All supported use cases are in the unit tests
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any, Literal, TextIO, overload
|
||||
|
||||
from .json_parser import JSONParser
|
||||
from .schema_repair import SchemaRepairer, load_schema_model, normalize_schema_repair_mode, schema_from_input
|
||||
from .utils.constants import JSONReturnType
|
||||
|
||||
|
||||
@overload
|
||||
def repair_json(
|
||||
json_str: str = "",
|
||||
return_objects: Literal[False] = False,
|
||||
skip_json_loads: bool = False,
|
||||
logging: Literal[False] = False,
|
||||
json_fd: TextIO | None = None,
|
||||
chunk_length: int = 0,
|
||||
stream_stable: bool = False,
|
||||
strict: bool = False,
|
||||
schema: Any | None = None,
|
||||
schema_repair_mode: Literal["standard", "salvage"] = "standard",
|
||||
**json_dumps_args: Any,
|
||||
) -> str: ...
|
||||
|
||||
|
||||
@overload
|
||||
def repair_json(
|
||||
json_str: str = "",
|
||||
return_objects: Literal[True] = True,
|
||||
skip_json_loads: bool = False,
|
||||
logging: Literal[False] = False,
|
||||
json_fd: TextIO | None = None,
|
||||
chunk_length: int = 0,
|
||||
stream_stable: bool = False,
|
||||
strict: bool = False,
|
||||
schema: Any | None = None,
|
||||
schema_repair_mode: Literal["standard", "salvage"] = "standard",
|
||||
**json_dumps_args: Any,
|
||||
) -> JSONReturnType: ...
|
||||
|
||||
|
||||
@overload
|
||||
def repair_json(
|
||||
json_str: str = "",
|
||||
return_objects: bool = False,
|
||||
skip_json_loads: bool = False,
|
||||
logging: Literal[True] = True,
|
||||
json_fd: TextIO | None = None,
|
||||
chunk_length: int = 0,
|
||||
stream_stable: bool = False,
|
||||
strict: bool = False,
|
||||
schema: Any | None = None,
|
||||
schema_repair_mode: Literal["standard", "salvage"] = "standard",
|
||||
**json_dumps_args: Any,
|
||||
) -> tuple[JSONReturnType, list[dict[str, str]]]: ...
|
||||
|
||||
|
||||
@overload
|
||||
def repair_json(
|
||||
json_str: str = "",
|
||||
return_objects: bool = False,
|
||||
skip_json_loads: bool = False,
|
||||
logging: bool = False,
|
||||
json_fd: TextIO | None = None,
|
||||
chunk_length: int = 0,
|
||||
stream_stable: bool = False,
|
||||
strict: bool = False,
|
||||
schema: Any | None = None,
|
||||
schema_repair_mode: Literal["standard", "salvage"] = "standard",
|
||||
**json_dumps_args: Any,
|
||||
) -> str | JSONReturnType | tuple[JSONReturnType, list[dict[str, str]]]: ...
|
||||
|
||||
|
||||
def repair_json(
|
||||
json_str: str = "",
|
||||
return_objects: bool = False,
|
||||
skip_json_loads: bool = False,
|
||||
logging: bool = False,
|
||||
json_fd: TextIO | None = None,
|
||||
chunk_length: int = 0,
|
||||
stream_stable: bool = False,
|
||||
strict: bool = False,
|
||||
schema: Any | None = None,
|
||||
schema_repair_mode: Literal["standard", "salvage"] = "standard",
|
||||
**json_dumps_args: Any,
|
||||
) -> JSONReturnType | tuple[JSONReturnType, list[dict[str, str]]]:
|
||||
"""
|
||||
Given a json formatted string, it will try to decode it and, if it fails, it will try to fix it.
|
||||
|
||||
Args:
|
||||
json_str (str, optional): The JSON string to repair. Defaults to an empty string.
|
||||
return_objects (bool, optional): If True, return the decoded data structure. Defaults to False.
|
||||
skip_json_loads (bool, optional): If True, skip calling the built-in json.loads() function to verify that the json is valid before attempting to repair. Defaults to False.
|
||||
logging (bool, optional): If True, return a tuple with the repaired json and a log of all repair actions. Defaults to False. When no repairs were required, the repair log will be an empty list.
|
||||
json_fd (Optional[TextIO], optional): File descriptor for JSON input. Do not use! Use `from_file` or `load` instead. Defaults to None.
|
||||
ensure_ascii (bool, optional): Set to False to avoid converting non-latin characters to ascii (for example when using chinese characters). Defaults to True. Ignored if `skip_json_loads` is True.
|
||||
chunk_length (int, optional): Size in bytes of the file chunks to read at once. Ignored if `json_fd` is None. Do not use! Use `from_file` or `load` instead. Defaults to 1MB.
|
||||
stream_stable (bool, optional): When the json to be repaired is the accumulation of streaming json at a certain moment.If this parameter to True will keep the repair results stable.
|
||||
strict (bool, optional): If True, surface structural problems (duplicate keys, missing separators, empty keys/values, etc.) as ValueError instead of repairing them.
|
||||
schema (Any, optional): JSON Schema dict, boolean schema, or pydantic v2 model used to guide repairs and validation for both valid and invalid JSON inputs.
|
||||
schema_repair_mode (Literal["standard", "salvage"], optional): Schema repair mode. "standard" keeps default schema behavior; "salvage" enables best-effort schema salvage heuristics for arrays/objects.
|
||||
Returns:
|
||||
Union[JSONReturnType, Tuple[JSONReturnType, List[Dict[str, str]]]]: The repaired JSON or a tuple with the repaired JSON and repair log when logging is True.
|
||||
"""
|
||||
schema_repair_mode = normalize_schema_repair_mode(schema_repair_mode)
|
||||
if schema is None and schema_repair_mode == "salvage":
|
||||
raise ValueError("schema_repair_mode='salvage' requires schema.")
|
||||
|
||||
# Schema-guided repairs and strict mode are mutually exclusive to avoid conflicting behavior.
|
||||
if schema is not None and strict:
|
||||
raise ValueError("schema and strict cannot be used together.")
|
||||
|
||||
parser: JSONParser | None = None
|
||||
repair_log: list[dict[str, str]] = []
|
||||
# ``skip_json_loads`` only skips the initial whole-input validation. Once
|
||||
# the parser has found a top-level value after a prefix, raw decoding that
|
||||
# value is still a safe, targeted fast path.
|
||||
try_valid_json_suffix = json_fd is None
|
||||
if json_fd is not None:
|
||||
parser = JSONParser(
|
||||
json_str,
|
||||
json_fd,
|
||||
logging,
|
||||
chunk_length,
|
||||
stream_stable,
|
||||
strict,
|
||||
)
|
||||
if logging:
|
||||
repair_log = parser.logger
|
||||
schema_obj = schema_from_input(schema) if schema is not None else None
|
||||
repairer = (
|
||||
SchemaRepairer(schema_obj, repair_log if logging else None, schema_repair_mode=schema_repair_mode)
|
||||
if schema_obj is not None
|
||||
else None
|
||||
)
|
||||
|
||||
# Fast path for valid JSON: schema-aware mode still applies repair+validation.
|
||||
parsed_json: JSONReturnType = None
|
||||
is_valid_json = False
|
||||
try:
|
||||
if not skip_json_loads:
|
||||
parsed_json = json.load(json_fd) if json_fd else json.loads(json_str)
|
||||
if repairer is not None and schema_obj is not None:
|
||||
# Validate here to ensure that we reject values that cannot satisfy the schema and fall back to the more expensive parser+schema repair if needed, instead of just returning the valid but schema-noncompliant JSON.
|
||||
try:
|
||||
if repairer.is_valid(parsed_json, schema_obj):
|
||||
is_valid_json = True
|
||||
else:
|
||||
try:
|
||||
# repair_value may mutate containers in place; if validate fails we still
|
||||
# fall back to parser.parse_with_schema, which fully replaces parsed_json.
|
||||
repaired_value = repairer.repair_value(parsed_json, schema_obj, "$")
|
||||
if repairer.is_valid(repaired_value, schema_obj):
|
||||
parsed_json = repaired_value
|
||||
is_valid_json = True
|
||||
except ValueError:
|
||||
pass
|
||||
except RecursionError as exc:
|
||||
raise ValueError("Input schema nesting exceeds the supported schema recursion depth.") from exc
|
||||
else:
|
||||
is_valid_json = True
|
||||
except (json.JSONDecodeError, TypeError, ValueError):
|
||||
pass
|
||||
if not is_valid_json:
|
||||
if parser is None:
|
||||
parser = JSONParser(
|
||||
json_str,
|
||||
json_fd,
|
||||
logging,
|
||||
chunk_length,
|
||||
stream_stable,
|
||||
strict,
|
||||
try_valid_json_suffix=try_valid_json_suffix,
|
||||
)
|
||||
if logging:
|
||||
parser.logger = repair_log
|
||||
try:
|
||||
if repairer is not None and schema_obj is not None:
|
||||
# If schema-guided, we want to attempt repairs even on valid JSON that fails schema validation.
|
||||
try:
|
||||
parsed_json = parser.parse_with_schema(repairer, schema_obj)
|
||||
repairer.validate(parsed_json, schema_obj)
|
||||
except RecursionError as exc:
|
||||
raise ValueError("Input schema nesting exceeds the supported schema recursion depth.") from exc
|
||||
else:
|
||||
# Otherwise, we can skip the more expensive schema-aware parsing and just do a normal parse.
|
||||
parsed_json = parser.parse()
|
||||
except RecursionError as exc:
|
||||
raise ValueError("Input nesting exceeds the supported parser recursion depth.") from exc
|
||||
|
||||
# It's useful to return the actual object instead of the json string,
|
||||
# it allows this lib to be a replacement of the json library
|
||||
if logging:
|
||||
return parsed_json, repair_log
|
||||
if return_objects:
|
||||
return parsed_json
|
||||
# Avoid returning only a pair of quotes if it's an empty string
|
||||
if parsed_json == "":
|
||||
return ""
|
||||
return json.dumps(parsed_json, **json_dumps_args)
|
||||
|
||||
|
||||
def loads(
|
||||
json_str: str,
|
||||
skip_json_loads: bool = False,
|
||||
logging: bool = False,
|
||||
stream_stable: bool = False,
|
||||
strict: bool = False,
|
||||
schema: Any | None = None,
|
||||
schema_repair_mode: Literal["standard", "salvage"] = "standard",
|
||||
) -> JSONReturnType | tuple[JSONReturnType, list[dict[str, str]]] | str:
|
||||
"""
|
||||
This function works like `json.loads()` except that it will fix your JSON in the process.
|
||||
It is a wrapper around the `repair_json()` function with `return_objects=True`.
|
||||
|
||||
Args:
|
||||
json_str (str): The JSON string to load and repair.
|
||||
skip_json_loads (bool, optional): If True, skip calling the built-in json.loads() function to verify that the json is valid before attempting to repair. Defaults to False.
|
||||
logging (bool, optional): If True, return a tuple with the repaired json and a log of all repair actions. Defaults to False.
|
||||
strict (bool, optional): If True, surface structural problems (duplicate keys, missing separators, empty keys/values, etc.) as ValueError instead of repairing them.
|
||||
schema (Any, optional): JSON Schema dict, boolean schema, or pydantic v2 model used to guide repairs and validation for both valid and invalid JSON inputs.
|
||||
schema_repair_mode (Literal["standard", "salvage"], optional): Schema repair mode. "salvage" requires schema.
|
||||
|
||||
Returns:
|
||||
Union[JSONReturnType, Tuple[JSONReturnType, List[Dict[str, str]]], str]: The repaired JSON object or a tuple with the repaired JSON object and repair log.
|
||||
"""
|
||||
return repair_json(
|
||||
json_str=json_str,
|
||||
return_objects=True,
|
||||
skip_json_loads=skip_json_loads,
|
||||
logging=logging,
|
||||
stream_stable=stream_stable,
|
||||
strict=strict,
|
||||
schema=schema,
|
||||
schema_repair_mode=schema_repair_mode,
|
||||
)
|
||||
|
||||
|
||||
def load(
|
||||
fd: TextIO,
|
||||
skip_json_loads: bool = False,
|
||||
logging: bool = False,
|
||||
chunk_length: int = 0,
|
||||
strict: bool = False,
|
||||
schema: Any | None = None,
|
||||
schema_repair_mode: Literal["standard", "salvage"] = "standard",
|
||||
) -> JSONReturnType | tuple[JSONReturnType, list[dict[str, str]]]:
|
||||
"""
|
||||
This function works like `json.load()` except that it will fix your JSON in the process.
|
||||
It is a wrapper around the `repair_json()` function with `json_fd=fd` and `return_objects=True`.
|
||||
|
||||
Args:
|
||||
fd (TextIO): File descriptor for JSON input.
|
||||
skip_json_loads (bool, optional): If True, skip calling the built-in json.loads() function to verify that the json is valid before attempting to repair. Defaults to False.
|
||||
logging (bool, optional): If True, return a tuple with the repaired json and a log of all repair actions. Defaults to False.
|
||||
chunk_length (int, optional): Size in bytes of the file chunks to read at once. Defaults to 1MB.
|
||||
strict (bool, optional): If True, surface structural problems (duplicate keys, missing separators, empty keys/values, etc.) as ValueError instead of repairing them.
|
||||
schema (Any, optional): JSON Schema dict, boolean schema, or pydantic v2 model used to guide repairs and validation for both valid and invalid JSON inputs.
|
||||
schema_repair_mode (Literal["standard", "salvage"], optional): Schema repair mode. "salvage" requires schema.
|
||||
|
||||
Returns:
|
||||
Union[JSONReturnType, Tuple[JSONReturnType, List[Dict[str, str]]]]: The repaired JSON object or a tuple with the repaired JSON object and repair log.
|
||||
"""
|
||||
return repair_json(
|
||||
json_fd=fd,
|
||||
chunk_length=chunk_length,
|
||||
return_objects=True,
|
||||
skip_json_loads=skip_json_loads,
|
||||
logging=logging,
|
||||
strict=strict,
|
||||
schema=schema,
|
||||
schema_repair_mode=schema_repair_mode,
|
||||
)
|
||||
|
||||
|
||||
def from_file(
|
||||
filename: str | Path,
|
||||
skip_json_loads: bool = False,
|
||||
logging: bool = False,
|
||||
chunk_length: int = 0,
|
||||
strict: bool = False,
|
||||
schema: Any | None = None,
|
||||
schema_repair_mode: Literal["standard", "salvage"] = "standard",
|
||||
) -> JSONReturnType | tuple[JSONReturnType, list[dict[str, str]]]:
|
||||
"""
|
||||
This function is a wrapper around `load()` so you can pass the filename as string
|
||||
|
||||
Args:
|
||||
filename (str | Path): The name of the file containing JSON data to load and repair.
|
||||
skip_json_loads (bool, optional): If True, skip calling the built-in json.loads() function to verify that the json is valid before attempting to repair. Defaults to False.
|
||||
logging (bool, optional): If True, return a tuple with the repaired json and a log of all repair actions. Defaults to False.
|
||||
chunk_length (int, optional): Size in bytes of the file chunks to read at once. Defaults to 1MB.
|
||||
strict (bool, optional): If True, surface structural problems (duplicate keys, missing separators, empty keys/values, etc.) as ValueError instead of repairing them.
|
||||
schema (Any, optional): JSON Schema dict, boolean schema, or pydantic v2 model used to guide repairs and validation for both valid and invalid JSON inputs.
|
||||
schema_repair_mode (Literal["standard", "salvage"], optional): Schema repair mode. "salvage" requires schema.
|
||||
|
||||
Returns:
|
||||
Union[JSONReturnType, Tuple[JSONReturnType, List[Dict[str, str]]]]: The repaired JSON object or a tuple with the repaired JSON object and repair log.
|
||||
"""
|
||||
with Path(filename).open() as fd:
|
||||
return load(
|
||||
fd=fd,
|
||||
skip_json_loads=skip_json_loads,
|
||||
logging=logging,
|
||||
chunk_length=chunk_length,
|
||||
strict=strict,
|
||||
schema=schema,
|
||||
schema_repair_mode=schema_repair_mode,
|
||||
)
|
||||
|
||||
|
||||
def cli(inline_args: list[str] | None = None) -> int:
|
||||
"""
|
||||
Command-line interface for repairing and parsing JSON files.
|
||||
|
||||
Args:
|
||||
inline_args (Optional[List[str]]): List of command-line arguments for testing purposes. Defaults to None.
|
||||
- filename (str): The JSON file to repair. If omitted, the JSON is read from stdin.
|
||||
- -i, --inline (bool): Replace the file inline instead of returning the output to stdout.
|
||||
- -o, --output TARGET (str): If specified, the output will be written to TARGET filename instead of stdout.
|
||||
- --ensure_ascii (bool): Pass ensure_ascii=True to json.dumps(). Will pass False otherwise.
|
||||
- --indent INDENT (int): Number of spaces for indentation (Default 2).
|
||||
- --skip-json-loads (bool): Skip initial json.loads validation.
|
||||
- --schema SCHEMA (str): Path to a JSON Schema file that guides repairs.
|
||||
- --schema-model MODEL (str): Pydantic v2 model in 'module:ClassName' form that guides repairs.
|
||||
- --strict (bool): Raise on duplicate keys, missing separators, empty keys/values, and other unrecoverable structures instead of repairing them.
|
||||
|
||||
Returns:
|
||||
int: Exit code of the CLI operation.
|
||||
|
||||
Raises:
|
||||
Exception: Any exception that occurs during file processing.
|
||||
|
||||
Example:
|
||||
>>> cli(['example.json', '--indent', '4'])
|
||||
>>> cat json.txt | json_repair
|
||||
"""
|
||||
parser = argparse.ArgumentParser(description="Repair and parse JSON files.")
|
||||
# Make the filename argument optional; if omitted, we will read from stdin.
|
||||
parser.add_argument(
|
||||
"filename",
|
||||
nargs="?",
|
||||
help="The JSON file to repair (if omitted, reads from stdin)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"-i",
|
||||
"--inline",
|
||||
action="store_true",
|
||||
help="Replace the file inline instead of returning the output to stdout",
|
||||
)
|
||||
parser.add_argument(
|
||||
"-o",
|
||||
"--output",
|
||||
metavar="TARGET",
|
||||
help="If specified, the output will be written to TARGET filename instead of stdout",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--ensure_ascii",
|
||||
action="store_true",
|
||||
help="Pass ensure_ascii=True to json.dumps()",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--indent",
|
||||
type=int,
|
||||
default=2,
|
||||
help="Number of spaces for indentation (Default 2)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--skip-json-loads",
|
||||
action="store_true",
|
||||
help="Skip initial json.loads validation",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--schema",
|
||||
metavar="SCHEMA",
|
||||
help="Path to a JSON Schema file that guides repairs",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--schema-model",
|
||||
metavar="MODEL",
|
||||
help="Pydantic v2 model in 'module:ClassName' form that guides repairs",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--strict",
|
||||
action="store_true",
|
||||
help="Raise on duplicate keys, missing separators, empty keys/values, and other unrecoverable structures instead of repairing them",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--schema-repair-mode",
|
||||
choices=["standard", "salvage"],
|
||||
default="standard",
|
||||
help="Schema repair mode: 'standard' (default) or 'salvage' (best-effort array/object salvage).",
|
||||
)
|
||||
|
||||
args = parser.parse_args(inline_args)
|
||||
|
||||
# Inline mode requires a filename, so error out if none was provided.
|
||||
if args.inline and not args.filename: # pragma: no cover
|
||||
print("Error: Inline mode requires a filename", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
if args.inline and args.output: # pragma: no cover
|
||||
print("Error: You cannot pass both --inline and --output", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
if args.schema and args.schema_model:
|
||||
print("Error: You cannot pass both --schema and --schema-model", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
if args.strict and (args.schema or args.schema_model):
|
||||
print("Error: --strict cannot be used with --schema or --schema-model", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
if args.schema_repair_mode == "salvage" and not (args.schema or args.schema_model):
|
||||
print("Error: --schema-repair-mode salvage requires --schema or --schema-model", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
ensure_ascii = args.ensure_ascii
|
||||
|
||||
try:
|
||||
schema = None
|
||||
if args.schema:
|
||||
with Path(args.schema).open() as fd:
|
||||
schema = json.load(fd)
|
||||
elif args.schema_model:
|
||||
schema = load_schema_model(args.schema_model)
|
||||
|
||||
# Use from_file if a filename is provided; otherwise read from stdin.
|
||||
if args.filename:
|
||||
result = from_file(
|
||||
args.filename,
|
||||
skip_json_loads=args.skip_json_loads,
|
||||
strict=args.strict,
|
||||
schema=schema,
|
||||
schema_repair_mode=args.schema_repair_mode,
|
||||
)
|
||||
else:
|
||||
data = sys.stdin.read()
|
||||
result = loads(
|
||||
data,
|
||||
skip_json_loads=args.skip_json_loads,
|
||||
strict=args.strict,
|
||||
schema=schema,
|
||||
schema_repair_mode=args.schema_repair_mode,
|
||||
)
|
||||
if args.inline or args.output:
|
||||
with Path(args.output or args.filename).open(mode="w") as fd:
|
||||
json.dump(result, fd, indent=args.indent, ensure_ascii=ensure_ascii)
|
||||
else:
|
||||
print(json.dumps(result, indent=args.indent, ensure_ascii=ensure_ascii))
|
||||
except (OSError, TypeError, ValueError) as e: # pragma: no cover
|
||||
print(f"Error: {e!s}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
return 0 # Success
|
||||
|
||||
|
||||
if __name__ == "__main__": # pragma: no cover
|
||||
sys.exit(cli())
|
||||
@@ -0,0 +1,119 @@
|
||||
from typing import TYPE_CHECKING, Any, cast
|
||||
|
||||
from .parser_schema import ArraySchemaConfig, resolve_parser_array_schema
|
||||
from .utils.constants import STRING_DELIMITERS, JSONReturnType
|
||||
from .utils.json_context import ContextValues
|
||||
from .utils.object_comparer import ObjectComparer
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from .json_parser import JSONParser
|
||||
|
||||
|
||||
def _resolve_array_item_schema(
|
||||
schema_config: ArraySchemaConfig | None,
|
||||
idx: int,
|
||||
) -> tuple[dict[str, Any] | bool | None, bool]:
|
||||
if schema_config is None:
|
||||
return None, False
|
||||
|
||||
item_schema: dict[str, Any] | bool | None = None
|
||||
drop_item = False
|
||||
if isinstance(schema_config.items_schema, list):
|
||||
if idx < len(schema_config.items_schema):
|
||||
raw_schema = schema_config.items_schema[idx]
|
||||
if raw_schema is not None and not isinstance(raw_schema, (dict, bool)):
|
||||
raise ValueError("Schema must be an object.")
|
||||
item_schema = cast("dict[str, Any] | bool | None", raw_schema)
|
||||
elif schema_config.additional_items is False:
|
||||
drop_item = True
|
||||
elif isinstance(schema_config.additional_items, dict):
|
||||
item_schema = cast("dict[str, Any]", schema_config.additional_items)
|
||||
else:
|
||||
item_schema = True
|
||||
elif isinstance(schema_config.items_schema, dict):
|
||||
item_schema = cast("dict[str, Any]", schema_config.items_schema)
|
||||
else:
|
||||
item_schema = True
|
||||
|
||||
return item_schema, drop_item
|
||||
|
||||
|
||||
def parse_array(
|
||||
self: "JSONParser",
|
||||
schema: dict[str, Any] | bool | None = None,
|
||||
path: str = "$",
|
||||
closing_delimiter: str = "]",
|
||||
) -> list[JSONReturnType]:
|
||||
# <array> ::= '[' [ <json> *(', ' <json>) ] ']' ; A sequence of JSON values separated by commas
|
||||
schema_repairer, _schema, schema_config = resolve_parser_array_schema(self.schema_repairer, schema)
|
||||
salvage_mode = schema_repairer is not None and schema_repairer.schema_repair_mode == "salvage"
|
||||
array_is_object_value = self.context.current == ContextValues.OBJECT_VALUE
|
||||
closed_before_parent_member = False
|
||||
|
||||
arr: list[JSONReturnType] = []
|
||||
with self.context.enter(ContextValues.ARRAY):
|
||||
self.skip_whitespaces()
|
||||
char = self.get_char_at()
|
||||
idx = 0
|
||||
while char and char not in [closing_delimiter, "}"]:
|
||||
item_schema, drop_item = _resolve_array_item_schema(schema_config, idx)
|
||||
item_path = f"{path}[{idx}]"
|
||||
active_schema_repairer = (
|
||||
schema_repairer if schema_repairer is not None and not drop_item and not salvage_mode else None
|
||||
)
|
||||
|
||||
if char in STRING_DELIMITERS:
|
||||
# A string followed by ':' is often a missing object start; treat it as an object.
|
||||
i = 1
|
||||
i = self.skip_to_character(char, i)
|
||||
i = self.scroll_whitespaces(idx=i + 1)
|
||||
if self.get_char_at(i) == ":":
|
||||
if array_is_object_value and arr and all(not isinstance(item, (dict, list)) for item in arr):
|
||||
self.log("Closed unclosed array before object member after scalar items")
|
||||
closed_before_parent_member = True
|
||||
break
|
||||
if active_schema_repairer is not None:
|
||||
# Schema-guided object parsing, then enforce schema on the parsed object.
|
||||
value = self.parse_object(item_schema, item_path)
|
||||
value = active_schema_repairer.repair_value(value, item_schema, item_path)
|
||||
else:
|
||||
# No schema (or dropping): still parse to keep the cursor in sync.
|
||||
value = self.parse_object()
|
||||
else:
|
||||
value = self.parse_string()
|
||||
if active_schema_repairer is not None:
|
||||
# Apply schema constraints/coercions to scalar values when configured.
|
||||
value = active_schema_repairer.repair_value(value, item_schema, item_path)
|
||||
else:
|
||||
# Use schema-aware parsing to guide nested repairs when configured.
|
||||
value = (
|
||||
self.parse_json(item_schema, item_path) if active_schema_repairer is not None else self.parse_json()
|
||||
)
|
||||
|
||||
if ObjectComparer.is_strictly_empty(value) and self.get_char_at() not in [closing_delimiter, ","]:
|
||||
self.index += 1
|
||||
elif value == "..." and self.get_char_at(-1) == ".":
|
||||
self.log(
|
||||
"While parsing an array, found a stray '...'; ignoring it",
|
||||
)
|
||||
elif not drop_item:
|
||||
arr.append(value)
|
||||
elif schema_repairer is not None:
|
||||
# Record drops for visibility when schema forbids extra tuple items.
|
||||
schema_repairer._log("Dropped extra array item not covered by schema", item_path)
|
||||
|
||||
idx += 1
|
||||
char = self.get_char_at()
|
||||
while char and char != closing_delimiter and (char.isspace() or char == ","):
|
||||
self.index += 1
|
||||
char = self.get_char_at()
|
||||
|
||||
if char != closing_delimiter:
|
||||
self.log(
|
||||
f"While parsing an array we missed the closing {closing_delimiter}, ignoring it",
|
||||
)
|
||||
|
||||
if not closed_before_parent_member:
|
||||
self.index += 1
|
||||
|
||||
return arr
|
||||
@@ -0,0 +1,77 @@
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from .utils.constants import JSONReturnType
|
||||
from .utils.json_context import ContextValues
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from .json_parser import JSONParser
|
||||
|
||||
|
||||
def parse_comment(self: "JSONParser", record_top_level_value: bool = False) -> JSONReturnType:
|
||||
"""
|
||||
Parse code-like comments:
|
||||
|
||||
- "# comment": A line comment that continues until a newline.
|
||||
- "// comment": A line comment that continues until a newline.
|
||||
- "/* comment */": A block comment that continues until the closing delimiter "*/".
|
||||
|
||||
The comment is skipped over and an empty string is returned so that comments do not interfere
|
||||
with the actual JSON elements.
|
||||
"""
|
||||
while True:
|
||||
char = self.get_char_at()
|
||||
termination_characters = ["\n", "\r"]
|
||||
if ContextValues.ARRAY in self.context.context:
|
||||
termination_characters.append("]")
|
||||
if ContextValues.OBJECT_VALUE in self.context.context:
|
||||
termination_characters.append("}")
|
||||
if ContextValues.OBJECT_KEY in self.context.context:
|
||||
termination_characters.append(":")
|
||||
# Line comment starting with #
|
||||
if char == "#":
|
||||
comment = ""
|
||||
while char and char not in termination_characters:
|
||||
comment += char
|
||||
self.index += 1
|
||||
char = self.get_char_at()
|
||||
self.log(f"Found line comment: {comment}, ignoring")
|
||||
# Comments starting with '/'
|
||||
elif char == "/":
|
||||
next_char = self.get_char_at(1)
|
||||
# Handle line comment starting with //
|
||||
if next_char == "/":
|
||||
comment = "//"
|
||||
self.index += 2 # Skip both slashes.
|
||||
char = self.get_char_at()
|
||||
while char and char not in ["\n", "\r"]:
|
||||
comment += char
|
||||
self.index += 1
|
||||
char = self.get_char_at()
|
||||
self.log(f"Found line comment: {comment}, ignoring")
|
||||
# Handle block comment starting with /*
|
||||
elif next_char == "*":
|
||||
comment = "/*"
|
||||
self.index += 2 # Skip '/*'
|
||||
while True:
|
||||
char = self.get_char_at()
|
||||
if not char:
|
||||
self.log("Reached end-of-string while parsing block comment; unclosed block comment.")
|
||||
break
|
||||
comment += char
|
||||
self.index += 1
|
||||
if comment.endswith("*/"):
|
||||
break
|
||||
self.log(f"Found block comment: {comment}, ignoring")
|
||||
else:
|
||||
# Skip standalone '/' characters that are not part of a comment
|
||||
# to avoid getting stuck in an infinite loop
|
||||
self.index += 1
|
||||
if self.context.empty:
|
||||
# Avoid a parse_json -> parse_comment -> parse_json chain for long runs of
|
||||
# top-level comments; re-enter only once after we have consumed them all.
|
||||
self.skip_whitespaces()
|
||||
if self.get_char_at() in ["#", "/"]:
|
||||
continue
|
||||
return self.parse_json(record_top_level_value=record_top_level_value)
|
||||
break
|
||||
return ""
|
||||
@@ -0,0 +1,43 @@
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from .utils.constants import JSONReturnType
|
||||
from .utils.json_context import ContextValues
|
||||
|
||||
NUMBER_CHARS: set[str] = set("0123456789-.eE/,_")
|
||||
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from .json_parser import JSONParser
|
||||
|
||||
|
||||
def parse_number(self: "JSONParser") -> JSONReturnType:
|
||||
# <number> is a valid real number expressed in one of a number of given formats
|
||||
number_start_index = self.index
|
||||
number_str = ""
|
||||
char = self.get_char_at()
|
||||
is_array = self.context.current == ContextValues.ARRAY
|
||||
while (
|
||||
char
|
||||
and (char in NUMBER_CHARS or (char == "+" and number_str.endswith(("e", "E"))))
|
||||
and (not is_array or char != ",")
|
||||
):
|
||||
if char != "_":
|
||||
number_str += char
|
||||
self.index += 1
|
||||
char = self.get_char_at()
|
||||
if (self.get_char_at() or "").isalpha():
|
||||
# this was a string instead, sorry
|
||||
self.index = number_start_index
|
||||
return self.parse_string()
|
||||
if number_str and number_str[-1] in "+-eE/,":
|
||||
# The number ends with a non valid character for a number/currency, rolling back one
|
||||
number_str = number_str[:-1]
|
||||
self.index -= 1
|
||||
try:
|
||||
if "," in number_str:
|
||||
return number_str
|
||||
if "." in number_str or "e" in number_str or "E" in number_str:
|
||||
return float(number_str)
|
||||
return int(number_str)
|
||||
except ValueError:
|
||||
return number_str
|
||||
@@ -0,0 +1,512 @@
|
||||
from typing import TYPE_CHECKING, Any, cast
|
||||
|
||||
from .parser_schema import ObjectSchemaConfig, resolve_parser_object_schema
|
||||
from .utils.constants import MISSING_VALUE, STRING_DELIMITERS, JSONReturnType
|
||||
from .utils.json_context import ContextValues
|
||||
from .utils.pattern_properties import match_pattern_properties
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from .json_parser import JSONParser
|
||||
from .schema_repair import SchemaRepairer
|
||||
|
||||
|
||||
def _finalize_object(
|
||||
obj: dict[str, JSONReturnType],
|
||||
schema: dict[str, Any] | bool | None,
|
||||
schema_repairer: "SchemaRepairer | None",
|
||||
schema_config: ObjectSchemaConfig | None,
|
||||
path: str,
|
||||
) -> dict[str, JSONReturnType]:
|
||||
if schema_repairer is None or schema_config is None:
|
||||
return obj
|
||||
|
||||
resolved_schema = schema_repairer.resolve_schema(schema)
|
||||
property_aliases = (
|
||||
resolved_schema.get("x-json-repair-property-aliases", {}) if isinstance(resolved_schema, dict) else {}
|
||||
)
|
||||
aliases_with_field_name = set()
|
||||
if isinstance(property_aliases, dict):
|
||||
aliases_with_field_name = {
|
||||
alias
|
||||
for alias, field_names in property_aliases.items()
|
||||
if isinstance(alias, str)
|
||||
and isinstance(field_names, list)
|
||||
and any(isinstance(name, str) and name in obj for name in field_names)
|
||||
}
|
||||
|
||||
missing_required = [key for key in schema_config.required if key not in obj and key not in aliases_with_field_name]
|
||||
if missing_required and schema_repairer.schema_repair_mode != "salvage":
|
||||
raise ValueError(f"Missing required properties at {path}: {', '.join(missing_required)}")
|
||||
|
||||
for key, prop_schema in schema_config.properties.items():
|
||||
if key in obj or key in schema_config.required or key in aliases_with_field_name:
|
||||
continue
|
||||
if isinstance(prop_schema, dict) and "default" in prop_schema:
|
||||
obj[key] = schema_repairer._copy_json_value(prop_schema["default"], f"{path}.{key}", "default")
|
||||
schema_repairer._log("Inserted default value for missing property", f"{path}.{key}")
|
||||
return obj
|
||||
|
||||
|
||||
def _strip_comments_for_empty_object_classification(body: str) -> str:
|
||||
stripped = []
|
||||
in_quote: str | None = None
|
||||
backslashes = 0
|
||||
index = 0
|
||||
while index < len(body):
|
||||
char = body[index]
|
||||
next_char = body[index + 1] if index + 1 < len(body) else ""
|
||||
|
||||
if char == "\\":
|
||||
backslashes += 1
|
||||
stripped.append(char)
|
||||
index += 1
|
||||
continue
|
||||
if in_quote is not None:
|
||||
stripped.append(char)
|
||||
if char == in_quote and backslashes % 2 == 0:
|
||||
in_quote = None
|
||||
backslashes = 0
|
||||
index += 1
|
||||
continue
|
||||
if char in STRING_DELIMITERS and backslashes % 2 == 0:
|
||||
in_quote = char
|
||||
stripped.append(char)
|
||||
backslashes = 0
|
||||
index += 1
|
||||
continue
|
||||
backslashes = 0
|
||||
|
||||
if char == "#" or (char == "/" and next_char == "/"):
|
||||
index += 2 if char == "/" else 1
|
||||
while index < len(body) and body[index] not in ["\n", "\r"]:
|
||||
index += 1
|
||||
continue
|
||||
if char == "/" and next_char == "*":
|
||||
index += 2
|
||||
while index < len(body) - 1 and body[index : index + 2] != "*/":
|
||||
index += 1
|
||||
index = min(index + 2, len(body))
|
||||
continue
|
||||
|
||||
stripped.append(char)
|
||||
index += 1
|
||||
|
||||
return "".join(stripped)
|
||||
|
||||
|
||||
def _classify_empty_object_repair(
|
||||
self: "JSONParser",
|
||||
start_index: int,
|
||||
schema: dict[str, Any] | bool | None,
|
||||
schema_repairer: "SchemaRepairer | None",
|
||||
) -> tuple[str, str | None]:
|
||||
attempted_object = self.json_str[start_index - 1 : self.index + 1]
|
||||
body = attempted_object[1:]
|
||||
body = body.removesuffix("}")
|
||||
body = body.lstrip()
|
||||
if not body:
|
||||
return "keep", None
|
||||
if (body.startswith('\\"') and '\\":' in body) or (body.startswith("\\'") and "\\':" in body):
|
||||
normalized_object = attempted_object.replace('\\"', '"').replace("\\'", "'")
|
||||
self.log(
|
||||
"Parsed object is empty but the input starts like an escaped object key, normalizing and reparsing it as an object",
|
||||
)
|
||||
return "object", normalized_object
|
||||
body = _strip_comments_for_empty_object_classification(body).lstrip()
|
||||
if not body:
|
||||
return "keep", None
|
||||
|
||||
in_quote: str | None = None
|
||||
backslashes = 0
|
||||
for char in body:
|
||||
if char == "\\":
|
||||
backslashes += 1
|
||||
continue
|
||||
if in_quote is not None:
|
||||
if char == in_quote and backslashes % 2 == 0:
|
||||
in_quote = None
|
||||
elif char in STRING_DELIMITERS and backslashes % 2 == 0:
|
||||
in_quote = char
|
||||
elif char == ":" and backslashes % 2 == 0:
|
||||
self.log(
|
||||
"Parsed object is empty but the input still contains an object-style separator, keeping object repair",
|
||||
)
|
||||
return "keep", None
|
||||
backslashes = 0
|
||||
if (
|
||||
schema_repairer is not None
|
||||
and schema_repairer.schema_repair_mode == "salvage"
|
||||
and isinstance(schema, dict)
|
||||
and schema_repairer.is_object_schema(schema)
|
||||
and not schema_repairer.is_array_schema(schema)
|
||||
):
|
||||
return "schema_set_object", None
|
||||
return "array", None
|
||||
|
||||
|
||||
def _merge_object_array_continuation(
|
||||
self: "JSONParser",
|
||||
obj: dict[str, JSONReturnType],
|
||||
) -> bool:
|
||||
prev_key = list(obj.keys())[-1] if obj else None
|
||||
if not prev_key or not isinstance(obj[prev_key], list) or self.strict:
|
||||
return False
|
||||
|
||||
self.index += 1
|
||||
new_array = self.parse_array()
|
||||
if isinstance(new_array, list):
|
||||
prev_value = obj[prev_key]
|
||||
if isinstance(prev_value, list):
|
||||
list_lengths = [len(item) for item in prev_value if isinstance(item, list)]
|
||||
expected_len = (
|
||||
list_lengths[0] if list_lengths and all(length == list_lengths[0] for length in list_lengths) else None
|
||||
)
|
||||
if expected_len:
|
||||
tail = []
|
||||
while prev_value and not isinstance(prev_value[-1], list):
|
||||
tail.append(prev_value.pop())
|
||||
if tail:
|
||||
tail.reverse()
|
||||
if len(tail) % expected_len == 0:
|
||||
self.log(
|
||||
"While parsing an object we found row values without an inner array, grouping them into rows",
|
||||
)
|
||||
for i in range(0, len(tail), expected_len):
|
||||
prev_value.append(tail[i : i + expected_len])
|
||||
else:
|
||||
prev_value.extend(tail)
|
||||
if new_array:
|
||||
if all(isinstance(item, list) for item in new_array):
|
||||
self.log(
|
||||
"While parsing an object we found additional rows, appending them without flattening",
|
||||
)
|
||||
prev_value.extend(new_array)
|
||||
else:
|
||||
prev_value.append(new_array)
|
||||
else:
|
||||
prev_value.extend(new_array[0] if len(new_array) == 1 and isinstance(new_array[0], list) else new_array)
|
||||
|
||||
self.skip_whitespaces()
|
||||
if self.get_char_at() == ",":
|
||||
self.index += 1
|
||||
self.skip_whitespaces()
|
||||
return True
|
||||
|
||||
|
||||
def _parse_object_key(
|
||||
self: "JSONParser",
|
||||
obj: dict[str, JSONReturnType],
|
||||
) -> tuple[str, int]:
|
||||
key = ""
|
||||
rollback_index = self.index
|
||||
self.context.set(ContextValues.OBJECT_KEY)
|
||||
try:
|
||||
while self.get_char_at():
|
||||
rollback_index = self.index
|
||||
if self.get_char_at() == "[" and key == "" and _merge_object_array_continuation(self, obj):
|
||||
continue
|
||||
|
||||
raw_key = self.parse_string()
|
||||
assert isinstance(raw_key, str)
|
||||
key = raw_key
|
||||
if key == "":
|
||||
self.skip_whitespaces()
|
||||
if key != "" or (key == "" and self.get_char_at() in [":", "}"]):
|
||||
if key == "" and self.strict:
|
||||
self.log(
|
||||
"Empty key found in strict mode while parsing object, raising an error",
|
||||
)
|
||||
raise ValueError("Empty key found in strict mode while parsing object.")
|
||||
break
|
||||
finally:
|
||||
self.context.reset()
|
||||
return key, rollback_index
|
||||
|
||||
|
||||
def _should_split_duplicate_object(self: "JSONParser", rollback_index: int) -> bool:
|
||||
lookback_idx = rollback_index - self.index - 1
|
||||
prev_non_whitespace = self.get_char_at(lookback_idx)
|
||||
while prev_non_whitespace and prev_non_whitespace.isspace():
|
||||
lookback_idx -= 1
|
||||
prev_non_whitespace = self.get_char_at(lookback_idx)
|
||||
key_start_char = self.get_char_at(rollback_index - self.index)
|
||||
next_non_whitespace = self.get_char_at(self.scroll_whitespaces())
|
||||
return not (key_start_char in STRING_DELIMITERS and prev_non_whitespace == "," and next_non_whitespace == ":")
|
||||
|
||||
|
||||
def _split_object_on_duplicate_key(self: "JSONParser", rollback_index: int) -> None:
|
||||
self.index = rollback_index - 1
|
||||
self.json_str = self.json_str[: self.index + 1] + "{" + self.json_str[self.index + 1 :]
|
||||
|
||||
|
||||
def _resolve_object_property_schema(
|
||||
self: "JSONParser",
|
||||
schema_repairer: "SchemaRepairer | None",
|
||||
schema: dict[str, Any] | bool | None,
|
||||
schema_config: ObjectSchemaConfig | None,
|
||||
key: str,
|
||||
) -> tuple[dict[str, Any] | bool | None, list[dict[str, Any] | bool | None], bool]:
|
||||
if schema_repairer is None or schema_config is None:
|
||||
return None, [], False
|
||||
|
||||
prop_schema: dict[str, Any] | bool | None = None
|
||||
extra_schemas: list[dict[str, Any] | bool | None] = []
|
||||
if key in schema_config.properties:
|
||||
schema_value = schema_config.properties[key]
|
||||
if schema_value is not None and not isinstance(schema_value, (dict, bool)):
|
||||
raise ValueError("Schema must be an object.")
|
||||
prop_schema = cast("dict[str, Any] | bool | None", schema_value)
|
||||
return prop_schema, extra_schemas, False
|
||||
|
||||
resolved_schema = schema_repairer.resolve_schema(schema)
|
||||
property_aliases = (
|
||||
resolved_schema.get("x-json-repair-property-aliases", {}) if isinstance(resolved_schema, dict) else {}
|
||||
)
|
||||
if isinstance(property_aliases, dict):
|
||||
for alias, field_names in property_aliases.items():
|
||||
if isinstance(alias, str) and isinstance(field_names, list) and key in field_names:
|
||||
schema_value = schema_config.properties.get(alias)
|
||||
return cast("dict[str, Any] | bool | None", schema_value), extra_schemas, False
|
||||
|
||||
matched: list[Any] = []
|
||||
unsupported_patterns: list[str] = []
|
||||
if schema_config.pattern_properties:
|
||||
matched, unsupported_patterns = match_pattern_properties(schema_config.pattern_properties, key)
|
||||
for pattern in unsupported_patterns:
|
||||
self.log(
|
||||
f"Skipped unsupported patternProperties regex '{pattern}' while parsing object key '{key}'",
|
||||
)
|
||||
if matched:
|
||||
primary_schema = matched[0]
|
||||
if primary_schema is not None and not isinstance(primary_schema, (dict, bool)):
|
||||
raise ValueError("Schema must be an object.")
|
||||
prop_schema = cast("dict[str, Any] | bool | None", primary_schema)
|
||||
for extra_schema in matched[1:]:
|
||||
if extra_schema is not None and not isinstance(extra_schema, (dict, bool)):
|
||||
raise ValueError("Schema must be an object.")
|
||||
extra_schemas.append(cast("dict[str, Any] | bool | None", extra_schema))
|
||||
return prop_schema, extra_schemas, False
|
||||
|
||||
if schema_config.additional_properties is False:
|
||||
return None, [], True
|
||||
if isinstance(schema_config.additional_properties, dict):
|
||||
return cast("dict[str, Any]", schema_config.additional_properties), [], False
|
||||
return True, [], False
|
||||
|
||||
|
||||
def _parse_object_value(
|
||||
self: "JSONParser",
|
||||
schema_repairer: "SchemaRepairer | None",
|
||||
prop_schema: dict[str, Any] | bool | None,
|
||||
key_path: str,
|
||||
) -> JSONReturnType:
|
||||
self.context.set(ContextValues.OBJECT_VALUE)
|
||||
try:
|
||||
self.skip_whitespaces()
|
||||
char = self.get_char_at()
|
||||
if char in [",", "}"]:
|
||||
self.log(
|
||||
f"While parsing an object value we found a stray {char}, ignoring it",
|
||||
)
|
||||
if schema_repairer is not None:
|
||||
return schema_repairer.repair_value(MISSING_VALUE, prop_schema, key_path)
|
||||
return ""
|
||||
|
||||
if schema_repairer is not None:
|
||||
return self.parse_json(prop_schema, key_path)
|
||||
return self.parse_json()
|
||||
finally:
|
||||
self.context.reset()
|
||||
|
||||
|
||||
def _repair_empty_object_result(
|
||||
self: "JSONParser",
|
||||
obj: dict[str, JSONReturnType],
|
||||
start_index: int,
|
||||
schema: dict[str, Any] | bool | None,
|
||||
path: str,
|
||||
schema_repairer: "SchemaRepairer | None",
|
||||
) -> tuple[bool, JSONReturnType]:
|
||||
if obj or self.index - start_index <= 2:
|
||||
return False, None
|
||||
|
||||
if self.strict:
|
||||
self.log(
|
||||
"Parsed object is empty but contains extra characters in strict mode, raising an error",
|
||||
)
|
||||
raise ValueError("Parsed object is empty but contains extra characters in strict mode.")
|
||||
|
||||
empty_object_repair, normalized_object = _classify_empty_object_repair(self, start_index, schema, schema_repairer)
|
||||
if empty_object_repair == "object" and normalized_object is not None:
|
||||
end_index = self.index + 1
|
||||
self.json_str = self.json_str[: start_index - 1] + normalized_object + self.json_str[end_index:]
|
||||
self.index = start_index
|
||||
with self.context.enter(ContextValues.OBJECT_KEY):
|
||||
repaired_value = self.parse_object(schema, path)
|
||||
self.deferred_contexts.append(ContextValues.OBJECT_KEY)
|
||||
return True, repaired_value
|
||||
if empty_object_repair == "schema_set_object":
|
||||
self.log(
|
||||
"Parsed object is empty but salvage schema expects an object, reparsing set-like members as null-valued object keys",
|
||||
)
|
||||
self.index = start_index
|
||||
with self.context.enter(ContextValues.OBJECT_KEY):
|
||||
set_items = self.parse_array()
|
||||
self.deferred_contexts.append(ContextValues.OBJECT_KEY)
|
||||
if isinstance(set_items, list):
|
||||
key_candidates: list[str] = [item for item in set_items if isinstance(item, str) and item]
|
||||
if len(key_candidates) == len(set_items):
|
||||
return True, cast("JSONReturnType", dict.fromkeys(key_candidates))
|
||||
return True, set_items
|
||||
if empty_object_repair == "array":
|
||||
self.log("Parsed object is empty, we will try to parse this as an array instead")
|
||||
self.index = start_index
|
||||
with self.context.enter(ContextValues.OBJECT_KEY):
|
||||
repaired_array = self.parse_array()
|
||||
self.deferred_contexts.append(ContextValues.OBJECT_KEY)
|
||||
return True, repaired_array
|
||||
return False, None
|
||||
|
||||
|
||||
def _complete_object_parse(
|
||||
self: "JSONParser",
|
||||
obj: dict[str, JSONReturnType],
|
||||
schema: dict[str, Any] | bool | None,
|
||||
path: str,
|
||||
schema_repairer: "SchemaRepairer | None",
|
||||
schema_config: ObjectSchemaConfig | None,
|
||||
) -> JSONReturnType:
|
||||
if not self.context.empty:
|
||||
if self.get_char_at() == "}" and self.context.current not in [
|
||||
ContextValues.OBJECT_KEY,
|
||||
ContextValues.OBJECT_VALUE,
|
||||
]:
|
||||
self.log(
|
||||
"Found an extra closing brace that shouldn't be there, skipping it",
|
||||
)
|
||||
self.index += 1
|
||||
return obj
|
||||
|
||||
self.skip_whitespaces()
|
||||
if self.get_char_at() == ",":
|
||||
self.index += 1
|
||||
self.skip_whitespaces()
|
||||
if self.get_char_at() in STRING_DELIMITERS and not self.strict:
|
||||
self.log(
|
||||
"Found a comma and string delimiter after object closing brace, checking for additional key-value pairs",
|
||||
)
|
||||
additional_obj = self.parse_object(schema, path)
|
||||
if isinstance(additional_obj, dict):
|
||||
obj.update(additional_obj)
|
||||
|
||||
return _finalize_object(obj, schema, schema_repairer, schema_config, path)
|
||||
|
||||
|
||||
def parse_object(
|
||||
self: "JSONParser",
|
||||
schema: dict[str, Any] | bool | None = None,
|
||||
path: str = "$",
|
||||
) -> JSONReturnType:
|
||||
# <object> ::= '{' [ <member> *(', ' <member>) ] '}' ; A sequence of 'members'
|
||||
obj: dict[str, JSONReturnType] = {}
|
||||
start_index = self.index
|
||||
parsing_object_value = self.context.current == ContextValues.OBJECT_VALUE
|
||||
schema_repairer, schema, schema_config = resolve_parser_object_schema(self.schema_repairer, schema)
|
||||
|
||||
while (self.get_char_at() or "}") != "}":
|
||||
self.skip_whitespaces()
|
||||
|
||||
if self.get_char_at() == ":":
|
||||
self.log(
|
||||
"While parsing an object we found a : before a key, ignoring",
|
||||
)
|
||||
self.index += 1
|
||||
|
||||
key, rollback_index = _parse_object_key(self, obj)
|
||||
if ContextValues.ARRAY in self.context.context and key in obj:
|
||||
if self.strict:
|
||||
self.log("Duplicate key found in strict mode while parsing object, raising an error")
|
||||
raise ValueError("Duplicate key found in strict mode while parsing object.")
|
||||
if not parsing_object_value:
|
||||
if _should_split_duplicate_object(self, rollback_index):
|
||||
self.log(
|
||||
"While parsing an object we found a duplicate key, closing the object here and rolling back the index",
|
||||
)
|
||||
_split_object_on_duplicate_key(self, rollback_index)
|
||||
break
|
||||
self.log(
|
||||
"While parsing an object we found a duplicate key with a normal comma separator, keeping duplicate-key overwrite behavior",
|
||||
)
|
||||
|
||||
self.skip_whitespaces()
|
||||
if (self.get_char_at() or "}") == "}":
|
||||
continue
|
||||
|
||||
self.skip_whitespaces()
|
||||
if self.get_char_at() != ":":
|
||||
if self.strict:
|
||||
self.log(
|
||||
"Missing ':' after key in strict mode while parsing object, raising an error",
|
||||
)
|
||||
raise ValueError("Missing ':' after key in strict mode while parsing object.")
|
||||
self.log(
|
||||
"While parsing an object we missed a : after a key",
|
||||
)
|
||||
|
||||
self.index += 1
|
||||
prop_schema, extra_schemas, drop_property = _resolve_object_property_schema(
|
||||
self,
|
||||
schema_repairer,
|
||||
schema,
|
||||
schema_config,
|
||||
key,
|
||||
)
|
||||
key_path = f"{path}.{key}"
|
||||
value = _parse_object_value(self, schema_repairer, prop_schema, key_path)
|
||||
|
||||
if schema_repairer is not None:
|
||||
for extra_schema in extra_schemas:
|
||||
value = schema_repairer.repair_value(value, extra_schema, key_path)
|
||||
|
||||
if schema_repairer is None and value == "" and self.strict and self.get_char_at(-1) not in STRING_DELIMITERS:
|
||||
self.log(
|
||||
"Parsed value is empty in strict mode while parsing object, raising an error",
|
||||
)
|
||||
raise ValueError("Parsed value is empty in strict mode while parsing object.")
|
||||
|
||||
if schema_repairer is None or not drop_property:
|
||||
obj[key] = value
|
||||
else:
|
||||
schema_repairer._log("Dropped extra property not covered by schema", key_path)
|
||||
|
||||
if self.get_char_at() in [",", "'", '"']:
|
||||
self.index += 1
|
||||
if self.get_char_at() == "]" and ContextValues.ARRAY in self.context.context:
|
||||
self.log(
|
||||
"While parsing an object we found a closing array bracket, closing the object here and rolling back the index"
|
||||
)
|
||||
self.index -= 1
|
||||
break
|
||||
self.skip_whitespaces()
|
||||
|
||||
self.index += 1
|
||||
|
||||
repaired_empty_object, repaired_value = _repair_empty_object_result(
|
||||
self,
|
||||
obj,
|
||||
start_index,
|
||||
schema,
|
||||
path,
|
||||
schema_repairer,
|
||||
)
|
||||
if repaired_empty_object:
|
||||
return repaired_value
|
||||
|
||||
return _complete_object_parse(
|
||||
self,
|
||||
obj,
|
||||
schema,
|
||||
path,
|
||||
schema_repairer,
|
||||
schema_config,
|
||||
)
|
||||
@@ -0,0 +1,942 @@
|
||||
from dataclasses import dataclass, field
|
||||
from typing import TYPE_CHECKING, cast
|
||||
|
||||
from .parse_string_helpers.object_value_context import classify_object_value_comma, update_inline_container_stack
|
||||
from .parse_string_helpers.parse_boolean_or_null import parse_boolean_or_null
|
||||
from .parse_string_helpers.parse_json_llm_block import parse_json_llm_block
|
||||
from .utils.constants import STRING_DELIMITERS, JSONReturnType
|
||||
from .utils.json_context import ContextValues
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from .json_parser import JSONParser
|
||||
|
||||
|
||||
NO_DIRECT_RESULT = object()
|
||||
INLINE_CONTAINER_CLOSING_DELIMITERS = {"[": "]", "{": "}", "(": ")"}
|
||||
INLINE_CONTAINER_OPENERS = tuple(INLINE_CONTAINER_CLOSING_DELIMITERS)
|
||||
LOW_SMART_QUOTE_SENTINEL = "\0"
|
||||
|
||||
|
||||
@dataclass
|
||||
class StringParseState:
|
||||
missing_quotes: bool = False
|
||||
doubled_quotes: bool = False
|
||||
lstring_delimiter: str = '"'
|
||||
rstring_delimiter: str = '"'
|
||||
string_acc: str = ""
|
||||
unmatched_delimiter: bool = False
|
||||
pending_inline_container: bool = False
|
||||
inline_container_stack: list[str] = field(default_factory=list)
|
||||
object_value_has_no_future_delimiter: bool = False
|
||||
lookahead_cache: dict[tuple[str, ...], tuple[int, int | None]] = field(default_factory=dict)
|
||||
object_value_unmatched_opening_braces: int = 0
|
||||
regex_character_class_start: int | None = None
|
||||
|
||||
|
||||
def _outer_rstring_delimiter(state: StringParseState) -> str:
|
||||
return state.rstring_delimiter[0]
|
||||
|
||||
|
||||
def _active_rstring_delimiter(state: StringParseState) -> str:
|
||||
return state.rstring_delimiter[-1]
|
||||
|
||||
|
||||
def _in_low_smart_quote_span(state: StringParseState) -> bool:
|
||||
return _active_rstring_delimiter(state) == LOW_SMART_QUOTE_SENTINEL
|
||||
|
||||
|
||||
def _push_low_smart_quote_span(state: StringParseState) -> None:
|
||||
state.rstring_delimiter += LOW_SMART_QUOTE_SENTINEL
|
||||
|
||||
|
||||
def _pop_low_smart_quote_span(state: StringParseState) -> None:
|
||||
state.rstring_delimiter = state.rstring_delimiter[:-1]
|
||||
|
||||
|
||||
def _try_parse_simple_quoted_string(self: "JSONParser") -> str | None:
|
||||
if self.get_char_at() != '"':
|
||||
return None
|
||||
|
||||
start = self.index + 1
|
||||
json_str = self.json_str
|
||||
if isinstance(json_str, str):
|
||||
end = json_str.find('"', start)
|
||||
if end == -1:
|
||||
return None
|
||||
value = json_str[start:end]
|
||||
if "\\" in value or "\n" in value or "\r" in value:
|
||||
return None
|
||||
else:
|
||||
end = start
|
||||
limit = len(json_str)
|
||||
while end < limit:
|
||||
char = json_str[end]
|
||||
if char == '"':
|
||||
break
|
||||
if char in {"\\", "\n", "\r"}:
|
||||
return None
|
||||
end += 1
|
||||
if end >= limit:
|
||||
return None
|
||||
value = json_str[start:end]
|
||||
|
||||
next_index = end + 1
|
||||
limit = len(json_str)
|
||||
while next_index < limit and self.json_str[next_index].isspace():
|
||||
next_index += 1
|
||||
next_char = self.json_str[next_index] if next_index < limit else None
|
||||
|
||||
current_context = self.context.current
|
||||
if current_context == ContextValues.OBJECT_KEY:
|
||||
if next_char != ":":
|
||||
return None
|
||||
elif current_context == ContextValues.OBJECT_VALUE:
|
||||
if next_char not in {",", "}", None}:
|
||||
return None
|
||||
elif current_context == ContextValues.ARRAY:
|
||||
if next_char not in {",", "]", None}:
|
||||
return None
|
||||
elif next_char is not None:
|
||||
return None
|
||||
|
||||
self.index = end + 1
|
||||
return value
|
||||
|
||||
|
||||
def _append_literal_char(
|
||||
self: "JSONParser",
|
||||
state: StringParseState,
|
||||
current_char: str,
|
||||
) -> str | None:
|
||||
_append_string_content(state, current_char)
|
||||
self.index += 1
|
||||
return self.get_char_at()
|
||||
|
||||
|
||||
def _append_string_content(state: StringParseState, content: str) -> None:
|
||||
start_index = len(state.string_acc)
|
||||
state.string_acc += content
|
||||
for offset, char in enumerate(content):
|
||||
if char == "{":
|
||||
state.object_value_unmatched_opening_braces += 1
|
||||
elif char == "}" and state.object_value_unmatched_opening_braces:
|
||||
state.object_value_unmatched_opening_braces -= 1
|
||||
elif char == "[":
|
||||
state.regex_character_class_start = start_index + offset + 1
|
||||
elif char == "]":
|
||||
state.regex_character_class_start = None
|
||||
|
||||
|
||||
def _quote_belongs_to_regex_character_class(
|
||||
self: "JSONParser",
|
||||
state: StringParseState,
|
||||
) -> bool:
|
||||
"""Return whether the current quote is inside a compact ``[...]`` character class."""
|
||||
start = state.regex_character_class_start
|
||||
if start is None or any(char.isspace() for char in state.string_acc[start:]):
|
||||
return False
|
||||
|
||||
closing_bracket_idx = self.skip_to_character("]", idx=1)
|
||||
return self.get_char_at(closing_bracket_idx) == "]"
|
||||
|
||||
|
||||
def _rebuild_unmatched_opening_braces(state: StringParseState) -> None:
|
||||
state.object_value_unmatched_opening_braces = 0
|
||||
state.regex_character_class_start = None
|
||||
for index, char in enumerate(state.string_acc):
|
||||
if char == "{":
|
||||
state.object_value_unmatched_opening_braces += 1
|
||||
elif char == "}" and state.object_value_unmatched_opening_braces:
|
||||
state.object_value_unmatched_opening_braces -= 1
|
||||
elif char == "[":
|
||||
state.regex_character_class_start = index + 1
|
||||
elif char == "]":
|
||||
state.regex_character_class_start = None
|
||||
|
||||
|
||||
def _cached_skip_to_character(
|
||||
self: "JSONParser",
|
||||
state: StringParseState,
|
||||
character: str | list[str],
|
||||
idx: int = 0,
|
||||
) -> int:
|
||||
targets = (character,) if isinstance(character, str) else tuple(character)
|
||||
start_index = self.index + idx
|
||||
cached = state.lookahead_cache.get(targets)
|
||||
if cached is not None:
|
||||
cached_start, cached_match = cached
|
||||
if cached_match is None and start_index >= cached_start:
|
||||
return len(self.json_str) - self.index
|
||||
if cached_match is not None and cached_start <= start_index <= cached_match:
|
||||
return cached_match - self.index
|
||||
|
||||
match_offset = self.skip_to_character(character, idx)
|
||||
match = self.get_char_at(match_offset)
|
||||
if not match:
|
||||
state.lookahead_cache[targets] = (start_index, None)
|
||||
return match_offset
|
||||
|
||||
match_index = self.index + match_offset
|
||||
if match_index == 0 or self.json_str[match_index - 1] != "\\":
|
||||
state.lookahead_cache[targets] = (start_index, match_index)
|
||||
return match_offset
|
||||
|
||||
|
||||
def _prepare_string_entry(
|
||||
self: "JSONParser",
|
||||
) -> tuple[StringParseState | None, object]:
|
||||
char = self.get_char_at()
|
||||
if char in ["#", "/"]:
|
||||
return None, self.parse_comment()
|
||||
|
||||
while char and char not in STRING_DELIMITERS and not char.isalnum():
|
||||
self.index += 1
|
||||
char = self.get_char_at()
|
||||
|
||||
if not char:
|
||||
return None, ""
|
||||
|
||||
fast_path_value = _try_parse_simple_quoted_string(self)
|
||||
if fast_path_value is not None:
|
||||
return None, fast_path_value
|
||||
|
||||
state = StringParseState()
|
||||
|
||||
if char == "'":
|
||||
state.lstring_delimiter = state.rstring_delimiter = "'"
|
||||
elif char == "“":
|
||||
state.lstring_delimiter = "“"
|
||||
state.rstring_delimiter = "”"
|
||||
elif char.isalnum():
|
||||
if char.lower() in ["t", "f", "n"] and self.context.current != ContextValues.OBJECT_KEY:
|
||||
found_literal, value = parse_boolean_or_null(self)
|
||||
if found_literal:
|
||||
return state, value
|
||||
self.log(
|
||||
"While parsing a string, we found a literal instead of a quote",
|
||||
)
|
||||
state.missing_quotes = True
|
||||
|
||||
if not state.missing_quotes:
|
||||
self.index += 1
|
||||
if self.get_char_at() == "`":
|
||||
ret_val = parse_json_llm_block(self)
|
||||
if ret_val is not False:
|
||||
return state, ret_val
|
||||
self.log(
|
||||
"While parsing a string, we found code fences but they did not enclose valid JSON, continuing parsing the string",
|
||||
)
|
||||
|
||||
if self.get_char_at() == state.lstring_delimiter:
|
||||
if (
|
||||
(self.context.current == ContextValues.OBJECT_KEY and self.get_char_at(1) == ":")
|
||||
or (self.context.current == ContextValues.OBJECT_VALUE and self.get_char_at(1) in [",", "}"])
|
||||
or (self.context.current == ContextValues.ARRAY and self.get_char_at(1) in [",", "]"])
|
||||
):
|
||||
self.index += 1
|
||||
return state, ""
|
||||
if self.get_char_at(1) == state.lstring_delimiter:
|
||||
self.log(
|
||||
"While parsing a string, we found a doubled quote and then a quote again, ignoring it",
|
||||
)
|
||||
if self.strict:
|
||||
raise ValueError("Found doubled quotes followed by another quote.")
|
||||
return state, ""
|
||||
i = self.skip_to_character(character=state.rstring_delimiter, idx=1)
|
||||
if self.get_char_at(i + 1) == state.rstring_delimiter:
|
||||
self.log(
|
||||
"While parsing a string, we found a valid starting doubled quote",
|
||||
)
|
||||
state.doubled_quotes = True
|
||||
self.index += 1
|
||||
else:
|
||||
if self.get_char_at(i) == state.rstring_delimiter:
|
||||
outer_quote_idx = self.skip_to_character(character=state.rstring_delimiter, idx=i + 1)
|
||||
after_outer_quote_idx = self.scroll_whitespaces(idx=outer_quote_idx + 1)
|
||||
after_outer_quote = self.get_char_at(after_outer_quote_idx)
|
||||
if (
|
||||
self.context.current == ContextValues.OBJECT_VALUE
|
||||
and not _only_whitespace_until(self, i)
|
||||
and self.get_char_at(outer_quote_idx) == state.rstring_delimiter
|
||||
and after_outer_quote in [",", "}", None]
|
||||
):
|
||||
self.log(
|
||||
"While parsing a string, we found a leading quote that starts a quoted span, keeping it",
|
||||
)
|
||||
_append_string_content(state, state.lstring_delimiter)
|
||||
state.unmatched_delimiter = True
|
||||
self.index += 1
|
||||
return state, NO_DIRECT_RESULT
|
||||
i = self.scroll_whitespaces(idx=1)
|
||||
next_c = self.get_char_at(i)
|
||||
if next_c in [*STRING_DELIMITERS, "{", "["]:
|
||||
self.log(
|
||||
"While parsing a string, we found a doubled quote but also another quote afterwards, ignoring it",
|
||||
)
|
||||
if self.strict:
|
||||
raise ValueError(
|
||||
"Found doubled quotes followed by another quote while parsing a string.",
|
||||
)
|
||||
self.index += 1
|
||||
return state, ""
|
||||
if next_c not in [",", "]", "}"]:
|
||||
self.log(
|
||||
"While parsing a string, we found a doubled quote but it was a mistake, removing one quote",
|
||||
)
|
||||
self.index += 1
|
||||
|
||||
return state, NO_DIRECT_RESULT
|
||||
|
||||
|
||||
def _normalize_escape_sequence(
|
||||
self: "JSONParser",
|
||||
state: StringParseState,
|
||||
char: str,
|
||||
) -> tuple[bool, str | None]:
|
||||
self.log("Found a stray escape sequence, normalizing it")
|
||||
active_rstring_delimiter = _active_rstring_delimiter(state)
|
||||
if _in_low_smart_quote_span(state) and char == '"':
|
||||
state.string_acc = state.string_acc[:-1] + char
|
||||
_rebuild_unmatched_opening_braces(state)
|
||||
_pop_low_smart_quote_span(state)
|
||||
self.index += 1
|
||||
return True, self.get_char_at()
|
||||
if char == "\\":
|
||||
run_start = self.index - 1
|
||||
run_end = self.index + 1
|
||||
while run_end < len(self.json_str) and self.json_str[run_end] == "\\":
|
||||
run_end += 1
|
||||
run_length = run_end - run_start
|
||||
next_char = self.get_char_at(run_end - self.index)
|
||||
if run_length % 2 == 0 and next_char != active_rstring_delimiter:
|
||||
state.string_acc = state.string_acc[:-1] + ("\\" * (run_length // 2))
|
||||
_rebuild_unmatched_opening_braces(state)
|
||||
self.index = run_end
|
||||
return True, self.get_char_at()
|
||||
if char in [active_rstring_delimiter, "t", "n", "r", "b", "\\"]:
|
||||
state.string_acc = state.string_acc[:-1]
|
||||
escape_seqs = {"t": "\t", "n": "\n", "r": "\r", "b": "\b"}
|
||||
state.string_acc += escape_seqs.get(char, char)
|
||||
_rebuild_unmatched_opening_braces(state)
|
||||
self.index += 1
|
||||
next_char = self.get_char_at()
|
||||
while (
|
||||
next_char
|
||||
and state.string_acc
|
||||
and state.string_acc[-1] == "\\"
|
||||
and next_char in [active_rstring_delimiter, "\\"]
|
||||
):
|
||||
state.string_acc = state.string_acc[:-1] + next_char
|
||||
_rebuild_unmatched_opening_braces(state)
|
||||
self.index += 1
|
||||
next_char = self.get_char_at()
|
||||
return True, next_char
|
||||
if char in ["u", "x"]:
|
||||
num_chars = 4 if char == "u" else 2
|
||||
next_chars = self.json_str[self.index + 1 : self.index + 1 + num_chars]
|
||||
if len(next_chars) == num_chars and all(c in "0123456789abcdefABCDEF" for c in next_chars):
|
||||
self.log("Found a unicode escape sequence, normalizing it")
|
||||
state.string_acc = state.string_acc[:-1] + chr(int(next_chars, 16))
|
||||
_rebuild_unmatched_opening_braces(state)
|
||||
self.index += 1 + num_chars
|
||||
return True, self.get_char_at()
|
||||
elif char == "„" or char in STRING_DELIMITERS and char != active_rstring_delimiter:
|
||||
self.log("Found a delimiter that was escaped but shouldn't be escaped, removing the escape")
|
||||
state.string_acc = state.string_acc[:-1] + char
|
||||
_rebuild_unmatched_opening_braces(state)
|
||||
self.index += 1
|
||||
return True, self.get_char_at()
|
||||
return False, char
|
||||
|
||||
|
||||
def _brace_before_code_fence_belongs_to_string(
|
||||
self: "JSONParser",
|
||||
state: StringParseState,
|
||||
fence_idx: int,
|
||||
) -> bool:
|
||||
# Distinguish trailing wrapper fences from literal fenced snippets inside the current string.
|
||||
quote_search_idx = fence_idx + 3
|
||||
next_content_idx = _scroll_comment_prefixed_member_start(self, quote_search_idx)
|
||||
keep_post_fence_container = False
|
||||
if self.get_char_at(next_content_idx) in INLINE_CONTAINER_OPENERS:
|
||||
container_end_idx = _skip_inline_container(self, next_content_idx)
|
||||
if container_end_idx is not None:
|
||||
if _post_fence_container_starts_next_member(self, container_end_idx):
|
||||
return False
|
||||
keep_post_fence_container = True
|
||||
quote_search_idx = container_end_idx
|
||||
|
||||
outer_rstring_delimiter = _outer_rstring_delimiter(state)
|
||||
quote_idx = self.skip_to_character(character=outer_rstring_delimiter, idx=quote_search_idx)
|
||||
while self.get_char_at(quote_idx) == outer_rstring_delimiter:
|
||||
after_quote_idx = self.scroll_whitespaces(idx=quote_idx + 1)
|
||||
after_quote = self.get_char_at(after_quote_idx)
|
||||
if after_quote in [",", "}", "]", None]:
|
||||
if keep_post_fence_container:
|
||||
state.pending_inline_container = True
|
||||
return True
|
||||
quote_idx = self.skip_to_character(character=outer_rstring_delimiter, idx=quote_idx + 1)
|
||||
return False
|
||||
|
||||
|
||||
def _matching_string_delimiter(delimiter: str) -> str:
|
||||
return "”" if delimiter == "“" else delimiter
|
||||
|
||||
|
||||
def _bare_key_is_followed_by_colon(
|
||||
self: "JSONParser",
|
||||
key_idx: int,
|
||||
) -> bool:
|
||||
key_char = self.get_char_at(key_idx)
|
||||
if not key_char or not (key_char.isalnum() or key_char == "_"):
|
||||
return False
|
||||
|
||||
while True:
|
||||
key_char = self.get_char_at(key_idx)
|
||||
if not key_char or not (key_char.isalnum() or key_char in ["_", "-"]):
|
||||
break
|
||||
key_idx += 1
|
||||
|
||||
key_idx = self.scroll_whitespaces(idx=key_idx)
|
||||
return self.get_char_at(key_idx) == ":"
|
||||
|
||||
|
||||
def _post_fence_container_starts_next_member(
|
||||
self: "JSONParser",
|
||||
container_end_idx: int,
|
||||
) -> bool:
|
||||
after_container_idx = self.scroll_whitespaces(idx=container_end_idx)
|
||||
after_container = self.get_char_at(after_container_idx)
|
||||
if after_container in ["}", None]:
|
||||
return True
|
||||
if after_container != ",":
|
||||
return False
|
||||
|
||||
next_member_idx = _scroll_comment_prefixed_member_start(self, after_container_idx + 1)
|
||||
return self.get_char_at(next_member_idx) in ["}", None] or _object_member_starts_at(self, next_member_idx)
|
||||
|
||||
|
||||
def _starts_nested_inline_container(
|
||||
self: "JSONParser",
|
||||
idx: int,
|
||||
) -> bool:
|
||||
opening_delimiter = self.get_char_at(idx)
|
||||
prev_idx = idx - 1
|
||||
while prev_idx >= 0:
|
||||
prev_char = self.get_char_at(prev_idx)
|
||||
if prev_char is None:
|
||||
return True
|
||||
if not prev_char.isspace():
|
||||
if prev_char in INLINE_CONTAINER_OPENERS:
|
||||
return True
|
||||
if prev_char not in [",", ":"]:
|
||||
return False
|
||||
|
||||
next_idx = self.scroll_whitespaces(idx=idx + 1)
|
||||
next_char = self.get_char_at(next_idx)
|
||||
if opening_delimiter in ["[", "("]:
|
||||
return next_char in ["]", ")", *STRING_DELIMITERS, "-", *INLINE_CONTAINER_OPENERS, "t", "f", "n"] or (
|
||||
next_char is not None and next_char.isdigit()
|
||||
)
|
||||
if opening_delimiter != "{":
|
||||
return False
|
||||
if next_char in ["}", *STRING_DELIMITERS]:
|
||||
return True
|
||||
return prev_char == ":" and _bare_key_is_followed_by_colon(self, next_idx)
|
||||
prev_idx -= 1
|
||||
return True
|
||||
|
||||
|
||||
def _skip_inline_container(
|
||||
self: "JSONParser",
|
||||
idx: int,
|
||||
) -> int | None:
|
||||
opening_delimiter = self.get_char_at(idx)
|
||||
if opening_delimiter not in INLINE_CONTAINER_CLOSING_DELIMITERS:
|
||||
return idx
|
||||
|
||||
stack = [INLINE_CONTAINER_CLOSING_DELIMITERS[opening_delimiter]]
|
||||
i = idx + 1
|
||||
while stack:
|
||||
char = self.get_char_at(i)
|
||||
if not char:
|
||||
return None
|
||||
if char in STRING_DELIMITERS:
|
||||
end_delimiter = _matching_string_delimiter(char)
|
||||
i = self.skip_to_character(character=end_delimiter, idx=i + 1)
|
||||
if self.get_char_at(i) != end_delimiter:
|
||||
return None
|
||||
elif char in INLINE_CONTAINER_CLOSING_DELIMITERS and _starts_nested_inline_container(self, i):
|
||||
stack.append(INLINE_CONTAINER_CLOSING_DELIMITERS[char])
|
||||
elif char == stack[-1]:
|
||||
stack.pop()
|
||||
if not stack:
|
||||
return i + 1
|
||||
i += 1
|
||||
|
||||
return None # pragma: no cover
|
||||
|
||||
|
||||
def _scroll_comment_prefixed_member_start(
|
||||
self: "JSONParser",
|
||||
idx: int,
|
||||
) -> int:
|
||||
idx = self.scroll_whitespaces(idx=idx)
|
||||
while True:
|
||||
char = self.get_char_at(idx)
|
||||
if char == "#":
|
||||
while char and char not in ["\n", "\r"]:
|
||||
idx += 1
|
||||
char = self.get_char_at(idx)
|
||||
idx = self.scroll_whitespaces(idx=idx)
|
||||
continue
|
||||
if char == "/":
|
||||
next_char = self.get_char_at(idx + 1)
|
||||
if next_char == "/":
|
||||
idx += 2
|
||||
char = self.get_char_at(idx)
|
||||
while char and char not in ["\n", "\r"]:
|
||||
idx += 1
|
||||
char = self.get_char_at(idx)
|
||||
idx = self.scroll_whitespaces(idx=idx)
|
||||
continue
|
||||
if next_char == "*":
|
||||
idx += 2
|
||||
while True:
|
||||
char = self.get_char_at(idx)
|
||||
if not char:
|
||||
return idx
|
||||
if char == "*" and self.get_char_at(idx + 1) == "/":
|
||||
idx += 2
|
||||
break
|
||||
idx += 1
|
||||
idx = self.scroll_whitespaces(idx=idx)
|
||||
continue
|
||||
return idx
|
||||
|
||||
|
||||
def _quoted_object_member_follows(
|
||||
self: "JSONParser",
|
||||
quote_idx: int,
|
||||
) -> bool:
|
||||
comma_idx = self.scroll_whitespaces(idx=quote_idx + 1)
|
||||
if self.get_char_at(comma_idx) != ",":
|
||||
return False
|
||||
|
||||
next_member_idx = _scroll_comment_prefixed_member_start(self, comma_idx + 1)
|
||||
return _object_member_starts_at(self, next_member_idx)
|
||||
|
||||
|
||||
def _object_member_starts_at(
|
||||
self: "JSONParser",
|
||||
next_member_idx: int,
|
||||
) -> bool:
|
||||
if self.get_char_at(next_member_idx) in ["}", None]:
|
||||
return False
|
||||
|
||||
next_member = self.get_char_at(next_member_idx)
|
||||
if next_member in STRING_DELIMITERS:
|
||||
key_end_delimiter = _matching_string_delimiter(next_member)
|
||||
key_end_idx = self.skip_to_character(character=key_end_delimiter, idx=next_member_idx + 1)
|
||||
if self.get_char_at(key_end_idx) != key_end_delimiter:
|
||||
return False
|
||||
after_key_idx = self.scroll_whitespaces(idx=key_end_idx + 1)
|
||||
return self.get_char_at(after_key_idx) == ":"
|
||||
|
||||
if next_member and (next_member.isalnum() or next_member == "_"):
|
||||
return _bare_key_is_followed_by_colon(self, next_member_idx)
|
||||
|
||||
return False
|
||||
|
||||
|
||||
def _handle_right_delimiter_candidate(
|
||||
self: "JSONParser",
|
||||
state: StringParseState,
|
||||
char: str,
|
||||
) -> tuple[bool, str | None, bool]:
|
||||
outer_rstring_delimiter = _outer_rstring_delimiter(state)
|
||||
|
||||
if state.doubled_quotes and self.get_char_at(1) == outer_rstring_delimiter:
|
||||
self.log("While parsing a string, we found a doubled quote, ignoring it")
|
||||
self.index += 1
|
||||
return True, char, False
|
||||
|
||||
if state.missing_quotes and self.context.current == ContextValues.OBJECT_VALUE:
|
||||
i = 1
|
||||
next_c = self.get_char_at(i)
|
||||
while next_c and next_c not in [
|
||||
outer_rstring_delimiter,
|
||||
state.lstring_delimiter,
|
||||
]:
|
||||
i += 1
|
||||
next_c = self.get_char_at(i)
|
||||
if next_c:
|
||||
i += 1
|
||||
i = self.scroll_whitespaces(idx=i)
|
||||
if self.get_char_at(i) == ":":
|
||||
self.index -= 1
|
||||
next_char = self.get_char_at()
|
||||
self.log(
|
||||
"In a string with missing quotes and object value context, I found a delimeter but it turns out it was the beginning on the next key. Stopping here.",
|
||||
)
|
||||
return False, next_char, True
|
||||
return False, char, False
|
||||
|
||||
if state.unmatched_delimiter:
|
||||
state.unmatched_delimiter = False
|
||||
next_char = _append_literal_char(self, state, char)
|
||||
return True, next_char, False
|
||||
|
||||
i = 1
|
||||
next_c = self.get_char_at(i)
|
||||
check_comma_in_object_value = True
|
||||
while next_c and next_c not in [
|
||||
outer_rstring_delimiter,
|
||||
state.lstring_delimiter,
|
||||
]:
|
||||
if check_comma_in_object_value and next_c.isalpha():
|
||||
check_comma_in_object_value = False
|
||||
if (
|
||||
(ContextValues.OBJECT_KEY in self.context.context and next_c in [":", "}"])
|
||||
or (ContextValues.OBJECT_VALUE in self.context.context and next_c == "}")
|
||||
or (ContextValues.ARRAY in self.context.context and next_c in ["]", ","])
|
||||
or (check_comma_in_object_value and self.context.current == ContextValues.OBJECT_VALUE and next_c == ",")
|
||||
):
|
||||
break
|
||||
i += 1
|
||||
next_c = self.get_char_at(i)
|
||||
if next_c == "," and self.context.current == ContextValues.OBJECT_VALUE:
|
||||
i += 1
|
||||
i = self.skip_to_character(character=outer_rstring_delimiter, idx=i)
|
||||
next_c = self.get_char_at(i)
|
||||
i += 1
|
||||
i = self.scroll_whitespaces(idx=i)
|
||||
next_c = self.get_char_at(i)
|
||||
if next_c in ["}", ","]:
|
||||
self.log(
|
||||
"While parsing a string, we found a misplaced quote that would have closed the string but has a different meaning here, ignoring it",
|
||||
)
|
||||
next_char = _append_literal_char(self, state, char)
|
||||
return True, next_char, False
|
||||
elif next_c == outer_rstring_delimiter and self.get_char_at(i - 1) != "\\":
|
||||
if _only_whitespace_until(self, i) and not (
|
||||
self.context.current == ContextValues.OBJECT_VALUE and _quoted_object_member_follows(self, i)
|
||||
):
|
||||
return False, char, True
|
||||
if self.context.current == ContextValues.OBJECT_VALUE:
|
||||
if _quoted_object_member_follows(self, i):
|
||||
self.log(
|
||||
"While parsing a string, we found a misplaced quote that would have closed the string but has a different meaning here, ignoring it",
|
||||
)
|
||||
next_char = _append_literal_char(self, state, char)
|
||||
return True, next_char, False
|
||||
i = self.skip_to_character(character=outer_rstring_delimiter, idx=i + 1)
|
||||
i += 1
|
||||
next_c = self.get_char_at(i)
|
||||
while next_c and next_c != ":":
|
||||
if next_c in [",", "]", "}"] or (next_c == outer_rstring_delimiter and self.get_char_at(i - 1) != "\\"):
|
||||
break
|
||||
i += 1
|
||||
next_c = self.get_char_at(i)
|
||||
if next_c != ":":
|
||||
self.log(
|
||||
"While parsing a string, we found a misplaced quote that would have closed the string but has a different meaning here, ignoring it",
|
||||
)
|
||||
state.unmatched_delimiter = not state.unmatched_delimiter
|
||||
next_char = _append_literal_char(self, state, char)
|
||||
return True, next_char, False
|
||||
elif self.context.current == ContextValues.ARRAY:
|
||||
even_delimiters = next_c == outer_rstring_delimiter
|
||||
while next_c == outer_rstring_delimiter:
|
||||
i = self.skip_to_character(character=[outer_rstring_delimiter, "]"], idx=i + 1)
|
||||
next_c = self.get_char_at(i)
|
||||
if next_c != outer_rstring_delimiter:
|
||||
even_delimiters = False
|
||||
break
|
||||
i = self.skip_to_character(character=[outer_rstring_delimiter, "]"], idx=i + 1)
|
||||
next_c = self.get_char_at(i)
|
||||
if even_delimiters:
|
||||
self.log(
|
||||
"While parsing a string in Array context, we detected a quoted section that would have closed the string but has a different meaning here, ignoring it",
|
||||
)
|
||||
state.unmatched_delimiter = not state.unmatched_delimiter
|
||||
next_char = _append_literal_char(self, state, char)
|
||||
return True, next_char, False
|
||||
return False, char, True
|
||||
elif self.context.current == ContextValues.OBJECT_KEY:
|
||||
self.log(
|
||||
"While parsing a string in Object Key context, we detected a quoted section that would have closed the string but has a different meaning here, ignoring it",
|
||||
)
|
||||
next_char = _append_literal_char(self, state, char)
|
||||
return True, next_char, False
|
||||
|
||||
return False, char, False
|
||||
|
||||
|
||||
def _scan_string_body(
|
||||
self: "JSONParser",
|
||||
state: StringParseState,
|
||||
) -> str | None:
|
||||
outer_rstring_delimiter = _outer_rstring_delimiter(state)
|
||||
|
||||
def cached_skip_to_character(character: str | list[str], idx: int = 0) -> int:
|
||||
return _cached_skip_to_character(self, state, character, idx)
|
||||
|
||||
char = self.get_char_at()
|
||||
while char and (char != outer_rstring_delimiter or _in_low_smart_quote_span(state)):
|
||||
if state.missing_quotes:
|
||||
if self.context.current == ContextValues.OBJECT_KEY and (char == ":" or char.isspace()):
|
||||
self.log(
|
||||
"While parsing a string missing the left delimiter in object key context, we found a :, stopping here",
|
||||
)
|
||||
break
|
||||
if self.context.current == ContextValues.ARRAY and char in ["]", ","]:
|
||||
self.log(
|
||||
"While parsing a string missing the left delimiter in array context, we found a ] or ,, stopping here",
|
||||
)
|
||||
break
|
||||
if char == "„" and (not state.string_acc or state.string_acc[-1] != "\\"):
|
||||
_push_low_smart_quote_span(state)
|
||||
char = _append_literal_char(self, state, char)
|
||||
continue
|
||||
if _in_low_smart_quote_span(state) and char == "”":
|
||||
_pop_low_smart_quote_span(state)
|
||||
char = _append_literal_char(self, state, char)
|
||||
continue
|
||||
if (
|
||||
(
|
||||
state.pending_inline_container
|
||||
or (
|
||||
self.context.current == ContextValues.OBJECT_VALUE
|
||||
and char == "{"
|
||||
and self.get_char_at(-1) != "\\"
|
||||
and _bare_key_is_followed_by_colon(self, self.scroll_whitespaces(idx=1))
|
||||
)
|
||||
)
|
||||
and char in INLINE_CONTAINER_OPENERS
|
||||
and (not state.string_acc or state.string_acc[-1] != "\\")
|
||||
):
|
||||
container_end_idx = _skip_inline_container(self, 0)
|
||||
if container_end_idx is not None:
|
||||
self.log(
|
||||
"While parsing a string in object value context, we found a balanced inline container that belongs to the string, keeping it",
|
||||
)
|
||||
state.pending_inline_container = False
|
||||
state.inline_container_stack.clear()
|
||||
_append_string_content(state, self.json_str[self.index : self.index + container_end_idx])
|
||||
self.index += container_end_idx
|
||||
char = self.get_char_at()
|
||||
continue
|
||||
if (
|
||||
not self.stream_stable
|
||||
and self.context.current == ContextValues.OBJECT_VALUE
|
||||
and char == ","
|
||||
and not state.pending_inline_container
|
||||
and not state.inline_container_stack
|
||||
):
|
||||
comma_classification = (
|
||||
"string"
|
||||
if state.object_value_has_no_future_delimiter
|
||||
else classify_object_value_comma(
|
||||
self,
|
||||
cached_skip_to_character,
|
||||
)
|
||||
)
|
||||
if comma_classification == "member":
|
||||
self.log(
|
||||
"While parsing a string missing the right delimiter in object value context, we found a comma that starts the next object member. Stopping here",
|
||||
)
|
||||
break
|
||||
if comma_classification == "string_no_future_delimiter":
|
||||
state.object_value_has_no_future_delimiter = True
|
||||
state.pending_inline_container = comma_classification == "container"
|
||||
self.log(
|
||||
"While parsing a string in object value context, we found a comma that belongs to the string, keeping it",
|
||||
)
|
||||
char = _append_literal_char(self, state, char)
|
||||
continue
|
||||
state.pending_inline_container, keep_inline_container_char = update_inline_container_stack(
|
||||
char,
|
||||
state.pending_inline_container,
|
||||
state.inline_container_stack,
|
||||
)
|
||||
if keep_inline_container_char:
|
||||
char = _append_literal_char(self, state, char)
|
||||
continue
|
||||
if (
|
||||
not self.stream_stable
|
||||
and self.context.current == ContextValues.OBJECT_VALUE
|
||||
and char == "}"
|
||||
and (not state.string_acc or state.string_acc[-1] != outer_rstring_delimiter)
|
||||
):
|
||||
if state.object_value_unmatched_opening_braces:
|
||||
char = _append_literal_char(self, state, char)
|
||||
continue
|
||||
rstring_delimiter_missing = True
|
||||
self.skip_whitespaces()
|
||||
if self.get_char_at(1) == "\\":
|
||||
rstring_delimiter_missing = False
|
||||
i = _cached_skip_to_character(self, state, outer_rstring_delimiter, idx=1)
|
||||
next_c = self.get_char_at(i)
|
||||
if next_c:
|
||||
i += 1
|
||||
i = self.scroll_whitespaces(idx=i)
|
||||
next_c = self.get_char_at(i)
|
||||
if not next_c or next_c in [",", "}"]:
|
||||
rstring_delimiter_missing = False
|
||||
else:
|
||||
i = self.skip_to_character(character=state.lstring_delimiter, idx=i)
|
||||
next_c = self.get_char_at(i)
|
||||
if not next_c:
|
||||
rstring_delimiter_missing = False
|
||||
else:
|
||||
i = self.scroll_whitespaces(idx=i + 1)
|
||||
next_c = self.get_char_at(i)
|
||||
if next_c and next_c != ":":
|
||||
rstring_delimiter_missing = False
|
||||
else:
|
||||
i = self.skip_to_character(character=":", idx=1)
|
||||
next_c = self.get_char_at(i)
|
||||
if next_c:
|
||||
break
|
||||
i = self.scroll_whitespaces(idx=1)
|
||||
j = self.skip_to_character(character="}", idx=i)
|
||||
if j - i > 1:
|
||||
rstring_delimiter_missing = False
|
||||
if rstring_delimiter_missing:
|
||||
self.log(
|
||||
"While parsing a string missing the left delimiter in object value context, we found a , or } and we couldn't determine that a right delimiter was present. Stopping here",
|
||||
)
|
||||
break
|
||||
if (
|
||||
not self.stream_stable
|
||||
and char == "]"
|
||||
and ContextValues.ARRAY in self.context.context
|
||||
and (not state.string_acc or state.string_acc[-1] != outer_rstring_delimiter)
|
||||
):
|
||||
i = self.skip_to_character(outer_rstring_delimiter)
|
||||
if not self.get_char_at(i):
|
||||
break
|
||||
if self.context.current == ContextValues.OBJECT_VALUE and char == "}":
|
||||
i = self.scroll_whitespaces(idx=1)
|
||||
next_c = self.get_char_at(i)
|
||||
if next_c == "`" and self.get_char_at(i + 1) == "`" and self.get_char_at(i + 2) == "`":
|
||||
if _brace_before_code_fence_belongs_to_string(self, state, i):
|
||||
self.log(
|
||||
"While parsing a string in object value context, we found a literal fenced snippet after }, keeping it in the string",
|
||||
)
|
||||
char = _append_literal_char(self, state, char)
|
||||
continue
|
||||
self.log(
|
||||
"While parsing a string in object value context, we found a } that closes the object before code fences, stopping here",
|
||||
)
|
||||
break
|
||||
if not next_c:
|
||||
self.log(
|
||||
"While parsing a string in object value context, we found a } that closes the object, stopping here",
|
||||
)
|
||||
break
|
||||
assert char is not None
|
||||
_append_string_content(state, char)
|
||||
self.index += 1
|
||||
char = self.get_char_at()
|
||||
if char is None:
|
||||
if self.stream_stable and state.string_acc and state.string_acc[-1] == "\\":
|
||||
state.string_acc = state.string_acc[:-1]
|
||||
_rebuild_unmatched_opening_braces(state)
|
||||
break
|
||||
if state.string_acc and state.string_acc[-1] == "\\":
|
||||
handled_escape, char = _normalize_escape_sequence(self, state, char)
|
||||
if handled_escape:
|
||||
continue
|
||||
if char == ":" and not state.missing_quotes and self.context.current == ContextValues.OBJECT_KEY:
|
||||
i = self.skip_to_character(character=state.lstring_delimiter, idx=1)
|
||||
next_c = self.get_char_at(i)
|
||||
if next_c:
|
||||
i += 1
|
||||
i = self.skip_to_character(character=outer_rstring_delimiter, idx=i)
|
||||
next_c = self.get_char_at(i)
|
||||
if next_c:
|
||||
i += 1
|
||||
i = self.scroll_whitespaces(idx=i)
|
||||
ch = self.get_char_at(i)
|
||||
if ch in [",", "}"]:
|
||||
self.log(
|
||||
f"While parsing a string missing the right delimiter in object key context, we found a {ch} stopping here",
|
||||
)
|
||||
break
|
||||
else:
|
||||
self.log(
|
||||
"While parsing a string missing the right delimiter in object key context, we found a :, stopping here",
|
||||
)
|
||||
break
|
||||
if _in_low_smart_quote_span(state) and char == '"':
|
||||
_pop_low_smart_quote_span(state)
|
||||
char = _append_literal_char(self, state, char)
|
||||
continue
|
||||
if (
|
||||
char == outer_rstring_delimiter
|
||||
and self.context.current == ContextValues.OBJECT_VALUE
|
||||
and _quote_belongs_to_regex_character_class(self, state)
|
||||
):
|
||||
self.log(
|
||||
"While parsing a string, we found a bare quote inside a regex character class, keeping it",
|
||||
)
|
||||
assert char is not None
|
||||
char = _append_literal_char(self, state, char)
|
||||
continue
|
||||
if char == outer_rstring_delimiter and state.string_acc and state.string_acc[-1] != "\\":
|
||||
assert char is not None
|
||||
handled_delimiter, char, should_break = _handle_right_delimiter_candidate(self, state, char)
|
||||
if should_break:
|
||||
break
|
||||
if handled_delimiter:
|
||||
continue
|
||||
return char
|
||||
|
||||
|
||||
def _finalize_string_result(
|
||||
self: "JSONParser",
|
||||
state: StringParseState,
|
||||
char: str | None,
|
||||
) -> str:
|
||||
outer_rstring_delimiter = _outer_rstring_delimiter(state)
|
||||
if char and state.missing_quotes and self.context.current == ContextValues.OBJECT_KEY and char.isspace():
|
||||
self.log(
|
||||
"While parsing a string, handling an extreme corner case in which the LLM added a comment instead of valid string, invalidate the string and return an empty value",
|
||||
)
|
||||
self.skip_whitespaces()
|
||||
if self.get_char_at() not in [":", ","]:
|
||||
return ""
|
||||
|
||||
if char != outer_rstring_delimiter:
|
||||
if not self.stream_stable:
|
||||
self.log(
|
||||
"While parsing a string, we missed the closing quote, ignoring",
|
||||
)
|
||||
state.string_acc = state.string_acc.rstrip()
|
||||
else:
|
||||
self.index += 1
|
||||
|
||||
if not self.stream_stable and (state.missing_quotes or (state.string_acc and state.string_acc[-1] == "\n")):
|
||||
state.string_acc = state.string_acc.rstrip()
|
||||
|
||||
return state.string_acc
|
||||
|
||||
|
||||
def parse_string(self: "JSONParser") -> JSONReturnType:
|
||||
state, direct_result = _prepare_string_entry(self)
|
||||
if direct_result is not NO_DIRECT_RESULT:
|
||||
return cast("JSONReturnType", direct_result)
|
||||
assert state is not None
|
||||
|
||||
char = _scan_string_body(self, state)
|
||||
return _finalize_string_result(self, state, char)
|
||||
|
||||
|
||||
def _only_whitespace_until(self: "JSONParser", end: int) -> bool:
|
||||
for j in range(1, end):
|
||||
c = self.get_char_at(j)
|
||||
if c is not None and not c.isspace():
|
||||
return False
|
||||
return True
|
||||
@@ -0,0 +1 @@
|
||||
"""Helpers used by string parsing."""
|
||||
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
@@ -0,0 +1,115 @@
|
||||
from collections.abc import Callable
|
||||
from typing import TYPE_CHECKING, Literal
|
||||
|
||||
from ..utils.constants import STRING_DELIMITERS # noqa: TID252
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from ..json_parser import JSONParser # noqa: TID252
|
||||
|
||||
|
||||
ObjectValueCommaClassification = Literal["container", "member", "string", "string_no_future_delimiter"]
|
||||
|
||||
|
||||
def _bare_member_has_recoverable_value(
|
||||
parser: "JSONParser",
|
||||
value_idx: int,
|
||||
skip_to_character: Callable[[str | list[str], int], int],
|
||||
) -> bool:
|
||||
value_start_idx = parser.scroll_whitespaces(idx=value_idx)
|
||||
value_start = parser.get_char_at(value_start_idx)
|
||||
if value_start in [*STRING_DELIMITERS, "{", "[", "-"]:
|
||||
return True
|
||||
if value_start and value_start.isdigit():
|
||||
return True
|
||||
|
||||
for literal in ["true", "false", "null"]:
|
||||
if all(parser.get_char_at(value_start_idx + offset) == char for offset, char in enumerate(literal)):
|
||||
value_end = parser.get_char_at(value_start_idx + len(literal))
|
||||
if value_end is None or value_end.isspace() or value_end in [",", "}", "]"]:
|
||||
return True
|
||||
|
||||
# An unquoted value is only a safe member boundary when its object closes
|
||||
# before the current string can close. Otherwise prose such as
|
||||
# `, floof: explanation` is more likely to belong to the string.
|
||||
value_end_idx = skip_to_character([*STRING_DELIMITERS, "}"], value_start_idx)
|
||||
return parser.get_char_at(value_end_idx) == "}"
|
||||
|
||||
|
||||
def classify_object_value_comma(
|
||||
parser: "JSONParser",
|
||||
cached_skip_to_character: Callable[[str | list[str], int], int] | None = None,
|
||||
) -> ObjectValueCommaClassification:
|
||||
skip_to_character = cached_skip_to_character or parser.skip_to_character
|
||||
next_idx = parser.scroll_whitespaces(idx=1)
|
||||
next_c = parser.get_char_at(next_idx)
|
||||
if next_c in ["}", None]:
|
||||
return "member"
|
||||
|
||||
if next_c in STRING_DELIMITERS:
|
||||
key_end_idx = parser.skip_to_character(character=next_c, idx=next_idx + 1)
|
||||
if not parser.get_char_at(key_end_idx):
|
||||
return "string"
|
||||
key_end_idx = parser.scroll_whitespaces(idx=key_end_idx + 1)
|
||||
return "member" if parser.get_char_at(key_end_idx) == ":" else "string"
|
||||
|
||||
if next_c == "`":
|
||||
bare_key_idx = next_idx + 1
|
||||
while True:
|
||||
key_char = parser.get_char_at(bare_key_idx)
|
||||
if not key_char or not (key_char.isalnum() or key_char in ["_", "-"]):
|
||||
break
|
||||
bare_key_idx += 1
|
||||
bare_key_idx = parser.scroll_whitespaces(idx=bare_key_idx)
|
||||
return "member" if parser.get_char_at(bare_key_idx) == ":" else "string"
|
||||
|
||||
if next_c and (next_c.isalnum() or next_c == "_"):
|
||||
bare_key_idx = next_idx
|
||||
while True:
|
||||
key_char = parser.get_char_at(bare_key_idx)
|
||||
if not key_char or not (key_char.isalnum() or key_char in ["_", "-"]):
|
||||
break
|
||||
bare_key_idx += 1
|
||||
bare_key_idx = parser.scroll_whitespaces(idx=bare_key_idx)
|
||||
if parser.get_char_at(bare_key_idx) == ":" and _bare_member_has_recoverable_value(
|
||||
parser,
|
||||
bare_key_idx + 1,
|
||||
skip_to_character,
|
||||
):
|
||||
return "member"
|
||||
|
||||
if next_c in ["{", "["]:
|
||||
return "container"
|
||||
|
||||
next_special_idx = skip_to_character([*STRING_DELIMITERS, "{", "["], next_idx)
|
||||
next_special = parser.get_char_at(next_special_idx)
|
||||
if not next_special:
|
||||
return "string_no_future_delimiter"
|
||||
if next_special in ["{", "["]:
|
||||
return "string"
|
||||
|
||||
key_end_idx = skip_to_character(next_special, next_special_idx + 1)
|
||||
if not parser.get_char_at(key_end_idx):
|
||||
return "string"
|
||||
key_end_idx = parser.scroll_whitespaces(idx=key_end_idx + 1)
|
||||
return "member" if parser.get_char_at(key_end_idx) == ":" else "string"
|
||||
|
||||
|
||||
def update_inline_container_stack(
|
||||
char: str,
|
||||
pending_inline_container: bool,
|
||||
inline_container_stack: list[str],
|
||||
) -> tuple[bool, bool]:
|
||||
if char in ["{", "["]:
|
||||
if pending_inline_container:
|
||||
inline_container_stack.append(char)
|
||||
return False, False
|
||||
if inline_container_stack:
|
||||
inline_container_stack.append(char)
|
||||
|
||||
if inline_container_stack and (
|
||||
(char == "}" and inline_container_stack[-1] == "{") or (char == "]" and inline_container_stack[-1] == "[")
|
||||
):
|
||||
inline_container_stack.pop()
|
||||
return pending_inline_container, True
|
||||
|
||||
return pending_inline_container, False
|
||||
@@ -0,0 +1,35 @@
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from ..json_parser import JSONParser # noqa: TID252
|
||||
|
||||
|
||||
LITERAL_VALUES: dict[str, bool | None] = {
|
||||
"true": True,
|
||||
"false": False,
|
||||
"null": None,
|
||||
"none": None,
|
||||
}
|
||||
|
||||
|
||||
def parse_boolean_or_null(parser: "JSONParser") -> tuple[bool, bool | None]:
|
||||
"""Return whether an unquoted complete literal was found and its value."""
|
||||
char = (parser.get_char_at() or "").lower()
|
||||
starting_index = parser.index
|
||||
for literal, value in LITERAL_VALUES.items():
|
||||
if char != literal[0]:
|
||||
continue
|
||||
parser.index = starting_index
|
||||
for expected_char in literal:
|
||||
if (parser.get_char_at() or "").lower() != expected_char:
|
||||
break
|
||||
parser.index += 1
|
||||
else:
|
||||
next_char = parser.get_char_at()
|
||||
if next_char is None or next_char.isspace() or next_char in {",", ")", "]", "}"}:
|
||||
if literal == "none":
|
||||
parser.log("Converted unquoted Python None literal to JSON null")
|
||||
return True, value
|
||||
|
||||
parser.index = starting_index
|
||||
return False, None
|
||||
@@ -0,0 +1,19 @@
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from ..utils.constants import JSONReturnType # noqa: TID252
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from ..json_parser import JSONParser # noqa: TID252
|
||||
|
||||
|
||||
def parse_json_llm_block(parser: "JSONParser") -> JSONReturnType:
|
||||
"""
|
||||
Extracts and normalizes JSON enclosed in ```json ... ``` blocks.
|
||||
"""
|
||||
# Try to find a ```json ... ``` block
|
||||
if parser.json_str[parser.index : parser.index + 7] == "```json":
|
||||
i = parser.skip_to_character("`", idx=7)
|
||||
if parser.json_str[parser.index + i : parser.index + i + 3] == "```":
|
||||
parser.index += 7 # Move past ```json
|
||||
return parser.parse_json()
|
||||
return False
|
||||
@@ -0,0 +1,169 @@
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from .utils.constants import STRING_DELIMITERS
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from .json_parser import JSONParser
|
||||
|
||||
|
||||
def parenthesized_is_explicit_tuple(parser: "JSONParser") -> bool:
|
||||
"""
|
||||
Return True when the current '(' starts an explicit Python tuple literal.
|
||||
|
||||
Empty parentheses count as a tuple. A single grouped value like ``(1)`` does not.
|
||||
"""
|
||||
i = parser.index + 1
|
||||
n = len(parser.json_str)
|
||||
nested_parentheses = 0
|
||||
square_brackets = 0
|
||||
braces = 0
|
||||
in_quote: str | None = None
|
||||
backslashes = 0
|
||||
saw_top_level_content = False
|
||||
|
||||
while i < n:
|
||||
ch = parser.json_str[i]
|
||||
|
||||
if ch == "\\":
|
||||
backslashes += 1
|
||||
i += 1
|
||||
continue
|
||||
|
||||
if in_quote is not None:
|
||||
if ch == in_quote and backslashes % 2 == 0:
|
||||
in_quote = None
|
||||
backslashes = 0
|
||||
i += 1
|
||||
continue
|
||||
|
||||
if ch in STRING_DELIMITERS and backslashes % 2 == 0:
|
||||
in_quote = ch
|
||||
saw_top_level_content = saw_top_level_content or (
|
||||
nested_parentheses == 0 and square_brackets == 0 and braces == 0
|
||||
)
|
||||
backslashes = 0
|
||||
i += 1
|
||||
continue
|
||||
|
||||
backslashes = 0
|
||||
|
||||
if (
|
||||
not ch.isspace()
|
||||
and ch not in [",", ")"]
|
||||
and nested_parentheses == 0
|
||||
and square_brackets == 0
|
||||
and braces == 0
|
||||
):
|
||||
saw_top_level_content = True
|
||||
|
||||
if ch == "(":
|
||||
nested_parentheses += 1
|
||||
elif ch == ")":
|
||||
if nested_parentheses == 0 and square_brackets == 0 and braces == 0:
|
||||
return not saw_top_level_content
|
||||
if nested_parentheses > 0:
|
||||
nested_parentheses -= 1
|
||||
elif ch == "[":
|
||||
square_brackets += 1
|
||||
elif ch == "]" and square_brackets > 0:
|
||||
square_brackets -= 1
|
||||
elif ch == "{":
|
||||
braces += 1
|
||||
elif ch == "}" and braces > 0:
|
||||
braces -= 1
|
||||
elif ch == "," and nested_parentheses == 0 and square_brackets == 0 and braces == 0:
|
||||
return True
|
||||
|
||||
i += 1
|
||||
|
||||
return not saw_top_level_content
|
||||
|
||||
|
||||
def top_level_parenthesized_can_start_value(parser: "JSONParser") -> bool:
|
||||
"""
|
||||
Return True when a top-level '(' looks like a standalone value rather than inline prose.
|
||||
|
||||
This keeps tuple support available for direct inputs and fenced blocks while avoiding
|
||||
regressions on surrounding explanatory text like ``foo (clarification): {...}``.
|
||||
"""
|
||||
i = parser.index - 1
|
||||
while i >= 0:
|
||||
ch = parser.json_str[i]
|
||||
if ch in "\n\r":
|
||||
break
|
||||
if not ch.isspace():
|
||||
return False
|
||||
i -= 1
|
||||
|
||||
idx = parser.scroll_whitespaces(idx=1)
|
||||
first_inner_char = parser.get_char_at(idx)
|
||||
if first_inner_char is None:
|
||||
return False
|
||||
|
||||
inner_text = parser.json_str[parser.index + idx :].lower()
|
||||
if (
|
||||
first_inner_char not in [")", "{", "[", "(", *STRING_DELIMITERS]
|
||||
and not first_inner_char.isdigit()
|
||||
and first_inner_char not in ["-", "."]
|
||||
and inner_text[:4] not in ["true", "null", "none"]
|
||||
and inner_text[:5] != "false"
|
||||
):
|
||||
return False
|
||||
|
||||
i = parser.index + 1
|
||||
n = len(parser.json_str)
|
||||
nested_parentheses = 0
|
||||
square_brackets = 0
|
||||
braces = 0
|
||||
in_quote: str | None = None
|
||||
backslashes = 0
|
||||
|
||||
while i < n:
|
||||
ch = parser.json_str[i]
|
||||
|
||||
if ch == "\\":
|
||||
backslashes += 1
|
||||
i += 1
|
||||
continue
|
||||
|
||||
if in_quote is not None:
|
||||
if ch == in_quote and backslashes % 2 == 0:
|
||||
in_quote = None
|
||||
backslashes = 0
|
||||
i += 1
|
||||
continue
|
||||
|
||||
if ch in STRING_DELIMITERS and backslashes % 2 == 0:
|
||||
in_quote = ch
|
||||
backslashes = 0
|
||||
i += 1
|
||||
continue
|
||||
|
||||
backslashes = 0
|
||||
|
||||
if ch == "(":
|
||||
nested_parentheses += 1
|
||||
elif ch == ")":
|
||||
if nested_parentheses == 0 and square_brackets == 0 and braces == 0:
|
||||
i += 1
|
||||
while i < n:
|
||||
trailer = parser.json_str[i]
|
||||
if trailer in "\n\r":
|
||||
return True
|
||||
if not trailer.isspace():
|
||||
return False
|
||||
i += 1
|
||||
return True
|
||||
nested_parentheses -= 1
|
||||
elif ch == "[":
|
||||
square_brackets += 1
|
||||
elif ch == "]" and square_brackets > 0:
|
||||
square_brackets -= 1
|
||||
elif ch == "{":
|
||||
braces += 1
|
||||
elif ch == "}" and braces > 0:
|
||||
braces -= 1
|
||||
|
||||
i += 1
|
||||
|
||||
return True
|
||||
@@ -0,0 +1,78 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from .schema_repair import SchemaRepairer
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ObjectSchemaConfig:
|
||||
properties: dict[str, Any]
|
||||
pattern_properties: dict[str, Any]
|
||||
additional_properties: object | None
|
||||
required: set[str]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ArraySchemaConfig:
|
||||
items_schema: object | None
|
||||
additional_items: object | None
|
||||
|
||||
|
||||
def resolve_parser_object_schema(
|
||||
repairer: SchemaRepairer | None,
|
||||
schema: dict[str, Any] | bool | None,
|
||||
) -> tuple[SchemaRepairer | None, dict[str, Any] | bool | None, ObjectSchemaConfig | None]:
|
||||
if repairer is None or schema in (None, True):
|
||||
return None, schema, None
|
||||
|
||||
schema = repairer.resolve_schema(schema)
|
||||
if schema is False:
|
||||
raise ValueError("Schema does not allow any values.")
|
||||
if schema is True or not repairer.is_object_schema(schema):
|
||||
return None, schema, None
|
||||
return repairer, schema, object_schema_config(schema)
|
||||
|
||||
|
||||
def resolve_parser_array_schema(
|
||||
repairer: SchemaRepairer | None,
|
||||
schema: dict[str, Any] | bool | None,
|
||||
) -> tuple[SchemaRepairer | None, dict[str, Any] | bool | None, ArraySchemaConfig | None]:
|
||||
if repairer is None or schema in (None, True):
|
||||
return None, schema, None
|
||||
|
||||
schema = repairer.resolve_schema(schema)
|
||||
if schema is False:
|
||||
raise ValueError("Schema does not allow any values.")
|
||||
if schema is True or not repairer.is_array_schema(schema):
|
||||
return None, schema, None
|
||||
return repairer, schema, array_schema_config(schema)
|
||||
|
||||
|
||||
def object_schema_config(schema: dict[str, Any]) -> ObjectSchemaConfig:
|
||||
properties = schema.get("properties", {})
|
||||
if not isinstance(properties, dict):
|
||||
properties = {}
|
||||
pattern_properties = schema.get("patternProperties", {})
|
||||
if not isinstance(pattern_properties, dict):
|
||||
pattern_properties = {}
|
||||
return ObjectSchemaConfig(
|
||||
properties=properties,
|
||||
pattern_properties=pattern_properties,
|
||||
additional_properties=schema.get("additionalProperties"),
|
||||
required=set(schema.get("required", [])),
|
||||
)
|
||||
|
||||
|
||||
def array_schema_config(schema: dict[str, Any]) -> ArraySchemaConfig:
|
||||
if isinstance(schema.get("prefixItems"), list):
|
||||
return ArraySchemaConfig(
|
||||
items_schema=schema["prefixItems"],
|
||||
additional_items=schema.get("items"),
|
||||
)
|
||||
return ArraySchemaConfig(
|
||||
items_schema=schema.get("items"),
|
||||
additional_items=schema.get("additionalItems"),
|
||||
)
|
||||
Whitespace-only changes.
@@ -0,0 +1,782 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import copy
|
||||
import importlib
|
||||
import json
|
||||
from typing import Any, Literal, cast
|
||||
|
||||
from .parser_schema import array_schema_config, object_schema_config
|
||||
from .utils.constants import MISSING_VALUE, JSONReturnType, MissingValueType
|
||||
from .utils.pattern_properties import match_pattern_properties
|
||||
|
||||
SchemaRepairMode = Literal["standard", "salvage"]
|
||||
SUPPORTED_SCHEMA_REPAIR_MODES: tuple[SchemaRepairMode, ...] = ("standard", "salvage")
|
||||
|
||||
|
||||
class SchemaDefinitionError(ValueError):
|
||||
"""Raised when schema metadata is invalid or unsupported."""
|
||||
|
||||
|
||||
def normalize_schema_repair_mode(mode: str | None) -> SchemaRepairMode:
|
||||
if mode is None:
|
||||
return "standard"
|
||||
if mode == "standard":
|
||||
return "standard"
|
||||
if mode == "salvage":
|
||||
return "salvage"
|
||||
expected = ", ".join(SUPPORTED_SCHEMA_REPAIR_MODES)
|
||||
raise ValueError(f"schema_repair_mode must be one of: {expected}.")
|
||||
|
||||
|
||||
def _require_jsonschema() -> Any:
|
||||
try:
|
||||
return importlib.import_module("jsonschema")
|
||||
except ImportError as exc: # pragma: no cover - optional dependency
|
||||
raise ValueError("jsonschema is required when using schema-aware repair.") from exc
|
||||
|
||||
|
||||
def _require_pydantic() -> Any:
|
||||
try:
|
||||
return importlib.import_module("pydantic")
|
||||
except ImportError as exc: # pragma: no cover - optional dependency
|
||||
raise ValueError("pydantic is required when using schema models.") from exc
|
||||
|
||||
|
||||
def _add_pydantic_field_metadata(schema_dict: dict[str, Any], model: type[Any]) -> None:
|
||||
properties = schema_dict.setdefault("properties", {})
|
||||
if not isinstance(properties, dict):
|
||||
properties = {}
|
||||
schema_dict["properties"] = properties
|
||||
model_config = getattr(model, "model_config", {})
|
||||
accepts_field_names = model_config.get("populate_by_name", False) or model_config.get("validate_by_name", False)
|
||||
property_aliases: dict[str, list[str]] = {}
|
||||
for name, field in model.model_fields.items():
|
||||
validation_alias = getattr(field, "validation_alias", None)
|
||||
alias = validation_alias if isinstance(validation_alias, str) else getattr(field, "alias", None)
|
||||
candidates = [candidate for candidate in (alias, name) if isinstance(candidate, str)]
|
||||
property_name = next((candidate for candidate in candidates if candidate in properties), None)
|
||||
if property_name is None:
|
||||
property_name = candidates[0] if candidates else name
|
||||
if accepts_field_names and property_name != name:
|
||||
property_aliases.setdefault(property_name, []).append(name)
|
||||
if field.is_required():
|
||||
continue
|
||||
property_schema = properties.setdefault(property_name, {})
|
||||
if not isinstance(property_schema, dict):
|
||||
property_schema = {}
|
||||
properties[property_name] = property_schema
|
||||
if "default" not in property_schema:
|
||||
if field.default_factory is not None:
|
||||
property_schema["default"] = field.default_factory()
|
||||
else:
|
||||
property_schema["default"] = field.default
|
||||
if property_aliases:
|
||||
schema_dict["x-json-repair-property-aliases"] = property_aliases
|
||||
|
||||
|
||||
def _prepare_schema_for_validation_node(node: Any) -> Any:
|
||||
if isinstance(node, dict):
|
||||
normalized = {key: _prepare_schema_for_validation_node(value) for key, value in node.items()}
|
||||
items = normalized.get("items")
|
||||
if isinstance(items, list):
|
||||
normalized.pop("items", None)
|
||||
normalized["prefixItems"] = items
|
||||
additional_items = normalized.pop("additionalItems", None)
|
||||
if additional_items is False:
|
||||
normalized["items"] = False
|
||||
elif isinstance(additional_items, dict):
|
||||
normalized["items"] = additional_items
|
||||
return normalized
|
||||
if isinstance(node, list):
|
||||
return [_prepare_schema_for_validation_node(item) for item in node]
|
||||
return node
|
||||
|
||||
|
||||
def load_schema_model(path: str) -> type[Any]:
|
||||
if ":" not in path:
|
||||
raise ValueError("Schema model must be in the form 'module:ClassName'.")
|
||||
module_name, class_name = path.split(":", 1)
|
||||
module = importlib.import_module(module_name)
|
||||
model: object | None = module.__dict__.get(class_name)
|
||||
if model is None or not isinstance(model, type):
|
||||
raise ValueError(f"Schema model '{class_name}' not found in module '{module_name}'.")
|
||||
return model
|
||||
|
||||
|
||||
def normalize_missing_values(value: object) -> JSONReturnType:
|
||||
if value is MISSING_VALUE or isinstance(value, MissingValueType):
|
||||
return ""
|
||||
if isinstance(value, dict):
|
||||
normalized: dict[str, JSONReturnType] = {}
|
||||
for key, item in value.items():
|
||||
if not isinstance(key, str):
|
||||
raise ValueError("Object keys must be strings.")
|
||||
normalized[key] = normalize_missing_values(item)
|
||||
return normalized
|
||||
if isinstance(value, list):
|
||||
return [normalize_missing_values(item) for item in value]
|
||||
if value is None or isinstance(value, (str, int, float, bool)):
|
||||
return value
|
||||
raise ValueError("Value is not JSON compatible.")
|
||||
|
||||
|
||||
def schema_from_input(schema: Any) -> dict[str, Any] | bool:
|
||||
if isinstance(schema, dict):
|
||||
return schema
|
||||
if schema is True or schema is False:
|
||||
return schema
|
||||
if hasattr(schema, "model_json_schema"):
|
||||
pydantic = _require_pydantic()
|
||||
version = getattr(pydantic, "VERSION", getattr(pydantic, "__version__", "0"))
|
||||
if int(version.split(".")[0]) < 2:
|
||||
raise ValueError("pydantic v2 is required for schema models.")
|
||||
schema_dict: dict[str, Any] = schema.model_json_schema()
|
||||
if hasattr(schema, "model_fields"):
|
||||
_add_pydantic_field_metadata(schema_dict, schema)
|
||||
return schema_dict
|
||||
raise ValueError("Schema must be a JSON Schema dict, boolean schema, or pydantic v2 model.")
|
||||
|
||||
|
||||
class SchemaRepairer:
|
||||
def __init__(
|
||||
self,
|
||||
schema: dict[str, Any] | bool,
|
||||
log: list[dict[str, str]] | None,
|
||||
schema_repair_mode: str = "standard",
|
||||
) -> None:
|
||||
self.root_schema = schema
|
||||
self.log = log
|
||||
self.schema_repair_mode = normalize_schema_repair_mode(schema_repair_mode)
|
||||
self._validator_cache: dict[int, tuple[dict[str, Any], Any]] = {}
|
||||
self._root_validator: Any | None = None
|
||||
|
||||
def _log(self, text: str, path: str) -> None:
|
||||
if self.log is not None:
|
||||
self.log.append({"text": text, "context": path})
|
||||
|
||||
def _get_validator(self, schema: dict[str, Any]) -> Any:
|
||||
cache_key = id(schema)
|
||||
cached_validator = self._validator_cache.get(cache_key)
|
||||
if cached_validator is not None and cached_validator[0] is schema:
|
||||
return cached_validator[1]
|
||||
|
||||
prepared_schema = self._prepare_schema_for_validation(schema)
|
||||
root_validator = self._get_root_validator()
|
||||
validator = root_validator if schema is self.root_schema else root_validator.evolve(schema=prepared_schema)
|
||||
self._validator_cache[cache_key] = (schema, validator)
|
||||
return validator
|
||||
|
||||
def _get_root_validator(self) -> Any:
|
||||
if self._root_validator is not None:
|
||||
return self._root_validator
|
||||
|
||||
prepared_root_schema = self._prepare_schema_for_validation(self.root_schema)
|
||||
jsonschema = _require_jsonschema()
|
||||
validator_cls = jsonschema.validators.validator_for(prepared_root_schema)
|
||||
self._root_validator = validator_cls(prepared_root_schema)
|
||||
return self._root_validator
|
||||
|
||||
def is_valid(self, value: JSONReturnType, schema: dict[str, Any] | bool) -> bool:
|
||||
schema = self.resolve_schema(schema)
|
||||
if schema is True:
|
||||
return True
|
||||
if schema is False:
|
||||
return False
|
||||
validator = self._get_validator(schema)
|
||||
return bool(validator.is_valid(value))
|
||||
|
||||
def validate(self, value: JSONReturnType, schema: dict[str, Any] | bool) -> None:
|
||||
schema = self.resolve_schema(schema)
|
||||
if schema is True:
|
||||
return
|
||||
if schema is False:
|
||||
raise ValueError("Schema does not allow any values.")
|
||||
jsonschema = _require_jsonschema()
|
||||
validator = self._get_validator(schema)
|
||||
try:
|
||||
validator.validate(value)
|
||||
except jsonschema.exceptions.ValidationError as exc:
|
||||
raise ValueError(exc.message) from exc
|
||||
|
||||
def resolve_schema(self, schema: object | None) -> dict[str, Any] | bool:
|
||||
if schema is None:
|
||||
return True
|
||||
if isinstance(schema, bool):
|
||||
return schema
|
||||
if not isinstance(schema, dict):
|
||||
raise SchemaDefinitionError("Schema must be an object.")
|
||||
for key in schema:
|
||||
if not isinstance(key, str):
|
||||
raise SchemaDefinitionError("Schema keys must be strings.")
|
||||
schema_dict = cast("dict[str, Any]", schema)
|
||||
seen_schema_ids: set[int] = set()
|
||||
while "$ref" in schema_dict:
|
||||
ref = schema_dict["$ref"]
|
||||
if not isinstance(ref, str):
|
||||
raise SchemaDefinitionError("$ref must be a string.")
|
||||
schema_id = id(schema_dict)
|
||||
if schema_id in seen_schema_ids:
|
||||
raise SchemaDefinitionError(f"Circular $ref detected: {ref}")
|
||||
seen_schema_ids.add(schema_id)
|
||||
resolved = self._resolve_ref(ref)
|
||||
if isinstance(resolved, bool):
|
||||
return resolved
|
||||
schema_dict = resolved
|
||||
return schema_dict
|
||||
|
||||
def is_object_schema(self, schema: dict[str, Any] | bool | None) -> bool:
|
||||
schema = self.resolve_schema(schema)
|
||||
if not isinstance(schema, dict):
|
||||
return False
|
||||
schema_type = schema.get("type")
|
||||
if schema_type == "object":
|
||||
return True
|
||||
if isinstance(schema_type, list) and "object" in schema_type:
|
||||
return True
|
||||
return any(key in schema for key in ("properties", "patternProperties", "additionalProperties", "required"))
|
||||
|
||||
def is_array_schema(self, schema: dict[str, Any] | bool | None) -> bool:
|
||||
schema = self.resolve_schema(schema)
|
||||
if not isinstance(schema, dict):
|
||||
return False
|
||||
schema_type = schema.get("type")
|
||||
if schema_type == "array":
|
||||
return True
|
||||
if isinstance(schema_type, list) and "array" in schema_type:
|
||||
return True
|
||||
return "items" in schema or "prefixItems" in schema
|
||||
|
||||
def _allows_schema_type(self, schema: dict[str, Any], schema_type: str) -> bool:
|
||||
declared_type = schema.get("type")
|
||||
if isinstance(declared_type, str):
|
||||
return declared_type == schema_type
|
||||
if isinstance(declared_type, list):
|
||||
return schema_type in declared_type
|
||||
if schema_type == "object":
|
||||
return self.is_object_schema(schema)
|
||||
# This helper is only used for object/array checks in _can_salvage_list_as_object.
|
||||
return self.is_array_schema(schema)
|
||||
|
||||
def _can_salvage_list_as_object(self, schema: dict[str, Any]) -> bool:
|
||||
return self._allows_schema_type(schema, "object") and not self._allows_schema_type(schema, "array")
|
||||
|
||||
def repair_value(self, value: Any, schema: dict[str, Any] | bool | None, path: str) -> JSONReturnType:
|
||||
"""Apply schema rules to a parsed value, including unions, coercions, and defaults."""
|
||||
schema = self.resolve_schema(schema)
|
||||
if schema is True:
|
||||
return normalize_missing_values(value)
|
||||
if schema is False:
|
||||
raise ValueError("Schema does not allow any values.")
|
||||
if not schema:
|
||||
return normalize_missing_values(value)
|
||||
|
||||
if value is MISSING_VALUE:
|
||||
return self._fill_missing(schema, path)
|
||||
|
||||
if "allOf" in schema:
|
||||
subschemas = schema["allOf"]
|
||||
if not subschemas:
|
||||
return normalize_missing_values(value)
|
||||
repaired = self.repair_value(value, subschemas[0], path)
|
||||
for subschema in subschemas[1:]:
|
||||
repaired = self.repair_value(repaired, subschema, path)
|
||||
return repaired
|
||||
|
||||
if "oneOf" in schema:
|
||||
return self._repair_union(value, schema["oneOf"], path)
|
||||
if "anyOf" in schema:
|
||||
return self._repair_union(value, schema["anyOf"], path)
|
||||
|
||||
expected_type = schema.get("type")
|
||||
if expected_type is None:
|
||||
if self.is_object_schema(schema):
|
||||
expected_type = "object"
|
||||
elif self.is_array_schema(schema):
|
||||
expected_type = "array"
|
||||
|
||||
if isinstance(expected_type, list):
|
||||
return self._repair_type_union(value, expected_type, schema, path)
|
||||
|
||||
if expected_type == "object":
|
||||
repaired = self._repair_object(value, schema, path)
|
||||
elif expected_type == "array":
|
||||
repaired = self._repair_array(value, schema, path)
|
||||
elif isinstance(expected_type, str):
|
||||
repaired = self._coerce_scalar(value, expected_type, path)
|
||||
else:
|
||||
repaired = normalize_missing_values(value)
|
||||
|
||||
return self._apply_enum_const(repaired, schema, path)
|
||||
|
||||
def _repair_union(self, value: Any, schemas: list[dict[str, Any] | bool], path: str) -> JSONReturnType:
|
||||
last_error: Exception | None = None
|
||||
for subschema in schemas:
|
||||
try:
|
||||
candidate = self.repair_value(copy.deepcopy(value), subschema, path)
|
||||
self.validate(candidate, subschema)
|
||||
except ValueError as exc:
|
||||
last_error = exc
|
||||
else:
|
||||
return candidate
|
||||
if last_error:
|
||||
raise ValueError(str(last_error)) from last_error
|
||||
raise ValueError("No schema matched the value.")
|
||||
|
||||
def _repair_type_union(
|
||||
self,
|
||||
value: Any,
|
||||
types: list[str],
|
||||
schema: dict[str, Any],
|
||||
path: str,
|
||||
) -> JSONReturnType:
|
||||
last_error: Exception | None = None
|
||||
for schema_type in types:
|
||||
branch_schema = {**schema, "type": schema_type}
|
||||
try:
|
||||
# Keep structural schema context for repair heuristics, but validate against the narrowed branch type.
|
||||
candidate = self._repair_by_type(copy.deepcopy(value), schema_type, schema, path)
|
||||
candidate = self._apply_enum_const(candidate, branch_schema, path)
|
||||
self.validate(candidate, branch_schema)
|
||||
except ValueError as exc:
|
||||
last_error = exc
|
||||
else:
|
||||
return candidate
|
||||
if last_error:
|
||||
raise ValueError(str(last_error)) from last_error
|
||||
raise ValueError("No schema type matched the value.")
|
||||
|
||||
def _repair_by_type(self, value: Any, schema_type: str, schema: dict[str, Any], path: str) -> JSONReturnType:
|
||||
if schema_type == "array":
|
||||
return self._repair_array(value, schema, path)
|
||||
if schema_type == "object":
|
||||
return self._repair_object(value, schema, path)
|
||||
return self._coerce_scalar(value, schema_type, path)
|
||||
|
||||
def _load_json_string_container(
|
||||
self,
|
||||
value: Any,
|
||||
expected_type: type[object],
|
||||
path: str,
|
||||
unwrap_log_message: str,
|
||||
salvage_log_message: str,
|
||||
) -> Any:
|
||||
if not isinstance(value, str):
|
||||
return value
|
||||
try:
|
||||
parsed = json.loads(value)
|
||||
except json.JSONDecodeError:
|
||||
if self.schema_repair_mode != "salvage":
|
||||
return value
|
||||
json_repair_module = importlib.import_module(f"{__package__}.json_repair")
|
||||
repaired = json_repair_module.loads(value, skip_json_loads=True)
|
||||
if isinstance(repaired, expected_type):
|
||||
self._log(salvage_log_message, path)
|
||||
return repaired
|
||||
return value
|
||||
if isinstance(parsed, expected_type):
|
||||
self._log(unwrap_log_message, path)
|
||||
return parsed
|
||||
return value
|
||||
|
||||
def _repair_array(self, value: Any, schema: dict[str, Any], path: str) -> JSONReturnType:
|
||||
value = self._load_json_string_container(
|
||||
value,
|
||||
list,
|
||||
path,
|
||||
"Unwrapped JSON string to array to match schema",
|
||||
"Repaired malformed JSON string to array to match schema",
|
||||
)
|
||||
if isinstance(value, list):
|
||||
items: list[JSONReturnType] = value
|
||||
else:
|
||||
self._log("Wrapped value in array to match schema", path)
|
||||
items = [normalize_missing_values(value)]
|
||||
salvage_mode = self.schema_repair_mode == "salvage"
|
||||
schema_config = array_schema_config(schema)
|
||||
|
||||
def repair_or_drop(raw_item: Any, item_schema: Any, item_path: str) -> tuple[bool, JSONReturnType]:
|
||||
try:
|
||||
return True, self.repair_value(raw_item, item_schema, item_path)
|
||||
except SchemaDefinitionError:
|
||||
raise
|
||||
except ValueError:
|
||||
if not salvage_mode:
|
||||
raise
|
||||
self._log("Dropped invalid array item while salvaging", item_path)
|
||||
return False, None
|
||||
|
||||
if schema_config.items_schema is not None:
|
||||
if isinstance(schema_config.items_schema, list):
|
||||
repaired_items: list[JSONReturnType] = []
|
||||
for idx, item_schema in enumerate(schema_config.items_schema):
|
||||
if idx >= len(items):
|
||||
break
|
||||
item_path = f"{path}[{idx}]"
|
||||
keep_item, repaired_value = repair_or_drop(items[idx], item_schema, item_path)
|
||||
if keep_item:
|
||||
repaired_items.append(repaired_value)
|
||||
if len(items) > len(schema_config.items_schema):
|
||||
tail = items[len(schema_config.items_schema) :]
|
||||
if isinstance(schema_config.additional_items, dict):
|
||||
for offset, item in enumerate(tail, start=len(schema_config.items_schema)):
|
||||
item_path = f"{path}[{offset}]"
|
||||
keep_item, repaired_value = repair_or_drop(item, schema_config.additional_items, item_path)
|
||||
if keep_item:
|
||||
repaired_items.append(repaired_value)
|
||||
elif schema_config.additional_items is True or schema_config.additional_items is None:
|
||||
repaired_items.extend(normalize_missing_values(item) for item in tail)
|
||||
else:
|
||||
for offset, _item in enumerate(tail, start=len(schema_config.items_schema)):
|
||||
self._log("Dropped extra array item not covered by schema", f"{path}[{offset}]")
|
||||
items = repaired_items
|
||||
else:
|
||||
repaired_items = []
|
||||
for idx, item in enumerate(items):
|
||||
item_path = f"{path}[{idx}]"
|
||||
keep_item, repaired_value = repair_or_drop(item, schema_config.items_schema, item_path)
|
||||
if keep_item:
|
||||
repaired_items.append(repaired_value)
|
||||
items = repaired_items
|
||||
min_items = schema.get("minItems")
|
||||
if min_items is not None and len(items) < min_items:
|
||||
raise ValueError(f"Array at {path} does not meet minItems.")
|
||||
return items
|
||||
|
||||
def _repair_object(self, value: Any, schema: dict[str, Any], path: str) -> JSONReturnType:
|
||||
if (
|
||||
self.schema_repair_mode == "salvage"
|
||||
and isinstance(value, list)
|
||||
and self._can_salvage_list_as_object(schema)
|
||||
):
|
||||
mapped = self._map_list_to_object(value, schema, path)
|
||||
if mapped is not None:
|
||||
value = mapped
|
||||
elif path == "$" and len(value) == 1 and isinstance(value[0], dict):
|
||||
# Conservatively unwrap the common root wrapper shape: [{...}] -> {...}.
|
||||
value = value[0]
|
||||
self._log("Unwrapped single-item root array to object while salvaging", path)
|
||||
value = self._load_json_string_container(
|
||||
value,
|
||||
dict,
|
||||
path,
|
||||
"Unwrapped JSON string to object to match schema",
|
||||
"Repaired malformed JSON string to object to match schema",
|
||||
)
|
||||
if not isinstance(value, dict):
|
||||
raise ValueError(f"Expected object at {path}, got {type(value).__name__}.")
|
||||
|
||||
property_aliases = schema.get("x-json-repair-property-aliases", {})
|
||||
if isinstance(property_aliases, dict):
|
||||
normalized_value = dict(value)
|
||||
for alias, field_names in property_aliases.items():
|
||||
if not isinstance(alias, str) or not isinstance(field_names, list):
|
||||
continue
|
||||
provided_names = [name for name in field_names if isinstance(name, str) and name in normalized_value]
|
||||
if alias in normalized_value:
|
||||
for name in provided_names:
|
||||
normalized_value.pop(name)
|
||||
self._log(f"Dropped field-name key because alias '{alias}' is present", f"{path}.{name}")
|
||||
continue
|
||||
if provided_names:
|
||||
name = provided_names[0]
|
||||
normalized_value[alias] = normalized_value.pop(name)
|
||||
self._log(f"Mapped field name to validation alias '{alias}'", f"{path}.{name}")
|
||||
value = normalized_value
|
||||
|
||||
schema_config = object_schema_config(schema)
|
||||
|
||||
if self.schema_repair_mode == "salvage" and schema_config.required:
|
||||
value_with_salvage_fills = dict(value)
|
||||
for key in schema_config.required:
|
||||
if key in value_with_salvage_fills:
|
||||
continue
|
||||
prop_schema = schema_config.properties.get(key)
|
||||
if prop_schema is None:
|
||||
continue
|
||||
key_path = f"{path}.{key}"
|
||||
filled, filled_value = self._fill_missing_required_for_salvage(prop_schema, key_path)
|
||||
if filled:
|
||||
value_with_salvage_fills[key] = filled_value
|
||||
self._log("Filled missing required property while salvaging", key_path)
|
||||
value = value_with_salvage_fills
|
||||
|
||||
missing_required = [key for key in schema_config.required if key not in value]
|
||||
if missing_required:
|
||||
raise ValueError(f"Missing required properties at {path}: {', '.join(missing_required)}")
|
||||
|
||||
repaired: dict[str, JSONReturnType] = {}
|
||||
|
||||
for key, prop_schema in schema_config.properties.items():
|
||||
key_path = f"{path}.{key}"
|
||||
if key in value:
|
||||
repaired[key] = self.repair_value(value[key], prop_schema, key_path)
|
||||
elif isinstance(prop_schema, dict) and "default" in prop_schema and key not in schema_config.required:
|
||||
repaired[key] = self._copy_json_value(prop_schema["default"], key_path, "default")
|
||||
self._log("Inserted default value for missing property", key_path)
|
||||
|
||||
for key, raw_value in value.items():
|
||||
if key in schema_config.properties:
|
||||
continue
|
||||
key_path = f"{path}.{key}"
|
||||
matched: list[Any] = []
|
||||
unsupported_patterns: list[str] = []
|
||||
if schema_config.pattern_properties:
|
||||
matched, unsupported_patterns = match_pattern_properties(schema_config.pattern_properties, key)
|
||||
for pattern in unsupported_patterns:
|
||||
self._log(f"Skipped unsupported patternProperties regex '{pattern}'", key_path)
|
||||
if matched:
|
||||
repaired_value = self.repair_value(raw_value, matched[0], key_path)
|
||||
for prop_schema in matched[1:]:
|
||||
repaired_value = self.repair_value(repaired_value, prop_schema, key_path)
|
||||
repaired[key] = repaired_value
|
||||
continue
|
||||
if isinstance(schema_config.additional_properties, dict):
|
||||
repaired[key] = self.repair_value(
|
||||
raw_value,
|
||||
cast("dict[str, Any]", schema_config.additional_properties),
|
||||
key_path,
|
||||
)
|
||||
continue
|
||||
if schema_config.additional_properties is True or schema_config.additional_properties is None:
|
||||
repaired[key] = normalize_missing_values(raw_value)
|
||||
continue
|
||||
self._log("Dropped extra property not covered by schema", key_path)
|
||||
|
||||
min_properties = schema.get("minProperties")
|
||||
if min_properties is not None and len(repaired) < min_properties:
|
||||
raise ValueError(f"Object at {path} does not meet minProperties.")
|
||||
return repaired
|
||||
|
||||
def _map_list_to_object(
|
||||
self, value: list[Any], schema: dict[str, Any], path: str
|
||||
) -> dict[str, JSONReturnType] | None:
|
||||
properties = schema.get("properties")
|
||||
if not isinstance(properties, dict) or not properties:
|
||||
return None
|
||||
|
||||
typed_properties: dict[str, Any] = {}
|
||||
for key, prop_schema in properties.items():
|
||||
if not isinstance(key, str):
|
||||
raise SchemaDefinitionError("Schema object property names must be strings.")
|
||||
typed_properties[key] = prop_schema
|
||||
|
||||
keys = list(typed_properties.keys())
|
||||
if len(value) != len(keys):
|
||||
return None
|
||||
|
||||
mapped: dict[str, JSONReturnType] = {}
|
||||
for idx, key in enumerate(keys):
|
||||
key_path = f"{path}.{key}"
|
||||
try:
|
||||
mapped[key] = self.repair_value(value[idx], typed_properties[key], key_path)
|
||||
except SchemaDefinitionError:
|
||||
raise
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
self._log("Mapped array to object by schema property order", path)
|
||||
return mapped
|
||||
|
||||
def _fill_missing_required_for_salvage(self, schema: object, path: str) -> tuple[bool, JSONReturnType]:
|
||||
resolved_schema = self.resolve_schema(schema)
|
||||
if not isinstance(resolved_schema, dict):
|
||||
return False, ""
|
||||
|
||||
if "default" in resolved_schema:
|
||||
return True, self._copy_json_value(resolved_schema["default"], path, "default")
|
||||
if "const" in resolved_schema:
|
||||
return True, self._copy_json_value(resolved_schema["const"], path, "const")
|
||||
enum_values = resolved_schema.get("enum")
|
||||
if enum_values:
|
||||
return True, self._copy_json_value(enum_values[0], path, "enum")
|
||||
|
||||
expected_type = resolved_schema.get("type")
|
||||
if expected_type is None:
|
||||
if self.is_array_schema(resolved_schema):
|
||||
expected_type = "array"
|
||||
elif self.is_object_schema(resolved_schema):
|
||||
expected_type = "object"
|
||||
|
||||
if expected_type == "array" and not resolved_schema.get("minItems"):
|
||||
return True, []
|
||||
if expected_type == "object" and not resolved_schema.get("minProperties"):
|
||||
return True, {}
|
||||
|
||||
return False, ""
|
||||
|
||||
def _fill_missing(self, schema: dict[str, Any], path: str) -> JSONReturnType:
|
||||
if "const" in schema:
|
||||
# Const/enum/default have priority over type inference.
|
||||
self._log("Filled missing value with const", path)
|
||||
return self._copy_json_value(schema["const"], path, "const")
|
||||
if "enum" in schema:
|
||||
enum_values = schema["enum"]
|
||||
if not enum_values:
|
||||
raise ValueError(f"Enum at {path} has no values.")
|
||||
self._log("Filled missing value with first enum value", path)
|
||||
return self._copy_json_value(enum_values[0], path, "enum")
|
||||
if "default" in schema:
|
||||
self._log("Filled missing value with default", path)
|
||||
return self._copy_json_value(schema["default"], path, "default")
|
||||
|
||||
expected_type = schema.get("type")
|
||||
if isinstance(expected_type, list):
|
||||
for schema_type in expected_type:
|
||||
try:
|
||||
return self._fill_missing({**schema, "type": schema_type}, path)
|
||||
except ValueError:
|
||||
continue
|
||||
raise ValueError(f"Cannot infer missing value at {path}.")
|
||||
|
||||
if expected_type is None:
|
||||
# Infer container types based on schema shape if type is omitted.
|
||||
if self.is_object_schema(schema):
|
||||
expected_type = "object"
|
||||
elif self.is_array_schema(schema):
|
||||
expected_type = "array"
|
||||
|
||||
if expected_type == "string":
|
||||
self._log("Filled missing value with empty string", path)
|
||||
return ""
|
||||
if expected_type in ("integer", "number"):
|
||||
self._log("Filled missing value with 0", path)
|
||||
return 0
|
||||
if expected_type == "boolean":
|
||||
self._log("Filled missing value with false", path)
|
||||
return False
|
||||
if expected_type == "array":
|
||||
min_items = schema.get("minItems")
|
||||
if min_items:
|
||||
raise ValueError(f"Array at {path} requires at least {min_items} items.")
|
||||
self._log("Filled missing value with empty array", path)
|
||||
return []
|
||||
if expected_type == "object":
|
||||
min_properties = schema.get("minProperties")
|
||||
if min_properties:
|
||||
raise ValueError(f"Object at {path} requires at least {min_properties} properties.")
|
||||
self._log("Filled missing value with empty object", path)
|
||||
return {}
|
||||
if expected_type == "null":
|
||||
self._log("Filled missing value with null", path)
|
||||
return None
|
||||
|
||||
raise ValueError(f"Cannot infer missing value at {path}.")
|
||||
|
||||
def _coerce_scalar(self, value: Any, schema_type: str, path: str) -> JSONReturnType:
|
||||
if schema_type == "string":
|
||||
if isinstance(value, str):
|
||||
return value
|
||||
if isinstance(value, (int, float)) and not isinstance(value, bool):
|
||||
self._log("Coerced number to string", path)
|
||||
return str(value)
|
||||
raise ValueError(f"Expected string at {path}.")
|
||||
|
||||
if schema_type == "integer":
|
||||
if isinstance(value, bool):
|
||||
raise ValueError(f"Expected integer at {path}.")
|
||||
if isinstance(value, int):
|
||||
return value
|
||||
if isinstance(value, float):
|
||||
if value.is_integer():
|
||||
self._log("Coerced number to integer", path)
|
||||
return int(value)
|
||||
raise ValueError(f"Expected integer at {path}.")
|
||||
if isinstance(value, str):
|
||||
try:
|
||||
int_value = int(value)
|
||||
except ValueError:
|
||||
int_value = None
|
||||
if int_value is not None:
|
||||
self._log("Coerced string to integer", path)
|
||||
return int_value
|
||||
try:
|
||||
num = float(value)
|
||||
except ValueError as exc:
|
||||
raise ValueError(f"Expected integer at {path}.") from exc
|
||||
if not num.is_integer():
|
||||
raise ValueError(f"Expected integer at {path}.")
|
||||
self._log("Coerced number to integer", path)
|
||||
return int(num)
|
||||
raise ValueError(f"Expected integer at {path}.")
|
||||
|
||||
if schema_type == "number":
|
||||
if isinstance(value, bool):
|
||||
raise ValueError(f"Expected number at {path}.")
|
||||
if isinstance(value, (int, float)):
|
||||
return value
|
||||
if isinstance(value, str):
|
||||
try:
|
||||
float_value = float(value)
|
||||
except ValueError as exc:
|
||||
raise ValueError(f"Expected number at {path}.") from exc
|
||||
self._log("Coerced string to number", path)
|
||||
return float_value
|
||||
raise ValueError(f"Expected number at {path}.")
|
||||
|
||||
if schema_type == "boolean":
|
||||
if isinstance(value, bool):
|
||||
return value
|
||||
if isinstance(value, str):
|
||||
lowered = value.lower()
|
||||
if lowered in ("true", "yes", "y", "on", "1"):
|
||||
self._log("Coerced string to boolean", path)
|
||||
return True
|
||||
if lowered in ("false", "no", "n", "off", "0"):
|
||||
self._log("Coerced string to boolean", path)
|
||||
return False
|
||||
if isinstance(value, (int, float)) and not isinstance(value, bool) and value in (0, 1):
|
||||
self._log("Coerced number to boolean", path)
|
||||
return bool(value)
|
||||
raise ValueError(f"Expected boolean at {path}.")
|
||||
|
||||
if schema_type == "null":
|
||||
if value is None:
|
||||
return None
|
||||
raise ValueError(f"Expected null at {path}.")
|
||||
|
||||
raise SchemaDefinitionError(f"Unsupported schema type {schema_type} at {path}.")
|
||||
|
||||
def _apply_enum_const(self, value: JSONReturnType, schema: dict[str, Any], path: str) -> JSONReturnType:
|
||||
if "const" in schema and value != schema["const"]:
|
||||
raise ValueError(f"Value at {path} does not match const.")
|
||||
if "enum" in schema and value not in schema["enum"]:
|
||||
raise ValueError(f"Value at {path} does not match enum.")
|
||||
return value
|
||||
|
||||
def _resolve_ref(self, ref: str) -> dict[str, Any] | bool:
|
||||
if not ref.startswith("#/"):
|
||||
raise SchemaDefinitionError(f"Unsupported $ref: {ref}")
|
||||
parts = ref.lstrip("#/").split("/")
|
||||
current: Any = self.root_schema
|
||||
for part in parts:
|
||||
resolved_part = part.replace("~1", "/").replace("~0", "~")
|
||||
if not isinstance(current, dict) or resolved_part not in current:
|
||||
raise SchemaDefinitionError(f"Unresolvable $ref: {ref}")
|
||||
current = current[resolved_part]
|
||||
if isinstance(current, dict):
|
||||
return current
|
||||
if current is True:
|
||||
return True
|
||||
if current is False:
|
||||
return False
|
||||
raise SchemaDefinitionError(f"Unresolvable $ref: {ref}")
|
||||
|
||||
def _copy_json_value(self, value: Any, path: str, label: str) -> JSONReturnType:
|
||||
if value is None or isinstance(value, (str, int, float, bool)):
|
||||
return value
|
||||
if isinstance(value, list):
|
||||
return [self._copy_json_value(item, f"{path}[{idx}]", label) for idx, item in enumerate(value)]
|
||||
if isinstance(value, dict):
|
||||
copied: dict[str, JSONReturnType] = {}
|
||||
for key, item in value.items():
|
||||
if not isinstance(key, str):
|
||||
raise ValueError(f"{label.capitalize()} value at {path} contains a non-string key.")
|
||||
copied[key] = self._copy_json_value(item, f"{path}.{key}", label)
|
||||
return copied
|
||||
raise ValueError(f"{label.capitalize()} value at {path} is not JSON compatible.")
|
||||
|
||||
def _prepare_schema_for_validation(self, schema: object) -> dict[str, Any]:
|
||||
normalized = _prepare_schema_for_validation_node(schema)
|
||||
if not isinstance(normalized, dict):
|
||||
raise ValueError("Schema must be an object.")
|
||||
return normalized
|
||||
@@ -0,0 +1 @@
|
||||
"""Internal utility helpers for json_repair."""
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
@@ -0,0 +1,15 @@
|
||||
from typing import Any
|
||||
|
||||
|
||||
class MissingValueType:
|
||||
def __repr__(self) -> str:
|
||||
return "<MISSING_VALUE>"
|
||||
|
||||
def __deepcopy__(self, memo: dict[int, Any]) -> "MissingValueType":
|
||||
return self
|
||||
|
||||
|
||||
MISSING_VALUE = MissingValueType()
|
||||
|
||||
JSONReturnType = dict[str, Any] | list[Any] | str | float | int | bool | None
|
||||
STRING_DELIMITERS: list[str] = ['"', "'", "“", "”"]
|
||||
@@ -0,0 +1,78 @@
|
||||
from enum import Enum, auto
|
||||
from types import TracebackType
|
||||
from typing import Literal
|
||||
|
||||
|
||||
class ContextValues(Enum):
|
||||
OBJECT_KEY = auto()
|
||||
OBJECT_VALUE = auto()
|
||||
ARRAY = auto()
|
||||
|
||||
|
||||
class _JsonContextEntry:
|
||||
__slots__ = ("context", "value")
|
||||
|
||||
def __init__(self, context: "JsonContext", value: ContextValues) -> None:
|
||||
self.context = context
|
||||
self.value = value
|
||||
|
||||
def __enter__(self) -> None:
|
||||
self.context.set(self.value)
|
||||
|
||||
def __exit__(
|
||||
self,
|
||||
_exc_type: type[BaseException] | None,
|
||||
_exc: BaseException | None,
|
||||
_traceback: TracebackType | None,
|
||||
) -> Literal[False]:
|
||||
self.context.reset()
|
||||
return False
|
||||
|
||||
|
||||
class JsonContext:
|
||||
def __init__(self) -> None:
|
||||
self.context: list[ContextValues] = []
|
||||
self.current: ContextValues | None = None
|
||||
self.empty: bool = True
|
||||
|
||||
def enter(self, value: ContextValues) -> _JsonContextEntry:
|
||||
return _JsonContextEntry(self, value)
|
||||
|
||||
def set(self, value: ContextValues) -> None:
|
||||
"""
|
||||
Set a new context value.
|
||||
|
||||
Args:
|
||||
value (ContextValues): The context value to be added.
|
||||
|
||||
Returns:
|
||||
None
|
||||
"""
|
||||
self.context.append(value)
|
||||
self.current = value
|
||||
self.empty = False
|
||||
|
||||
def reset(self) -> None:
|
||||
"""
|
||||
Remove the most recent context value.
|
||||
|
||||
Returns:
|
||||
None
|
||||
"""
|
||||
try:
|
||||
self.context.pop()
|
||||
self.current = self.context[-1]
|
||||
except IndexError:
|
||||
self.current = None
|
||||
self.empty = True
|
||||
|
||||
def clear(self) -> None:
|
||||
"""
|
||||
Remove all context values.
|
||||
|
||||
Returns:
|
||||
None
|
||||
"""
|
||||
self.context.clear()
|
||||
self.current = None
|
||||
self.empty = True
|
||||
@@ -0,0 +1,47 @@
|
||||
from typing import Any
|
||||
|
||||
|
||||
class ObjectComparer: # pragma: no cover
|
||||
def __init__(self) -> None:
|
||||
pass # No operation performed in the constructor
|
||||
|
||||
@staticmethod
|
||||
def is_same_object(obj1: Any, obj2: Any) -> bool:
|
||||
"""
|
||||
Recursively compares two objects and ensures that:
|
||||
- Their types match
|
||||
- Their keys/structure match
|
||||
"""
|
||||
if type(obj1) is not type(obj2):
|
||||
# Fail immediately if the types don't match
|
||||
return False
|
||||
|
||||
if isinstance(obj1, dict):
|
||||
# Check that both are dicts and same length
|
||||
if not isinstance(obj2, dict) or len(obj1) != len(obj2):
|
||||
return False
|
||||
for key in obj1:
|
||||
if key not in obj2:
|
||||
return False
|
||||
# Recursively compare each value
|
||||
if not ObjectComparer.is_same_object(obj1[key], obj2[key]):
|
||||
return False
|
||||
return True
|
||||
|
||||
if isinstance(obj1, list):
|
||||
# Check that both are lists and same length
|
||||
if not isinstance(obj2, list) or len(obj1) != len(obj2):
|
||||
return False
|
||||
# Recursively compare each item
|
||||
return all(ObjectComparer.is_same_object(obj1[i], obj2[i]) for i in range(len(obj1)))
|
||||
|
||||
# For atomic values: types already match, so return True
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
def is_strictly_empty(value: Any) -> bool:
|
||||
"""
|
||||
Returns True if value is an empty container (str, list, dict, set, tuple).
|
||||
Returns False for non-containers like None, 0, False, etc.
|
||||
"""
|
||||
return isinstance(value, str | list | dict | set | tuple) and len(value) == 0
|
||||
@@ -0,0 +1,50 @@
|
||||
from collections.abc import Mapping
|
||||
from typing import Any
|
||||
|
||||
_UNSUPPORTED_REGEX_TOKENS = frozenset({".", "^", "$", "*", "+", "?", "{", "}", "[", "]", "|", "(", ")", "\\"})
|
||||
|
||||
|
||||
def match_pattern_properties(
|
||||
pattern_properties: Mapping[str, Any],
|
||||
key: str,
|
||||
) -> tuple[list[Any], list[str]]:
|
||||
"""Match JSON Schema patternProperties using a safe literal+anchor subset.
|
||||
|
||||
Supported forms:
|
||||
- "token" -> key contains token
|
||||
- "^token" -> key starts with token
|
||||
- "token$" -> key ends with token
|
||||
- "^token$" -> key equals token
|
||||
|
||||
Any pattern using additional regex tokens is treated as unsupported and skipped.
|
||||
The caller can log unsupported patterns using the returned list.
|
||||
"""
|
||||
|
||||
if not pattern_properties:
|
||||
return [], []
|
||||
|
||||
matched_schemas: list[Any] = []
|
||||
unsupported_patterns: list[str] = []
|
||||
|
||||
for pattern, schema in pattern_properties.items():
|
||||
anchored_start = pattern.startswith("^")
|
||||
anchored_end = pattern.endswith("$")
|
||||
literal = pattern[1 if anchored_start else 0 : -1 if anchored_end else None]
|
||||
|
||||
if any(token in literal for token in _UNSUPPORTED_REGEX_TOKENS):
|
||||
unsupported_patterns.append(pattern)
|
||||
continue
|
||||
|
||||
if anchored_start and anchored_end:
|
||||
is_match = key == literal
|
||||
elif anchored_start:
|
||||
is_match = key.startswith(literal)
|
||||
elif anchored_end:
|
||||
is_match = key.endswith(literal)
|
||||
else:
|
||||
is_match = literal in key
|
||||
|
||||
if is_match:
|
||||
matched_schemas.append(schema)
|
||||
|
||||
return matched_schemas, unsupported_patterns
|
||||
@@ -0,0 +1,183 @@
|
||||
import os
|
||||
from typing import TextIO
|
||||
|
||||
|
||||
class StringFileWrapper:
|
||||
# This is a trick to simplify the code, transform the filedescriptor handling into a string handling
|
||||
def __init__(self, fd: TextIO, chunk_length: int) -> None:
|
||||
"""
|
||||
Initialize the StringFileWrapper with a file descriptor and chunk length.
|
||||
|
||||
Args:
|
||||
fd (TextIO): The file descriptor to wrap.
|
||||
CHUNK_LENGTH (int): The length of each chunk to read from the file.
|
||||
|
||||
Attributes:
|
||||
fd (TextIO): The wrapped file descriptor.
|
||||
length (int): The total length of the file content.
|
||||
buffers (dict[int, str]): Dictionary to store chunks of file content.
|
||||
buffer_length (int): The length of each buffer chunk.
|
||||
"""
|
||||
self.fd = fd
|
||||
# Buffers are chunks of text read from the file and cached to reduce disk access.
|
||||
self.buffers: dict[int, str] = {}
|
||||
if not chunk_length or chunk_length < 2:
|
||||
chunk_length = 1_000_000
|
||||
# chunk_length now refers to the number of characters per chunk.
|
||||
self.buffer_length = chunk_length
|
||||
# Keep track of the starting file position ("cookie") for each chunk so we can
|
||||
# seek safely without landing in the middle of a multibyte code point.
|
||||
self._initial_position = fd.tell()
|
||||
self._chunk_positions: list[int] = [self._initial_position]
|
||||
self.length: int | None = None
|
||||
|
||||
def get_buffer(self, index: int) -> str:
|
||||
"""
|
||||
Retrieve or load a buffer chunk from the file.
|
||||
|
||||
Args:
|
||||
index (int): The index of the buffer chunk to retrieve.
|
||||
|
||||
Returns:
|
||||
str: The buffer chunk at the specified index.
|
||||
"""
|
||||
if index < 0:
|
||||
raise IndexError("Negative indexing is not supported")
|
||||
|
||||
cached = self.buffers.get(index)
|
||||
if cached is not None:
|
||||
return cached
|
||||
|
||||
self._ensure_chunk_position(index)
|
||||
start_pos = self._chunk_positions[index]
|
||||
self.fd.seek(start_pos)
|
||||
chunk = self.fd.read(self.buffer_length)
|
||||
if not chunk:
|
||||
raise IndexError("Chunk index out of range")
|
||||
end_pos = self.fd.tell()
|
||||
if len(self._chunk_positions) <= index + 1:
|
||||
self._chunk_positions.append(end_pos)
|
||||
if len(chunk) < self.buffer_length:
|
||||
self.length = index * self.buffer_length + len(chunk)
|
||||
|
||||
self.buffers[index] = chunk
|
||||
# Save memory by keeping max 2MB buffer chunks and min 2 chunks
|
||||
max_buffers = max(2, int(2_000_000 / self.buffer_length))
|
||||
if len(self.buffers) > max_buffers:
|
||||
oldest_key = next(iter(self.buffers))
|
||||
if oldest_key != index:
|
||||
self.buffers.pop(oldest_key)
|
||||
return chunk
|
||||
|
||||
def __getitem__(self, index: int | slice) -> str:
|
||||
"""
|
||||
Retrieve a character or a slice of characters from the file.
|
||||
|
||||
Args:
|
||||
index (Union[int, slice]): The index or slice of characters to retrieve.
|
||||
|
||||
Returns:
|
||||
str: The character(s) at the specified index or slice.
|
||||
"""
|
||||
# The buffer is an array that is seek like a RAM:
|
||||
# self.buffers[index]: the row in the array of length 1MB, index is `i` modulo CHUNK_LENGTH
|
||||
# self.buffures[index][j]: the column of the row that is `i` remainder CHUNK_LENGTH
|
||||
if isinstance(index, slice):
|
||||
start, stop, step = self._normalize_slice(index)
|
||||
|
||||
if step == 0:
|
||||
raise ValueError("slice step cannot be zero")
|
||||
if step != 1:
|
||||
return "".join(self[i] for i in range(start, stop, step))
|
||||
|
||||
if start >= stop:
|
||||
return ""
|
||||
return self._slice_from_buffers(start, stop)
|
||||
if index < 0:
|
||||
index += len(self)
|
||||
if index < 0:
|
||||
raise IndexError("string index out of range")
|
||||
buffer_index = index // self.buffer_length
|
||||
buffer = self.get_buffer(buffer_index)
|
||||
return buffer[index % self.buffer_length]
|
||||
|
||||
def __len__(self) -> int:
|
||||
"""
|
||||
Get the total length of the file.
|
||||
|
||||
Returns:
|
||||
int: The total number of characters in the file.
|
||||
"""
|
||||
if self.length is None:
|
||||
while self.length is None:
|
||||
chunk_index = len(self._chunk_positions)
|
||||
self._ensure_chunk_position(chunk_index)
|
||||
assert self.length is not None
|
||||
return self.length
|
||||
|
||||
def _normalize_slice(self, index: slice) -> tuple[int, int, int]:
|
||||
total_len = len(self)
|
||||
start = 0 if index.start is None else index.start
|
||||
stop = total_len if index.stop is None else index.stop
|
||||
step = 1 if index.step is None else index.step
|
||||
|
||||
if start < 0:
|
||||
start += total_len
|
||||
if stop < 0:
|
||||
stop += total_len
|
||||
|
||||
start = max(start, 0)
|
||||
stop = min(stop, total_len)
|
||||
return start, stop, step
|
||||
|
||||
def _slice_from_buffers(self, start: int, stop: int) -> str:
|
||||
buffer_index = start // self.buffer_length
|
||||
buffer_end = (stop - 1) // self.buffer_length
|
||||
start_mod = start % self.buffer_length
|
||||
stop_mod = stop % self.buffer_length
|
||||
if stop_mod == 0 and stop > start:
|
||||
stop_mod = self.buffer_length
|
||||
if buffer_index == buffer_end:
|
||||
buffer = self.get_buffer(buffer_index)
|
||||
return buffer[start_mod:stop_mod]
|
||||
|
||||
start_slice = self.get_buffer(buffer_index)[start_mod:]
|
||||
end_slice = self.get_buffer(buffer_end)[:stop_mod]
|
||||
middle_slices = [self.get_buffer(i) for i in range(buffer_index + 1, buffer_end)]
|
||||
return start_slice + "".join(middle_slices) + end_slice
|
||||
|
||||
def __setitem__(self, index: int | slice, value: str) -> None: # pragma: no cover
|
||||
"""
|
||||
Set a character or a slice of characters in the file.
|
||||
|
||||
Args:
|
||||
index (slice): The slice of characters to set.
|
||||
value (str): The value to set at the specified index or slice.
|
||||
"""
|
||||
start = index.start or 0 if isinstance(index, slice) else index or 0
|
||||
|
||||
if start < 0:
|
||||
start += len(self)
|
||||
|
||||
current_position = self.fd.tell()
|
||||
self.fd.seek(self._initial_position + start)
|
||||
self.fd.write(value)
|
||||
self.fd.seek(current_position)
|
||||
|
||||
def _ensure_chunk_position(self, chunk_index: int) -> None:
|
||||
"""
|
||||
Ensure that we know the starting file position for the given chunk index.
|
||||
"""
|
||||
while len(self._chunk_positions) <= chunk_index:
|
||||
prev_index = len(self._chunk_positions) - 1
|
||||
start_pos = self._chunk_positions[-1]
|
||||
self.fd.seek(start_pos, os.SEEK_SET)
|
||||
chunk = self.fd.read(self.buffer_length)
|
||||
end_pos = self.fd.tell()
|
||||
if len(chunk) < self.buffer_length:
|
||||
self.length = prev_index * self.buffer_length + len(chunk)
|
||||
self._chunk_positions.append(end_pos)
|
||||
if not chunk:
|
||||
break
|
||||
if len(self._chunk_positions) <= chunk_index:
|
||||
raise IndexError("Chunk index out of range")
|
||||
Reference in new issue
Block a user