Initial commit

This commit is contained in:
Alex committed 2026-11-02 22:54:14 +08:00
commit 57fc3525eb
7311 files changed
+1291398

No files matched your search

@@ -0,0 +1,4 @@
from .json_repair import from_file, load, loads, repair_json
from .utils.constants import JSONReturnType
__all__ = ["JSONReturnType", "from_file", "load", "loads", "repair_json"]
@@ -0,0 +1,6 @@
import sys
from .json_repair import cli
if __name__ == "__main__":
sys.exit(cli())
@@ -0,0 +1,481 @@
import json
from collections.abc import Callable
from contextlib import ExitStack
from typing import TYPE_CHECKING, Any, TextIO
from .parse_array import parse_array as _parse_array
from .parse_comment import parse_comment as _parse_comment
from .parse_number import parse_number as _parse_number
from .parse_object import parse_object as _parse_object
from .parse_string import parse_string as _parse_string
from .parser_parenthesized import parenthesized_is_explicit_tuple, top_level_parenthesized_can_start_value
from .utils.constants import STRING_DELIMITERS, JSONReturnType
from .utils.json_context import ContextValues, JsonContext
from .utils.object_comparer import ObjectComparer
from .utils.string_file_wrapper import StringFileWrapper
if TYPE_CHECKING:
from .schema_repair import SchemaRepairer
class JSONParser:
# Split the parse methods into separate files because this one was like 3000 lines
def parse_array(
self,
schema: dict[str, Any] | bool | None = None,
path: str = "$",
closing_delimiter: str = "]",
) -> list[JSONReturnType]:
return _parse_array(self, schema, path, closing_delimiter)
def parse_comment(self, record_top_level_value: bool = False) -> JSONReturnType:
return _parse_comment(self, record_top_level_value=record_top_level_value)
def parse_number(self) -> JSONReturnType:
return _parse_number(self)
def parse_object(
self,
schema: dict[str, Any] | bool | None = None,
path: str = "$",
) -> JSONReturnType:
return _parse_object(self, schema, path)
def parse_string(self) -> JSONReturnType:
return _parse_string(self)
def __init__(
self,
json_str: str | StringFileWrapper,
json_fd: TextIO | None,
logging: bool | None,
json_fd_chunk_length: int = 0,
stream_stable: bool = False,
strict: bool = False,
try_valid_json_suffix: bool = False,
) -> None:
# The string to parse
self.json_str: str | StringFileWrapper = json_str
# Alternatively, the file description with a json file in it
if json_fd:
# This is a trick we do to treat the file wrapper as an array
self.json_str = StringFileWrapper(json_fd, json_fd_chunk_length)
# Index is our iterator that will keep track of which character we are looking at right now
self.index: int = 0
# This is used in the object member parsing to manage the special cases of missing quotes in key or value
self.context = JsonContext()
self.deferred_contexts: list[ContextValues] = []
# Use this to log the activity, but only if logging is active
# This is a trick but a beautiful one. We call self.log in the code over and over even if it's not needed.
# We could add a guard in the code for each call but that would make this code unreadable, so here's this neat trick
# Replace self.log with a noop
self.logging = logging
self.logger: list[dict[str, str]] = []
if logging:
self.log = self._log
else:
# No-op
self.log = lambda *args, **kwargs: None # noqa: ARG005
# When the json to be repaired is the accumulation of streaming json at a certain moment.
# e.g. json obtained from llm response.
# If this parameter to True will keep the repair results stable. For example:
# case 1: '{"key": "val\\' => '{"key": "val"}'
# case 2: '{"key": "val\\n' => '{"key": "val\\n"}'
# case 3: '{"key": "val\\n123,`key2:value2' => '{"key": "val\\n123,`key2:value2"}'
# case 4: '{"key": "val\\n123,`key2:value2`"}' => '{"key": "val\\n123,`key2:value2`"}'
self.stream_stable = stream_stable
# Over time the library got more and more complex heuristics to repair JSON. Some of these heuristics
# may not be desirable in some use cases and the user would prefer json_repair to return an exception.
# So strict mode was added to disable some of those heuristics.
self.strict = strict
self.try_valid_json_suffix = try_valid_json_suffix
self.has_tried_valid_json_suffix = False
self._last_parse_found_value = False
self.schema_repairer: SchemaRepairer | None = None
def parse(
self,
) -> JSONReturnType:
return self._parse_top_level(self.parse_json)
def parse_with_schema(
self,
repairer: "SchemaRepairer",
schema: dict[str, Any] | bool,
) -> JSONReturnType:
"""Parse with schema guidance enabled for all nested values."""
self.schema_repairer = repairer
if repairer.schema_repair_mode == "salvage":
return self._parse_top_level_salvage_with_schema(repairer, schema)
return self._parse_top_level(lambda: self.parse_json(schema, "$"))
def _parse_top_level_salvage_with_schema(
self,
repairer: "SchemaRepairer",
schema: dict[str, Any] | bool,
) -> JSONReturnType:
"""Return the first schema-valid top-level fragment while salvaging."""
last_error: ValueError | None = None
while self.index < len(self.json_str):
self.context.clear()
self.deferred_contexts.clear()
value = self.parse_json(schema, "$", finalize_schema=False, record_top_level_value=True)
if not self._last_parse_found_value:
break
try:
repaired = repairer.repair_value(value, schema, "$")
repairer.validate(repaired, schema)
except ValueError as exc:
last_error = exc
if self.index >= len(self.json_str):
break
self.log("Skipped top-level fragment that did not match schema while salvaging")
continue
else:
return repaired
if last_error is not None:
raise last_error
return ""
# Consolidate top-level parsing so we handle multiple sequential JSON values consistently
# (including update semantics and strict-mode validation).
def _parse_top_level(self, parse_element: Callable[[], JSONReturnType]) -> JSONReturnType:
json = parse_element()
if self.index < len(self.json_str):
self.log(
"The parser returned early, checking if there's more json elements",
)
json = [json]
while self.index < len(self.json_str):
self.context.clear()
self.deferred_contexts.clear()
is_comma_separated = self._next_top_level_value_is_comma_separated()
element_start_index = self.index
j = parse_element()
if self.strict and self.index > element_start_index:
self.log(
"Multiple top-level JSON elements found in strict mode, raising an error",
)
raise ValueError("Multiple top-level JSON elements found in strict mode.")
if j:
if not is_comma_separated and ObjectComparer.is_same_object(json[-1], j):
# Treat repeated objects as updates: keep the newest value.
json.pop()
else:
if not json[-1]:
json.pop()
json.append(j)
else:
self.index += 1
if len(json) == 1:
self.log(
"There were no more elements, returning the element without the array",
)
json = json[0]
return json
def _next_top_level_value_is_comma_separated(self) -> bool:
idx = self.scroll_whitespaces()
if self.get_char_at(idx) == ",":
return True
idx = self.index - 1
while idx >= 0 and self.json_str[idx].isspace():
idx -= 1
return idx >= 0 and self.json_str[idx] == ","
def _initial_container_has_non_comma_trailing_content(self) -> bool:
if self.index != 0 or not isinstance(self.json_str, str):
return True
containers: list[str] = []
in_string = False
escaped = False
closing_delimiters = {"{": "}", "[": "]"}
for index, char in enumerate(self.json_str):
if in_string:
if escaped:
escaped = False
elif char == "\\":
escaped = True
elif char == '"':
in_string = False
continue
if char == '"':
in_string = True
elif char in closing_delimiters:
containers.append(char)
elif char in ["}", "]"]:
if not containers or char != closing_delimiters[containers.pop()]:
return False
if not containers:
trailing_index = index + 1
while trailing_index < len(self.json_str) and self.json_str[trailing_index].isspace():
trailing_index += 1
return trailing_index < len(self.json_str) and self.json_str[trailing_index] != ","
return False
def _try_parse_valid_json_value(self) -> tuple[bool, JSONReturnType]:
if (
not self.try_valid_json_suffix
or self.has_tried_valid_json_suffix
or not self.context.empty
or not isinstance(self.json_str, str)
):
return False, ""
self.has_tried_valid_json_suffix = True
try:
value, end_idx = json.JSONDecoder().raw_decode(self.json_str[self.index :])
except json.JSONDecodeError:
return False, ""
self.index += end_idx
return True, value
def _try_parse_missing_opening_object(self, delimiter: str) -> tuple[bool, JSONReturnType]:
if self.scroll_whitespaces(-self.index) != 0:
return False, ""
offset = self.skip_to_character(delimiter, idx=1)
if self.get_char_at(offset) is None:
return False, ""
offset = self.scroll_whitespaces(offset + 1)
if self.get_char_at(offset) != ":":
return False, ""
end_offset = self.scroll_whitespaces(len(self.json_str) - self.index - 1, backwards=True)
if self.get_char_at(end_offset) != "}":
return False, ""
# Require a single insertion to explain the whole input, rather than
# treating a quoted prose prefix or an inner container as an object.
candidate = "{" + self.json_str[:]
try:
value, end_idx = json.JSONDecoder().raw_decode(candidate)
except ValueError:
# Decoder limits (such as oversized integers) also mean the
# insertion is insufficient; leave these inputs to normal repair.
return False, ""
if candidate[end_idx:].strip(" \t\r\n"):
return False, ""
if self.strict:
raise ValueError("Missing opening object brace in strict mode.")
self.log("Inserted missing opening object brace")
self.index = len(self.json_str)
return True, value
def parse_json(
self,
schema: dict[str, Any] | bool | None = None,
path: str = "$",
finalize_schema: bool = True,
record_top_level_value: bool = False,
) -> JSONReturnType:
"""Parse the next JSON value and, when configured, enforce schema constraints."""
if record_top_level_value:
self._last_parse_found_value = False
if self.deferred_contexts:
deferred_contexts, self.deferred_contexts = self.deferred_contexts, []
with ExitStack() as stack:
for context_value in deferred_contexts:
stack.enter_context(self.context.enter(context_value))
return self.parse_json(schema, path, finalize_schema, record_top_level_value)
repairer, schema = self._resolve_schema_for_parse(schema)
check_initial_quote = self.index == 0 and self.context.empty
while True:
char = self.get_char_at()
# None means that we are at the end of the string provided
if char is None:
return ""
if (
self.try_valid_json_suffix
and char in ["{", "["]
and self._initial_container_has_non_comma_trailing_content()
):
parsed_suffix, value = self._try_parse_valid_json_value()
if parsed_suffix:
return self._finalize_parsed_value(
value, repairer, schema, path, finalize_schema, record_top_level_value
)
# <object> starts with '{'
if char == "{":
self.index += 1
value = self.parse_object(schema, path) if repairer else self.parse_object()
return self._finalize_parsed_value(
value, repairer, schema, path, finalize_schema, record_top_level_value
)
# <array> starts with '['
if char == "[":
self.index += 1
value = self.parse_array(schema, path) if repairer else self.parse_array()
return self._finalize_parsed_value(
value, repairer, schema, path, finalize_schema, record_top_level_value
)
# Python tuple literals and grouped values start with '('
if char == "(":
# Keep top-level tuple detection conservative so inline prose like
# "note (clarification):" does not hijack later JSON blocks.
if not self.context.empty or self.top_level_parenthesized_can_start_value():
value = self.parse_parenthesized(schema, path) if repairer else self.parse_parenthesized()
return self._finalize_parsed_value(
value, repairer, schema, path, finalize_schema, record_top_level_value
)
self.index += 1
continue
# <string> starts with a quote
if not self.context.empty and (char in STRING_DELIMITERS or char.isalpha()):
value = self.parse_string()
return self._finalize_parsed_value(
value, repairer, schema, path, finalize_schema, record_top_level_value
)
# <number> starts with [0-9] or minus
if not self.context.empty and (char.isdigit() or char == "-" or char == "."):
value = self.parse_number()
return self._finalize_parsed_value(
value, repairer, schema, path, finalize_schema, record_top_level_value
)
if char in ["#", "/"]:
value = self.parse_comment(record_top_level_value)
return self._finalize_parsed_value(value, repairer, schema, path, finalize_schema)
# Only probe a missing brace when normal dispatch would discard the
# initial quote; later quoted prose must not start this recovery.
if char in STRING_DELIMITERS and check_initial_quote:
check_initial_quote = False
parsed_object, value = self._try_parse_missing_opening_object(char)
if parsed_object:
return self._finalize_parsed_value(
value, repairer, schema, path, finalize_schema, record_top_level_value
)
# If everything else fails, we just ignore and move on
self.index += 1
def _resolve_schema_for_parse(
self,
schema: dict[str, Any] | bool | None,
) -> tuple["SchemaRepairer | None", dict[str, Any] | bool | None]:
repairer = self.schema_repairer if self.schema_repairer is not None and schema not in (None, True) else None
if repairer is None:
return None, schema
schema = repairer.resolve_schema(schema)
if schema is True:
return None, schema
if schema is False:
raise ValueError("Schema does not allow any values.")
return repairer, schema
def _finalize_parsed_value(
self,
value: JSONReturnType,
repairer: "SchemaRepairer | None",
schema: dict[str, Any] | bool | None,
path: str,
finalize_schema: bool = True,
record_top_level_value: bool = False,
) -> JSONReturnType:
if record_top_level_value:
self._last_parse_found_value = True
if repairer is None or not finalize_schema:
return value
return repairer.repair_value(value, schema, path)
def get_char_at(self, count: int = 0) -> str | None:
# Why not use something simpler? Because try/except in python is a faster alternative to an "if" statement that is often True
try:
return self.json_str[self.index + count]
except IndexError:
return None
def skip_whitespaces(self) -> None:
"""
This function quickly iterates on whitespaces, moving the self.index forward
"""
try:
char = self.json_str[self.index]
while char.isspace():
self.index += 1
char = self.json_str[self.index]
except IndexError:
pass
def scroll_whitespaces(self, idx: int = 0, *, backwards: bool = False) -> int:
"""
Skip whitespace at idx without moving self.index, returning the relative offset.
Set backwards to scan toward the start.
"""
step = -1 if backwards else 1
try:
char = self.json_str[self.index + idx]
while self.index + idx >= 0 and char.isspace():
idx += step
char = self.json_str[self.index + idx]
except IndexError:
pass
return idx
def skip_to_character(self, character: str | list[str], idx: int = 0) -> int:
"""
Advance from (self.index + idx) until we hit an *unescaped* target character.
Returns the offset (idx) from self.index to that position, or the distance to the end if not found.
"""
targets = set(character) if isinstance(character, list) else {character}
i = self.index + idx
n = len(self.json_str)
backslashes = 0 # count of consecutive '\' immediately before current char
while i < n:
ch = self.json_str[i]
if ch == "\\":
backslashes += 1
i += 1
continue
# ch is not a backslash; if it's a target and not escaped (even backslashes), we're done
if ch in targets and (backslashes % 2 == 0):
return i - self.index
# reset backslash run when we see a non-backslash
backslashes = 0
i += 1
# not found; return distance to end
return n - self.index
def parenthesized_is_explicit_tuple(self) -> bool:
return parenthesized_is_explicit_tuple(self)
def top_level_parenthesized_can_start_value(self) -> bool:
return top_level_parenthesized_can_start_value(self)
def parse_parenthesized(
self,
schema: dict[str, Any] | bool | None = None,
path: str = "$",
) -> JSONReturnType:
explicit_tuple = self.parenthesized_is_explicit_tuple()
self.index += 1
values = self.parse_array(schema, path, closing_delimiter=")")
if explicit_tuple or len(values) != 1:
return values
return values[0]
def _log(self, text: str) -> None:
window: int = 10
start: int = max(self.index - window, 0)
end: int = min(self.index + window, len(self.json_str))
context: str = self.json_str[start:end]
self.logger.append(
{
"text": text,
"context": context,
}
)
@@ -0,0 +1,484 @@
"""
This module will parse the JSON file following the BNF definition:
<json> ::= <container>
<primitive> ::= <number> | <string> | <boolean>
; Where:
; <number> is a valid real number expressed in one of a number of given formats
; <string> is a string of valid characters enclosed in quotes
; <boolean> is one of the literal strings 'true', 'false', or 'null' (unquoted)
<container> ::= <object> | <array>
<array> ::= '[' [ <json> *(', ' <json>) ] ']' ; A sequence of JSON values separated by commas
<object> ::= '{' [ <member> *(', ' <member>) ] '}' ; A sequence of 'members'
<member> ::= <string> ': ' <json> ; A pair consisting of a name, and a JSON value
If something is wrong (a missing parentheses or quotes for example) it will use a few simple heuristics to fix the JSON string:
- Add the missing parentheses if the parser believes that the array or object should be closed
- Quote strings or add missing single quotes
- Adjust whitespaces and remove line breaks
All supported use cases are in the unit tests
"""
import argparse
import json
import sys
from pathlib import Path
from typing import Any, Literal, TextIO, overload
from .json_parser import JSONParser
from .schema_repair import SchemaRepairer, load_schema_model, normalize_schema_repair_mode, schema_from_input
from .utils.constants import JSONReturnType
@overload
def repair_json(
json_str: str = "",
return_objects: Literal[False] = False,
skip_json_loads: bool = False,
logging: Literal[False] = False,
json_fd: TextIO | None = None,
chunk_length: int = 0,
stream_stable: bool = False,
strict: bool = False,
schema: Any | None = None,
schema_repair_mode: Literal["standard", "salvage"] = "standard",
**json_dumps_args: Any,
) -> str: ...
@overload
def repair_json(
json_str: str = "",
return_objects: Literal[True] = True,
skip_json_loads: bool = False,
logging: Literal[False] = False,
json_fd: TextIO | None = None,
chunk_length: int = 0,
stream_stable: bool = False,
strict: bool = False,
schema: Any | None = None,
schema_repair_mode: Literal["standard", "salvage"] = "standard",
**json_dumps_args: Any,
) -> JSONReturnType: ...
@overload
def repair_json(
json_str: str = "",
return_objects: bool = False,
skip_json_loads: bool = False,
logging: Literal[True] = True,
json_fd: TextIO | None = None,
chunk_length: int = 0,
stream_stable: bool = False,
strict: bool = False,
schema: Any | None = None,
schema_repair_mode: Literal["standard", "salvage"] = "standard",
**json_dumps_args: Any,
) -> tuple[JSONReturnType, list[dict[str, str]]]: ...
@overload
def repair_json(
json_str: str = "",
return_objects: bool = False,
skip_json_loads: bool = False,
logging: bool = False,
json_fd: TextIO | None = None,
chunk_length: int = 0,
stream_stable: bool = False,
strict: bool = False,
schema: Any | None = None,
schema_repair_mode: Literal["standard", "salvage"] = "standard",
**json_dumps_args: Any,
) -> str | JSONReturnType | tuple[JSONReturnType, list[dict[str, str]]]: ...
def repair_json(
json_str: str = "",
return_objects: bool = False,
skip_json_loads: bool = False,
logging: bool = False,
json_fd: TextIO | None = None,
chunk_length: int = 0,
stream_stable: bool = False,
strict: bool = False,
schema: Any | None = None,
schema_repair_mode: Literal["standard", "salvage"] = "standard",
**json_dumps_args: Any,
) -> JSONReturnType | tuple[JSONReturnType, list[dict[str, str]]]:
"""
Given a json formatted string, it will try to decode it and, if it fails, it will try to fix it.
Args:
json_str (str, optional): The JSON string to repair. Defaults to an empty string.
return_objects (bool, optional): If True, return the decoded data structure. Defaults to False.
skip_json_loads (bool, optional): If True, skip calling the built-in json.loads() function to verify that the json is valid before attempting to repair. Defaults to False.
logging (bool, optional): If True, return a tuple with the repaired json and a log of all repair actions. Defaults to False. When no repairs were required, the repair log will be an empty list.
json_fd (Optional[TextIO], optional): File descriptor for JSON input. Do not use! Use `from_file` or `load` instead. Defaults to None.
ensure_ascii (bool, optional): Set to False to avoid converting non-latin characters to ascii (for example when using chinese characters). Defaults to True. Ignored if `skip_json_loads` is True.
chunk_length (int, optional): Size in bytes of the file chunks to read at once. Ignored if `json_fd` is None. Do not use! Use `from_file` or `load` instead. Defaults to 1MB.
stream_stable (bool, optional): When the json to be repaired is the accumulation of streaming json at a certain moment.If this parameter to True will keep the repair results stable.
strict (bool, optional): If True, surface structural problems (duplicate keys, missing separators, empty keys/values, etc.) as ValueError instead of repairing them.
schema (Any, optional): JSON Schema dict, boolean schema, or pydantic v2 model used to guide repairs and validation for both valid and invalid JSON inputs.
schema_repair_mode (Literal["standard", "salvage"], optional): Schema repair mode. "standard" keeps default schema behavior; "salvage" enables best-effort schema salvage heuristics for arrays/objects.
Returns:
Union[JSONReturnType, Tuple[JSONReturnType, List[Dict[str, str]]]]: The repaired JSON or a tuple with the repaired JSON and repair log when logging is True.
"""
schema_repair_mode = normalize_schema_repair_mode(schema_repair_mode)
if schema is None and schema_repair_mode == "salvage":
raise ValueError("schema_repair_mode='salvage' requires schema.")
# Schema-guided repairs and strict mode are mutually exclusive to avoid conflicting behavior.
if schema is not None and strict:
raise ValueError("schema and strict cannot be used together.")
parser: JSONParser | None = None
repair_log: list[dict[str, str]] = []
# ``skip_json_loads`` only skips the initial whole-input validation. Once
# the parser has found a top-level value after a prefix, raw decoding that
# value is still a safe, targeted fast path.
try_valid_json_suffix = json_fd is None
if json_fd is not None:
parser = JSONParser(
json_str,
json_fd,
logging,
chunk_length,
stream_stable,
strict,
)
if logging:
repair_log = parser.logger
schema_obj = schema_from_input(schema) if schema is not None else None
repairer = (
SchemaRepairer(schema_obj, repair_log if logging else None, schema_repair_mode=schema_repair_mode)
if schema_obj is not None
else None
)
# Fast path for valid JSON: schema-aware mode still applies repair+validation.
parsed_json: JSONReturnType = None
is_valid_json = False
try:
if not skip_json_loads:
parsed_json = json.load(json_fd) if json_fd else json.loads(json_str)
if repairer is not None and schema_obj is not None:
# Validate here to ensure that we reject values that cannot satisfy the schema and fall back to the more expensive parser+schema repair if needed, instead of just returning the valid but schema-noncompliant JSON.
try:
if repairer.is_valid(parsed_json, schema_obj):
is_valid_json = True
else:
try:
# repair_value may mutate containers in place; if validate fails we still
# fall back to parser.parse_with_schema, which fully replaces parsed_json.
repaired_value = repairer.repair_value(parsed_json, schema_obj, "$")
if repairer.is_valid(repaired_value, schema_obj):
parsed_json = repaired_value
is_valid_json = True
except ValueError:
pass
except RecursionError as exc:
raise ValueError("Input schema nesting exceeds the supported schema recursion depth.") from exc
else:
is_valid_json = True
except (json.JSONDecodeError, TypeError, ValueError):
pass
if not is_valid_json:
if parser is None:
parser = JSONParser(
json_str,
json_fd,
logging,
chunk_length,
stream_stable,
strict,
try_valid_json_suffix=try_valid_json_suffix,
)
if logging:
parser.logger = repair_log
try:
if repairer is not None and schema_obj is not None:
# If schema-guided, we want to attempt repairs even on valid JSON that fails schema validation.
try:
parsed_json = parser.parse_with_schema(repairer, schema_obj)
repairer.validate(parsed_json, schema_obj)
except RecursionError as exc:
raise ValueError("Input schema nesting exceeds the supported schema recursion depth.") from exc
else:
# Otherwise, we can skip the more expensive schema-aware parsing and just do a normal parse.
parsed_json = parser.parse()
except RecursionError as exc:
raise ValueError("Input nesting exceeds the supported parser recursion depth.") from exc
# It's useful to return the actual object instead of the json string,
# it allows this lib to be a replacement of the json library
if logging:
return parsed_json, repair_log
if return_objects:
return parsed_json
# Avoid returning only a pair of quotes if it's an empty string
if parsed_json == "":
return ""
return json.dumps(parsed_json, **json_dumps_args)
def loads(
json_str: str,
skip_json_loads: bool = False,
logging: bool = False,
stream_stable: bool = False,
strict: bool = False,
schema: Any | None = None,
schema_repair_mode: Literal["standard", "salvage"] = "standard",
) -> JSONReturnType | tuple[JSONReturnType, list[dict[str, str]]] | str:
"""
This function works like `json.loads()` except that it will fix your JSON in the process.
It is a wrapper around the `repair_json()` function with `return_objects=True`.
Args:
json_str (str): The JSON string to load and repair.
skip_json_loads (bool, optional): If True, skip calling the built-in json.loads() function to verify that the json is valid before attempting to repair. Defaults to False.
logging (bool, optional): If True, return a tuple with the repaired json and a log of all repair actions. Defaults to False.
strict (bool, optional): If True, surface structural problems (duplicate keys, missing separators, empty keys/values, etc.) as ValueError instead of repairing them.
schema (Any, optional): JSON Schema dict, boolean schema, or pydantic v2 model used to guide repairs and validation for both valid and invalid JSON inputs.
schema_repair_mode (Literal["standard", "salvage"], optional): Schema repair mode. "salvage" requires schema.
Returns:
Union[JSONReturnType, Tuple[JSONReturnType, List[Dict[str, str]]], str]: The repaired JSON object or a tuple with the repaired JSON object and repair log.
"""
return repair_json(
json_str=json_str,
return_objects=True,
skip_json_loads=skip_json_loads,
logging=logging,
stream_stable=stream_stable,
strict=strict,
schema=schema,
schema_repair_mode=schema_repair_mode,
)
def load(
fd: TextIO,
skip_json_loads: bool = False,
logging: bool = False,
chunk_length: int = 0,
strict: bool = False,
schema: Any | None = None,
schema_repair_mode: Literal["standard", "salvage"] = "standard",
) -> JSONReturnType | tuple[JSONReturnType, list[dict[str, str]]]:
"""
This function works like `json.load()` except that it will fix your JSON in the process.
It is a wrapper around the `repair_json()` function with `json_fd=fd` and `return_objects=True`.
Args:
fd (TextIO): File descriptor for JSON input.
skip_json_loads (bool, optional): If True, skip calling the built-in json.loads() function to verify that the json is valid before attempting to repair. Defaults to False.
logging (bool, optional): If True, return a tuple with the repaired json and a log of all repair actions. Defaults to False.
chunk_length (int, optional): Size in bytes of the file chunks to read at once. Defaults to 1MB.
strict (bool, optional): If True, surface structural problems (duplicate keys, missing separators, empty keys/values, etc.) as ValueError instead of repairing them.
schema (Any, optional): JSON Schema dict, boolean schema, or pydantic v2 model used to guide repairs and validation for both valid and invalid JSON inputs.
schema_repair_mode (Literal["standard", "salvage"], optional): Schema repair mode. "salvage" requires schema.
Returns:
Union[JSONReturnType, Tuple[JSONReturnType, List[Dict[str, str]]]]: The repaired JSON object or a tuple with the repaired JSON object and repair log.
"""
return repair_json(
json_fd=fd,
chunk_length=chunk_length,
return_objects=True,
skip_json_loads=skip_json_loads,
logging=logging,
strict=strict,
schema=schema,
schema_repair_mode=schema_repair_mode,
)
def from_file(
filename: str | Path,
skip_json_loads: bool = False,
logging: bool = False,
chunk_length: int = 0,
strict: bool = False,
schema: Any | None = None,
schema_repair_mode: Literal["standard", "salvage"] = "standard",
) -> JSONReturnType | tuple[JSONReturnType, list[dict[str, str]]]:
"""
This function is a wrapper around `load()` so you can pass the filename as string
Args:
filename (str | Path): The name of the file containing JSON data to load and repair.
skip_json_loads (bool, optional): If True, skip calling the built-in json.loads() function to verify that the json is valid before attempting to repair. Defaults to False.
logging (bool, optional): If True, return a tuple with the repaired json and a log of all repair actions. Defaults to False.
chunk_length (int, optional): Size in bytes of the file chunks to read at once. Defaults to 1MB.
strict (bool, optional): If True, surface structural problems (duplicate keys, missing separators, empty keys/values, etc.) as ValueError instead of repairing them.
schema (Any, optional): JSON Schema dict, boolean schema, or pydantic v2 model used to guide repairs and validation for both valid and invalid JSON inputs.
schema_repair_mode (Literal["standard", "salvage"], optional): Schema repair mode. "salvage" requires schema.
Returns:
Union[JSONReturnType, Tuple[JSONReturnType, List[Dict[str, str]]]]: The repaired JSON object or a tuple with the repaired JSON object and repair log.
"""
with Path(filename).open() as fd:
return load(
fd=fd,
skip_json_loads=skip_json_loads,
logging=logging,
chunk_length=chunk_length,
strict=strict,
schema=schema,
schema_repair_mode=schema_repair_mode,
)
def cli(inline_args: list[str] | None = None) -> int:
"""
Command-line interface for repairing and parsing JSON files.
Args:
inline_args (Optional[List[str]]): List of command-line arguments for testing purposes. Defaults to None.
- filename (str): The JSON file to repair. If omitted, the JSON is read from stdin.
- -i, --inline (bool): Replace the file inline instead of returning the output to stdout.
- -o, --output TARGET (str): If specified, the output will be written to TARGET filename instead of stdout.
- --ensure_ascii (bool): Pass ensure_ascii=True to json.dumps(). Will pass False otherwise.
- --indent INDENT (int): Number of spaces for indentation (Default 2).
- --skip-json-loads (bool): Skip initial json.loads validation.
- --schema SCHEMA (str): Path to a JSON Schema file that guides repairs.
- --schema-model MODEL (str): Pydantic v2 model in 'module:ClassName' form that guides repairs.
- --strict (bool): Raise on duplicate keys, missing separators, empty keys/values, and other unrecoverable structures instead of repairing them.
Returns:
int: Exit code of the CLI operation.
Raises:
Exception: Any exception that occurs during file processing.
Example:
>>> cli(['example.json', '--indent', '4'])
>>> cat json.txt | json_repair
"""
parser = argparse.ArgumentParser(description="Repair and parse JSON files.")
# Make the filename argument optional; if omitted, we will read from stdin.
parser.add_argument(
"filename",
nargs="?",
help="The JSON file to repair (if omitted, reads from stdin)",
)
parser.add_argument(
"-i",
"--inline",
action="store_true",
help="Replace the file inline instead of returning the output to stdout",
)
parser.add_argument(
"-o",
"--output",
metavar="TARGET",
help="If specified, the output will be written to TARGET filename instead of stdout",
)
parser.add_argument(
"--ensure_ascii",
action="store_true",
help="Pass ensure_ascii=True to json.dumps()",
)
parser.add_argument(
"--indent",
type=int,
default=2,
help="Number of spaces for indentation (Default 2)",
)
parser.add_argument(
"--skip-json-loads",
action="store_true",
help="Skip initial json.loads validation",
)
parser.add_argument(
"--schema",
metavar="SCHEMA",
help="Path to a JSON Schema file that guides repairs",
)
parser.add_argument(
"--schema-model",
metavar="MODEL",
help="Pydantic v2 model in 'module:ClassName' form that guides repairs",
)
parser.add_argument(
"--strict",
action="store_true",
help="Raise on duplicate keys, missing separators, empty keys/values, and other unrecoverable structures instead of repairing them",
)
parser.add_argument(
"--schema-repair-mode",
choices=["standard", "salvage"],
default="standard",
help="Schema repair mode: 'standard' (default) or 'salvage' (best-effort array/object salvage).",
)
args = parser.parse_args(inline_args)
# Inline mode requires a filename, so error out if none was provided.
if args.inline and not args.filename: # pragma: no cover
print("Error: Inline mode requires a filename", file=sys.stderr)
sys.exit(1)
if args.inline and args.output: # pragma: no cover
print("Error: You cannot pass both --inline and --output", file=sys.stderr)
sys.exit(1)
if args.schema and args.schema_model:
print("Error: You cannot pass both --schema and --schema-model", file=sys.stderr)
sys.exit(1)
if args.strict and (args.schema or args.schema_model):
print("Error: --strict cannot be used with --schema or --schema-model", file=sys.stderr)
sys.exit(1)
if args.schema_repair_mode == "salvage" and not (args.schema or args.schema_model):
print("Error: --schema-repair-mode salvage requires --schema or --schema-model", file=sys.stderr)
sys.exit(1)
ensure_ascii = args.ensure_ascii
try:
schema = None
if args.schema:
with Path(args.schema).open() as fd:
schema = json.load(fd)
elif args.schema_model:
schema = load_schema_model(args.schema_model)
# Use from_file if a filename is provided; otherwise read from stdin.
if args.filename:
result = from_file(
args.filename,
skip_json_loads=args.skip_json_loads,
strict=args.strict,
schema=schema,
schema_repair_mode=args.schema_repair_mode,
)
else:
data = sys.stdin.read()
result = loads(
data,
skip_json_loads=args.skip_json_loads,
strict=args.strict,
schema=schema,
schema_repair_mode=args.schema_repair_mode,
)
if args.inline or args.output:
with Path(args.output or args.filename).open(mode="w") as fd:
json.dump(result, fd, indent=args.indent, ensure_ascii=ensure_ascii)
else:
print(json.dumps(result, indent=args.indent, ensure_ascii=ensure_ascii))
except (OSError, TypeError, ValueError) as e: # pragma: no cover
print(f"Error: {e!s}", file=sys.stderr)
return 1
return 0 # Success
if __name__ == "__main__": # pragma: no cover
sys.exit(cli())
@@ -0,0 +1,119 @@
from typing import TYPE_CHECKING, Any, cast
from .parser_schema import ArraySchemaConfig, resolve_parser_array_schema
from .utils.constants import STRING_DELIMITERS, JSONReturnType
from .utils.json_context import ContextValues
from .utils.object_comparer import ObjectComparer
if TYPE_CHECKING:
from .json_parser import JSONParser
def _resolve_array_item_schema(
schema_config: ArraySchemaConfig | None,
idx: int,
) -> tuple[dict[str, Any] | bool | None, bool]:
if schema_config is None:
return None, False
item_schema: dict[str, Any] | bool | None = None
drop_item = False
if isinstance(schema_config.items_schema, list):
if idx < len(schema_config.items_schema):
raw_schema = schema_config.items_schema[idx]
if raw_schema is not None and not isinstance(raw_schema, (dict, bool)):
raise ValueError("Schema must be an object.")
item_schema = cast("dict[str, Any] | bool | None", raw_schema)
elif schema_config.additional_items is False:
drop_item = True
elif isinstance(schema_config.additional_items, dict):
item_schema = cast("dict[str, Any]", schema_config.additional_items)
else:
item_schema = True
elif isinstance(schema_config.items_schema, dict):
item_schema = cast("dict[str, Any]", schema_config.items_schema)
else:
item_schema = True
return item_schema, drop_item
def parse_array(
self: "JSONParser",
schema: dict[str, Any] | bool | None = None,
path: str = "$",
closing_delimiter: str = "]",
) -> list[JSONReturnType]:
# <array> ::= '[' [ <json> *(', ' <json>) ] ']' ; A sequence of JSON values separated by commas
schema_repairer, _schema, schema_config = resolve_parser_array_schema(self.schema_repairer, schema)
salvage_mode = schema_repairer is not None and schema_repairer.schema_repair_mode == "salvage"
array_is_object_value = self.context.current == ContextValues.OBJECT_VALUE
closed_before_parent_member = False
arr: list[JSONReturnType] = []
with self.context.enter(ContextValues.ARRAY):
self.skip_whitespaces()
char = self.get_char_at()
idx = 0
while char and char not in [closing_delimiter, "}"]:
item_schema, drop_item = _resolve_array_item_schema(schema_config, idx)
item_path = f"{path}[{idx}]"
active_schema_repairer = (
schema_repairer if schema_repairer is not None and not drop_item and not salvage_mode else None
)
if char in STRING_DELIMITERS:
# A string followed by ':' is often a missing object start; treat it as an object.
i = 1
i = self.skip_to_character(char, i)
i = self.scroll_whitespaces(idx=i + 1)
if self.get_char_at(i) == ":":
if array_is_object_value and arr and all(not isinstance(item, (dict, list)) for item in arr):
self.log("Closed unclosed array before object member after scalar items")
closed_before_parent_member = True
break
if active_schema_repairer is not None:
# Schema-guided object parsing, then enforce schema on the parsed object.
value = self.parse_object(item_schema, item_path)
value = active_schema_repairer.repair_value(value, item_schema, item_path)
else:
# No schema (or dropping): still parse to keep the cursor in sync.
value = self.parse_object()
else:
value = self.parse_string()
if active_schema_repairer is not None:
# Apply schema constraints/coercions to scalar values when configured.
value = active_schema_repairer.repair_value(value, item_schema, item_path)
else:
# Use schema-aware parsing to guide nested repairs when configured.
value = (
self.parse_json(item_schema, item_path) if active_schema_repairer is not None else self.parse_json()
)
if ObjectComparer.is_strictly_empty(value) and self.get_char_at() not in [closing_delimiter, ","]:
self.index += 1
elif value == "..." and self.get_char_at(-1) == ".":
self.log(
"While parsing an array, found a stray '...'; ignoring it",
)
elif not drop_item:
arr.append(value)
elif schema_repairer is not None:
# Record drops for visibility when schema forbids extra tuple items.
schema_repairer._log("Dropped extra array item not covered by schema", item_path)
idx += 1
char = self.get_char_at()
while char and char != closing_delimiter and (char.isspace() or char == ","):
self.index += 1
char = self.get_char_at()
if char != closing_delimiter:
self.log(
f"While parsing an array we missed the closing {closing_delimiter}, ignoring it",
)
if not closed_before_parent_member:
self.index += 1
return arr
@@ -0,0 +1,77 @@
from typing import TYPE_CHECKING
from .utils.constants import JSONReturnType
from .utils.json_context import ContextValues
if TYPE_CHECKING:
from .json_parser import JSONParser
def parse_comment(self: "JSONParser", record_top_level_value: bool = False) -> JSONReturnType:
"""
Parse code-like comments:
- "# comment": A line comment that continues until a newline.
- "// comment": A line comment that continues until a newline.
- "/* comment */": A block comment that continues until the closing delimiter "*/".
The comment is skipped over and an empty string is returned so that comments do not interfere
with the actual JSON elements.
"""
while True:
char = self.get_char_at()
termination_characters = ["\n", "\r"]
if ContextValues.ARRAY in self.context.context:
termination_characters.append("]")
if ContextValues.OBJECT_VALUE in self.context.context:
termination_characters.append("}")
if ContextValues.OBJECT_KEY in self.context.context:
termination_characters.append(":")
# Line comment starting with #
if char == "#":
comment = ""
while char and char not in termination_characters:
comment += char
self.index += 1
char = self.get_char_at()
self.log(f"Found line comment: {comment}, ignoring")
# Comments starting with '/'
elif char == "/":
next_char = self.get_char_at(1)
# Handle line comment starting with //
if next_char == "/":
comment = "//"
self.index += 2 # Skip both slashes.
char = self.get_char_at()
while char and char not in ["\n", "\r"]:
comment += char
self.index += 1
char = self.get_char_at()
self.log(f"Found line comment: {comment}, ignoring")
# Handle block comment starting with /*
elif next_char == "*":
comment = "/*"
self.index += 2 # Skip '/*'
while True:
char = self.get_char_at()
if not char:
self.log("Reached end-of-string while parsing block comment; unclosed block comment.")
break
comment += char
self.index += 1
if comment.endswith("*/"):
break
self.log(f"Found block comment: {comment}, ignoring")
else:
# Skip standalone '/' characters that are not part of a comment
# to avoid getting stuck in an infinite loop
self.index += 1
if self.context.empty:
# Avoid a parse_json -> parse_comment -> parse_json chain for long runs of
# top-level comments; re-enter only once after we have consumed them all.
self.skip_whitespaces()
if self.get_char_at() in ["#", "/"]:
continue
return self.parse_json(record_top_level_value=record_top_level_value)
break
return ""
@@ -0,0 +1,43 @@
from typing import TYPE_CHECKING
from .utils.constants import JSONReturnType
from .utils.json_context import ContextValues
NUMBER_CHARS: set[str] = set("0123456789-.eE/,_")
if TYPE_CHECKING:
from .json_parser import JSONParser
def parse_number(self: "JSONParser") -> JSONReturnType:
# <number> is a valid real number expressed in one of a number of given formats
number_start_index = self.index
number_str = ""
char = self.get_char_at()
is_array = self.context.current == ContextValues.ARRAY
while (
char
and (char in NUMBER_CHARS or (char == "+" and number_str.endswith(("e", "E"))))
and (not is_array or char != ",")
):
if char != "_":
number_str += char
self.index += 1
char = self.get_char_at()
if (self.get_char_at() or "").isalpha():
# this was a string instead, sorry
self.index = number_start_index
return self.parse_string()
if number_str and number_str[-1] in "+-eE/,":
# The number ends with a non valid character for a number/currency, rolling back one
number_str = number_str[:-1]
self.index -= 1
try:
if "," in number_str:
return number_str
if "." in number_str or "e" in number_str or "E" in number_str:
return float(number_str)
return int(number_str)
except ValueError:
return number_str
@@ -0,0 +1,512 @@
from typing import TYPE_CHECKING, Any, cast
from .parser_schema import ObjectSchemaConfig, resolve_parser_object_schema
from .utils.constants import MISSING_VALUE, STRING_DELIMITERS, JSONReturnType
from .utils.json_context import ContextValues
from .utils.pattern_properties import match_pattern_properties
if TYPE_CHECKING:
from .json_parser import JSONParser
from .schema_repair import SchemaRepairer
def _finalize_object(
obj: dict[str, JSONReturnType],
schema: dict[str, Any] | bool | None,
schema_repairer: "SchemaRepairer | None",
schema_config: ObjectSchemaConfig | None,
path: str,
) -> dict[str, JSONReturnType]:
if schema_repairer is None or schema_config is None:
return obj
resolved_schema = schema_repairer.resolve_schema(schema)
property_aliases = (
resolved_schema.get("x-json-repair-property-aliases", {}) if isinstance(resolved_schema, dict) else {}
)
aliases_with_field_name = set()
if isinstance(property_aliases, dict):
aliases_with_field_name = {
alias
for alias, field_names in property_aliases.items()
if isinstance(alias, str)
and isinstance(field_names, list)
and any(isinstance(name, str) and name in obj for name in field_names)
}
missing_required = [key for key in schema_config.required if key not in obj and key not in aliases_with_field_name]
if missing_required and schema_repairer.schema_repair_mode != "salvage":
raise ValueError(f"Missing required properties at {path}: {', '.join(missing_required)}")
for key, prop_schema in schema_config.properties.items():
if key in obj or key in schema_config.required or key in aliases_with_field_name:
continue
if isinstance(prop_schema, dict) and "default" in prop_schema:
obj[key] = schema_repairer._copy_json_value(prop_schema["default"], f"{path}.{key}", "default")
schema_repairer._log("Inserted default value for missing property", f"{path}.{key}")
return obj
def _strip_comments_for_empty_object_classification(body: str) -> str:
stripped = []
in_quote: str | None = None
backslashes = 0
index = 0
while index < len(body):
char = body[index]
next_char = body[index + 1] if index + 1 < len(body) else ""
if char == "\\":
backslashes += 1
stripped.append(char)
index += 1
continue
if in_quote is not None:
stripped.append(char)
if char == in_quote and backslashes % 2 == 0:
in_quote = None
backslashes = 0
index += 1
continue
if char in STRING_DELIMITERS and backslashes % 2 == 0:
in_quote = char
stripped.append(char)
backslashes = 0
index += 1
continue
backslashes = 0
if char == "#" or (char == "/" and next_char == "/"):
index += 2 if char == "/" else 1
while index < len(body) and body[index] not in ["\n", "\r"]:
index += 1
continue
if char == "/" and next_char == "*":
index += 2
while index < len(body) - 1 and body[index : index + 2] != "*/":
index += 1
index = min(index + 2, len(body))
continue
stripped.append(char)
index += 1
return "".join(stripped)
def _classify_empty_object_repair(
self: "JSONParser",
start_index: int,
schema: dict[str, Any] | bool | None,
schema_repairer: "SchemaRepairer | None",
) -> tuple[str, str | None]:
attempted_object = self.json_str[start_index - 1 : self.index + 1]
body = attempted_object[1:]
body = body.removesuffix("}")
body = body.lstrip()
if not body:
return "keep", None
if (body.startswith('\\"') and '\\":' in body) or (body.startswith("\\'") and "\\':" in body):
normalized_object = attempted_object.replace('\\"', '"').replace("\\'", "'")
self.log(
"Parsed object is empty but the input starts like an escaped object key, normalizing and reparsing it as an object",
)
return "object", normalized_object
body = _strip_comments_for_empty_object_classification(body).lstrip()
if not body:
return "keep", None
in_quote: str | None = None
backslashes = 0
for char in body:
if char == "\\":
backslashes += 1
continue
if in_quote is not None:
if char == in_quote and backslashes % 2 == 0:
in_quote = None
elif char in STRING_DELIMITERS and backslashes % 2 == 0:
in_quote = char
elif char == ":" and backslashes % 2 == 0:
self.log(
"Parsed object is empty but the input still contains an object-style separator, keeping object repair",
)
return "keep", None
backslashes = 0
if (
schema_repairer is not None
and schema_repairer.schema_repair_mode == "salvage"
and isinstance(schema, dict)
and schema_repairer.is_object_schema(schema)
and not schema_repairer.is_array_schema(schema)
):
return "schema_set_object", None
return "array", None
def _merge_object_array_continuation(
self: "JSONParser",
obj: dict[str, JSONReturnType],
) -> bool:
prev_key = list(obj.keys())[-1] if obj else None
if not prev_key or not isinstance(obj[prev_key], list) or self.strict:
return False
self.index += 1
new_array = self.parse_array()
if isinstance(new_array, list):
prev_value = obj[prev_key]
if isinstance(prev_value, list):
list_lengths = [len(item) for item in prev_value if isinstance(item, list)]
expected_len = (
list_lengths[0] if list_lengths and all(length == list_lengths[0] for length in list_lengths) else None
)
if expected_len:
tail = []
while prev_value and not isinstance(prev_value[-1], list):
tail.append(prev_value.pop())
if tail:
tail.reverse()
if len(tail) % expected_len == 0:
self.log(
"While parsing an object we found row values without an inner array, grouping them into rows",
)
for i in range(0, len(tail), expected_len):
prev_value.append(tail[i : i + expected_len])
else:
prev_value.extend(tail)
if new_array:
if all(isinstance(item, list) for item in new_array):
self.log(
"While parsing an object we found additional rows, appending them without flattening",
)
prev_value.extend(new_array)
else:
prev_value.append(new_array)
else:
prev_value.extend(new_array[0] if len(new_array) == 1 and isinstance(new_array[0], list) else new_array)
self.skip_whitespaces()
if self.get_char_at() == ",":
self.index += 1
self.skip_whitespaces()
return True
def _parse_object_key(
self: "JSONParser",
obj: dict[str, JSONReturnType],
) -> tuple[str, int]:
key = ""
rollback_index = self.index
self.context.set(ContextValues.OBJECT_KEY)
try:
while self.get_char_at():
rollback_index = self.index
if self.get_char_at() == "[" and key == "" and _merge_object_array_continuation(self, obj):
continue
raw_key = self.parse_string()
assert isinstance(raw_key, str)
key = raw_key
if key == "":
self.skip_whitespaces()
if key != "" or (key == "" and self.get_char_at() in [":", "}"]):
if key == "" and self.strict:
self.log(
"Empty key found in strict mode while parsing object, raising an error",
)
raise ValueError("Empty key found in strict mode while parsing object.")
break
finally:
self.context.reset()
return key, rollback_index
def _should_split_duplicate_object(self: "JSONParser", rollback_index: int) -> bool:
lookback_idx = rollback_index - self.index - 1
prev_non_whitespace = self.get_char_at(lookback_idx)
while prev_non_whitespace and prev_non_whitespace.isspace():
lookback_idx -= 1
prev_non_whitespace = self.get_char_at(lookback_idx)
key_start_char = self.get_char_at(rollback_index - self.index)
next_non_whitespace = self.get_char_at(self.scroll_whitespaces())
return not (key_start_char in STRING_DELIMITERS and prev_non_whitespace == "," and next_non_whitespace == ":")
def _split_object_on_duplicate_key(self: "JSONParser", rollback_index: int) -> None:
self.index = rollback_index - 1
self.json_str = self.json_str[: self.index + 1] + "{" + self.json_str[self.index + 1 :]
def _resolve_object_property_schema(
self: "JSONParser",
schema_repairer: "SchemaRepairer | None",
schema: dict[str, Any] | bool | None,
schema_config: ObjectSchemaConfig | None,
key: str,
) -> tuple[dict[str, Any] | bool | None, list[dict[str, Any] | bool | None], bool]:
if schema_repairer is None or schema_config is None:
return None, [], False
prop_schema: dict[str, Any] | bool | None = None
extra_schemas: list[dict[str, Any] | bool | None] = []
if key in schema_config.properties:
schema_value = schema_config.properties[key]
if schema_value is not None and not isinstance(schema_value, (dict, bool)):
raise ValueError("Schema must be an object.")
prop_schema = cast("dict[str, Any] | bool | None", schema_value)
return prop_schema, extra_schemas, False
resolved_schema = schema_repairer.resolve_schema(schema)
property_aliases = (
resolved_schema.get("x-json-repair-property-aliases", {}) if isinstance(resolved_schema, dict) else {}
)
if isinstance(property_aliases, dict):
for alias, field_names in property_aliases.items():
if isinstance(alias, str) and isinstance(field_names, list) and key in field_names:
schema_value = schema_config.properties.get(alias)
return cast("dict[str, Any] | bool | None", schema_value), extra_schemas, False
matched: list[Any] = []
unsupported_patterns: list[str] = []
if schema_config.pattern_properties:
matched, unsupported_patterns = match_pattern_properties(schema_config.pattern_properties, key)
for pattern in unsupported_patterns:
self.log(
f"Skipped unsupported patternProperties regex '{pattern}' while parsing object key '{key}'",
)
if matched:
primary_schema = matched[0]
if primary_schema is not None and not isinstance(primary_schema, (dict, bool)):
raise ValueError("Schema must be an object.")
prop_schema = cast("dict[str, Any] | bool | None", primary_schema)
for extra_schema in matched[1:]:
if extra_schema is not None and not isinstance(extra_schema, (dict, bool)):
raise ValueError("Schema must be an object.")
extra_schemas.append(cast("dict[str, Any] | bool | None", extra_schema))
return prop_schema, extra_schemas, False
if schema_config.additional_properties is False:
return None, [], True
if isinstance(schema_config.additional_properties, dict):
return cast("dict[str, Any]", schema_config.additional_properties), [], False
return True, [], False
def _parse_object_value(
self: "JSONParser",
schema_repairer: "SchemaRepairer | None",
prop_schema: dict[str, Any] | bool | None,
key_path: str,
) -> JSONReturnType:
self.context.set(ContextValues.OBJECT_VALUE)
try:
self.skip_whitespaces()
char = self.get_char_at()
if char in [",", "}"]:
self.log(
f"While parsing an object value we found a stray {char}, ignoring it",
)
if schema_repairer is not None:
return schema_repairer.repair_value(MISSING_VALUE, prop_schema, key_path)
return ""
if schema_repairer is not None:
return self.parse_json(prop_schema, key_path)
return self.parse_json()
finally:
self.context.reset()
def _repair_empty_object_result(
self: "JSONParser",
obj: dict[str, JSONReturnType],
start_index: int,
schema: dict[str, Any] | bool | None,
path: str,
schema_repairer: "SchemaRepairer | None",
) -> tuple[bool, JSONReturnType]:
if obj or self.index - start_index <= 2:
return False, None
if self.strict:
self.log(
"Parsed object is empty but contains extra characters in strict mode, raising an error",
)
raise ValueError("Parsed object is empty but contains extra characters in strict mode.")
empty_object_repair, normalized_object = _classify_empty_object_repair(self, start_index, schema, schema_repairer)
if empty_object_repair == "object" and normalized_object is not None:
end_index = self.index + 1
self.json_str = self.json_str[: start_index - 1] + normalized_object + self.json_str[end_index:]
self.index = start_index
with self.context.enter(ContextValues.OBJECT_KEY):
repaired_value = self.parse_object(schema, path)
self.deferred_contexts.append(ContextValues.OBJECT_KEY)
return True, repaired_value
if empty_object_repair == "schema_set_object":
self.log(
"Parsed object is empty but salvage schema expects an object, reparsing set-like members as null-valued object keys",
)
self.index = start_index
with self.context.enter(ContextValues.OBJECT_KEY):
set_items = self.parse_array()
self.deferred_contexts.append(ContextValues.OBJECT_KEY)
if isinstance(set_items, list):
key_candidates: list[str] = [item for item in set_items if isinstance(item, str) and item]
if len(key_candidates) == len(set_items):
return True, cast("JSONReturnType", dict.fromkeys(key_candidates))
return True, set_items
if empty_object_repair == "array":
self.log("Parsed object is empty, we will try to parse this as an array instead")
self.index = start_index
with self.context.enter(ContextValues.OBJECT_KEY):
repaired_array = self.parse_array()
self.deferred_contexts.append(ContextValues.OBJECT_KEY)
return True, repaired_array
return False, None
def _complete_object_parse(
self: "JSONParser",
obj: dict[str, JSONReturnType],
schema: dict[str, Any] | bool | None,
path: str,
schema_repairer: "SchemaRepairer | None",
schema_config: ObjectSchemaConfig | None,
) -> JSONReturnType:
if not self.context.empty:
if self.get_char_at() == "}" and self.context.current not in [
ContextValues.OBJECT_KEY,
ContextValues.OBJECT_VALUE,
]:
self.log(
"Found an extra closing brace that shouldn't be there, skipping it",
)
self.index += 1
return obj
self.skip_whitespaces()
if self.get_char_at() == ",":
self.index += 1
self.skip_whitespaces()
if self.get_char_at() in STRING_DELIMITERS and not self.strict:
self.log(
"Found a comma and string delimiter after object closing brace, checking for additional key-value pairs",
)
additional_obj = self.parse_object(schema, path)
if isinstance(additional_obj, dict):
obj.update(additional_obj)
return _finalize_object(obj, schema, schema_repairer, schema_config, path)
def parse_object(
self: "JSONParser",
schema: dict[str, Any] | bool | None = None,
path: str = "$",
) -> JSONReturnType:
# <object> ::= '{' [ <member> *(', ' <member>) ] '}' ; A sequence of 'members'
obj: dict[str, JSONReturnType] = {}
start_index = self.index
parsing_object_value = self.context.current == ContextValues.OBJECT_VALUE
schema_repairer, schema, schema_config = resolve_parser_object_schema(self.schema_repairer, schema)
while (self.get_char_at() or "}") != "}":
self.skip_whitespaces()
if self.get_char_at() == ":":
self.log(
"While parsing an object we found a : before a key, ignoring",
)
self.index += 1
key, rollback_index = _parse_object_key(self, obj)
if ContextValues.ARRAY in self.context.context and key in obj:
if self.strict:
self.log("Duplicate key found in strict mode while parsing object, raising an error")
raise ValueError("Duplicate key found in strict mode while parsing object.")
if not parsing_object_value:
if _should_split_duplicate_object(self, rollback_index):
self.log(
"While parsing an object we found a duplicate key, closing the object here and rolling back the index",
)
_split_object_on_duplicate_key(self, rollback_index)
break
self.log(
"While parsing an object we found a duplicate key with a normal comma separator, keeping duplicate-key overwrite behavior",
)
self.skip_whitespaces()
if (self.get_char_at() or "}") == "}":
continue
self.skip_whitespaces()
if self.get_char_at() != ":":
if self.strict:
self.log(
"Missing ':' after key in strict mode while parsing object, raising an error",
)
raise ValueError("Missing ':' after key in strict mode while parsing object.")
self.log(
"While parsing an object we missed a : after a key",
)
self.index += 1
prop_schema, extra_schemas, drop_property = _resolve_object_property_schema(
self,
schema_repairer,
schema,
schema_config,
key,
)
key_path = f"{path}.{key}"
value = _parse_object_value(self, schema_repairer, prop_schema, key_path)
if schema_repairer is not None:
for extra_schema in extra_schemas:
value = schema_repairer.repair_value(value, extra_schema, key_path)
if schema_repairer is None and value == "" and self.strict and self.get_char_at(-1) not in STRING_DELIMITERS:
self.log(
"Parsed value is empty in strict mode while parsing object, raising an error",
)
raise ValueError("Parsed value is empty in strict mode while parsing object.")
if schema_repairer is None or not drop_property:
obj[key] = value
else:
schema_repairer._log("Dropped extra property not covered by schema", key_path)
if self.get_char_at() in [",", "'", '"']:
self.index += 1
if self.get_char_at() == "]" and ContextValues.ARRAY in self.context.context:
self.log(
"While parsing an object we found a closing array bracket, closing the object here and rolling back the index"
)
self.index -= 1
break
self.skip_whitespaces()
self.index += 1
repaired_empty_object, repaired_value = _repair_empty_object_result(
self,
obj,
start_index,
schema,
path,
schema_repairer,
)
if repaired_empty_object:
return repaired_value
return _complete_object_parse(
self,
obj,
schema,
path,
schema_repairer,
schema_config,
)
@@ -0,0 +1,942 @@
from dataclasses import dataclass, field
from typing import TYPE_CHECKING, cast
from .parse_string_helpers.object_value_context import classify_object_value_comma, update_inline_container_stack
from .parse_string_helpers.parse_boolean_or_null import parse_boolean_or_null
from .parse_string_helpers.parse_json_llm_block import parse_json_llm_block
from .utils.constants import STRING_DELIMITERS, JSONReturnType
from .utils.json_context import ContextValues
if TYPE_CHECKING:
from .json_parser import JSONParser
NO_DIRECT_RESULT = object()
INLINE_CONTAINER_CLOSING_DELIMITERS = {"[": "]", "{": "}", "(": ")"}
INLINE_CONTAINER_OPENERS = tuple(INLINE_CONTAINER_CLOSING_DELIMITERS)
LOW_SMART_QUOTE_SENTINEL = "\0"
@dataclass
class StringParseState:
missing_quotes: bool = False
doubled_quotes: bool = False
lstring_delimiter: str = '"'
rstring_delimiter: str = '"'
string_acc: str = ""
unmatched_delimiter: bool = False
pending_inline_container: bool = False
inline_container_stack: list[str] = field(default_factory=list)
object_value_has_no_future_delimiter: bool = False
lookahead_cache: dict[tuple[str, ...], tuple[int, int | None]] = field(default_factory=dict)
object_value_unmatched_opening_braces: int = 0
regex_character_class_start: int | None = None
def _outer_rstring_delimiter(state: StringParseState) -> str:
return state.rstring_delimiter[0]
def _active_rstring_delimiter(state: StringParseState) -> str:
return state.rstring_delimiter[-1]
def _in_low_smart_quote_span(state: StringParseState) -> bool:
return _active_rstring_delimiter(state) == LOW_SMART_QUOTE_SENTINEL
def _push_low_smart_quote_span(state: StringParseState) -> None:
state.rstring_delimiter += LOW_SMART_QUOTE_SENTINEL
def _pop_low_smart_quote_span(state: StringParseState) -> None:
state.rstring_delimiter = state.rstring_delimiter[:-1]
def _try_parse_simple_quoted_string(self: "JSONParser") -> str | None:
if self.get_char_at() != '"':
return None
start = self.index + 1
json_str = self.json_str
if isinstance(json_str, str):
end = json_str.find('"', start)
if end == -1:
return None
value = json_str[start:end]
if "\\" in value or "\n" in value or "\r" in value:
return None
else:
end = start
limit = len(json_str)
while end < limit:
char = json_str[end]
if char == '"':
break
if char in {"\\", "\n", "\r"}:
return None
end += 1
if end >= limit:
return None
value = json_str[start:end]
next_index = end + 1
limit = len(json_str)
while next_index < limit and self.json_str[next_index].isspace():
next_index += 1
next_char = self.json_str[next_index] if next_index < limit else None
current_context = self.context.current
if current_context == ContextValues.OBJECT_KEY:
if next_char != ":":
return None
elif current_context == ContextValues.OBJECT_VALUE:
if next_char not in {",", "}", None}:
return None
elif current_context == ContextValues.ARRAY:
if next_char not in {",", "]", None}:
return None
elif next_char is not None:
return None
self.index = end + 1
return value
def _append_literal_char(
self: "JSONParser",
state: StringParseState,
current_char: str,
) -> str | None:
_append_string_content(state, current_char)
self.index += 1
return self.get_char_at()
def _append_string_content(state: StringParseState, content: str) -> None:
start_index = len(state.string_acc)
state.string_acc += content
for offset, char in enumerate(content):
if char == "{":
state.object_value_unmatched_opening_braces += 1
elif char == "}" and state.object_value_unmatched_opening_braces:
state.object_value_unmatched_opening_braces -= 1
elif char == "[":
state.regex_character_class_start = start_index + offset + 1
elif char == "]":
state.regex_character_class_start = None
def _quote_belongs_to_regex_character_class(
self: "JSONParser",
state: StringParseState,
) -> bool:
"""Return whether the current quote is inside a compact ``[...]`` character class."""
start = state.regex_character_class_start
if start is None or any(char.isspace() for char in state.string_acc[start:]):
return False
closing_bracket_idx = self.skip_to_character("]", idx=1)
return self.get_char_at(closing_bracket_idx) == "]"
def _rebuild_unmatched_opening_braces(state: StringParseState) -> None:
state.object_value_unmatched_opening_braces = 0
state.regex_character_class_start = None
for index, char in enumerate(state.string_acc):
if char == "{":
state.object_value_unmatched_opening_braces += 1
elif char == "}" and state.object_value_unmatched_opening_braces:
state.object_value_unmatched_opening_braces -= 1
elif char == "[":
state.regex_character_class_start = index + 1
elif char == "]":
state.regex_character_class_start = None
def _cached_skip_to_character(
self: "JSONParser",
state: StringParseState,
character: str | list[str],
idx: int = 0,
) -> int:
targets = (character,) if isinstance(character, str) else tuple(character)
start_index = self.index + idx
cached = state.lookahead_cache.get(targets)
if cached is not None:
cached_start, cached_match = cached
if cached_match is None and start_index >= cached_start:
return len(self.json_str) - self.index
if cached_match is not None and cached_start <= start_index <= cached_match:
return cached_match - self.index
match_offset = self.skip_to_character(character, idx)
match = self.get_char_at(match_offset)
if not match:
state.lookahead_cache[targets] = (start_index, None)
return match_offset
match_index = self.index + match_offset
if match_index == 0 or self.json_str[match_index - 1] != "\\":
state.lookahead_cache[targets] = (start_index, match_index)
return match_offset
def _prepare_string_entry(
self: "JSONParser",
) -> tuple[StringParseState | None, object]:
char = self.get_char_at()
if char in ["#", "/"]:
return None, self.parse_comment()
while char and char not in STRING_DELIMITERS and not char.isalnum():
self.index += 1
char = self.get_char_at()
if not char:
return None, ""
fast_path_value = _try_parse_simple_quoted_string(self)
if fast_path_value is not None:
return None, fast_path_value
state = StringParseState()
if char == "'":
state.lstring_delimiter = state.rstring_delimiter = "'"
elif char == "“":
state.lstring_delimiter = "“"
state.rstring_delimiter = "”"
elif char.isalnum():
if char.lower() in ["t", "f", "n"] and self.context.current != ContextValues.OBJECT_KEY:
found_literal, value = parse_boolean_or_null(self)
if found_literal:
return state, value
self.log(
"While parsing a string, we found a literal instead of a quote",
)
state.missing_quotes = True
if not state.missing_quotes:
self.index += 1
if self.get_char_at() == "`":
ret_val = parse_json_llm_block(self)
if ret_val is not False:
return state, ret_val
self.log(
"While parsing a string, we found code fences but they did not enclose valid JSON, continuing parsing the string",
)
if self.get_char_at() == state.lstring_delimiter:
if (
(self.context.current == ContextValues.OBJECT_KEY and self.get_char_at(1) == ":")
or (self.context.current == ContextValues.OBJECT_VALUE and self.get_char_at(1) in [",", "}"])
or (self.context.current == ContextValues.ARRAY and self.get_char_at(1) in [",", "]"])
):
self.index += 1
return state, ""
if self.get_char_at(1) == state.lstring_delimiter:
self.log(
"While parsing a string, we found a doubled quote and then a quote again, ignoring it",
)
if self.strict:
raise ValueError("Found doubled quotes followed by another quote.")
return state, ""
i = self.skip_to_character(character=state.rstring_delimiter, idx=1)
if self.get_char_at(i + 1) == state.rstring_delimiter:
self.log(
"While parsing a string, we found a valid starting doubled quote",
)
state.doubled_quotes = True
self.index += 1
else:
if self.get_char_at(i) == state.rstring_delimiter:
outer_quote_idx = self.skip_to_character(character=state.rstring_delimiter, idx=i + 1)
after_outer_quote_idx = self.scroll_whitespaces(idx=outer_quote_idx + 1)
after_outer_quote = self.get_char_at(after_outer_quote_idx)
if (
self.context.current == ContextValues.OBJECT_VALUE
and not _only_whitespace_until(self, i)
and self.get_char_at(outer_quote_idx) == state.rstring_delimiter
and after_outer_quote in [",", "}", None]
):
self.log(
"While parsing a string, we found a leading quote that starts a quoted span, keeping it",
)
_append_string_content(state, state.lstring_delimiter)
state.unmatched_delimiter = True
self.index += 1
return state, NO_DIRECT_RESULT
i = self.scroll_whitespaces(idx=1)
next_c = self.get_char_at(i)
if next_c in [*STRING_DELIMITERS, "{", "["]:
self.log(
"While parsing a string, we found a doubled quote but also another quote afterwards, ignoring it",
)
if self.strict:
raise ValueError(
"Found doubled quotes followed by another quote while parsing a string.",
)
self.index += 1
return state, ""
if next_c not in [",", "]", "}"]:
self.log(
"While parsing a string, we found a doubled quote but it was a mistake, removing one quote",
)
self.index += 1
return state, NO_DIRECT_RESULT
def _normalize_escape_sequence(
self: "JSONParser",
state: StringParseState,
char: str,
) -> tuple[bool, str | None]:
self.log("Found a stray escape sequence, normalizing it")
active_rstring_delimiter = _active_rstring_delimiter(state)
if _in_low_smart_quote_span(state) and char == '"':
state.string_acc = state.string_acc[:-1] + char
_rebuild_unmatched_opening_braces(state)
_pop_low_smart_quote_span(state)
self.index += 1
return True, self.get_char_at()
if char == "\\":
run_start = self.index - 1
run_end = self.index + 1
while run_end < len(self.json_str) and self.json_str[run_end] == "\\":
run_end += 1
run_length = run_end - run_start
next_char = self.get_char_at(run_end - self.index)
if run_length % 2 == 0 and next_char != active_rstring_delimiter:
state.string_acc = state.string_acc[:-1] + ("\\" * (run_length // 2))
_rebuild_unmatched_opening_braces(state)
self.index = run_end
return True, self.get_char_at()
if char in [active_rstring_delimiter, "t", "n", "r", "b", "\\"]:
state.string_acc = state.string_acc[:-1]
escape_seqs = {"t": "\t", "n": "\n", "r": "\r", "b": "\b"}
state.string_acc += escape_seqs.get(char, char)
_rebuild_unmatched_opening_braces(state)
self.index += 1
next_char = self.get_char_at()
while (
next_char
and state.string_acc
and state.string_acc[-1] == "\\"
and next_char in [active_rstring_delimiter, "\\"]
):
state.string_acc = state.string_acc[:-1] + next_char
_rebuild_unmatched_opening_braces(state)
self.index += 1
next_char = self.get_char_at()
return True, next_char
if char in ["u", "x"]:
num_chars = 4 if char == "u" else 2
next_chars = self.json_str[self.index + 1 : self.index + 1 + num_chars]
if len(next_chars) == num_chars and all(c in "0123456789abcdefABCDEF" for c in next_chars):
self.log("Found a unicode escape sequence, normalizing it")
state.string_acc = state.string_acc[:-1] + chr(int(next_chars, 16))
_rebuild_unmatched_opening_braces(state)
self.index += 1 + num_chars
return True, self.get_char_at()
elif char == "„" or char in STRING_DELIMITERS and char != active_rstring_delimiter:
self.log("Found a delimiter that was escaped but shouldn't be escaped, removing the escape")
state.string_acc = state.string_acc[:-1] + char
_rebuild_unmatched_opening_braces(state)
self.index += 1
return True, self.get_char_at()
return False, char
def _brace_before_code_fence_belongs_to_string(
self: "JSONParser",
state: StringParseState,
fence_idx: int,
) -> bool:
# Distinguish trailing wrapper fences from literal fenced snippets inside the current string.
quote_search_idx = fence_idx + 3
next_content_idx = _scroll_comment_prefixed_member_start(self, quote_search_idx)
keep_post_fence_container = False
if self.get_char_at(next_content_idx) in INLINE_CONTAINER_OPENERS:
container_end_idx = _skip_inline_container(self, next_content_idx)
if container_end_idx is not None:
if _post_fence_container_starts_next_member(self, container_end_idx):
return False
keep_post_fence_container = True
quote_search_idx = container_end_idx
outer_rstring_delimiter = _outer_rstring_delimiter(state)
quote_idx = self.skip_to_character(character=outer_rstring_delimiter, idx=quote_search_idx)
while self.get_char_at(quote_idx) == outer_rstring_delimiter:
after_quote_idx = self.scroll_whitespaces(idx=quote_idx + 1)
after_quote = self.get_char_at(after_quote_idx)
if after_quote in [",", "}", "]", None]:
if keep_post_fence_container:
state.pending_inline_container = True
return True
quote_idx = self.skip_to_character(character=outer_rstring_delimiter, idx=quote_idx + 1)
return False
def _matching_string_delimiter(delimiter: str) -> str:
return "”" if delimiter == "“" else delimiter
def _bare_key_is_followed_by_colon(
self: "JSONParser",
key_idx: int,
) -> bool:
key_char = self.get_char_at(key_idx)
if not key_char or not (key_char.isalnum() or key_char == "_"):
return False
while True:
key_char = self.get_char_at(key_idx)
if not key_char or not (key_char.isalnum() or key_char in ["_", "-"]):
break
key_idx += 1
key_idx = self.scroll_whitespaces(idx=key_idx)
return self.get_char_at(key_idx) == ":"
def _post_fence_container_starts_next_member(
self: "JSONParser",
container_end_idx: int,
) -> bool:
after_container_idx = self.scroll_whitespaces(idx=container_end_idx)
after_container = self.get_char_at(after_container_idx)
if after_container in ["}", None]:
return True
if after_container != ",":
return False
next_member_idx = _scroll_comment_prefixed_member_start(self, after_container_idx + 1)
return self.get_char_at(next_member_idx) in ["}", None] or _object_member_starts_at(self, next_member_idx)
def _starts_nested_inline_container(
self: "JSONParser",
idx: int,
) -> bool:
opening_delimiter = self.get_char_at(idx)
prev_idx = idx - 1
while prev_idx >= 0:
prev_char = self.get_char_at(prev_idx)
if prev_char is None:
return True
if not prev_char.isspace():
if prev_char in INLINE_CONTAINER_OPENERS:
return True
if prev_char not in [",", ":"]:
return False
next_idx = self.scroll_whitespaces(idx=idx + 1)
next_char = self.get_char_at(next_idx)
if opening_delimiter in ["[", "("]:
return next_char in ["]", ")", *STRING_DELIMITERS, "-", *INLINE_CONTAINER_OPENERS, "t", "f", "n"] or (
next_char is not None and next_char.isdigit()
)
if opening_delimiter != "{":
return False
if next_char in ["}", *STRING_DELIMITERS]:
return True
return prev_char == ":" and _bare_key_is_followed_by_colon(self, next_idx)
prev_idx -= 1
return True
def _skip_inline_container(
self: "JSONParser",
idx: int,
) -> int | None:
opening_delimiter = self.get_char_at(idx)
if opening_delimiter not in INLINE_CONTAINER_CLOSING_DELIMITERS:
return idx
stack = [INLINE_CONTAINER_CLOSING_DELIMITERS[opening_delimiter]]
i = idx + 1
while stack:
char = self.get_char_at(i)
if not char:
return None
if char in STRING_DELIMITERS:
end_delimiter = _matching_string_delimiter(char)
i = self.skip_to_character(character=end_delimiter, idx=i + 1)
if self.get_char_at(i) != end_delimiter:
return None
elif char in INLINE_CONTAINER_CLOSING_DELIMITERS and _starts_nested_inline_container(self, i):
stack.append(INLINE_CONTAINER_CLOSING_DELIMITERS[char])
elif char == stack[-1]:
stack.pop()
if not stack:
return i + 1
i += 1
return None # pragma: no cover
def _scroll_comment_prefixed_member_start(
self: "JSONParser",
idx: int,
) -> int:
idx = self.scroll_whitespaces(idx=idx)
while True:
char = self.get_char_at(idx)
if char == "#":
while char and char not in ["\n", "\r"]:
idx += 1
char = self.get_char_at(idx)
idx = self.scroll_whitespaces(idx=idx)
continue
if char == "/":
next_char = self.get_char_at(idx + 1)
if next_char == "/":
idx += 2
char = self.get_char_at(idx)
while char and char not in ["\n", "\r"]:
idx += 1
char = self.get_char_at(idx)
idx = self.scroll_whitespaces(idx=idx)
continue
if next_char == "*":
idx += 2
while True:
char = self.get_char_at(idx)
if not char:
return idx
if char == "*" and self.get_char_at(idx + 1) == "/":
idx += 2
break
idx += 1
idx = self.scroll_whitespaces(idx=idx)
continue
return idx
def _quoted_object_member_follows(
self: "JSONParser",
quote_idx: int,
) -> bool:
comma_idx = self.scroll_whitespaces(idx=quote_idx + 1)
if self.get_char_at(comma_idx) != ",":
return False
next_member_idx = _scroll_comment_prefixed_member_start(self, comma_idx + 1)
return _object_member_starts_at(self, next_member_idx)
def _object_member_starts_at(
self: "JSONParser",
next_member_idx: int,
) -> bool:
if self.get_char_at(next_member_idx) in ["}", None]:
return False
next_member = self.get_char_at(next_member_idx)
if next_member in STRING_DELIMITERS:
key_end_delimiter = _matching_string_delimiter(next_member)
key_end_idx = self.skip_to_character(character=key_end_delimiter, idx=next_member_idx + 1)
if self.get_char_at(key_end_idx) != key_end_delimiter:
return False
after_key_idx = self.scroll_whitespaces(idx=key_end_idx + 1)
return self.get_char_at(after_key_idx) == ":"
if next_member and (next_member.isalnum() or next_member == "_"):
return _bare_key_is_followed_by_colon(self, next_member_idx)
return False
def _handle_right_delimiter_candidate(
self: "JSONParser",
state: StringParseState,
char: str,
) -> tuple[bool, str | None, bool]:
outer_rstring_delimiter = _outer_rstring_delimiter(state)
if state.doubled_quotes and self.get_char_at(1) == outer_rstring_delimiter:
self.log("While parsing a string, we found a doubled quote, ignoring it")
self.index += 1
return True, char, False
if state.missing_quotes and self.context.current == ContextValues.OBJECT_VALUE:
i = 1
next_c = self.get_char_at(i)
while next_c and next_c not in [
outer_rstring_delimiter,
state.lstring_delimiter,
]:
i += 1
next_c = self.get_char_at(i)
if next_c:
i += 1
i = self.scroll_whitespaces(idx=i)
if self.get_char_at(i) == ":":
self.index -= 1
next_char = self.get_char_at()
self.log(
"In a string with missing quotes and object value context, I found a delimeter but it turns out it was the beginning on the next key. Stopping here.",
)
return False, next_char, True
return False, char, False
if state.unmatched_delimiter:
state.unmatched_delimiter = False
next_char = _append_literal_char(self, state, char)
return True, next_char, False
i = 1
next_c = self.get_char_at(i)
check_comma_in_object_value = True
while next_c and next_c not in [
outer_rstring_delimiter,
state.lstring_delimiter,
]:
if check_comma_in_object_value and next_c.isalpha():
check_comma_in_object_value = False
if (
(ContextValues.OBJECT_KEY in self.context.context and next_c in [":", "}"])
or (ContextValues.OBJECT_VALUE in self.context.context and next_c == "}")
or (ContextValues.ARRAY in self.context.context and next_c in ["]", ","])
or (check_comma_in_object_value and self.context.current == ContextValues.OBJECT_VALUE and next_c == ",")
):
break
i += 1
next_c = self.get_char_at(i)
if next_c == "," and self.context.current == ContextValues.OBJECT_VALUE:
i += 1
i = self.skip_to_character(character=outer_rstring_delimiter, idx=i)
next_c = self.get_char_at(i)
i += 1
i = self.scroll_whitespaces(idx=i)
next_c = self.get_char_at(i)
if next_c in ["}", ","]:
self.log(
"While parsing a string, we found a misplaced quote that would have closed the string but has a different meaning here, ignoring it",
)
next_char = _append_literal_char(self, state, char)
return True, next_char, False
elif next_c == outer_rstring_delimiter and self.get_char_at(i - 1) != "\\":
if _only_whitespace_until(self, i) and not (
self.context.current == ContextValues.OBJECT_VALUE and _quoted_object_member_follows(self, i)
):
return False, char, True
if self.context.current == ContextValues.OBJECT_VALUE:
if _quoted_object_member_follows(self, i):
self.log(
"While parsing a string, we found a misplaced quote that would have closed the string but has a different meaning here, ignoring it",
)
next_char = _append_literal_char(self, state, char)
return True, next_char, False
i = self.skip_to_character(character=outer_rstring_delimiter, idx=i + 1)
i += 1
next_c = self.get_char_at(i)
while next_c and next_c != ":":
if next_c in [",", "]", "}"] or (next_c == outer_rstring_delimiter and self.get_char_at(i - 1) != "\\"):
break
i += 1
next_c = self.get_char_at(i)
if next_c != ":":
self.log(
"While parsing a string, we found a misplaced quote that would have closed the string but has a different meaning here, ignoring it",
)
state.unmatched_delimiter = not state.unmatched_delimiter
next_char = _append_literal_char(self, state, char)
return True, next_char, False
elif self.context.current == ContextValues.ARRAY:
even_delimiters = next_c == outer_rstring_delimiter
while next_c == outer_rstring_delimiter:
i = self.skip_to_character(character=[outer_rstring_delimiter, "]"], idx=i + 1)
next_c = self.get_char_at(i)
if next_c != outer_rstring_delimiter:
even_delimiters = False
break
i = self.skip_to_character(character=[outer_rstring_delimiter, "]"], idx=i + 1)
next_c = self.get_char_at(i)
if even_delimiters:
self.log(
"While parsing a string in Array context, we detected a quoted section that would have closed the string but has a different meaning here, ignoring it",
)
state.unmatched_delimiter = not state.unmatched_delimiter
next_char = _append_literal_char(self, state, char)
return True, next_char, False
return False, char, True
elif self.context.current == ContextValues.OBJECT_KEY:
self.log(
"While parsing a string in Object Key context, we detected a quoted section that would have closed the string but has a different meaning here, ignoring it",
)
next_char = _append_literal_char(self, state, char)
return True, next_char, False
return False, char, False
def _scan_string_body(
self: "JSONParser",
state: StringParseState,
) -> str | None:
outer_rstring_delimiter = _outer_rstring_delimiter(state)
def cached_skip_to_character(character: str | list[str], idx: int = 0) -> int:
return _cached_skip_to_character(self, state, character, idx)
char = self.get_char_at()
while char and (char != outer_rstring_delimiter or _in_low_smart_quote_span(state)):
if state.missing_quotes:
if self.context.current == ContextValues.OBJECT_KEY and (char == ":" or char.isspace()):
self.log(
"While parsing a string missing the left delimiter in object key context, we found a :, stopping here",
)
break
if self.context.current == ContextValues.ARRAY and char in ["]", ","]:
self.log(
"While parsing a string missing the left delimiter in array context, we found a ] or ,, stopping here",
)
break
if char == "„" and (not state.string_acc or state.string_acc[-1] != "\\"):
_push_low_smart_quote_span(state)
char = _append_literal_char(self, state, char)
continue
if _in_low_smart_quote_span(state) and char == "”":
_pop_low_smart_quote_span(state)
char = _append_literal_char(self, state, char)
continue
if (
(
state.pending_inline_container
or (
self.context.current == ContextValues.OBJECT_VALUE
and char == "{"
and self.get_char_at(-1) != "\\"
and _bare_key_is_followed_by_colon(self, self.scroll_whitespaces(idx=1))
)
)
and char in INLINE_CONTAINER_OPENERS
and (not state.string_acc or state.string_acc[-1] != "\\")
):
container_end_idx = _skip_inline_container(self, 0)
if container_end_idx is not None:
self.log(
"While parsing a string in object value context, we found a balanced inline container that belongs to the string, keeping it",
)
state.pending_inline_container = False
state.inline_container_stack.clear()
_append_string_content(state, self.json_str[self.index : self.index + container_end_idx])
self.index += container_end_idx
char = self.get_char_at()
continue
if (
not self.stream_stable
and self.context.current == ContextValues.OBJECT_VALUE
and char == ","
and not state.pending_inline_container
and not state.inline_container_stack
):
comma_classification = (
"string"
if state.object_value_has_no_future_delimiter
else classify_object_value_comma(
self,
cached_skip_to_character,
)
)
if comma_classification == "member":
self.log(
"While parsing a string missing the right delimiter in object value context, we found a comma that starts the next object member. Stopping here",
)
break
if comma_classification == "string_no_future_delimiter":
state.object_value_has_no_future_delimiter = True
state.pending_inline_container = comma_classification == "container"
self.log(
"While parsing a string in object value context, we found a comma that belongs to the string, keeping it",
)
char = _append_literal_char(self, state, char)
continue
state.pending_inline_container, keep_inline_container_char = update_inline_container_stack(
char,
state.pending_inline_container,
state.inline_container_stack,
)
if keep_inline_container_char:
char = _append_literal_char(self, state, char)
continue
if (
not self.stream_stable
and self.context.current == ContextValues.OBJECT_VALUE
and char == "}"
and (not state.string_acc or state.string_acc[-1] != outer_rstring_delimiter)
):
if state.object_value_unmatched_opening_braces:
char = _append_literal_char(self, state, char)
continue
rstring_delimiter_missing = True
self.skip_whitespaces()
if self.get_char_at(1) == "\\":
rstring_delimiter_missing = False
i = _cached_skip_to_character(self, state, outer_rstring_delimiter, idx=1)
next_c = self.get_char_at(i)
if next_c:
i += 1
i = self.scroll_whitespaces(idx=i)
next_c = self.get_char_at(i)
if not next_c or next_c in [",", "}"]:
rstring_delimiter_missing = False
else:
i = self.skip_to_character(character=state.lstring_delimiter, idx=i)
next_c = self.get_char_at(i)
if not next_c:
rstring_delimiter_missing = False
else:
i = self.scroll_whitespaces(idx=i + 1)
next_c = self.get_char_at(i)
if next_c and next_c != ":":
rstring_delimiter_missing = False
else:
i = self.skip_to_character(character=":", idx=1)
next_c = self.get_char_at(i)
if next_c:
break
i = self.scroll_whitespaces(idx=1)
j = self.skip_to_character(character="}", idx=i)
if j - i > 1:
rstring_delimiter_missing = False
if rstring_delimiter_missing:
self.log(
"While parsing a string missing the left delimiter in object value context, we found a , or } and we couldn't determine that a right delimiter was present. Stopping here",
)
break
if (
not self.stream_stable
and char == "]"
and ContextValues.ARRAY in self.context.context
and (not state.string_acc or state.string_acc[-1] != outer_rstring_delimiter)
):
i = self.skip_to_character(outer_rstring_delimiter)
if not self.get_char_at(i):
break
if self.context.current == ContextValues.OBJECT_VALUE and char == "}":
i = self.scroll_whitespaces(idx=1)
next_c = self.get_char_at(i)
if next_c == "`" and self.get_char_at(i + 1) == "`" and self.get_char_at(i + 2) == "`":
if _brace_before_code_fence_belongs_to_string(self, state, i):
self.log(
"While parsing a string in object value context, we found a literal fenced snippet after }, keeping it in the string",
)
char = _append_literal_char(self, state, char)
continue
self.log(
"While parsing a string in object value context, we found a } that closes the object before code fences, stopping here",
)
break
if not next_c:
self.log(
"While parsing a string in object value context, we found a } that closes the object, stopping here",
)
break
assert char is not None
_append_string_content(state, char)
self.index += 1
char = self.get_char_at()
if char is None:
if self.stream_stable and state.string_acc and state.string_acc[-1] == "\\":
state.string_acc = state.string_acc[:-1]
_rebuild_unmatched_opening_braces(state)
break
if state.string_acc and state.string_acc[-1] == "\\":
handled_escape, char = _normalize_escape_sequence(self, state, char)
if handled_escape:
continue
if char == ":" and not state.missing_quotes and self.context.current == ContextValues.OBJECT_KEY:
i = self.skip_to_character(character=state.lstring_delimiter, idx=1)
next_c = self.get_char_at(i)
if next_c:
i += 1
i = self.skip_to_character(character=outer_rstring_delimiter, idx=i)
next_c = self.get_char_at(i)
if next_c:
i += 1
i = self.scroll_whitespaces(idx=i)
ch = self.get_char_at(i)
if ch in [",", "}"]:
self.log(
f"While parsing a string missing the right delimiter in object key context, we found a {ch} stopping here",
)
break
else:
self.log(
"While parsing a string missing the right delimiter in object key context, we found a :, stopping here",
)
break
if _in_low_smart_quote_span(state) and char == '"':
_pop_low_smart_quote_span(state)
char = _append_literal_char(self, state, char)
continue
if (
char == outer_rstring_delimiter
and self.context.current == ContextValues.OBJECT_VALUE
and _quote_belongs_to_regex_character_class(self, state)
):
self.log(
"While parsing a string, we found a bare quote inside a regex character class, keeping it",
)
assert char is not None
char = _append_literal_char(self, state, char)
continue
if char == outer_rstring_delimiter and state.string_acc and state.string_acc[-1] != "\\":
assert char is not None
handled_delimiter, char, should_break = _handle_right_delimiter_candidate(self, state, char)
if should_break:
break
if handled_delimiter:
continue
return char
def _finalize_string_result(
self: "JSONParser",
state: StringParseState,
char: str | None,
) -> str:
outer_rstring_delimiter = _outer_rstring_delimiter(state)
if char and state.missing_quotes and self.context.current == ContextValues.OBJECT_KEY and char.isspace():
self.log(
"While parsing a string, handling an extreme corner case in which the LLM added a comment instead of valid string, invalidate the string and return an empty value",
)
self.skip_whitespaces()
if self.get_char_at() not in [":", ","]:
return ""
if char != outer_rstring_delimiter:
if not self.stream_stable:
self.log(
"While parsing a string, we missed the closing quote, ignoring",
)
state.string_acc = state.string_acc.rstrip()
else:
self.index += 1
if not self.stream_stable and (state.missing_quotes or (state.string_acc and state.string_acc[-1] == "\n")):
state.string_acc = state.string_acc.rstrip()
return state.string_acc
def parse_string(self: "JSONParser") -> JSONReturnType:
state, direct_result = _prepare_string_entry(self)
if direct_result is not NO_DIRECT_RESULT:
return cast("JSONReturnType", direct_result)
assert state is not None
char = _scan_string_body(self, state)
return _finalize_string_result(self, state, char)
def _only_whitespace_until(self: "JSONParser", end: int) -> bool:
for j in range(1, end):
c = self.get_char_at(j)
if c is not None and not c.isspace():
return False
return True
@@ -0,0 +1 @@
"""Helpers used by string parsing."""
@@ -0,0 +1,115 @@
from collections.abc import Callable
from typing import TYPE_CHECKING, Literal
from ..utils.constants import STRING_DELIMITERS # noqa: TID252
if TYPE_CHECKING:
from ..json_parser import JSONParser # noqa: TID252
ObjectValueCommaClassification = Literal["container", "member", "string", "string_no_future_delimiter"]
def _bare_member_has_recoverable_value(
parser: "JSONParser",
value_idx: int,
skip_to_character: Callable[[str | list[str], int], int],
) -> bool:
value_start_idx = parser.scroll_whitespaces(idx=value_idx)
value_start = parser.get_char_at(value_start_idx)
if value_start in [*STRING_DELIMITERS, "{", "[", "-"]:
return True
if value_start and value_start.isdigit():
return True
for literal in ["true", "false", "null"]:
if all(parser.get_char_at(value_start_idx + offset) == char for offset, char in enumerate(literal)):
value_end = parser.get_char_at(value_start_idx + len(literal))
if value_end is None or value_end.isspace() or value_end in [",", "}", "]"]:
return True
# An unquoted value is only a safe member boundary when its object closes
# before the current string can close. Otherwise prose such as
# `, floof: explanation` is more likely to belong to the string.
value_end_idx = skip_to_character([*STRING_DELIMITERS, "}"], value_start_idx)
return parser.get_char_at(value_end_idx) == "}"
def classify_object_value_comma(
parser: "JSONParser",
cached_skip_to_character: Callable[[str | list[str], int], int] | None = None,
) -> ObjectValueCommaClassification:
skip_to_character = cached_skip_to_character or parser.skip_to_character
next_idx = parser.scroll_whitespaces(idx=1)
next_c = parser.get_char_at(next_idx)
if next_c in ["}", None]:
return "member"
if next_c in STRING_DELIMITERS:
key_end_idx = parser.skip_to_character(character=next_c, idx=next_idx + 1)
if not parser.get_char_at(key_end_idx):
return "string"
key_end_idx = parser.scroll_whitespaces(idx=key_end_idx + 1)
return "member" if parser.get_char_at(key_end_idx) == ":" else "string"
if next_c == "`":
bare_key_idx = next_idx + 1
while True:
key_char = parser.get_char_at(bare_key_idx)
if not key_char or not (key_char.isalnum() or key_char in ["_", "-"]):
break
bare_key_idx += 1
bare_key_idx = parser.scroll_whitespaces(idx=bare_key_idx)
return "member" if parser.get_char_at(bare_key_idx) == ":" else "string"
if next_c and (next_c.isalnum() or next_c == "_"):
bare_key_idx = next_idx
while True:
key_char = parser.get_char_at(bare_key_idx)
if not key_char or not (key_char.isalnum() or key_char in ["_", "-"]):
break
bare_key_idx += 1
bare_key_idx = parser.scroll_whitespaces(idx=bare_key_idx)
if parser.get_char_at(bare_key_idx) == ":" and _bare_member_has_recoverable_value(
parser,
bare_key_idx + 1,
skip_to_character,
):
return "member"
if next_c in ["{", "["]:
return "container"
next_special_idx = skip_to_character([*STRING_DELIMITERS, "{", "["], next_idx)
next_special = parser.get_char_at(next_special_idx)
if not next_special:
return "string_no_future_delimiter"
if next_special in ["{", "["]:
return "string"
key_end_idx = skip_to_character(next_special, next_special_idx + 1)
if not parser.get_char_at(key_end_idx):
return "string"
key_end_idx = parser.scroll_whitespaces(idx=key_end_idx + 1)
return "member" if parser.get_char_at(key_end_idx) == ":" else "string"
def update_inline_container_stack(
char: str,
pending_inline_container: bool,
inline_container_stack: list[str],
) -> tuple[bool, bool]:
if char in ["{", "["]:
if pending_inline_container:
inline_container_stack.append(char)
return False, False
if inline_container_stack:
inline_container_stack.append(char)
if inline_container_stack and (
(char == "}" and inline_container_stack[-1] == "{") or (char == "]" and inline_container_stack[-1] == "[")
):
inline_container_stack.pop()
return pending_inline_container, True
return pending_inline_container, False
@@ -0,0 +1,35 @@
from typing import TYPE_CHECKING
if TYPE_CHECKING:
from ..json_parser import JSONParser # noqa: TID252
LITERAL_VALUES: dict[str, bool | None] = {
"true": True,
"false": False,
"null": None,
"none": None,
}
def parse_boolean_or_null(parser: "JSONParser") -> tuple[bool, bool | None]:
"""Return whether an unquoted complete literal was found and its value."""
char = (parser.get_char_at() or "").lower()
starting_index = parser.index
for literal, value in LITERAL_VALUES.items():
if char != literal[0]:
continue
parser.index = starting_index
for expected_char in literal:
if (parser.get_char_at() or "").lower() != expected_char:
break
parser.index += 1
else:
next_char = parser.get_char_at()
if next_char is None or next_char.isspace() or next_char in {",", ")", "]", "}"}:
if literal == "none":
parser.log("Converted unquoted Python None literal to JSON null")
return True, value
parser.index = starting_index
return False, None
@@ -0,0 +1,19 @@
from typing import TYPE_CHECKING
from ..utils.constants import JSONReturnType # noqa: TID252
if TYPE_CHECKING:
from ..json_parser import JSONParser # noqa: TID252
def parse_json_llm_block(parser: "JSONParser") -> JSONReturnType:
"""
Extracts and normalizes JSON enclosed in ```json ... ``` blocks.
"""
# Try to find a ```json ... ``` block
if parser.json_str[parser.index : parser.index + 7] == "```json":
i = parser.skip_to_character("`", idx=7)
if parser.json_str[parser.index + i : parser.index + i + 3] == "```":
parser.index += 7 # Move past ```json
return parser.parse_json()
return False
@@ -0,0 +1,169 @@
from typing import TYPE_CHECKING
from .utils.constants import STRING_DELIMITERS
if TYPE_CHECKING:
from .json_parser import JSONParser
def parenthesized_is_explicit_tuple(parser: "JSONParser") -> bool:
"""
Return True when the current '(' starts an explicit Python tuple literal.
Empty parentheses count as a tuple. A single grouped value like ``(1)`` does not.
"""
i = parser.index + 1
n = len(parser.json_str)
nested_parentheses = 0
square_brackets = 0
braces = 0
in_quote: str | None = None
backslashes = 0
saw_top_level_content = False
while i < n:
ch = parser.json_str[i]
if ch == "\\":
backslashes += 1
i += 1
continue
if in_quote is not None:
if ch == in_quote and backslashes % 2 == 0:
in_quote = None
backslashes = 0
i += 1
continue
if ch in STRING_DELIMITERS and backslashes % 2 == 0:
in_quote = ch
saw_top_level_content = saw_top_level_content or (
nested_parentheses == 0 and square_brackets == 0 and braces == 0
)
backslashes = 0
i += 1
continue
backslashes = 0
if (
not ch.isspace()
and ch not in [",", ")"]
and nested_parentheses == 0
and square_brackets == 0
and braces == 0
):
saw_top_level_content = True
if ch == "(":
nested_parentheses += 1
elif ch == ")":
if nested_parentheses == 0 and square_brackets == 0 and braces == 0:
return not saw_top_level_content
if nested_parentheses > 0:
nested_parentheses -= 1
elif ch == "[":
square_brackets += 1
elif ch == "]" and square_brackets > 0:
square_brackets -= 1
elif ch == "{":
braces += 1
elif ch == "}" and braces > 0:
braces -= 1
elif ch == "," and nested_parentheses == 0 and square_brackets == 0 and braces == 0:
return True
i += 1
return not saw_top_level_content
def top_level_parenthesized_can_start_value(parser: "JSONParser") -> bool:
"""
Return True when a top-level '(' looks like a standalone value rather than inline prose.
This keeps tuple support available for direct inputs and fenced blocks while avoiding
regressions on surrounding explanatory text like ``foo (clarification): {...}``.
"""
i = parser.index - 1
while i >= 0:
ch = parser.json_str[i]
if ch in "\n\r":
break
if not ch.isspace():
return False
i -= 1
idx = parser.scroll_whitespaces(idx=1)
first_inner_char = parser.get_char_at(idx)
if first_inner_char is None:
return False
inner_text = parser.json_str[parser.index + idx :].lower()
if (
first_inner_char not in [")", "{", "[", "(", *STRING_DELIMITERS]
and not first_inner_char.isdigit()
and first_inner_char not in ["-", "."]
and inner_text[:4] not in ["true", "null", "none"]
and inner_text[:5] != "false"
):
return False
i = parser.index + 1
n = len(parser.json_str)
nested_parentheses = 0
square_brackets = 0
braces = 0
in_quote: str | None = None
backslashes = 0
while i < n:
ch = parser.json_str[i]
if ch == "\\":
backslashes += 1
i += 1
continue
if in_quote is not None:
if ch == in_quote and backslashes % 2 == 0:
in_quote = None
backslashes = 0
i += 1
continue
if ch in STRING_DELIMITERS and backslashes % 2 == 0:
in_quote = ch
backslashes = 0
i += 1
continue
backslashes = 0
if ch == "(":
nested_parentheses += 1
elif ch == ")":
if nested_parentheses == 0 and square_brackets == 0 and braces == 0:
i += 1
while i < n:
trailer = parser.json_str[i]
if trailer in "\n\r":
return True
if not trailer.isspace():
return False
i += 1
return True
nested_parentheses -= 1
elif ch == "[":
square_brackets += 1
elif ch == "]" and square_brackets > 0:
square_brackets -= 1
elif ch == "{":
braces += 1
elif ch == "}" and braces > 0:
braces -= 1
i += 1
return True
@@ -0,0 +1,78 @@
from __future__ import annotations
from dataclasses import dataclass
from typing import TYPE_CHECKING, Any
if TYPE_CHECKING:
from .schema_repair import SchemaRepairer
@dataclass(frozen=True)
class ObjectSchemaConfig:
properties: dict[str, Any]
pattern_properties: dict[str, Any]
additional_properties: object | None
required: set[str]
@dataclass(frozen=True)
class ArraySchemaConfig:
items_schema: object | None
additional_items: object | None
def resolve_parser_object_schema(
repairer: SchemaRepairer | None,
schema: dict[str, Any] | bool | None,
) -> tuple[SchemaRepairer | None, dict[str, Any] | bool | None, ObjectSchemaConfig | None]:
if repairer is None or schema in (None, True):
return None, schema, None
schema = repairer.resolve_schema(schema)
if schema is False:
raise ValueError("Schema does not allow any values.")
if schema is True or not repairer.is_object_schema(schema):
return None, schema, None
return repairer, schema, object_schema_config(schema)
def resolve_parser_array_schema(
repairer: SchemaRepairer | None,
schema: dict[str, Any] | bool | None,
) -> tuple[SchemaRepairer | None, dict[str, Any] | bool | None, ArraySchemaConfig | None]:
if repairer is None or schema in (None, True):
return None, schema, None
schema = repairer.resolve_schema(schema)
if schema is False:
raise ValueError("Schema does not allow any values.")
if schema is True or not repairer.is_array_schema(schema):
return None, schema, None
return repairer, schema, array_schema_config(schema)
def object_schema_config(schema: dict[str, Any]) -> ObjectSchemaConfig:
properties = schema.get("properties", {})
if not isinstance(properties, dict):
properties = {}
pattern_properties = schema.get("patternProperties", {})
if not isinstance(pattern_properties, dict):
pattern_properties = {}
return ObjectSchemaConfig(
properties=properties,
pattern_properties=pattern_properties,
additional_properties=schema.get("additionalProperties"),
required=set(schema.get("required", [])),
)
def array_schema_config(schema: dict[str, Any]) -> ArraySchemaConfig:
if isinstance(schema.get("prefixItems"), list):
return ArraySchemaConfig(
items_schema=schema["prefixItems"],
additional_items=schema.get("items"),
)
return ArraySchemaConfig(
items_schema=schema.get("items"),
additional_items=schema.get("additionalItems"),
)
Whitespace-only changes.
@@ -0,0 +1,782 @@
from __future__ import annotations
import copy
import importlib
import json
from typing import Any, Literal, cast
from .parser_schema import array_schema_config, object_schema_config
from .utils.constants import MISSING_VALUE, JSONReturnType, MissingValueType
from .utils.pattern_properties import match_pattern_properties
SchemaRepairMode = Literal["standard", "salvage"]
SUPPORTED_SCHEMA_REPAIR_MODES: tuple[SchemaRepairMode, ...] = ("standard", "salvage")
class SchemaDefinitionError(ValueError):
"""Raised when schema metadata is invalid or unsupported."""
def normalize_schema_repair_mode(mode: str | None) -> SchemaRepairMode:
if mode is None:
return "standard"
if mode == "standard":
return "standard"
if mode == "salvage":
return "salvage"
expected = ", ".join(SUPPORTED_SCHEMA_REPAIR_MODES)
raise ValueError(f"schema_repair_mode must be one of: {expected}.")
def _require_jsonschema() -> Any:
try:
return importlib.import_module("jsonschema")
except ImportError as exc: # pragma: no cover - optional dependency
raise ValueError("jsonschema is required when using schema-aware repair.") from exc
def _require_pydantic() -> Any:
try:
return importlib.import_module("pydantic")
except ImportError as exc: # pragma: no cover - optional dependency
raise ValueError("pydantic is required when using schema models.") from exc
def _add_pydantic_field_metadata(schema_dict: dict[str, Any], model: type[Any]) -> None:
properties = schema_dict.setdefault("properties", {})
if not isinstance(properties, dict):
properties = {}
schema_dict["properties"] = properties
model_config = getattr(model, "model_config", {})
accepts_field_names = model_config.get("populate_by_name", False) or model_config.get("validate_by_name", False)
property_aliases: dict[str, list[str]] = {}
for name, field in model.model_fields.items():
validation_alias = getattr(field, "validation_alias", None)
alias = validation_alias if isinstance(validation_alias, str) else getattr(field, "alias", None)
candidates = [candidate for candidate in (alias, name) if isinstance(candidate, str)]
property_name = next((candidate for candidate in candidates if candidate in properties), None)
if property_name is None:
property_name = candidates[0] if candidates else name
if accepts_field_names and property_name != name:
property_aliases.setdefault(property_name, []).append(name)
if field.is_required():
continue
property_schema = properties.setdefault(property_name, {})
if not isinstance(property_schema, dict):
property_schema = {}
properties[property_name] = property_schema
if "default" not in property_schema:
if field.default_factory is not None:
property_schema["default"] = field.default_factory()
else:
property_schema["default"] = field.default
if property_aliases:
schema_dict["x-json-repair-property-aliases"] = property_aliases
def _prepare_schema_for_validation_node(node: Any) -> Any:
if isinstance(node, dict):
normalized = {key: _prepare_schema_for_validation_node(value) for key, value in node.items()}
items = normalized.get("items")
if isinstance(items, list):
normalized.pop("items", None)
normalized["prefixItems"] = items
additional_items = normalized.pop("additionalItems", None)
if additional_items is False:
normalized["items"] = False
elif isinstance(additional_items, dict):
normalized["items"] = additional_items
return normalized
if isinstance(node, list):
return [_prepare_schema_for_validation_node(item) for item in node]
return node
def load_schema_model(path: str) -> type[Any]:
if ":" not in path:
raise ValueError("Schema model must be in the form 'module:ClassName'.")
module_name, class_name = path.split(":", 1)
module = importlib.import_module(module_name)
model: object | None = module.__dict__.get(class_name)
if model is None or not isinstance(model, type):
raise ValueError(f"Schema model '{class_name}' not found in module '{module_name}'.")
return model
def normalize_missing_values(value: object) -> JSONReturnType:
if value is MISSING_VALUE or isinstance(value, MissingValueType):
return ""
if isinstance(value, dict):
normalized: dict[str, JSONReturnType] = {}
for key, item in value.items():
if not isinstance(key, str):
raise ValueError("Object keys must be strings.")
normalized[key] = normalize_missing_values(item)
return normalized
if isinstance(value, list):
return [normalize_missing_values(item) for item in value]
if value is None or isinstance(value, (str, int, float, bool)):
return value
raise ValueError("Value is not JSON compatible.")
def schema_from_input(schema: Any) -> dict[str, Any] | bool:
if isinstance(schema, dict):
return schema
if schema is True or schema is False:
return schema
if hasattr(schema, "model_json_schema"):
pydantic = _require_pydantic()
version = getattr(pydantic, "VERSION", getattr(pydantic, "__version__", "0"))
if int(version.split(".")[0]) < 2:
raise ValueError("pydantic v2 is required for schema models.")
schema_dict: dict[str, Any] = schema.model_json_schema()
if hasattr(schema, "model_fields"):
_add_pydantic_field_metadata(schema_dict, schema)
return schema_dict
raise ValueError("Schema must be a JSON Schema dict, boolean schema, or pydantic v2 model.")
class SchemaRepairer:
def __init__(
self,
schema: dict[str, Any] | bool,
log: list[dict[str, str]] | None,
schema_repair_mode: str = "standard",
) -> None:
self.root_schema = schema
self.log = log
self.schema_repair_mode = normalize_schema_repair_mode(schema_repair_mode)
self._validator_cache: dict[int, tuple[dict[str, Any], Any]] = {}
self._root_validator: Any | None = None
def _log(self, text: str, path: str) -> None:
if self.log is not None:
self.log.append({"text": text, "context": path})
def _get_validator(self, schema: dict[str, Any]) -> Any:
cache_key = id(schema)
cached_validator = self._validator_cache.get(cache_key)
if cached_validator is not None and cached_validator[0] is schema:
return cached_validator[1]
prepared_schema = self._prepare_schema_for_validation(schema)
root_validator = self._get_root_validator()
validator = root_validator if schema is self.root_schema else root_validator.evolve(schema=prepared_schema)
self._validator_cache[cache_key] = (schema, validator)
return validator
def _get_root_validator(self) -> Any:
if self._root_validator is not None:
return self._root_validator
prepared_root_schema = self._prepare_schema_for_validation(self.root_schema)
jsonschema = _require_jsonschema()
validator_cls = jsonschema.validators.validator_for(prepared_root_schema)
self._root_validator = validator_cls(prepared_root_schema)
return self._root_validator
def is_valid(self, value: JSONReturnType, schema: dict[str, Any] | bool) -> bool:
schema = self.resolve_schema(schema)
if schema is True:
return True
if schema is False:
return False
validator = self._get_validator(schema)
return bool(validator.is_valid(value))
def validate(self, value: JSONReturnType, schema: dict[str, Any] | bool) -> None:
schema = self.resolve_schema(schema)
if schema is True:
return
if schema is False:
raise ValueError("Schema does not allow any values.")
jsonschema = _require_jsonschema()
validator = self._get_validator(schema)
try:
validator.validate(value)
except jsonschema.exceptions.ValidationError as exc:
raise ValueError(exc.message) from exc
def resolve_schema(self, schema: object | None) -> dict[str, Any] | bool:
if schema is None:
return True
if isinstance(schema, bool):
return schema
if not isinstance(schema, dict):
raise SchemaDefinitionError("Schema must be an object.")
for key in schema:
if not isinstance(key, str):
raise SchemaDefinitionError("Schema keys must be strings.")
schema_dict = cast("dict[str, Any]", schema)
seen_schema_ids: set[int] = set()
while "$ref" in schema_dict:
ref = schema_dict["$ref"]
if not isinstance(ref, str):
raise SchemaDefinitionError("$ref must be a string.")
schema_id = id(schema_dict)
if schema_id in seen_schema_ids:
raise SchemaDefinitionError(f"Circular $ref detected: {ref}")
seen_schema_ids.add(schema_id)
resolved = self._resolve_ref(ref)
if isinstance(resolved, bool):
return resolved
schema_dict = resolved
return schema_dict
def is_object_schema(self, schema: dict[str, Any] | bool | None) -> bool:
schema = self.resolve_schema(schema)
if not isinstance(schema, dict):
return False
schema_type = schema.get("type")
if schema_type == "object":
return True
if isinstance(schema_type, list) and "object" in schema_type:
return True
return any(key in schema for key in ("properties", "patternProperties", "additionalProperties", "required"))
def is_array_schema(self, schema: dict[str, Any] | bool | None) -> bool:
schema = self.resolve_schema(schema)
if not isinstance(schema, dict):
return False
schema_type = schema.get("type")
if schema_type == "array":
return True
if isinstance(schema_type, list) and "array" in schema_type:
return True
return "items" in schema or "prefixItems" in schema
def _allows_schema_type(self, schema: dict[str, Any], schema_type: str) -> bool:
declared_type = schema.get("type")
if isinstance(declared_type, str):
return declared_type == schema_type
if isinstance(declared_type, list):
return schema_type in declared_type
if schema_type == "object":
return self.is_object_schema(schema)
# This helper is only used for object/array checks in _can_salvage_list_as_object.
return self.is_array_schema(schema)
def _can_salvage_list_as_object(self, schema: dict[str, Any]) -> bool:
return self._allows_schema_type(schema, "object") and not self._allows_schema_type(schema, "array")
def repair_value(self, value: Any, schema: dict[str, Any] | bool | None, path: str) -> JSONReturnType:
"""Apply schema rules to a parsed value, including unions, coercions, and defaults."""
schema = self.resolve_schema(schema)
if schema is True:
return normalize_missing_values(value)
if schema is False:
raise ValueError("Schema does not allow any values.")
if not schema:
return normalize_missing_values(value)
if value is MISSING_VALUE:
return self._fill_missing(schema, path)
if "allOf" in schema:
subschemas = schema["allOf"]
if not subschemas:
return normalize_missing_values(value)
repaired = self.repair_value(value, subschemas[0], path)
for subschema in subschemas[1:]:
repaired = self.repair_value(repaired, subschema, path)
return repaired
if "oneOf" in schema:
return self._repair_union(value, schema["oneOf"], path)
if "anyOf" in schema:
return self._repair_union(value, schema["anyOf"], path)
expected_type = schema.get("type")
if expected_type is None:
if self.is_object_schema(schema):
expected_type = "object"
elif self.is_array_schema(schema):
expected_type = "array"
if isinstance(expected_type, list):
return self._repair_type_union(value, expected_type, schema, path)
if expected_type == "object":
repaired = self._repair_object(value, schema, path)
elif expected_type == "array":
repaired = self._repair_array(value, schema, path)
elif isinstance(expected_type, str):
repaired = self._coerce_scalar(value, expected_type, path)
else:
repaired = normalize_missing_values(value)
return self._apply_enum_const(repaired, schema, path)
def _repair_union(self, value: Any, schemas: list[dict[str, Any] | bool], path: str) -> JSONReturnType:
last_error: Exception | None = None
for subschema in schemas:
try:
candidate = self.repair_value(copy.deepcopy(value), subschema, path)
self.validate(candidate, subschema)
except ValueError as exc:
last_error = exc
else:
return candidate
if last_error:
raise ValueError(str(last_error)) from last_error
raise ValueError("No schema matched the value.")
def _repair_type_union(
self,
value: Any,
types: list[str],
schema: dict[str, Any],
path: str,
) -> JSONReturnType:
last_error: Exception | None = None
for schema_type in types:
branch_schema = {**schema, "type": schema_type}
try:
# Keep structural schema context for repair heuristics, but validate against the narrowed branch type.
candidate = self._repair_by_type(copy.deepcopy(value), schema_type, schema, path)
candidate = self._apply_enum_const(candidate, branch_schema, path)
self.validate(candidate, branch_schema)
except ValueError as exc:
last_error = exc
else:
return candidate
if last_error:
raise ValueError(str(last_error)) from last_error
raise ValueError("No schema type matched the value.")
def _repair_by_type(self, value: Any, schema_type: str, schema: dict[str, Any], path: str) -> JSONReturnType:
if schema_type == "array":
return self._repair_array(value, schema, path)
if schema_type == "object":
return self._repair_object(value, schema, path)
return self._coerce_scalar(value, schema_type, path)
def _load_json_string_container(
self,
value: Any,
expected_type: type[object],
path: str,
unwrap_log_message: str,
salvage_log_message: str,
) -> Any:
if not isinstance(value, str):
return value
try:
parsed = json.loads(value)
except json.JSONDecodeError:
if self.schema_repair_mode != "salvage":
return value
json_repair_module = importlib.import_module(f"{__package__}.json_repair")
repaired = json_repair_module.loads(value, skip_json_loads=True)
if isinstance(repaired, expected_type):
self._log(salvage_log_message, path)
return repaired
return value
if isinstance(parsed, expected_type):
self._log(unwrap_log_message, path)
return parsed
return value
def _repair_array(self, value: Any, schema: dict[str, Any], path: str) -> JSONReturnType:
value = self._load_json_string_container(
value,
list,
path,
"Unwrapped JSON string to array to match schema",
"Repaired malformed JSON string to array to match schema",
)
if isinstance(value, list):
items: list[JSONReturnType] = value
else:
self._log("Wrapped value in array to match schema", path)
items = [normalize_missing_values(value)]
salvage_mode = self.schema_repair_mode == "salvage"
schema_config = array_schema_config(schema)
def repair_or_drop(raw_item: Any, item_schema: Any, item_path: str) -> tuple[bool, JSONReturnType]:
try:
return True, self.repair_value(raw_item, item_schema, item_path)
except SchemaDefinitionError:
raise
except ValueError:
if not salvage_mode:
raise
self._log("Dropped invalid array item while salvaging", item_path)
return False, None
if schema_config.items_schema is not None:
if isinstance(schema_config.items_schema, list):
repaired_items: list[JSONReturnType] = []
for idx, item_schema in enumerate(schema_config.items_schema):
if idx >= len(items):
break
item_path = f"{path}[{idx}]"
keep_item, repaired_value = repair_or_drop(items[idx], item_schema, item_path)
if keep_item:
repaired_items.append(repaired_value)
if len(items) > len(schema_config.items_schema):
tail = items[len(schema_config.items_schema) :]
if isinstance(schema_config.additional_items, dict):
for offset, item in enumerate(tail, start=len(schema_config.items_schema)):
item_path = f"{path}[{offset}]"
keep_item, repaired_value = repair_or_drop(item, schema_config.additional_items, item_path)
if keep_item:
repaired_items.append(repaired_value)
elif schema_config.additional_items is True or schema_config.additional_items is None:
repaired_items.extend(normalize_missing_values(item) for item in tail)
else:
for offset, _item in enumerate(tail, start=len(schema_config.items_schema)):
self._log("Dropped extra array item not covered by schema", f"{path}[{offset}]")
items = repaired_items
else:
repaired_items = []
for idx, item in enumerate(items):
item_path = f"{path}[{idx}]"
keep_item, repaired_value = repair_or_drop(item, schema_config.items_schema, item_path)
if keep_item:
repaired_items.append(repaired_value)
items = repaired_items
min_items = schema.get("minItems")
if min_items is not None and len(items) < min_items:
raise ValueError(f"Array at {path} does not meet minItems.")
return items
def _repair_object(self, value: Any, schema: dict[str, Any], path: str) -> JSONReturnType:
if (
self.schema_repair_mode == "salvage"
and isinstance(value, list)
and self._can_salvage_list_as_object(schema)
):
mapped = self._map_list_to_object(value, schema, path)
if mapped is not None:
value = mapped
elif path == "$" and len(value) == 1 and isinstance(value[0], dict):
# Conservatively unwrap the common root wrapper shape: [{...}] -> {...}.
value = value[0]
self._log("Unwrapped single-item root array to object while salvaging", path)
value = self._load_json_string_container(
value,
dict,
path,
"Unwrapped JSON string to object to match schema",
"Repaired malformed JSON string to object to match schema",
)
if not isinstance(value, dict):
raise ValueError(f"Expected object at {path}, got {type(value).__name__}.")
property_aliases = schema.get("x-json-repair-property-aliases", {})
if isinstance(property_aliases, dict):
normalized_value = dict(value)
for alias, field_names in property_aliases.items():
if not isinstance(alias, str) or not isinstance(field_names, list):
continue
provided_names = [name for name in field_names if isinstance(name, str) and name in normalized_value]
if alias in normalized_value:
for name in provided_names:
normalized_value.pop(name)
self._log(f"Dropped field-name key because alias '{alias}' is present", f"{path}.{name}")
continue
if provided_names:
name = provided_names[0]
normalized_value[alias] = normalized_value.pop(name)
self._log(f"Mapped field name to validation alias '{alias}'", f"{path}.{name}")
value = normalized_value
schema_config = object_schema_config(schema)
if self.schema_repair_mode == "salvage" and schema_config.required:
value_with_salvage_fills = dict(value)
for key in schema_config.required:
if key in value_with_salvage_fills:
continue
prop_schema = schema_config.properties.get(key)
if prop_schema is None:
continue
key_path = f"{path}.{key}"
filled, filled_value = self._fill_missing_required_for_salvage(prop_schema, key_path)
if filled:
value_with_salvage_fills[key] = filled_value
self._log("Filled missing required property while salvaging", key_path)
value = value_with_salvage_fills
missing_required = [key for key in schema_config.required if key not in value]
if missing_required:
raise ValueError(f"Missing required properties at {path}: {', '.join(missing_required)}")
repaired: dict[str, JSONReturnType] = {}
for key, prop_schema in schema_config.properties.items():
key_path = f"{path}.{key}"
if key in value:
repaired[key] = self.repair_value(value[key], prop_schema, key_path)
elif isinstance(prop_schema, dict) and "default" in prop_schema and key not in schema_config.required:
repaired[key] = self._copy_json_value(prop_schema["default"], key_path, "default")
self._log("Inserted default value for missing property", key_path)
for key, raw_value in value.items():
if key in schema_config.properties:
continue
key_path = f"{path}.{key}"
matched: list[Any] = []
unsupported_patterns: list[str] = []
if schema_config.pattern_properties:
matched, unsupported_patterns = match_pattern_properties(schema_config.pattern_properties, key)
for pattern in unsupported_patterns:
self._log(f"Skipped unsupported patternProperties regex '{pattern}'", key_path)
if matched:
repaired_value = self.repair_value(raw_value, matched[0], key_path)
for prop_schema in matched[1:]:
repaired_value = self.repair_value(repaired_value, prop_schema, key_path)
repaired[key] = repaired_value
continue
if isinstance(schema_config.additional_properties, dict):
repaired[key] = self.repair_value(
raw_value,
cast("dict[str, Any]", schema_config.additional_properties),
key_path,
)
continue
if schema_config.additional_properties is True or schema_config.additional_properties is None:
repaired[key] = normalize_missing_values(raw_value)
continue
self._log("Dropped extra property not covered by schema", key_path)
min_properties = schema.get("minProperties")
if min_properties is not None and len(repaired) < min_properties:
raise ValueError(f"Object at {path} does not meet minProperties.")
return repaired
def _map_list_to_object(
self, value: list[Any], schema: dict[str, Any], path: str
) -> dict[str, JSONReturnType] | None:
properties = schema.get("properties")
if not isinstance(properties, dict) or not properties:
return None
typed_properties: dict[str, Any] = {}
for key, prop_schema in properties.items():
if not isinstance(key, str):
raise SchemaDefinitionError("Schema object property names must be strings.")
typed_properties[key] = prop_schema
keys = list(typed_properties.keys())
if len(value) != len(keys):
return None
mapped: dict[str, JSONReturnType] = {}
for idx, key in enumerate(keys):
key_path = f"{path}.{key}"
try:
mapped[key] = self.repair_value(value[idx], typed_properties[key], key_path)
except SchemaDefinitionError:
raise
except ValueError:
return None
self._log("Mapped array to object by schema property order", path)
return mapped
def _fill_missing_required_for_salvage(self, schema: object, path: str) -> tuple[bool, JSONReturnType]:
resolved_schema = self.resolve_schema(schema)
if not isinstance(resolved_schema, dict):
return False, ""
if "default" in resolved_schema:
return True, self._copy_json_value(resolved_schema["default"], path, "default")
if "const" in resolved_schema:
return True, self._copy_json_value(resolved_schema["const"], path, "const")
enum_values = resolved_schema.get("enum")
if enum_values:
return True, self._copy_json_value(enum_values[0], path, "enum")
expected_type = resolved_schema.get("type")
if expected_type is None:
if self.is_array_schema(resolved_schema):
expected_type = "array"
elif self.is_object_schema(resolved_schema):
expected_type = "object"
if expected_type == "array" and not resolved_schema.get("minItems"):
return True, []
if expected_type == "object" and not resolved_schema.get("minProperties"):
return True, {}
return False, ""
def _fill_missing(self, schema: dict[str, Any], path: str) -> JSONReturnType:
if "const" in schema:
# Const/enum/default have priority over type inference.
self._log("Filled missing value with const", path)
return self._copy_json_value(schema["const"], path, "const")
if "enum" in schema:
enum_values = schema["enum"]
if not enum_values:
raise ValueError(f"Enum at {path} has no values.")
self._log("Filled missing value with first enum value", path)
return self._copy_json_value(enum_values[0], path, "enum")
if "default" in schema:
self._log("Filled missing value with default", path)
return self._copy_json_value(schema["default"], path, "default")
expected_type = schema.get("type")
if isinstance(expected_type, list):
for schema_type in expected_type:
try:
return self._fill_missing({**schema, "type": schema_type}, path)
except ValueError:
continue
raise ValueError(f"Cannot infer missing value at {path}.")
if expected_type is None:
# Infer container types based on schema shape if type is omitted.
if self.is_object_schema(schema):
expected_type = "object"
elif self.is_array_schema(schema):
expected_type = "array"
if expected_type == "string":
self._log("Filled missing value with empty string", path)
return ""
if expected_type in ("integer", "number"):
self._log("Filled missing value with 0", path)
return 0
if expected_type == "boolean":
self._log("Filled missing value with false", path)
return False
if expected_type == "array":
min_items = schema.get("minItems")
if min_items:
raise ValueError(f"Array at {path} requires at least {min_items} items.")
self._log("Filled missing value with empty array", path)
return []
if expected_type == "object":
min_properties = schema.get("minProperties")
if min_properties:
raise ValueError(f"Object at {path} requires at least {min_properties} properties.")
self._log("Filled missing value with empty object", path)
return {}
if expected_type == "null":
self._log("Filled missing value with null", path)
return None
raise ValueError(f"Cannot infer missing value at {path}.")
def _coerce_scalar(self, value: Any, schema_type: str, path: str) -> JSONReturnType:
if schema_type == "string":
if isinstance(value, str):
return value
if isinstance(value, (int, float)) and not isinstance(value, bool):
self._log("Coerced number to string", path)
return str(value)
raise ValueError(f"Expected string at {path}.")
if schema_type == "integer":
if isinstance(value, bool):
raise ValueError(f"Expected integer at {path}.")
if isinstance(value, int):
return value
if isinstance(value, float):
if value.is_integer():
self._log("Coerced number to integer", path)
return int(value)
raise ValueError(f"Expected integer at {path}.")
if isinstance(value, str):
try:
int_value = int(value)
except ValueError:
int_value = None
if int_value is not None:
self._log("Coerced string to integer", path)
return int_value
try:
num = float(value)
except ValueError as exc:
raise ValueError(f"Expected integer at {path}.") from exc
if not num.is_integer():
raise ValueError(f"Expected integer at {path}.")
self._log("Coerced number to integer", path)
return int(num)
raise ValueError(f"Expected integer at {path}.")
if schema_type == "number":
if isinstance(value, bool):
raise ValueError(f"Expected number at {path}.")
if isinstance(value, (int, float)):
return value
if isinstance(value, str):
try:
float_value = float(value)
except ValueError as exc:
raise ValueError(f"Expected number at {path}.") from exc
self._log("Coerced string to number", path)
return float_value
raise ValueError(f"Expected number at {path}.")
if schema_type == "boolean":
if isinstance(value, bool):
return value
if isinstance(value, str):
lowered = value.lower()
if lowered in ("true", "yes", "y", "on", "1"):
self._log("Coerced string to boolean", path)
return True
if lowered in ("false", "no", "n", "off", "0"):
self._log("Coerced string to boolean", path)
return False
if isinstance(value, (int, float)) and not isinstance(value, bool) and value in (0, 1):
self._log("Coerced number to boolean", path)
return bool(value)
raise ValueError(f"Expected boolean at {path}.")
if schema_type == "null":
if value is None:
return None
raise ValueError(f"Expected null at {path}.")
raise SchemaDefinitionError(f"Unsupported schema type {schema_type} at {path}.")
def _apply_enum_const(self, value: JSONReturnType, schema: dict[str, Any], path: str) -> JSONReturnType:
if "const" in schema and value != schema["const"]:
raise ValueError(f"Value at {path} does not match const.")
if "enum" in schema and value not in schema["enum"]:
raise ValueError(f"Value at {path} does not match enum.")
return value
def _resolve_ref(self, ref: str) -> dict[str, Any] | bool:
if not ref.startswith("#/"):
raise SchemaDefinitionError(f"Unsupported $ref: {ref}")
parts = ref.lstrip("#/").split("/")
current: Any = self.root_schema
for part in parts:
resolved_part = part.replace("~1", "/").replace("~0", "~")
if not isinstance(current, dict) or resolved_part not in current:
raise SchemaDefinitionError(f"Unresolvable $ref: {ref}")
current = current[resolved_part]
if isinstance(current, dict):
return current
if current is True:
return True
if current is False:
return False
raise SchemaDefinitionError(f"Unresolvable $ref: {ref}")
def _copy_json_value(self, value: Any, path: str, label: str) -> JSONReturnType:
if value is None or isinstance(value, (str, int, float, bool)):
return value
if isinstance(value, list):
return [self._copy_json_value(item, f"{path}[{idx}]", label) for idx, item in enumerate(value)]
if isinstance(value, dict):
copied: dict[str, JSONReturnType] = {}
for key, item in value.items():
if not isinstance(key, str):
raise ValueError(f"{label.capitalize()} value at {path} contains a non-string key.")
copied[key] = self._copy_json_value(item, f"{path}.{key}", label)
return copied
raise ValueError(f"{label.capitalize()} value at {path} is not JSON compatible.")
def _prepare_schema_for_validation(self, schema: object) -> dict[str, Any]:
normalized = _prepare_schema_for_validation_node(schema)
if not isinstance(normalized, dict):
raise ValueError("Schema must be an object.")
return normalized
@@ -0,0 +1 @@
"""Internal utility helpers for json_repair."""
@@ -0,0 +1,15 @@
from typing import Any
class MissingValueType:
def __repr__(self) -> str:
return "<MISSING_VALUE>"
def __deepcopy__(self, memo: dict[int, Any]) -> "MissingValueType":
return self
MISSING_VALUE = MissingValueType()
JSONReturnType = dict[str, Any] | list[Any] | str | float | int | bool | None
STRING_DELIMITERS: list[str] = ['"', "'", "“", "”"]
@@ -0,0 +1,78 @@
from enum import Enum, auto
from types import TracebackType
from typing import Literal
class ContextValues(Enum):
OBJECT_KEY = auto()
OBJECT_VALUE = auto()
ARRAY = auto()
class _JsonContextEntry:
__slots__ = ("context", "value")
def __init__(self, context: "JsonContext", value: ContextValues) -> None:
self.context = context
self.value = value
def __enter__(self) -> None:
self.context.set(self.value)
def __exit__(
self,
_exc_type: type[BaseException] | None,
_exc: BaseException | None,
_traceback: TracebackType | None,
) -> Literal[False]:
self.context.reset()
return False
class JsonContext:
def __init__(self) -> None:
self.context: list[ContextValues] = []
self.current: ContextValues | None = None
self.empty: bool = True
def enter(self, value: ContextValues) -> _JsonContextEntry:
return _JsonContextEntry(self, value)
def set(self, value: ContextValues) -> None:
"""
Set a new context value.
Args:
value (ContextValues): The context value to be added.
Returns:
None
"""
self.context.append(value)
self.current = value
self.empty = False
def reset(self) -> None:
"""
Remove the most recent context value.
Returns:
None
"""
try:
self.context.pop()
self.current = self.context[-1]
except IndexError:
self.current = None
self.empty = True
def clear(self) -> None:
"""
Remove all context values.
Returns:
None
"""
self.context.clear()
self.current = None
self.empty = True
@@ -0,0 +1,47 @@
from typing import Any
class ObjectComparer: # pragma: no cover
def __init__(self) -> None:
pass # No operation performed in the constructor
@staticmethod
def is_same_object(obj1: Any, obj2: Any) -> bool:
"""
Recursively compares two objects and ensures that:
- Their types match
- Their keys/structure match
"""
if type(obj1) is not type(obj2):
# Fail immediately if the types don't match
return False
if isinstance(obj1, dict):
# Check that both are dicts and same length
if not isinstance(obj2, dict) or len(obj1) != len(obj2):
return False
for key in obj1:
if key not in obj2:
return False
# Recursively compare each value
if not ObjectComparer.is_same_object(obj1[key], obj2[key]):
return False
return True
if isinstance(obj1, list):
# Check that both are lists and same length
if not isinstance(obj2, list) or len(obj1) != len(obj2):
return False
# Recursively compare each item
return all(ObjectComparer.is_same_object(obj1[i], obj2[i]) for i in range(len(obj1)))
# For atomic values: types already match, so return True
return True
@staticmethod
def is_strictly_empty(value: Any) -> bool:
"""
Returns True if value is an empty container (str, list, dict, set, tuple).
Returns False for non-containers like None, 0, False, etc.
"""
return isinstance(value, str | list | dict | set | tuple) and len(value) == 0
@@ -0,0 +1,50 @@
from collections.abc import Mapping
from typing import Any
_UNSUPPORTED_REGEX_TOKENS = frozenset({".", "^", "$", "*", "+", "?", "{", "}", "[", "]", "|", "(", ")", "\\"})
def match_pattern_properties(
pattern_properties: Mapping[str, Any],
key: str,
) -> tuple[list[Any], list[str]]:
"""Match JSON Schema patternProperties using a safe literal+anchor subset.
Supported forms:
- "token" -> key contains token
- "^token" -> key starts with token
- "token$" -> key ends with token
- "^token$" -> key equals token
Any pattern using additional regex tokens is treated as unsupported and skipped.
The caller can log unsupported patterns using the returned list.
"""
if not pattern_properties:
return [], []
matched_schemas: list[Any] = []
unsupported_patterns: list[str] = []
for pattern, schema in pattern_properties.items():
anchored_start = pattern.startswith("^")
anchored_end = pattern.endswith("$")
literal = pattern[1 if anchored_start else 0 : -1 if anchored_end else None]
if any(token in literal for token in _UNSUPPORTED_REGEX_TOKENS):
unsupported_patterns.append(pattern)
continue
if anchored_start and anchored_end:
is_match = key == literal
elif anchored_start:
is_match = key.startswith(literal)
elif anchored_end:
is_match = key.endswith(literal)
else:
is_match = literal in key
if is_match:
matched_schemas.append(schema)
return matched_schemas, unsupported_patterns
@@ -0,0 +1,183 @@
import os
from typing import TextIO
class StringFileWrapper:
# This is a trick to simplify the code, transform the filedescriptor handling into a string handling
def __init__(self, fd: TextIO, chunk_length: int) -> None:
"""
Initialize the StringFileWrapper with a file descriptor and chunk length.
Args:
fd (TextIO): The file descriptor to wrap.
CHUNK_LENGTH (int): The length of each chunk to read from the file.
Attributes:
fd (TextIO): The wrapped file descriptor.
length (int): The total length of the file content.
buffers (dict[int, str]): Dictionary to store chunks of file content.
buffer_length (int): The length of each buffer chunk.
"""
self.fd = fd
# Buffers are chunks of text read from the file and cached to reduce disk access.
self.buffers: dict[int, str] = {}
if not chunk_length or chunk_length < 2:
chunk_length = 1_000_000
# chunk_length now refers to the number of characters per chunk.
self.buffer_length = chunk_length
# Keep track of the starting file position ("cookie") for each chunk so we can
# seek safely without landing in the middle of a multibyte code point.
self._initial_position = fd.tell()
self._chunk_positions: list[int] = [self._initial_position]
self.length: int | None = None
def get_buffer(self, index: int) -> str:
"""
Retrieve or load a buffer chunk from the file.
Args:
index (int): The index of the buffer chunk to retrieve.
Returns:
str: The buffer chunk at the specified index.
"""
if index < 0:
raise IndexError("Negative indexing is not supported")
cached = self.buffers.get(index)
if cached is not None:
return cached
self._ensure_chunk_position(index)
start_pos = self._chunk_positions[index]
self.fd.seek(start_pos)
chunk = self.fd.read(self.buffer_length)
if not chunk:
raise IndexError("Chunk index out of range")
end_pos = self.fd.tell()
if len(self._chunk_positions) <= index + 1:
self._chunk_positions.append(end_pos)
if len(chunk) < self.buffer_length:
self.length = index * self.buffer_length + len(chunk)
self.buffers[index] = chunk
# Save memory by keeping max 2MB buffer chunks and min 2 chunks
max_buffers = max(2, int(2_000_000 / self.buffer_length))
if len(self.buffers) > max_buffers:
oldest_key = next(iter(self.buffers))
if oldest_key != index:
self.buffers.pop(oldest_key)
return chunk
def __getitem__(self, index: int | slice) -> str:
"""
Retrieve a character or a slice of characters from the file.
Args:
index (Union[int, slice]): The index or slice of characters to retrieve.
Returns:
str: The character(s) at the specified index or slice.
"""
# The buffer is an array that is seek like a RAM:
# self.buffers[index]: the row in the array of length 1MB, index is `i` modulo CHUNK_LENGTH
# self.buffures[index][j]: the column of the row that is `i` remainder CHUNK_LENGTH
if isinstance(index, slice):
start, stop, step = self._normalize_slice(index)
if step == 0:
raise ValueError("slice step cannot be zero")
if step != 1:
return "".join(self[i] for i in range(start, stop, step))
if start >= stop:
return ""
return self._slice_from_buffers(start, stop)
if index < 0:
index += len(self)
if index < 0:
raise IndexError("string index out of range")
buffer_index = index // self.buffer_length
buffer = self.get_buffer(buffer_index)
return buffer[index % self.buffer_length]
def __len__(self) -> int:
"""
Get the total length of the file.
Returns:
int: The total number of characters in the file.
"""
if self.length is None:
while self.length is None:
chunk_index = len(self._chunk_positions)
self._ensure_chunk_position(chunk_index)
assert self.length is not None
return self.length
def _normalize_slice(self, index: slice) -> tuple[int, int, int]:
total_len = len(self)
start = 0 if index.start is None else index.start
stop = total_len if index.stop is None else index.stop
step = 1 if index.step is None else index.step
if start < 0:
start += total_len
if stop < 0:
stop += total_len
start = max(start, 0)
stop = min(stop, total_len)
return start, stop, step
def _slice_from_buffers(self, start: int, stop: int) -> str:
buffer_index = start // self.buffer_length
buffer_end = (stop - 1) // self.buffer_length
start_mod = start % self.buffer_length
stop_mod = stop % self.buffer_length
if stop_mod == 0 and stop > start:
stop_mod = self.buffer_length
if buffer_index == buffer_end:
buffer = self.get_buffer(buffer_index)
return buffer[start_mod:stop_mod]
start_slice = self.get_buffer(buffer_index)[start_mod:]
end_slice = self.get_buffer(buffer_end)[:stop_mod]
middle_slices = [self.get_buffer(i) for i in range(buffer_index + 1, buffer_end)]
return start_slice + "".join(middle_slices) + end_slice
def __setitem__(self, index: int | slice, value: str) -> None: # pragma: no cover
"""
Set a character or a slice of characters in the file.
Args:
index (slice): The slice of characters to set.
value (str): The value to set at the specified index or slice.
"""
start = index.start or 0 if isinstance(index, slice) else index or 0
if start < 0:
start += len(self)
current_position = self.fd.tell()
self.fd.seek(self._initial_position + start)
self.fd.write(value)
self.fd.seek(current_position)
def _ensure_chunk_position(self, chunk_index: int) -> None:
"""
Ensure that we know the starting file position for the given chunk index.
"""
while len(self._chunk_positions) <= chunk_index:
prev_index = len(self._chunk_positions) - 1
start_pos = self._chunk_positions[-1]
self.fd.seek(start_pos, os.SEEK_SET)
chunk = self.fd.read(self.buffer_length)
end_pos = self.fd.tell()
if len(chunk) < self.buffer_length:
self.length = prev_index * self.buffer_length + len(chunk)
self._chunk_positions.append(end_pos)
if not chunk:
break
if len(self._chunk_positions) <= chunk_index:
raise IndexError("Chunk index out of range")