amorfo77/json
1
0 Fork 0
mirror of https://github.com/nlohmann/json.git synced 2026-10-28 00:35:43 +00:00
Code Issues Packages Projects Releases Wiki Activity

json_view: source positions in materialize() with JSON_DIAGNOSTIC_POSITIONS

Browse Source 
With JSON_DIAGNOSTIC_POSITIONS, materialize() now gives every value the
start_pos()/end_pos() that basic_json::parse() would give it, so exceptions
about materialized values name the same bytes (#5864).

- json_sax_dom_parser takes the lexer type as a defaulted template
  parameter; materialize() passes position_source, which reports the
  positions it derives from the index. diagnostic_positions treats npos as
  "no position" and keeps ranges all-or-nothing.
- Strings with escapes keep the offset of their opening quote plus one in
  `next` (only with the macro); materialize() scans from there to the closing
  quote. The end of an array or object is found by scanning from the end of
  its last element over whitespace, comments and a trailing comma.
- In editable documents, values written by edits, arrays and objects whose
  elements changed, and containers whose last element has no position get
  none. Images keep the flag is_new, and load() checks that `next` of a
  string is within the text.

Nothing changes on the parse path without the macro. The view's own
exceptions still carry no byte range, so the tests that compare them with
those of materialized values skip it under the macro, as they skip the path
under JSON_DIAGNOSTICS.

Signed-off-by: Niels Lohmann <mail@nlohmann.me>
This commit is contained in:
Niels Lohmann committed 2026-10-27 17:00:22 +01:00
1 parent ebf5a176c4
commit 4c130c33d9
16 files changed
+947 -69
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

include/nlohmann/detail/view/materialize.hpp
+225 -4
View File
@@ -26,6 +26,176 @@ namespace detail
namespace view
{
#if JSON_DIAGNOSTIC_POSITIONS
/*!
@brief the source positions of the values materialize() replays
Stands in for the lexer of json_sax_dom_parser, which reads the positions of
the values it stores from these three functions (diagnostic_positions). npos
is a value without a source position.
*/
struct position_source
{
/// the token, of which diagnostic_positions only reads the length
struct token
{
std::size_t length;
std::size_t size() const noexcept
{
return length;
}
};
std::size_t start = std::string::npos; ///< the first byte of the next value
std::size_t end = std::string::npos; ///< one past the next value (start_object/start_array: one past the bracket)
std::size_t get_chars_read_total() const noexcept
{
return end;
}
std::size_t get_token_start_position() const noexcept
{
return start;
}
token get_string() const noexcept
{
return token{end - start};
}
/// the range [first, last), or none if either is npos
void set(std::size_t first, std::size_t last) noexcept
{
const bool known = first != std::string::npos && last != std::string::npos;
start = known ? first : std::string::npos;
end = known ? last : std::string::npos;
}
/// one past the closing quote of the string whose opening quote is at
/// src[quote], or npos (the scan is bounded by size, as images can be
/// loaded without a check)
static std::size_t string_end(const char* src, std::size_t size, std::size_t quote) noexcept
{
for (std::size_t i = quote + 1; i < size; ++i)
{
if (src[i] == '\\')
{
++i;
}
else if (src[i] == '"')
{
return i + 1;
}
}
return std::string::npos;
}
/// one past the closing bracket of the array or object whose last element
/// ends at src[pos] (or whose opening bracket is at src[pos - 1]), or npos;
/// in between, there can only be whitespace, comments, and a trailing comma
/// (the source was validated when it was parsed)
static std::size_t container_end(const char* src, std::size_t size, std::size_t pos) noexcept
{
while (pos < size)
{
switch (src[pos])
{
case ' ':
case '\t':
case '\n':
case '\r':
case ',':
++pos;
break;
case ']':
case '}':
return pos + 1;
case '/':
if (pos + 1 < size && src[pos + 1] == '/')
{
pos += 2;
while (pos < size && src[pos] != '\n' && src[pos] != '\r')
{
++pos;
}
break;
}
if (pos + 1 < size && src[pos + 1] == '*')
{
pos += 2;
while (pos < size && !(src[pos] == '*' && pos + 1 < size && src[pos + 1] == '/'))
{
++pos;
}
pos += 2;
break;
}
return std::string::npos;
default:
return std::string::npos;
}
}
return std::string::npos;
}
/// the range of the scalar n of document d
void set_scalar(const document_data& d, const node& n) noexcept
{
const std::size_t size = d.size;
const std::size_t off = n.off;
std::size_t first = std::string::npos;
std::size_t last = std::string::npos;
if ((n.flags & node_flags::is_new) == 0)
{
switch (static_cast<value_t>(n.kind))
{
case value_t::string:
if ((n.flags & node_flags::storage) == 0)
{
// off is the first byte after the opening quote
if (off != 0)
{
first = off - 1;
last = off + std::size_t{n.len} + 1;
}
}
else if ((n.flags & node_flags::storage) == node_flags::escaped && n.next != 0)
{
// next is the offset of the opening quote plus one
first = n.next - 1;
last = string_end(d.base[0], size, first);
}
break;
case value_t::number_integer:
case value_t::number_unsigned:
case value_t::number_float:
if ((n.flags & node_flags::storage) == 0)
{
first = off;
last = off + number_length(n);
}
break;
case value_t::boolean:
first = off;
last = off + ((n.flags & node_flags::is_true) != 0 ? 4u : 5u);
break;
case value_t::null:
first = off;
last = off + 4u;
break;
case value_t::object:
case value_t::array:
case value_t::binary:
case value_t::discarded:
default:
break;
}
}
// (a crafted image can point past the text)
set(first, last != std::string::npos && last <= size ? last : std::string::npos);
}
};
#endif
/*!
@brief the basic_json value of the subtree at n
@@ -33,14 +203,22 @@ The subtree is replayed into the SAX handler that parse() uses to build its
values, so the result is the value parse() would produce: duplicate keys keep
the last value, and with JSON_DIAGNOSTICS the parent pointers are set. It is
iterative, so the nesting depth is limited by memory only, as for parse().
Without a lexer the handler records no source positions
(JSON_DIAGNOSTIC_POSITIONS).
With JSON_DIAGNOSTIC_POSITIONS, the values get their source positions as with
parse(): position_source stands in for the lexer. Values written by edits have
none (npos), and neither has an array or object whose elements changed, or
whose last element has none (the end of its closing bracket is found by
scanning from the end of the last element).
*/
template<typename BasicJsonType, bool Editable>
BasicJsonType materialize(const document_data& d, const node* n)
{
using string_t = typename BasicJsonType::string_t;
#if JSON_DIAGNOSTIC_POSITIONS
using sax_t = json_sax_dom_parser<BasicJsonType, iterator_input_adapter<const char*>, true, position_source>;
#else
using sax_t = json_sax_dom_parser<BasicJsonType, iterator_input_adapter<const char*>, true>;
#endif
using nav = navigation<Editable>;
// the open containers: their next element (or key of the next member),
@@ -52,9 +230,15 @@ BasicJsonType materialize(const document_data& d, const node* n)
const node* pos;
const node* end;
bool object;
#if JSON_DIAGNOSTIC_POSITIONS
std::size_t start; ///< the opening bracket (npos: unknown)
std::size_t last_end; ///< the end of the last element read (npos: unknown)
#endif
};
#if !JSON_DIAGNOSTIC_POSITIONS
// a scalar needs neither the handler nor the stack of open containers
// (with positions, it goes through the handler, which sets them)
switch (static_cast<value_t>(n->kind))
{
case value_t::string:
@@ -76,10 +260,16 @@ BasicJsonType materialize(const document_data& d, const node* n)
default:
break;
}
#endif
BasicJsonType result;
#if JSON_DIAGNOSTIC_POSITIONS
position_source positions;
sax_t sax(result, true, &positions);
#else
sax_t sax(result, true);
const string_t no_token{};
#endif
const string_t no_token {};
// (shallow subtrees on the stack: a call for a small value then allocates only for the value)
std::array<frame, 16> shallow; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init): filled as containers open
std::vector<frame> deep{};
@@ -94,6 +284,16 @@ BasicJsonType materialize(const document_data& d, const node* n)
std::size_t depth = 0;
for (;;)
{
#if JSON_DIAGNOSTIC_POSITIONS
if (!is_container(*n))
{
positions.set_scalar(d, *n);
if (depth != 0)
{
open[depth - 1].last_end = positions.end;
}
}
#endif
// false positive: n comes from nav::value(), which never returns null for a valid index
// @infer-ignore NULLPTR_DEREFERENCE
switch (static_cast<value_t>(n->kind))
@@ -102,6 +302,12 @@ BasicJsonType materialize(const document_data& d, const node* n)
case value_t::array:
{
const bool object = n->kind == node_kind::object;
#if JSON_DIAGNOSTIC_POSITIONS
// (moved containers are also new)
const std::size_t start = (n->flags & node_flags::is_new) == 0 && n->off < d.size ? std::size_t{n->off} :
std::string::npos;
positions.set(start, start + 1);
#endif
if (object)
{
sax.start_object(n->len);
@@ -118,8 +324,14 @@ BasicJsonType materialize(const document_data& d, const node* n)
open = deep.data();
capacity = deep.size();
}
open[depth++] = frame{nav::first(d, n), nav::end(d, n), object};
#if JSON_DIAGNOSTIC_POSITIONS
// (until an element is read, the container's own bracket is the last thing read)
open[depth++] = frame{nav::first(d, n), nav::end(d, n), object, start, positions.end};
break;
#else
open[depth++] = frame {nav::first(d, n), nav::end(d, n), object};
break;
#endif
}
case value_t::string:
{
@@ -159,6 +371,15 @@ BasicJsonType materialize(const document_data& d, const node* n)
frame& f = open[depth - 1];
if (f.pos == f.end)
{
#if JSON_DIAGNOSTIC_POSITIONS
const std::size_t end = f.start == std::string::npos || f.last_end == std::string::npos ? std::string::npos : position_source::container_end(d.base[0], d.size, f.last_end);
// (an end of npos clears the start, see diagnostic_positions)
positions.set(end == std::string::npos ? end : end - 1, end);
if (depth > 1)
{
open[depth - 2].last_end = end;
}
#endif
if (f.object)
{
sax.end_object();