diff --git a/docs/mkdocs/docs/api/basic_json_view/materialize.md b/docs/mkdocs/docs/api/basic_json_view/materialize.md
index ba57e4f47..982b1597f 100644
--- a/docs/mkdocs/docs/api/basic_json_view/materialize.md
+++ b/docs/mkdocs/docs/api/basic_json_view/materialize.md
@@ -43,9 +43,13 @@ and [`end_pos()`](../basic_json/end_pos.md) return `std::string::npos` for
- an array or object whose elements were changed (its elements keep theirs), and
- an array or object whose last element has no positions (its end is found from that element).
-A `basic_json` value, in contrast, keeps its positions when it is changed. Documents [loaded](../basic_json_document/load.md)
-from an image have the positions of the saved document, except that its strings with escapes have none if it was saved
-without `JSON_DIAGNOSTIC_POSITIONS`.
+This differs from a `basic_json` value, whose positions are not updated when it is changed: they stay as they were
+parsed, even if they no longer match the value (see [`start_pos()`](../basic_json/start_pos.md)).
+
+Documents [loaded](../basic_json_document/load.md) from an image have the positions of the saved document. Its strings
+with escapes have positions only if the document was parsed with `JSON_DIAGNOSTIC_POSITIONS`: the position of their
+opening quote is recorded while parsing, and `save()` and `load()` keep it whether or not they are compiled with the
+macro.
Calling `materialize()` on the same view repeatedly builds a new, independent `BasicJsonType` value each time; it
never caches the result.
diff --git a/docs/mkdocs/docs/home/architecture.md b/docs/mkdocs/docs/home/architecture.md
index f27b45b97..76b3e1af1 100644
--- a/docs/mkdocs/docs/home/architecture.md
+++ b/docs/mkdocs/docs/home/architecture.md
@@ -203,7 +203,7 @@ packet-beta
| 2-3 | `extra` | `uint16_t` | numbers: the number of integer digits (low byte) and fraction digits (high byte), 255 for more; objects: the number of their key hash index (1-based), or 0; otherwise 0 |
| 4-7 | `off` | `uint32_t` | where the value starts: the first byte after a string's opening quote (or its position in the buffer of decoded strings), the first byte of a number or literal, the bracket of an array or object |
| 8-11 | `len` | `uint32_t` | strings: the length after decoding; floats and literals: the length of the token; arrays and objects: the number of elements |
-| 12-15 | `next` | `uint32_t` | arrays and objects: the number of nodes of the subtree, including the node itself; strings with escapes: the position of the opening quote in the source plus one, or 0 (only with [`JSON_DIAGNOSTIC_POSITIONS`](../api/macros/json_diagnostic_positions.md)) |
+| 12-15 | `next` | `uint32_t` | arrays and objects: the number of nodes of the subtree, including the node itself; strings with escapes: the position of the opening quote in the source plus one if the document was parsed with [`JSON_DIAGNOSTIC_POSITIONS`](../api/macros/json_diagnostic_positions.md), else 0; otherwise 0 |
Fields of a node, by their byte offset within the 16-byte node. Integers use bytes 8-15 differently, see below.
@@ -219,8 +219,9 @@ packet-beta
- **Offsets** are 32 bits wide, so a document is limited to 4294967279 bytes, 4 GiB minus 16 bytes (a margin below
2^32 for positions one scanner step past the end of the text; `out_of_range.416`).
- **Source positions** ([`JSON_DIAGNOSTIC_POSITIONS`](../api/macros/json_diagnostic_positions.md)) are derived by
- [`materialize()`](../api/basic_json_view/materialize.md) and cost nothing while parsing: a scalar's range follows
- from `off` and its length, a string with escapes is scanned from its opening quote (`next`) to the closing one,
+ [`materialize()`](../api/basic_json_view/materialize.md) and cost almost nothing while parsing: the only work is
+ storing the offset of the opening quote of a string with escapes in its `next`, which is otherwise unused. A
+ scalar's range follows from `off` and its length, a string with escapes is scanned from its opening quote (`next`) to the closing one,
and the end of an array or object is found by scanning from the end of its last element over whitespace, comments,
and a trailing comma. `materialize()` hands these positions to the SAX builder of `parse()` in place of its lexer.
- **Large objects** (128 members or more) get a hash index for their keys after parsing
diff --git a/include/nlohmann/detail/input/json_sax.hpp b/include/nlohmann/detail/input/json_sax.hpp
index 1a528e36b..0ceec61b7 100644
--- a/include/nlohmann/detail/input/json_sax.hpp
+++ b/include/nlohmann/detail/input/json_sax.hpp
@@ -233,7 +233,7 @@ struct diagnostic_positions
// (npos: a value without a source position, see materialize() of
// json_view)
const std::size_t read = lexer->get_chars_read_total();
- v.start_position = read == std::string::npos ? read : read - 1;
+ v.start_position = read == std::string::npos ? std::string::npos : read - 1;
}
}
@@ -252,7 +252,7 @@ struct diagnostic_positions
{
// a range has both ends or none (exception::diagnostics()
// prints only complete ones)
- v.start_position = v.end_position;
+ v.start_position = std::string::npos;
}
}
}
@@ -273,7 +273,7 @@ struct diagnostic_positions
if (v.end_position == std::string::npos)
{
// a value without a source position
- v.start_position = v.end_position;
+ v.start_position = std::string::npos;
return;
}
diff --git a/include/nlohmann/detail/view/builder.hpp b/include/nlohmann/detail/view/builder.hpp
index 34fb0f304..aa8605966 100644
--- a/include/nlohmann/detail/view/builder.hpp
+++ b/include/nlohmann/detail/view/builder.hpp
@@ -1196,10 +1196,11 @@ indent_done:
#if JSON_DIAGNOSTIC_POSITIONS
// the decoded string is in the arena: next keeps the source offset
// of the opening quote plus one (0: unknown) for materialize()
- push_node(node_kind::string, node_flags::escaped, 0, r.start, static_cast(r.len) | (static_cast(s - b) << 32));
+ const std::uint64_t second = static_cast(r.len) | (static_cast(s - b) << 32);
#else
- push_node(node_kind::string, node_flags::escaped, 0, r.start, r.len);
+ const std::uint64_t second = r.len;
#endif
+ push_node(node_kind::string, node_flags::escaped, 0, r.start, second);
return true;
}
};
diff --git a/include/nlohmann/detail/view/materialize.hpp b/include/nlohmann/detail/view/materialize.hpp
index ef403dfdd..0b7d91d71 100644
--- a/include/nlohmann/detail/view/materialize.hpp
+++ b/include/nlohmann/detail/view/materialize.hpp
@@ -32,7 +32,9 @@ namespace view
Stands in for the lexer of json_sax_dom_parser, which reads the positions of
the values it stores from these three functions (diagnostic_positions). npos
-is a value without a source position.
+is a value without a source position. One position_source serves a whole
+materialize() call: before each value is handed to the handler, the range of
+that value is set (or cleared), overwriting that of the value before.
*/
struct position_source
{
@@ -65,9 +67,20 @@ struct position_source
/// the range [first, last), or none if either is npos
void set(std::size_t first, std::size_t last) noexcept
{
- const bool known = first != std::string::npos && last != std::string::npos;
- start = known ? first : std::string::npos;
- end = known ? last : std::string::npos;
+ if (first == std::string::npos || last == std::string::npos)
+ {
+ clear();
+ return;
+ }
+ start = first;
+ end = last;
+ }
+
+ /// no range: the next value has no source position
+ void clear() noexcept
+ {
+ start = std::string::npos;
+ end = std::string::npos;
}
/// one past the closing quote of the string whose opening quote is at
@@ -140,58 +153,64 @@ struct position_source
/// the range of the scalar n of document d
void set_scalar(const document_data& d, const node& n) noexcept
{
- const std::size_t size = d.size;
- const std::size_t off = n.off;
+ if ((n.flags & node_flags::is_new) != 0)
+ {
+ clear();
+ return;
+ }
std::size_t first = std::string::npos;
std::size_t last = std::string::npos;
- if ((n.flags & node_flags::is_new) == 0)
+ switch (n.kind)
{
- switch (static_cast(n.kind))
- {
- case value_t::string:
- if ((n.flags & node_flags::storage) == 0)
+ case node_kind::string:
+ if ((n.flags & node_flags::storage) == 0)
+ {
+ // off is the first byte after the opening quote
+ if (n.off != 0)
{
- // off is the first byte after the opening quote
- if (off != 0)
- {
- first = off - 1;
- last = off + std::size_t{n.len} + 1;
- }
+ first = n.off - 1;
+ last = n.off + std::size_t{n.len} + 1;
}
- else if ((n.flags & node_flags::storage) == node_flags::escaped && n.next != 0)
- {
- // next is the offset of the opening quote plus one
- first = n.next - 1;
- last = string_end(d.base[0], size, first);
- }
- break;
- case value_t::number_integer:
- case value_t::number_unsigned:
- case value_t::number_float:
- if ((n.flags & node_flags::storage) == 0)
- {
- first = off;
- last = off + number_length(n);
- }
- break;
- case value_t::boolean:
- first = off;
- last = off + ((n.flags & node_flags::is_true) != 0 ? 4u : 5u);
- break;
- case value_t::null:
- first = off;
- last = off + 4u;
- break;
- case value_t::object:
- case value_t::array:
- case value_t::binary:
- case value_t::discarded:
- default:
- break;
- }
+ }
+ else if ((n.flags & node_flags::storage) == node_flags::escaped && n.next != 0)
+ {
+ // next is the offset of the opening quote plus one
+ first = n.next - 1;
+ last = string_end(d.base[0], d.size, first);
+ }
+ break;
+ case node_kind::number_integer:
+ case node_kind::number_unsigned:
+ case node_kind::number_float:
+ if ((n.flags & node_flags::storage) == 0)
+ {
+ first = n.off;
+ last = n.off + number_length(n);
+ }
+ break;
+ case node_kind::boolean:
+ first = n.off;
+ last = n.off + ((n.flags & node_flags::is_true) != 0 ? 4u : 5u);
+ break;
+ case node_kind::null:
+ first = n.off;
+ last = n.off + 4u;
+ break;
+ case node_kind::object:
+ case node_kind::array:
+ case node_kind::link:
+ default:
+ break;
}
// (a crafted image can point past the text)
- set(first, last != std::string::npos && last <= size ? last : std::string::npos);
+ if (last == std::string::npos || last > d.size)
+ {
+ clear();
+ }
+ else
+ {
+ set(first, last);
+ }
}
};
#endif
@@ -214,11 +233,13 @@ template
BasicJsonType materialize(const document_data& d, const node* n)
{
using string_t = typename BasicJsonType::string_t;
+ using input_t = iterator_input_adapter;
#if JSON_DIAGNOSTIC_POSITIONS
- using sax_t = json_sax_dom_parser, true, position_source>;
+ using lexer_t = position_source;
#else
- using sax_t = json_sax_dom_parser, true>;
+ using lexer_t = lexer; // (the default: unused, as no lexer is passed)
#endif
+ using sax_t = json_sax_dom_parser;
using nav = navigation;
// the open containers: their next element (or key of the next member),
@@ -306,7 +327,14 @@ BasicJsonType materialize(const document_data& d, const node* n)
// (moved containers are also new)
const std::size_t start = (n->flags & node_flags::is_new) == 0 && n->off < d.size ? std::size_t{n->off} :
std::string::npos;
- positions.set(start, start + 1);
+ if (start == std::string::npos)
+ {
+ positions.clear();
+ }
+ else
+ {
+ positions.set(start, start + 1);
+ }
#endif
if (object)
{
@@ -324,14 +352,14 @@ BasicJsonType materialize(const document_data& d, const node* n)
open = deep.data();
capacity = deep.size();
}
+ // (with positions: until an element is read, the container's
+ // own bracket is the last thing read)
+ open[depth++] = frame{nav::first(d, n), nav::end(d, n), object
#if JSON_DIAGNOSTIC_POSITIONS
- // (until an element is read, the container's own bracket is the last thing read)
- open[depth++] = frame{nav::first(d, n), nav::end(d, n), object, start, positions.end};
- break;
-#else
- open[depth++] = frame {nav::first(d, n), nav::end(d, n), object};
- break;
+ , start, positions.end
#endif
+ };
+ break;
}
case value_t::string:
{
@@ -372,9 +400,19 @@ BasicJsonType materialize(const document_data& d, const node* n)
if (f.pos == f.end)
{
#if JSON_DIAGNOSTIC_POSITIONS
- const std::size_t end = f.start == std::string::npos || f.last_end == std::string::npos ? std::string::npos : position_source::container_end(d.base[0], d.size, f.last_end);
- // (an end of npos clears the start, see diagnostic_positions)
- positions.set(end == std::string::npos ? end : end - 1, end);
+ const std::size_t end = f.start == std::string::npos || f.last_end == std::string::npos ? std::string::npos :
+ position_source::container_end(d.base[0], d.size, f.last_end);
+ if (end == std::string::npos)
+ {
+ positions.clear();
+ }
+ else
+ {
+ // the closing bracket, as the lexer reports it (end_object()
+ // and end_array() only read its end; the start of the
+ // container was set when it was opened)
+ positions.set(end - 1, end);
+ }
if (depth > 1)
{
open[depth - 2].last_end = end;
diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp
index c04ca7ac6..5f829a88a 100644
--- a/single_include/nlohmann/json.hpp
+++ b/single_include/nlohmann/json.hpp
@@ -15632,7 +15632,7 @@ struct diagnostic_positions
// (npos: a value without a source position, see materialize() of
// json_view)
const std::size_t read = lexer->get_chars_read_total();
- v.start_position = read == std::string::npos ? read : read - 1;
+ v.start_position = read == std::string::npos ? std::string::npos : read - 1;
}
}
@@ -15651,7 +15651,7 @@ struct diagnostic_positions
{
// a range has both ends or none (exception::diagnostics()
// prints only complete ones)
- v.start_position = v.end_position;
+ v.start_position = std::string::npos;
}
}
}
@@ -15672,7 +15672,7 @@ struct diagnostic_positions
if (v.end_position == std::string::npos)
{
// a value without a source position
- v.start_position = v.end_position;
+ v.start_position = std::string::npos;
return;
}
diff --git a/single_include/nlohmann/json_view.hpp b/single_include/nlohmann/json_view.hpp
index e6b958f19..41eb15940 100644
--- a/single_include/nlohmann/json_view.hpp
+++ b/single_include/nlohmann/json_view.hpp
@@ -2405,10 +2405,11 @@ indent_done:
#if JSON_DIAGNOSTIC_POSITIONS
// the decoded string is in the arena: next keeps the source offset
// of the opening quote plus one (0: unknown) for materialize()
- push_node(node_kind::string, node_flags::escaped, 0, r.start, static_cast(r.len) | (static_cast(s - b) << 32));
+ const std::uint64_t second = static_cast(r.len) | (static_cast(s - b) << 32);
#else
- push_node(node_kind::string, node_flags::escaped, 0, r.start, r.len);
+ const std::uint64_t second = r.len;
#endif
+ push_node(node_kind::string, node_flags::escaped, 0, r.start, second);
return true;
}
};
@@ -6126,7 +6127,9 @@ namespace view
Stands in for the lexer of json_sax_dom_parser, which reads the positions of
the values it stores from these three functions (diagnostic_positions). npos
-is a value without a source position.
+is a value without a source position. One position_source serves a whole
+materialize() call: before each value is handed to the handler, the range of
+that value is set (or cleared), overwriting that of the value before.
*/
struct position_source
{
@@ -6159,9 +6162,20 @@ struct position_source
/// the range [first, last), or none if either is npos
void set(std::size_t first, std::size_t last) noexcept
{
- const bool known = first != std::string::npos && last != std::string::npos;
- start = known ? first : std::string::npos;
- end = known ? last : std::string::npos;
+ if (first == std::string::npos || last == std::string::npos)
+ {
+ clear();
+ return;
+ }
+ start = first;
+ end = last;
+ }
+
+ /// no range: the next value has no source position
+ void clear() noexcept
+ {
+ start = std::string::npos;
+ end = std::string::npos;
}
/// one past the closing quote of the string whose opening quote is at
@@ -6234,58 +6248,64 @@ struct position_source
/// the range of the scalar n of document d
void set_scalar(const document_data& d, const node& n) noexcept
{
- const std::size_t size = d.size;
- const std::size_t off = n.off;
+ if ((n.flags & node_flags::is_new) != 0)
+ {
+ clear();
+ return;
+ }
std::size_t first = std::string::npos;
std::size_t last = std::string::npos;
- if ((n.flags & node_flags::is_new) == 0)
+ switch (n.kind)
{
- switch (static_cast(n.kind))
- {
- case value_t::string:
- if ((n.flags & node_flags::storage) == 0)
+ case node_kind::string:
+ if ((n.flags & node_flags::storage) == 0)
+ {
+ // off is the first byte after the opening quote
+ if (n.off != 0)
{
- // off is the first byte after the opening quote
- if (off != 0)
- {
- first = off - 1;
- last = off + std::size_t{n.len} + 1;
- }
+ first = n.off - 1;
+ last = n.off + std::size_t{n.len} + 1;
}
- else if ((n.flags & node_flags::storage) == node_flags::escaped && n.next != 0)
- {
- // next is the offset of the opening quote plus one
- first = n.next - 1;
- last = string_end(d.base[0], size, first);
- }
- break;
- case value_t::number_integer:
- case value_t::number_unsigned:
- case value_t::number_float:
- if ((n.flags & node_flags::storage) == 0)
- {
- first = off;
- last = off + number_length(n);
- }
- break;
- case value_t::boolean:
- first = off;
- last = off + ((n.flags & node_flags::is_true) != 0 ? 4u : 5u);
- break;
- case value_t::null:
- first = off;
- last = off + 4u;
- break;
- case value_t::object:
- case value_t::array:
- case value_t::binary:
- case value_t::discarded:
- default:
- break;
- }
+ }
+ else if ((n.flags & node_flags::storage) == node_flags::escaped && n.next != 0)
+ {
+ // next is the offset of the opening quote plus one
+ first = n.next - 1;
+ last = string_end(d.base[0], d.size, first);
+ }
+ break;
+ case node_kind::number_integer:
+ case node_kind::number_unsigned:
+ case node_kind::number_float:
+ if ((n.flags & node_flags::storage) == 0)
+ {
+ first = n.off;
+ last = n.off + number_length(n);
+ }
+ break;
+ case node_kind::boolean:
+ first = n.off;
+ last = n.off + ((n.flags & node_flags::is_true) != 0 ? 4u : 5u);
+ break;
+ case node_kind::null:
+ first = n.off;
+ last = n.off + 4u;
+ break;
+ case node_kind::object:
+ case node_kind::array:
+ case node_kind::link:
+ default:
+ break;
}
// (a crafted image can point past the text)
- set(first, last != std::string::npos && last <= size ? last : std::string::npos);
+ if (last == std::string::npos || last > d.size)
+ {
+ clear();
+ }
+ else
+ {
+ set(first, last);
+ }
}
};
#endif
@@ -6308,11 +6328,13 @@ template
BasicJsonType materialize(const document_data& d, const node* n)
{
using string_t = typename BasicJsonType::string_t;
+ using input_t = iterator_input_adapter;
#if JSON_DIAGNOSTIC_POSITIONS
- using sax_t = json_sax_dom_parser, true, position_source>;
+ using lexer_t = position_source;
#else
- using sax_t = json_sax_dom_parser, true>;
+ using lexer_t = lexer; // (the default: unused, as no lexer is passed)
#endif
+ using sax_t = json_sax_dom_parser;
using nav = navigation;
// the open containers: their next element (or key of the next member),
@@ -6400,7 +6422,14 @@ BasicJsonType materialize(const document_data& d, const node* n)
// (moved containers are also new)
const std::size_t start = (n->flags & node_flags::is_new) == 0 && n->off < d.size ? std::size_t{n->off} :
std::string::npos;
- positions.set(start, start + 1);
+ if (start == std::string::npos)
+ {
+ positions.clear();
+ }
+ else
+ {
+ positions.set(start, start + 1);
+ }
#endif
if (object)
{
@@ -6418,14 +6447,14 @@ BasicJsonType materialize(const document_data& d, const node* n)
open = deep.data();
capacity = deep.size();
}
+ // (with positions: until an element is read, the container's
+ // own bracket is the last thing read)
+ open[depth++] = frame{nav::first(d, n), nav::end(d, n), object
#if JSON_DIAGNOSTIC_POSITIONS
- // (until an element is read, the container's own bracket is the last thing read)
- open[depth++] = frame{nav::first(d, n), nav::end(d, n), object, start, positions.end};
- break;
-#else
- open[depth++] = frame {nav::first(d, n), nav::end(d, n), object};
- break;
+ , start, positions.end
#endif
+ };
+ break;
}
case value_t::string:
{
@@ -6466,9 +6495,19 @@ BasicJsonType materialize(const document_data& d, const node* n)
if (f.pos == f.end)
{
#if JSON_DIAGNOSTIC_POSITIONS
- const std::size_t end = f.start == std::string::npos || f.last_end == std::string::npos ? std::string::npos : position_source::container_end(d.base[0], d.size, f.last_end);
- // (an end of npos clears the start, see diagnostic_positions)
- positions.set(end == std::string::npos ? end : end - 1, end);
+ const std::size_t end = f.start == std::string::npos || f.last_end == std::string::npos ? std::string::npos :
+ position_source::container_end(d.base[0], d.size, f.last_end);
+ if (end == std::string::npos)
+ {
+ positions.clear();
+ }
+ else
+ {
+ // the closing bracket, as the lexer reports it (end_object()
+ // and end_array() only read its end; the start of the
+ // container was set when it was opened)
+ positions.set(end - 1, end);
+ }
if (depth > 1)
{
open[depth - 2].last_end = end;