diff --git a/docs/mkdocs/docs/api/basic_json/patch_inplace.md b/docs/mkdocs/docs/api/basic_json/patch_inplace.md index 67c6e8814..ff65e8dbd 100644 --- a/docs/mkdocs/docs/api/basic_json/patch_inplace.md +++ b/docs/mkdocs/docs/api/basic_json/patch_inplace.md @@ -69,6 +69,19 @@ Duplicate member names inside an operation object are resolved by the parser bef is kept, see [`object_t`](object_t.md#behavior)), so `#!json {"op": "add", "op": "remove", "path": "/a"}` is a `remove` operation. +The JSON patch may be the JSON value itself (`#!cpp j.patch_inplace(j)`): the operations are then read from a copy of +the patch made before the first operation is applied, so the result is the same as that of +`#!cpp j = j.patch(j)`, and the exception safety is the same as above. + +If [`JSON_DIAGNOSTICS`](../macros/json_diagnostics.md) is enabled, the same holds for a JSON patch stored inside the +JSON value (e.g., `#!cpp j.patch_inplace(j["patch"])`): it is detected via the parent pointers. + +!!! danger "Undefined behavior" + + Without [`JSON_DIAGNOSTICS`](../macros/json_diagnostics.md), the JSON patch must not be a value stored inside the + JSON value (e.g., `#!cpp j.patch_inplace(j["patch"])`), as the operations may change or destroy it while it is + read. Pass a copy (`#!cpp j.patch_inplace(json(j["patch"]))`) or use [`patch`](patch.md) instead. + ## Examples ??? example "Example: apply a JSON patch in place" @@ -122,3 +135,5 @@ is kept, see [`object_t`](object_t.md#behavior)), so `#!json {"op": "add", "op": - Throws [`parse_error.109`](../../home/exceptions.md#jsonexceptionparse_error109) instead of [`out_of_range.404`](../../home/exceptions.md#jsonexceptionout_of_range404) for a one-character array index that is not a digit (e.g., `/x`) in version 3.13.0, as it already did for longer ones. +- Applying a JSON value as patch to itself (or, with `JSON_DIAGNOSTICS`, to a value stored inside it) no longer results + in undefined behavior in version 3.13.0. diff --git a/docs/mkdocs/docs/api/basic_json_view/materialize.md b/docs/mkdocs/docs/api/basic_json_view/materialize.md index f29a25cda..982b1597f 100644 --- a/docs/mkdocs/docs/api/basic_json_view/materialize.md +++ b/docs/mkdocs/docs/api/basic_json_view/materialize.md @@ -32,9 +32,24 @@ uses, so the result matches it exactly — including, for an object, that a repe last value. The replay is iterative, so it is not limited by the call stack the way a naive recursive conversion would be; the JSON nesting depth is limited only by available memory, as for `BasicJsonType::parse()` itself. -Unlike parsing with [`JSON_DIAGNOSTIC_POSITIONS`](../macros/json_diagnostic_positions.md) enabled, the values -produced by `materialize()` do not carry source positions. The index has the start offset of every value, but not the -end of arrays, objects, and escaped strings, which the positions also need. +With [`JSON_DIAGNOSTIC_POSITIONS`](../macros/json_diagnostic_positions.md) enabled, the values get the source +positions [`BasicJsonType::parse()`](../basic_json/parse.md) would give them, also for the subtree of a view (the +positions are those in the document's whole source text), and exceptions about them name the same bytes. For values of +an [editable document](../json_editable_document.md), positions are all-or-nothing, and [`start_pos()`](../basic_json/start_pos.md) +and [`end_pos()`](../basic_json/end_pos.md) return `std::string::npos` for + +- a value written by an edit (a copy of an array or object of the document is also new; copies of its scalars keep + their positions), +- an array or object whose elements were changed (its elements keep theirs), and +- an array or object whose last element has no positions (its end is found from that element). + +This differs from a `basic_json` value, whose positions are not updated when it is changed: they stay as they were +parsed, even if they no longer match the value (see [`start_pos()`](../basic_json/start_pos.md)). + +Documents [loaded](../basic_json_document/load.md) from an image have the positions of the saved document. Its strings +with escapes have positions only if the document was parsed with `JSON_DIAGNOSTIC_POSITIONS`: the position of their +opening quote is recorded while parsing, and `save()` and `load()` keep it whether or not they are compiled with the +macro. Calling `materialize()` on the same view repeatedly builds a new, independent `BasicJsonType` value each time; it never caches the result. @@ -62,8 +77,7 @@ never caches the result. - [`root`](../basic_json_document/root.md) - the view of a document's root value - [`BasicJsonType::parse`](../basic_json/parse.md) - build a `BasicJsonType` value directly from a JSON text -- [`JSON_DIAGNOSTIC_POSITIONS`](../macros/json_diagnostic_positions.md) - source positions on parsed values (not - produced by `materialize()`) +- [`JSON_DIAGNOSTIC_POSITIONS`](../macros/json_diagnostic_positions.md) - source positions on parsed values - [Zero-copy JSON views](../../features/json_view.md#getting-values-out-without-copying) - when to build a basic_json value from a view ## Version history diff --git a/docs/mkdocs/docs/api/json_pointer/operator_slasheq.md b/docs/mkdocs/docs/api/json_pointer/operator_slasheq.md index e0d4560ec..2f2820d4f 100644 --- a/docs/mkdocs/docs/api/json_pointer/operator_slasheq.md +++ b/docs/mkdocs/docs/api/json_pointer/operator_slasheq.md @@ -18,7 +18,8 @@ json_pointer& operator/=(std::size_t array_idx) ## Parameters `ptr` (in) -: JSON pointer to append +: JSON pointer to append; may be the JSON pointer itself (`#!cpp p /= p`), which then appends a copy of its + reference tokens `token` (in) : reference token to append @@ -69,6 +70,7 @@ json_pointer& operator/=(std::size_t array_idx) ## Version history -1. Added in version 3.6.0. +1. Added in version 3.6.0. Appending a JSON pointer to itself no longer results in undefined behavior in version + 3.13.0. 2. Added in version 3.6.0. Changed type of `token` to `string_t` in version 3.11.0. 3. Added in version 3.6.0. diff --git a/docs/mkdocs/docs/api/macros/json_diagnostic_positions.md b/docs/mkdocs/docs/api/macros/json_diagnostic_positions.md index 56886ba86..8a772791a 100644 --- a/docs/mkdocs/docs/api/macros/json_diagnostic_positions.md +++ b/docs/mkdocs/docs/api/macros/json_diagnostic_positions.md @@ -113,6 +113,7 @@ When the macro is not defined, the library will define it to its default value. - [:simple-cmake: JSON_Diagnostic_Positions](../../integration/cmake.md#json_diagnostic_positions) - CMake option to control the macro - [`JSON_DIAGNOSTICS`](json_diagnostics.md) - macro to control extended diagnostics +- [`materialize()`](../basic_json_view/materialize.md) - values of a JSON view, with the same positions - [Parsing and Exceptions](../../features/parsing/parse_exceptions.md) - handling parse errors and locating them in the input ## Version history diff --git a/docs/mkdocs/docs/features/json_view.md b/docs/mkdocs/docs/features/json_view.md index f2660f539..09779dba2 100644 --- a/docs/mkdocs/docs/features/json_view.md +++ b/docs/mkdocs/docs/features/json_view.md @@ -149,10 +149,12 @@ document: `#!cpp auto v = json_document::parse(text).root();` does not compile. end must still fit. - **A stream is always read to its end.** There is no partial/streaming read of an `#!cpp std::istream`: the stream is read to its end into a buffer the document owns, and parsing starts after that. -- **No source positions on `materialize()`.** Even with +- **Fewer source positions after edits.** With [`JSON_DIAGNOSTIC_POSITIONS`](../api/macros/json_diagnostic_positions.md) enabled, - [`materialize()`](../api/basic_json_view/materialize.md) does not set them. The index has the start offset of every - value, but not the end of arrays, objects, and escaped strings, which the positions also need. + [`materialize()`](../api/basic_json_view/materialize.md) sets the same positions as `#!cpp basic_json::parse()`. + In an [editable document](../api/json_editable_document.md), a value written by an edit has none, and neither has + an array or object whose elements changed or whose last element has none; a `basic_json` value keeps its positions + when it is changed. See [`materialize()`](../api/basic_json_view/materialize.md#notes). - **Objects iterate in document order.** [`begin()`](../api/basic_json_view/begin.md)/ [`end()`](../api/basic_json_view/end.md) and [`items()`](../api/basic_json_view/items.md) visit an object's members in the order they appear in the source text. `basic_json`'s default `object_t` is a `std::map`, which diff --git a/docs/mkdocs/docs/home/architecture.md b/docs/mkdocs/docs/home/architecture.md index 82a660e3a..1b0d84dd7 100644 --- a/docs/mkdocs/docs/home/architecture.md +++ b/docs/mkdocs/docs/home/architecture.md @@ -203,7 +203,7 @@ packet-beta | 2-3 | `extra` | `uint16_t` | numbers: the number of integer digits (low byte) and fraction digits (high byte), 255 for more; objects: the number of their key hash index (1-based), or 0; otherwise 0 | | 4-7 | `off` | `uint32_t` | where the value starts: the first byte after a string's opening quote (or its position in the buffer of decoded strings), the first byte of a number or literal, the bracket of an array or object | | 8-11 | `len` | `uint32_t` | strings: the length after decoding; floats and literals: the length of the token; arrays and objects: the number of elements | -| 12-15 | `next` | `uint32_t` | arrays and objects: the number of nodes of the subtree, including the node itself | +| 12-15 | `next` | `uint32_t` | arrays and objects: the number of nodes of the subtree, including the node itself; strings with escapes: the position of the opening quote in the source plus one if the document was parsed with [`JSON_DIAGNOSTIC_POSITIONS`](../api/macros/json_diagnostic_positions.md), or 0 if it was not; all other values: 0 |

Fields of a node, by their byte offset within the 16-byte node. Integers use bytes 8-15 differently, see below.

@@ -218,6 +218,12 @@ packet-beta step from element to element this way and skip whole subtrees in constant time. - **Offsets** are 32 bits wide, so a document is limited to 4294967279 bytes, 4 GiB minus 16 bytes (a margin below 2^32 for positions one scanner step past the end of the text; `out_of_range.416`). +- **Source positions** ([`JSON_DIAGNOSTIC_POSITIONS`](../api/macros/json_diagnostic_positions.md)) are derived by + [`materialize()`](../api/basic_json_view/materialize.md) and cost almost nothing while parsing: the only work is + storing the offset of the opening quote of a string with escapes in its `next`, which is otherwise unused. A + scalar's range follows from `off` and its length, a string with escapes is scanned from its opening quote (`next`) to the closing one, + and the end of an array or object is found by scanning from the end of its last element over whitespace, comments, + and a trailing comma. `materialize()` hands these positions to the SAX builder of `parse()` in place of its lexer. - **Large objects** (128 members or more) get a hash index for their keys after parsing ([`detail/view/object_index.hpp`](https://github.com/nlohmann/json/blob/develop/include/nlohmann/detail/view/object_index.hpp)): an open-addressing table whose slots hold the distance from the object's node to a key's node, so that a lookup does @@ -277,7 +283,7 @@ Images ([`save`](../api/basic_json_document/save.md) and [`load`](../api/basic_j the nodes as they are: a 64-byte header (the magic bytes `NJVI`, a format version, the sizes, and reserved bytes that must be zero), the nodes, the text, and the decoded strings. An edited document is first written in document order, as the parser would have written it (without links), and the numbers of the hash indexes are cleared, since `load` -rebuilds the indexes. So a change of the node layout is a change of the image format: it must raise `image_version`, +rebuilds the indexes. The flag *new* is kept, as these values have no position in the text. So a change of the node layout is a change of the image format: it must raise `image_version`, and `load` then rejects images of other versions (`parse_error.116`) instead of misreading them. ## Input adapters diff --git a/include/nlohmann/detail/input/json_sax.hpp b/include/nlohmann/detail/input/json_sax.hpp index 04201efbe..0ceec61b7 100644 --- a/include/nlohmann/detail/input/json_sax.hpp +++ b/include/nlohmann/detail/input/json_sax.hpp @@ -230,7 +230,10 @@ struct diagnostic_positions { // Lexer has read the first character of the container, so // subtract 1 from the position to get the correct start position. - v.start_position = lexer->get_chars_read_total() - 1; + // (npos: a value without a source position, see materialize() of + // json_view) + const std::size_t read = lexer->get_chars_read_total(); + v.start_position = read == std::string::npos ? std::string::npos : read - 1; } } @@ -245,6 +248,12 @@ struct diagnostic_positions { // Lexer's position is past the closing brace or bracket, so set that as the end position. v.end_position = lexer->get_chars_read_total(); + if (v.end_position == std::string::npos) + { + // a range has both ends or none (exception::diagnostics() + // prints only complete ones) + v.start_position = std::string::npos; + } } } @@ -261,6 +270,12 @@ struct diagnostic_positions // The start position will be set below based on the length of the string representation // of the value. v.end_position = lexer->get_chars_read_total(); + if (v.end_position == std::string::npos) + { + // a value without a source position + v.start_position = std::string::npos; + return; + } switch (v.type()) { @@ -473,8 +488,12 @@ read up to it. @tparam Collect whether to collect the values of containers of unknown size on a stack (only for a caller that does not use the value after a parse error) +@tparam LexerType the type of the lexer that reports the source positions + (JSON_DIAGNOSTIC_POSITIONS); only diagnostic_positions uses + it, so any type with its three functions works (as + materialize() of json_view uses it) */ -template +template> class json_sax_dom_parser : public json_sax_dom_error_state { public: @@ -483,7 +502,7 @@ class json_sax_dom_parser : public json_sax_dom_error_state using number_float_t = typename BasicJsonType::number_float_t; using string_t = typename BasicJsonType::string_t; using binary_t = typename BasicJsonType::binary_t; - using lexer_t = lexer; + using lexer_t = LexerType; /*! @param[in,out] r reference to a JSON value that is manipulated while diff --git a/include/nlohmann/detail/input/parser.hpp b/include/nlohmann/detail/input/parser.hpp index 665052d73..5bf8f1b6b 100644 --- a/include/nlohmann/detail/input/parser.hpp +++ b/include/nlohmann/detail/input/parser.hpp @@ -56,8 +56,8 @@ using parser_callback_t = template struct sax_ignores_number_text : std::false_type {}; -template -struct sax_ignores_number_text> : std::true_type {}; +template +struct sax_ignores_number_text> : std::true_type {}; template struct sax_ignores_number_text> : std::true_type {}; diff --git a/include/nlohmann/detail/json_pointer.hpp b/include/nlohmann/detail/json_pointer.hpp index eef4ce0a4..6df49fbc0 100644 --- a/include/nlohmann/detail/json_pointer.hpp +++ b/include/nlohmann/detail/json_pointer.hpp @@ -266,6 +266,13 @@ class json_pointer /// @sa https://json.nlohmann.me/api/json_pointer/operator_slasheq/ json_pointer& operator/=(const json_pointer& ptr) { + // insert() must not be given iterators into the vector it inserts + // into: append a copy when a pointer is appended to itself + if (JSON_HEDLEY_UNLIKELY(&ptr == this)) + { + return *this /= json_pointer(ptr); + } + reference_tokens.insert(reference_tokens.end(), ptr.reference_tokens.begin(), ptr.reference_tokens.end()); diff --git a/include/nlohmann/detail/view/builder.hpp b/include/nlohmann/detail/view/builder.hpp index a0446a1e1..89b5af2ab 100644 --- a/include/nlohmann/detail/view/builder.hpp +++ b/include/nlohmann/detail/view/builder.hpp @@ -173,31 +173,16 @@ class builder } if (p[1] == '/') { - p += 2; - // (as in parse(), a null byte is the end of the input, so it is - // left for the caller to see) - while (p != m_end && *p != '\n' && *p != '\r' && !(NulIsEnd && *p == 0)) - { - ++p; - } - return p; + return skip_line_comment(p + 2, m_end); } if (p[1] == '*') { - p += 2; - for (;;) + p = find_block_comment_end(p + 2, m_end); + if (p == m_end || *p != '*') { - if (p == m_end || (NulIsEnd && *p == 0)) - { - return fail_at(error_code::comment_unterminated, p); - } - if (*p == '*' && p + 1 != m_end && p[1] == '/') - { - p += 2; - return p; - } - ++p; + return fail_at(error_code::comment_unterminated, p); } + return p + 2; } ++p; return fail_at(error_code::comment_start, p); @@ -1193,7 +1178,14 @@ indent_done: return false; } p = r.p; - push_node(node_kind::string, node_flags::escaped, 0, r.start, r.len); +#if JSON_DIAGNOSTIC_POSITIONS + // the decoded string is in the arena: next keeps the source offset + // of the opening quote plus one (0: unknown) for materialize() + const std::uint64_t second = static_cast(r.len) | (static_cast(s - b) << 32); +#else + const std::uint64_t second = r.len; +#endif + push_node(node_kind::string, node_flags::escaped, 0, r.start, second); return true; } }; diff --git a/include/nlohmann/detail/view/image.hpp b/include/nlohmann/detail/view/image.hpp index 637350949..545e04a6a 100644 --- a/include/nlohmann/detail/view/image.hpp +++ b/include/nlohmann/detail/view/image.hpp @@ -80,7 +80,8 @@ constexpr std::uint64_t image_limit = 0xFFFFFFF0u; /// Copy the current structure of an edited document into nodes in document /// order, as the parser would have written them. Text written by edits is /// appended to text_tail (number tokens) and arena_tail (strings); floats that -/// are not finite become null, as dump() writes them. +/// are not finite become null, as dump() writes them. Nodes keep the flag +/// is_new: they have no source position (JSON_DIAGNOSTIC_POSITIONS). inline void compact_nodes(const document_data& d, std::size_t arena_size, std::vector& out, std::string& text_tail, std::string& arena_tail) { struct frame @@ -103,16 +104,18 @@ inline void compact_nodes(const document_data& d, std::size_t arena_size, std::v arena_tail.append(d.str(s), s.len); r.flags = node_flags::escaped; } + r.flags = static_cast(r.flags | (s.flags & node_flags::is_new)); return r; }; const auto copy_node = [&](const node * v) { node r = *v; + const auto is_new = static_cast(v->flags & node_flags::is_new); switch (static_cast(v->kind)) { case value_t::object: case value_t::array: - r.flags = 0; + r.flags = is_new; r.extra = 0; r.off = (v->flags & (node_flags::moved | node_flags::is_new)) != 0 ? 0 : v->off; r.len = 0; // counted below @@ -129,7 +132,7 @@ inline void compact_nodes(const document_data& d, std::size_t arena_size, std::v r.off = static_cast(d.size + text_tail.size()); text_tail.append(d.str(*v), number_length(*v)); } - r.flags = 0; + r.flags = is_new; break; case value_t::number_float: if ((v->flags & node_flags::storage) == node_flags::edited) @@ -139,22 +142,23 @@ inline void compact_nodes(const document_data& d, std::size_t arena_size, std::v { r = node{}; // nan and infinity: null, as dump() writes them r.kind = node_kind::null; + r.flags = is_new; break; } r.off = static_cast(d.size + text_tail.size()); text_tail.append(t, v->len); r.extra = 0xFFFFu; // the digit layout is not recorded } - r.flags = 0; + r.flags = is_new; break; case value_t::boolean: - r.flags = static_cast(v->flags & node_flags::is_true); + r.flags = static_cast(v->flags & (node_flags::is_true | node_flags::is_new)); break; case value_t::null: case value_t::binary: case value_t::discarded: default: - r.flags = 0; + r.flags = is_new; break; } out.push_back(r); @@ -515,13 +519,15 @@ inline bool check_image(const node* nodes, std::size_t count, const unsigned cha std::vector floats; const auto check_string = [&](const node & n) -> bool { - if ((n.flags & ~node_flags::escaped) != 0 || n.extra != 0) + if ((n.flags & ~(node_flags::escaped | node_flags::is_new)) != 0 || n.extra != 0) { return false; } const bool decoded = (n.flags & node_flags::escaped) != 0; const std::size_t limit = decoded ? arena_size : text_size; - if (n.off > limit || n.len > limit - n.off) + // (next: the source offset of a decoded string's opening quote plus + // one, or 0; see node) + if (n.off > limit || n.len > limit - n.off || n.next > text_size) { return false; } @@ -593,13 +599,13 @@ inline bool check_image(const node* nodes, std::size_t count, const unsigned cha { case value_t::null: // (the offset of a literal is read to size the output of dump()) - if (n.flags != 0 || n.extra != 0 || n.off > text_size) + if ((n.flags & ~node_flags::is_new) != 0 || n.extra != 0 || n.off > text_size) { return false; } break; case value_t::boolean: - if ((n.flags & ~node_flags::is_true) != 0 || n.extra != 0 || n.off > text_size) + if ((n.flags & ~(node_flags::is_true | node_flags::is_new)) != 0 || n.extra != 0 || n.off > text_size) { return false; } @@ -613,7 +619,7 @@ inline bool check_image(const node* nodes, std::size_t count, const unsigned cha case value_t::number_integer: case value_t::number_unsigned: case value_t::number_float: - if (n.flags != 0 || !number_in_bounds(n)) + if ((n.flags & ~node_flags::is_new) != 0 || !number_in_bounds(n)) { return false; } @@ -635,7 +641,7 @@ inline bool check_image(const node* nodes, std::size_t count, const unsigned cha case value_t::object: { const std::size_t limit = stack.empty() ? count : stack.back().end; - if (n.flags != 0 || n.extra != 0 || n.next == 0 || n.next > limit - i || n.off > text_size) + if ((n.flags & ~node_flags::is_new) != 0 || n.extra != 0 || n.next == 0 || n.next > limit - i || n.off > text_size) { return false; } diff --git a/include/nlohmann/detail/view/materialize.hpp b/include/nlohmann/detail/view/materialize.hpp index 1bab0e896..1dcd1940c 100644 --- a/include/nlohmann/detail/view/materialize.hpp +++ b/include/nlohmann/detail/view/materialize.hpp @@ -19,6 +19,7 @@ #include #include #include +#include NLOHMANN_JSON_NAMESPACE_BEGIN namespace detail @@ -26,6 +27,189 @@ namespace detail namespace view { +#if JSON_DIAGNOSTIC_POSITIONS +/*! +@brief the source positions of the values materialize() replays + +Stands in for the lexer of json_sax_dom_parser, which reads the positions of +the values it stores from these three functions (diagnostic_positions). npos +is a value without a source position. One position_source serves a whole +materialize() call: before each value is handed to the handler, the range of +that value is set (or cleared), overwriting that of the value before. +*/ +struct position_source +{ + /// the token, of which diagnostic_positions only reads the length + struct token + { + std::size_t length; + std::size_t size() const noexcept + { + return length; + } + }; + + std::size_t start = std::string::npos; ///< the first byte of the next value + std::size_t end = std::string::npos; ///< one past the next value (start_object/start_array: one past the bracket) + + std::size_t get_chars_read_total() const noexcept + { + return end; + } + std::size_t get_token_start_position() const noexcept + { + return start; + } + token get_string() const noexcept + { + return token{end - start}; + } + + /// the range [first, last), or none if either is npos + void set(std::size_t first, std::size_t last) noexcept + { + if (first == std::string::npos || last == std::string::npos) + { + clear(); + return; + } + start = first; + end = last; + } + + /// no range: the next value has no source position + void clear() noexcept + { + start = std::string::npos; + end = std::string::npos; + } + + /// one past the closing quote of the string whose opening quote is at + /// src[quote], or npos (the scan is bounded by size, as images can be + /// loaded without a check) + static std::size_t string_end(const char* src, std::size_t size, std::size_t quote) noexcept + { + for (std::size_t i = quote + 1; i < size; ++i) + { + if (src[i] == '\\') + { + ++i; + } + else if (src[i] == '"') + { + return i + 1; + } + } + return std::string::npos; + } + + /// one past the closing bracket of the array or object whose last element + /// ends at src[pos] (or whose opening bracket is at src[pos - 1]), or npos; + /// in between, there can only be whitespace, comments, and a trailing comma + /// (the source was validated when it was parsed) + static std::size_t container_end(const char* src, std::size_t size, std::size_t pos) noexcept + { + const auto* const b = reinterpret_cast(src); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast): the text as bytes, as the builder reads it + const unsigned char* const e = b + size; + while (pos < size) + { + switch (src[pos]) + { + case ' ': + case '\t': + case '\n': + case '\r': + case ',': + ++pos; + break; + case ']': + case '}': + return pos + 1; + case '/': + if (pos + 1 < size && src[pos + 1] == '/') + { + pos = static_cast(skip_line_comment(b + pos + 2, e) - b); + break; + } + if (pos + 1 < size && src[pos + 1] == '*') + { + // (unterminated: pos is size, which ends the loop) + pos = static_cast(find_block_comment_end(b + pos + 2, e) - b) + 2; + break; + } + return std::string::npos; + default: + return std::string::npos; + } + } + return std::string::npos; + } + + /// the range of the scalar n of document d + void set_scalar(const document_data& d, const node& n) noexcept + { + if ((n.flags & node_flags::is_new) != 0) + { + clear(); + return; + } + std::size_t first = std::string::npos; + std::size_t last = std::string::npos; + switch (n.kind) + { + case node_kind::string: + if ((n.flags & node_flags::storage) == 0) + { + // off is the first byte after the opening quote + if (n.off != 0) + { + first = n.off - 1; + last = n.off + std::size_t{n.len} + 1; + } + } + else if ((n.flags & node_flags::storage) == node_flags::escaped && n.next != 0) + { + // next is the offset of the opening quote plus one + first = n.next - 1; + last = string_end(d.base[0], d.size, first); + } + break; + case node_kind::number_integer: + case node_kind::number_unsigned: + case node_kind::number_float: + if ((n.flags & node_flags::storage) == 0) + { + first = n.off; + last = n.off + number_length(n); + } + break; + case node_kind::boolean: + first = n.off; + last = n.off + ((n.flags & node_flags::is_true) != 0 ? 4u : 5u); + break; + case node_kind::null: + first = n.off; + last = n.off + 4u; + break; + case node_kind::object: + case node_kind::array: + case node_kind::link: + default: + break; + } + // (a crafted image can point past the text) + if (last == std::string::npos || last > d.size) + { + clear(); + } + else + { + set(first, last); + } + } +}; +#endif + /*! @brief the basic_json value of the subtree at n @@ -33,14 +217,24 @@ The subtree is replayed into the SAX handler that parse() uses to build its values, so the result is the value parse() would produce: duplicate keys keep the last value, and with JSON_DIAGNOSTICS the parent pointers are set. It is iterative, so the nesting depth is limited by memory only, as for parse(). -Without a lexer the handler records no source positions -(JSON_DIAGNOSTIC_POSITIONS). + +With JSON_DIAGNOSTIC_POSITIONS, the values get their source positions as with +parse(): position_source stands in for the lexer. Values written by edits have +none (npos), and neither has an array or object whose elements changed, or +whose last element has none (the end of its closing bracket is found by +scanning from the end of the last element). */ template BasicJsonType materialize(const document_data& d, const node* n) { using string_t = typename BasicJsonType::string_t; - using sax_t = json_sax_dom_parser, true>; + using input_t = iterator_input_adapter; +#if JSON_DIAGNOSTIC_POSITIONS + using lexer_t = position_source; +#else + using lexer_t = lexer; // (the default: unused, as no lexer is passed) +#endif + using sax_t = json_sax_dom_parser; using nav = navigation; // the open containers: their next element (or key of the next member), @@ -52,9 +246,15 @@ BasicJsonType materialize(const document_data& d, const node* n) const node* pos; const node* end; bool object; +#if JSON_DIAGNOSTIC_POSITIONS + std::size_t start; ///< the opening bracket (npos: unknown) + std::size_t last_end; ///< the end of the last element read (npos: unknown) +#endif }; +#if !JSON_DIAGNOSTIC_POSITIONS // a scalar needs neither the handler nor the stack of open containers + // (with positions, it goes through the handler, which sets them) switch (static_cast(n->kind)) { case value_t::string: @@ -76,10 +276,16 @@ BasicJsonType materialize(const document_data& d, const node* n) default: break; } +#endif BasicJsonType result; +#if JSON_DIAGNOSTIC_POSITIONS + position_source positions; + sax_t sax(result, true, &positions); +#else sax_t sax(result, true); - const string_t no_token{}; +#endif + const string_t no_token {}; // (shallow subtrees on the stack: a call for a small value then allocates only for the value) std::array shallow; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init): filled as containers open std::vector deep{}; @@ -94,6 +300,16 @@ BasicJsonType materialize(const document_data& d, const node* n) std::size_t depth = 0; for (;;) { +#if JSON_DIAGNOSTIC_POSITIONS + if (!is_container(*n)) + { + positions.set_scalar(d, *n); + if (depth != 0) + { + open[depth - 1].last_end = positions.end; + } + } +#endif // false positive: n comes from nav::value(), which never returns null for a valid index // @infer-ignore NULLPTR_DEREFERENCE switch (static_cast(n->kind)) @@ -102,6 +318,17 @@ BasicJsonType materialize(const document_data& d, const node* n) case value_t::array: { const bool object = n->kind == node_kind::object; +#if JSON_DIAGNOSTIC_POSITIONS + // (moved containers are also new) + if ((n->flags & node_flags::is_new) != 0 || n->off >= d.size) + { + positions.clear(); + } + else + { + positions.set(std::size_t{n->off}, std::size_t{n->off} + 1); + } +#endif if (object) { sax.start_object(n->len); @@ -118,7 +345,13 @@ BasicJsonType materialize(const document_data& d, const node* n) open = deep.data(); capacity = deep.size(); } - open[depth++] = frame{nav::first(d, n), nav::end(d, n), object}; + // (with positions: until an element is read, the container's + // own bracket is the last thing read) + open[depth++] = frame{nav::first(d, n), nav::end(d, n), object +#if JSON_DIAGNOSTIC_POSITIONS + , positions.start, positions.end +#endif + }; break; } case value_t::string: @@ -159,6 +392,31 @@ BasicJsonType materialize(const document_data& d, const node* n) frame& f = open[depth - 1]; if (f.pos == f.end) { +#if JSON_DIAGNOSTIC_POSITIONS + if (f.start == std::string::npos || f.last_end == std::string::npos) + { + positions.clear(); + } + else + { + const std::size_t end = position_source::container_end(d.base[0], d.size, f.last_end); + if (end == std::string::npos) + { + positions.clear(); + } + else + { + // the closing bracket, as the lexer reports it (end_object() + // and end_array() only read its end; the start of the + // container was set when it was opened) + positions.set(end - 1, end); + } + } + if (depth > 1) + { + open[depth - 2].last_end = positions.end; + } +#endif if (f.object) { sax.end_object(); diff --git a/include/nlohmann/detail/view/node.hpp b/include/nlohmann/detail/view/node.hpp index 8987a8860..e00ef9eb6 100644 --- a/include/nlohmann/detail/view/node.hpp +++ b/include/nlohmann/detail/view/node.hpp @@ -69,7 +69,7 @@ struct node std::uint16_t extra; ///< numbers: integer digits (low byte) and fraction digits (high byte), 255 = "many"; objects: number of the object's key hash index (1-based, 0 = none); otherwise 0 std::uint32_t off; ///< source offset (string content, number token, literal, bracket); arena offset if escaped/edited; number of the element sequence if moved std::uint32_t len; ///< string: decoded bytes; float: token bytes; array/object: element count - std::uint32_t next; ///< array/object: number of nodes of the subtree (its extent in the enclosing sequence) + std::uint32_t next; ///< array/object: number of nodes of the subtree (its extent in the enclosing sequence); escaped string: source offset of the opening quote plus one (JSON_DIAGNOSTIC_POSITIONS; 0: unknown) }; static_assert(sizeof(node) == 16, "node must stay 16 bytes"); diff --git a/include/nlohmann/detail/view/scan.hpp b/include/nlohmann/detail/view/scan.hpp index e230afbdb..b8d862203 100644 --- a/include/nlohmann/detail/view/scan.hpp +++ b/include/nlohmann/detail/view/scan.hpp @@ -172,6 +172,32 @@ NLOHMANN_VIEW_ALWAYS_INLINE const unsigned char* skip_digits(const unsigned char return p; } +/// advance over the text of a line comment (after its "//") to the line +/// break or the end (with NulIsEnd also a null byte, which is left for the +/// caller to see, as in parse()) +template +inline const unsigned char* skip_line_comment(const unsigned char* p, const unsigned char* e) noexcept +{ + while (p != e && *p != '\n' && *p != '\r' && !(NulIsEnd && *p == 0)) + { + ++p; + } + return p; +} + +/// the "*/" that ends a block comment whose text starts at p (after its "/*"); +/// if it is unterminated, the end (with NulIsEnd also a null byte), which is +/// where the caller reports the error +template +inline const unsigned char* find_block_comment_end(const unsigned char* p, const unsigned char* e) noexcept +{ + while (p != e && !(NulIsEnd && *p == 0) && !(*p == '*' && p + 1 != e && p[1] == '/')) + { + ++p; + } + return p; +} + /// powers of ten up to 10^19 as integers inline std::uint64_t int_pow10(unsigned k) noexcept { diff --git a/include/nlohmann/json.hpp b/include/nlohmann/json.hpp index c2dddd55a..4d4f0a1cd 100644 --- a/include/nlohmann/json.hpp +++ b/include/nlohmann/json.hpp @@ -159,7 +159,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec friend class ::nlohmann::detail::binary_writer; template friend class ::nlohmann::detail::binary_reader; - template + template friend class ::nlohmann::detail::json_sax_dom_parser; template friend class ::nlohmann::detail::json_sax_dom_binary_parser; @@ -6950,6 +6950,25 @@ public: /// @sa https://json.nlohmann.me/api/basic_json/patch/ void patch_inplace(const basic_json& json_patch) { + // the operations below change *this while iterating json_patch: apply + // a copy when a value is applied as patch to itself or, with + // JSON_DIAGNOSTICS, to a value stored inside it + // NOLINTNEXTLINE(misc-const-correctness): only the JSON_DIAGNOSTICS build modifies it + bool patch_aliases_this = (&json_patch == this); +#if JSON_DIAGNOSTICS + for (const basic_json* p = json_patch.m_parent; !patch_aliases_this && p != nullptr; p = p->m_parent) + { + patch_aliases_this = (p == this); + } +#endif + if (JSON_HEDLEY_UNLIKELY(patch_aliases_this)) + { + // NOLINTNEXTLINE(performance-unnecessary-copy-initialization): *this is modified below + const basic_json json_patch_copy = json_patch; + patch_inplace(json_patch_copy); + return; + } + basic_json& result = *this; // the valid JSON Patch operations enum class patch_operations {add, remove, replace, move, copy, test, invalid}; diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index 085b55afb..98d6045fa 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -15629,7 +15629,10 @@ struct diagnostic_positions { // Lexer has read the first character of the container, so // subtract 1 from the position to get the correct start position. - v.start_position = lexer->get_chars_read_total() - 1; + // (npos: a value without a source position, see materialize() of + // json_view) + const std::size_t read = lexer->get_chars_read_total(); + v.start_position = read == std::string::npos ? std::string::npos : read - 1; } } @@ -15644,6 +15647,12 @@ struct diagnostic_positions { // Lexer's position is past the closing brace or bracket, so set that as the end position. v.end_position = lexer->get_chars_read_total(); + if (v.end_position == std::string::npos) + { + // a range has both ends or none (exception::diagnostics() + // prints only complete ones) + v.start_position = std::string::npos; + } } } @@ -15660,6 +15669,12 @@ struct diagnostic_positions // The start position will be set below based on the length of the string representation // of the value. v.end_position = lexer->get_chars_read_total(); + if (v.end_position == std::string::npos) + { + // a value without a source position + v.start_position = std::string::npos; + return; + } switch (v.type()) { @@ -15872,8 +15887,12 @@ read up to it. @tparam Collect whether to collect the values of containers of unknown size on a stack (only for a caller that does not use the value after a parse error) +@tparam LexerType the type of the lexer that reports the source positions + (JSON_DIAGNOSTIC_POSITIONS); only diagnostic_positions uses + it, so any type with its three functions works (as + materialize() of json_view uses it) */ -template +template> class json_sax_dom_parser : public json_sax_dom_error_state { public: @@ -15882,7 +15901,7 @@ class json_sax_dom_parser : public json_sax_dom_error_state using number_float_t = typename BasicJsonType::number_float_t; using string_t = typename BasicJsonType::string_t; using binary_t = typename BasicJsonType::binary_t; - using lexer_t = lexer; + using lexer_t = LexerType; /*! @param[in,out] r reference to a JSON value that is manipulated while @@ -22515,8 +22534,8 @@ using parser_callback_t = template struct sax_ignores_number_text : std::false_type {}; -template -struct sax_ignores_number_text> : std::true_type {}; +template +struct sax_ignores_number_text> : std::true_type {}; template struct sax_ignores_number_text> : std::true_type {}; @@ -24331,6 +24350,13 @@ class json_pointer /// @sa https://json.nlohmann.me/api/json_pointer/operator_slasheq/ json_pointer& operator/=(const json_pointer& ptr) { + // insert() must not be given iterators into the vector it inserts + // into: append a copy when a pointer is appended to itself + if (JSON_HEDLEY_UNLIKELY(&ptr == this)) + { + return *this /= json_pointer(ptr); + } + reference_tokens.insert(reference_tokens.end(), ptr.reference_tokens.begin(), ptr.reference_tokens.end()); @@ -36910,7 +36936,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec friend class ::nlohmann::detail::binary_writer; template friend class ::nlohmann::detail::binary_reader; - template + template friend class ::nlohmann::detail::json_sax_dom_parser; template friend class ::nlohmann::detail::json_sax_dom_binary_parser; @@ -43701,6 +43727,25 @@ public: /// @sa https://json.nlohmann.me/api/basic_json/patch/ void patch_inplace(const basic_json& json_patch) { + // the operations below change *this while iterating json_patch: apply + // a copy when a value is applied as patch to itself or, with + // JSON_DIAGNOSTICS, to a value stored inside it + // NOLINTNEXTLINE(misc-const-correctness): only the JSON_DIAGNOSTICS build modifies it + bool patch_aliases_this = (&json_patch == this); +#if JSON_DIAGNOSTICS + for (const basic_json* p = json_patch.m_parent; !patch_aliases_this && p != nullptr; p = p->m_parent) + { + patch_aliases_this = (p == this); + } +#endif + if (JSON_HEDLEY_UNLIKELY(patch_aliases_this)) + { + // NOLINTNEXTLINE(performance-unnecessary-copy-initialization): *this is modified below + const basic_json json_patch_copy = json_patch; + patch_inplace(json_patch_copy); + return; + } + basic_json& result = *this; // the valid JSON Patch operations enum class patch_operations {add, remove, replace, move, copy, test, invalid}; diff --git a/single_include/nlohmann/json_view.hpp b/single_include/nlohmann/json_view.hpp index c7eca0ddb..1bc932c14 100644 --- a/single_include/nlohmann/json_view.hpp +++ b/single_include/nlohmann/json_view.hpp @@ -269,7 +269,7 @@ struct node std::uint16_t extra; ///< numbers: integer digits (low byte) and fraction digits (high byte), 255 = "many"; objects: number of the object's key hash index (1-based, 0 = none); otherwise 0 std::uint32_t off; ///< source offset (string content, number token, literal, bracket); arena offset if escaped/edited; number of the element sequence if moved std::uint32_t len; ///< string: decoded bytes; float: token bytes; array/object: element count - std::uint32_t next; ///< array/object: number of nodes of the subtree (its extent in the enclosing sequence) + std::uint32_t next; ///< array/object: number of nodes of the subtree (its extent in the enclosing sequence); escaped string: source offset of the opening quote plus one (JSON_DIAGNOSTIC_POSITIONS; 0: unknown) }; static_assert(sizeof(node) == 16, "node must stay 16 bytes"); @@ -1185,6 +1185,32 @@ NLOHMANN_VIEW_ALWAYS_INLINE const unsigned char* skip_digits(const unsigned char return p; } +/// advance over the text of a line comment (after its "//") to the line +/// break or the end (with NulIsEnd also a null byte, which is left for the +/// caller to see, as in parse()) +template +inline const unsigned char* skip_line_comment(const unsigned char* p, const unsigned char* e) noexcept +{ + while (p != e && *p != '\n' && *p != '\r' && !(NulIsEnd && *p == 0)) + { + ++p; + } + return p; +} + +/// the "*/" that ends a block comment whose text starts at p (after its "/*"); +/// if it is unterminated, the end (with NulIsEnd also a null byte), which is +/// where the caller reports the error +template +inline const unsigned char* find_block_comment_end(const unsigned char* p, const unsigned char* e) noexcept +{ + while (p != e && !(NulIsEnd && *p == 0) && !(*p == '*' && p + 1 != e && p[1] == '/')) + { + ++p; + } + return p; +} + /// powers of ten up to 10^19 as integers inline std::uint64_t int_pow10(unsigned k) noexcept { @@ -1382,31 +1408,16 @@ class builder } if (p[1] == '/') { - p += 2; - // (as in parse(), a null byte is the end of the input, so it is - // left for the caller to see) - while (p != m_end && *p != '\n' && *p != '\r' && !(NulIsEnd && *p == 0)) - { - ++p; - } - return p; + return skip_line_comment(p + 2, m_end); } if (p[1] == '*') { - p += 2; - for (;;) + p = find_block_comment_end(p + 2, m_end); + if (p == m_end || *p != '*') { - if (p == m_end || (NulIsEnd && *p == 0)) - { - return fail_at(error_code::comment_unterminated, p); - } - if (*p == '*' && p + 1 != m_end && p[1] == '/') - { - p += 2; - return p; - } - ++p; + return fail_at(error_code::comment_unterminated, p); } + return p + 2; } ++p; return fail_at(error_code::comment_start, p); @@ -2402,7 +2413,14 @@ indent_done: return false; } p = r.p; - push_node(node_kind::string, node_flags::escaped, 0, r.start, r.len); +#if JSON_DIAGNOSTIC_POSITIONS + // the decoded string is in the arena: next keeps the source offset + // of the opening quote plus one (0: unknown) for materialize() + const std::uint64_t second = static_cast(r.len) | (static_cast(s - b) << 32); +#else + const std::uint64_t second = r.len; +#endif + push_node(node_kind::string, node_flags::escaped, 0, r.start, second); return true; } }; @@ -5099,7 +5117,8 @@ constexpr std::uint64_t image_limit = 0xFFFFFFF0u; /// Copy the current structure of an edited document into nodes in document /// order, as the parser would have written them. Text written by edits is /// appended to text_tail (number tokens) and arena_tail (strings); floats that -/// are not finite become null, as dump() writes them. +/// are not finite become null, as dump() writes them. Nodes keep the flag +/// is_new: they have no source position (JSON_DIAGNOSTIC_POSITIONS). inline void compact_nodes(const document_data& d, std::size_t arena_size, std::vector& out, std::string& text_tail, std::string& arena_tail) { struct frame @@ -5122,16 +5141,18 @@ inline void compact_nodes(const document_data& d, std::size_t arena_size, std::v arena_tail.append(d.str(s), s.len); r.flags = node_flags::escaped; } + r.flags = static_cast(r.flags | (s.flags & node_flags::is_new)); return r; }; const auto copy_node = [&](const node * v) { node r = *v; + const auto is_new = static_cast(v->flags & node_flags::is_new); switch (static_cast(v->kind)) { case value_t::object: case value_t::array: - r.flags = 0; + r.flags = is_new; r.extra = 0; r.off = (v->flags & (node_flags::moved | node_flags::is_new)) != 0 ? 0 : v->off; r.len = 0; // counted below @@ -5148,7 +5169,7 @@ inline void compact_nodes(const document_data& d, std::size_t arena_size, std::v r.off = static_cast(d.size + text_tail.size()); text_tail.append(d.str(*v), number_length(*v)); } - r.flags = 0; + r.flags = is_new; break; case value_t::number_float: if ((v->flags & node_flags::storage) == node_flags::edited) @@ -5158,22 +5179,23 @@ inline void compact_nodes(const document_data& d, std::size_t arena_size, std::v { r = node{}; // nan and infinity: null, as dump() writes them r.kind = node_kind::null; + r.flags = is_new; break; } r.off = static_cast(d.size + text_tail.size()); text_tail.append(t, v->len); r.extra = 0xFFFFu; // the digit layout is not recorded } - r.flags = 0; + r.flags = is_new; break; case value_t::boolean: - r.flags = static_cast(v->flags & node_flags::is_true); + r.flags = static_cast(v->flags & (node_flags::is_true | node_flags::is_new)); break; case value_t::null: case value_t::binary: case value_t::discarded: default: - r.flags = 0; + r.flags = is_new; break; } out.push_back(r); @@ -5534,13 +5556,15 @@ inline bool check_image(const node* nodes, std::size_t count, const unsigned cha std::vector floats; const auto check_string = [&](const node & n) -> bool { - if ((n.flags & ~node_flags::escaped) != 0 || n.extra != 0) + if ((n.flags & ~(node_flags::escaped | node_flags::is_new)) != 0 || n.extra != 0) { return false; } const bool decoded = (n.flags & node_flags::escaped) != 0; const std::size_t limit = decoded ? arena_size : text_size; - if (n.off > limit || n.len > limit - n.off) + // (next: the source offset of a decoded string's opening quote plus + // one, or 0; see node) + if (n.off > limit || n.len > limit - n.off || n.next > text_size) { return false; } @@ -5612,13 +5636,13 @@ inline bool check_image(const node* nodes, std::size_t count, const unsigned cha { case value_t::null: // (the offset of a literal is read to size the output of dump()) - if (n.flags != 0 || n.extra != 0 || n.off > text_size) + if ((n.flags & ~node_flags::is_new) != 0 || n.extra != 0 || n.off > text_size) { return false; } break; case value_t::boolean: - if ((n.flags & ~node_flags::is_true) != 0 || n.extra != 0 || n.off > text_size) + if ((n.flags & ~(node_flags::is_true | node_flags::is_new)) != 0 || n.extra != 0 || n.off > text_size) { return false; } @@ -5632,7 +5656,7 @@ inline bool check_image(const node* nodes, std::size_t count, const unsigned cha case value_t::number_integer: case value_t::number_unsigned: case value_t::number_float: - if (n.flags != 0 || !number_in_bounds(n)) + if ((n.flags & ~node_flags::is_new) != 0 || !number_in_bounds(n)) { return false; } @@ -5654,7 +5678,7 @@ inline bool check_image(const node* nodes, std::size_t count, const unsigned cha case value_t::object: { const std::size_t limit = stack.empty() ? count : stack.back().end; - if (n.flags != 0 || n.extra != 0 || n.next == 0 || n.next > limit - i || n.off > text_size) + if ((n.flags & ~node_flags::is_new) != 0 || n.extra != 0 || n.next == 0 || n.next > limit - i || n.off > text_size) { return false; } @@ -6257,6 +6281,8 @@ NLOHMANN_JSON_NAMESPACE_END // #include +// #include + NLOHMANN_JSON_NAMESPACE_BEGIN namespace detail @@ -6264,6 +6290,189 @@ namespace detail namespace view { +#if JSON_DIAGNOSTIC_POSITIONS +/*! +@brief the source positions of the values materialize() replays + +Stands in for the lexer of json_sax_dom_parser, which reads the positions of +the values it stores from these three functions (diagnostic_positions). npos +is a value without a source position. One position_source serves a whole +materialize() call: before each value is handed to the handler, the range of +that value is set (or cleared), overwriting that of the value before. +*/ +struct position_source +{ + /// the token, of which diagnostic_positions only reads the length + struct token + { + std::size_t length; + std::size_t size() const noexcept + { + return length; + } + }; + + std::size_t start = std::string::npos; ///< the first byte of the next value + std::size_t end = std::string::npos; ///< one past the next value (start_object/start_array: one past the bracket) + + std::size_t get_chars_read_total() const noexcept + { + return end; + } + std::size_t get_token_start_position() const noexcept + { + return start; + } + token get_string() const noexcept + { + return token{end - start}; + } + + /// the range [first, last), or none if either is npos + void set(std::size_t first, std::size_t last) noexcept + { + if (first == std::string::npos || last == std::string::npos) + { + clear(); + return; + } + start = first; + end = last; + } + + /// no range: the next value has no source position + void clear() noexcept + { + start = std::string::npos; + end = std::string::npos; + } + + /// one past the closing quote of the string whose opening quote is at + /// src[quote], or npos (the scan is bounded by size, as images can be + /// loaded without a check) + static std::size_t string_end(const char* src, std::size_t size, std::size_t quote) noexcept + { + for (std::size_t i = quote + 1; i < size; ++i) + { + if (src[i] == '\\') + { + ++i; + } + else if (src[i] == '"') + { + return i + 1; + } + } + return std::string::npos; + } + + /// one past the closing bracket of the array or object whose last element + /// ends at src[pos] (or whose opening bracket is at src[pos - 1]), or npos; + /// in between, there can only be whitespace, comments, and a trailing comma + /// (the source was validated when it was parsed) + static std::size_t container_end(const char* src, std::size_t size, std::size_t pos) noexcept + { + const auto* const b = reinterpret_cast(src); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast): the text as bytes, as the builder reads it + const unsigned char* const e = b + size; + while (pos < size) + { + switch (src[pos]) + { + case ' ': + case '\t': + case '\n': + case '\r': + case ',': + ++pos; + break; + case ']': + case '}': + return pos + 1; + case '/': + if (pos + 1 < size && src[pos + 1] == '/') + { + pos = static_cast(skip_line_comment(b + pos + 2, e) - b); + break; + } + if (pos + 1 < size && src[pos + 1] == '*') + { + // (unterminated: pos is size, which ends the loop) + pos = static_cast(find_block_comment_end(b + pos + 2, e) - b) + 2; + break; + } + return std::string::npos; + default: + return std::string::npos; + } + } + return std::string::npos; + } + + /// the range of the scalar n of document d + void set_scalar(const document_data& d, const node& n) noexcept + { + if ((n.flags & node_flags::is_new) != 0) + { + clear(); + return; + } + std::size_t first = std::string::npos; + std::size_t last = std::string::npos; + switch (n.kind) + { + case node_kind::string: + if ((n.flags & node_flags::storage) == 0) + { + // off is the first byte after the opening quote + if (n.off != 0) + { + first = n.off - 1; + last = n.off + std::size_t{n.len} + 1; + } + } + else if ((n.flags & node_flags::storage) == node_flags::escaped && n.next != 0) + { + // next is the offset of the opening quote plus one + first = n.next - 1; + last = string_end(d.base[0], d.size, first); + } + break; + case node_kind::number_integer: + case node_kind::number_unsigned: + case node_kind::number_float: + if ((n.flags & node_flags::storage) == 0) + { + first = n.off; + last = n.off + number_length(n); + } + break; + case node_kind::boolean: + first = n.off; + last = n.off + ((n.flags & node_flags::is_true) != 0 ? 4u : 5u); + break; + case node_kind::null: + first = n.off; + last = n.off + 4u; + break; + case node_kind::object: + case node_kind::array: + case node_kind::link: + default: + break; + } + // (a crafted image can point past the text) + if (last == std::string::npos || last > d.size) + { + clear(); + } + else + { + set(first, last); + } + } +}; +#endif + /*! @brief the basic_json value of the subtree at n @@ -6271,14 +6480,24 @@ The subtree is replayed into the SAX handler that parse() uses to build its values, so the result is the value parse() would produce: duplicate keys keep the last value, and with JSON_DIAGNOSTICS the parent pointers are set. It is iterative, so the nesting depth is limited by memory only, as for parse(). -Without a lexer the handler records no source positions -(JSON_DIAGNOSTIC_POSITIONS). + +With JSON_DIAGNOSTIC_POSITIONS, the values get their source positions as with +parse(): position_source stands in for the lexer. Values written by edits have +none (npos), and neither has an array or object whose elements changed, or +whose last element has none (the end of its closing bracket is found by +scanning from the end of the last element). */ template BasicJsonType materialize(const document_data& d, const node* n) { using string_t = typename BasicJsonType::string_t; - using sax_t = json_sax_dom_parser, true>; + using input_t = iterator_input_adapter; +#if JSON_DIAGNOSTIC_POSITIONS + using lexer_t = position_source; +#else + using lexer_t = lexer; // (the default: unused, as no lexer is passed) +#endif + using sax_t = json_sax_dom_parser; using nav = navigation; // the open containers: their next element (or key of the next member), @@ -6290,9 +6509,15 @@ BasicJsonType materialize(const document_data& d, const node* n) const node* pos; const node* end; bool object; +#if JSON_DIAGNOSTIC_POSITIONS + std::size_t start; ///< the opening bracket (npos: unknown) + std::size_t last_end; ///< the end of the last element read (npos: unknown) +#endif }; +#if !JSON_DIAGNOSTIC_POSITIONS // a scalar needs neither the handler nor the stack of open containers + // (with positions, it goes through the handler, which sets them) switch (static_cast(n->kind)) { case value_t::string: @@ -6314,10 +6539,16 @@ BasicJsonType materialize(const document_data& d, const node* n) default: break; } +#endif BasicJsonType result; +#if JSON_DIAGNOSTIC_POSITIONS + position_source positions; + sax_t sax(result, true, &positions); +#else sax_t sax(result, true); - const string_t no_token{}; +#endif + const string_t no_token {}; // (shallow subtrees on the stack: a call for a small value then allocates only for the value) std::array shallow; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init): filled as containers open std::vector deep{}; @@ -6332,6 +6563,16 @@ BasicJsonType materialize(const document_data& d, const node* n) std::size_t depth = 0; for (;;) { +#if JSON_DIAGNOSTIC_POSITIONS + if (!is_container(*n)) + { + positions.set_scalar(d, *n); + if (depth != 0) + { + open[depth - 1].last_end = positions.end; + } + } +#endif // false positive: n comes from nav::value(), which never returns null for a valid index // @infer-ignore NULLPTR_DEREFERENCE switch (static_cast(n->kind)) @@ -6340,6 +6581,17 @@ BasicJsonType materialize(const document_data& d, const node* n) case value_t::array: { const bool object = n->kind == node_kind::object; +#if JSON_DIAGNOSTIC_POSITIONS + // (moved containers are also new) + if ((n->flags & node_flags::is_new) != 0 || n->off >= d.size) + { + positions.clear(); + } + else + { + positions.set(std::size_t{n->off}, std::size_t{n->off} + 1); + } +#endif if (object) { sax.start_object(n->len); @@ -6356,7 +6608,13 @@ BasicJsonType materialize(const document_data& d, const node* n) open = deep.data(); capacity = deep.size(); } - open[depth++] = frame{nav::first(d, n), nav::end(d, n), object}; + // (with positions: until an element is read, the container's + // own bracket is the last thing read) + open[depth++] = frame{nav::first(d, n), nav::end(d, n), object +#if JSON_DIAGNOSTIC_POSITIONS + , positions.start, positions.end +#endif + }; break; } case value_t::string: @@ -6397,6 +6655,31 @@ BasicJsonType materialize(const document_data& d, const node* n) frame& f = open[depth - 1]; if (f.pos == f.end) { +#if JSON_DIAGNOSTIC_POSITIONS + if (f.start == std::string::npos || f.last_end == std::string::npos) + { + positions.clear(); + } + else + { + const std::size_t end = position_source::container_end(d.base[0], d.size, f.last_end); + if (end == std::string::npos) + { + positions.clear(); + } + else + { + // the closing bracket, as the lexer reports it (end_object() + // and end_array() only read its end; the start of the + // container was set when it was opened) + positions.set(end - 1, end); + } + } + if (depth > 1) + { + open[depth - 2].last_end = positions.end; + } +#endif if (f.object) { sax.end_object(); diff --git a/tests/src/unit-json_patch.cpp b/tests/src/unit-json_patch.cpp index cc1b26a40..83df15350 100644 --- a/tests/src/unit-json_patch.cpp +++ b/tests/src/unit-json_patch.cpp @@ -826,6 +826,129 @@ TEST_CASE("JSON patch") CHECK(doc.at("foo") == "bar"); } #endif // !defined(JSON_NOEXCEPTION) + + SECTION("applying a value as patch to itself") + { + // a JSON patch is an array, so an array of operations can be + // applied to itself; the operations are read from the patch as + // it was before the first operation, as patch() does it + const json copy_op = {{"op", "copy"}, {"from", "/0"}, {"path", "/-"}}; + json copy_ops = json::array(); + for (int i = 0; i < 5; ++i) + { + copy_ops.push_back(copy_op); + } + + json expected_copies = copy_ops; + for (int i = 0; i < 5; ++i) + { + expected_copies.push_back(copy_op); + } + + SECTION("copy operations without spare capacity") + { + // a copy has exactly as much capacity as elements + json doc = copy_ops; + CHECK(doc.patch(doc) == expected_copies); + + doc.patch_inplace(doc); + CHECK(doc == expected_copies); + } + +#if JSON_DIAGNOSTICS + SECTION("patch stored inside the patched value") + { + // the parent pointers of JSON_DIAGNOSTICS make the aliasing detectable + const json op = {{"op", "replace"}, {"path", "/patch"}, {"value", 1}}; + json doc = {{"a", 1}, {"patch", json::array({op})}}; + const json expected = doc.patch(doc["patch"]); + + doc.patch_inplace(doc["patch"]); + CHECK(doc == expected); + } +#endif + + SECTION("copy operations with spare capacity") + { + // reserve while the array is empty: reallocating non-empty + // elements behind the library's back leaves stale parent + // pointers with JSON_DIAGNOSTICS + json doc = json::array(); + doc.get_ref().reserve(100); + for (int i = 0; i < 5; ++i) + { + doc.push_back(copy_op); + } + CHECK(doc.patch(doc) == expected_copies); + + doc.patch_inplace(doc); + CHECK(doc == expected_copies); + } + + SECTION("remove operations") + { + json doc = R"([ + {"op": "remove", "path": "/0"}, + {"op": "remove", "path": "/0"}, + {"op": "remove", "path": "/0"} + ])"_json; + CHECK(doc.patch(doc) == json::array()); + + doc.patch_inplace(doc); + CHECK(doc == json::array()); + } + + SECTION("operations that read a changed element") + { + // the second operation is still a "test" operation, although + // the first one replaced it in the document + json doc = R"([ + {"op": "replace", "path": "/1", "value": 1}, + {"op": "test", "path": "/1", "value": 1}, + {"op": "add", "path": "/-", "value": 2} + ])"_json; + const json expected = R"([ + {"op": "replace", "path": "/1", "value": 1}, + 1, + {"op": "add", "path": "/-", "value": 2}, + 2 + ])"_json; + CHECK(doc.patch(doc) == expected); + + doc.patch_inplace(doc); + CHECK(doc == expected); + } + + SECTION("replacing the root") + { + json doc = R"([ + {"op": "add", "path": "", "value": [1, 2]}, + {"op": "add", "path": "/-", "value": 3}, + {"op": "replace", "path": "", "value": {"a": 4}} + ])"_json; + const json expected = {{"a", 4}}; + CHECK(doc.patch(doc) == expected); + + doc.patch_inplace(doc); + CHECK(doc == expected); + } + +#if !defined(JSON_NOEXCEPTION) + SECTION("failing operation") + { + // the operations before the failing one remain applied + json doc = R"([ + {"op": "add", "path": "/-", "value": 1}, + {"op": "test", "path": "/0/op", "value": "remove"} + ])"_json; + CHECK_THROWS_AS(doc.patch(doc), json::other_error&); + + CHECK_THROWS_AS(doc.patch_inplace(doc), json::other_error&); + CHECK(doc.size() == 3); + CHECK(doc.back() == 1); + } +#endif // !defined(JSON_NOEXCEPTION) + } } SECTION("errors") diff --git a/tests/src/unit-json_pointer.cpp b/tests/src/unit-json_pointer.cpp index 81c588544..0e225b86e 100644 --- a/tests/src/unit-json_pointer.cpp +++ b/tests/src/unit-json_pointer.cpp @@ -917,6 +917,46 @@ TEST_CASE("JSON pointers") CHECK(ptr.to_string() == "/object/~1"); } + SECTION("appending a pointer to itself") + { + // operator/= must not pass iterators into its own token vector to + // insert(); each pointer is appended to itself and must yield the + // pointer whose string representation is the original one twice + const std::vector pointers = + { + "", + "/", + "/a", + "/a/b", + "/foo/0/-", + "/a~1b/~0c/~01", + "/1/2/3/4/5/6/7/8/9/10/11/12/13/14/15/16/17" + }; + + for (const auto& s : pointers) + { + CAPTURE(s) + json::json_pointer ptr(s); + const json::json_pointer expected(s + s); + + const json::json_pointer& ret = (ptr /= ptr); + CHECK(&ret == &ptr); + CHECK(ptr == expected); + CHECK(ptr.to_string() == s + s); + + // appending to itself again doubles the pointer once more + ptr /= ptr; + std::string quadruple = s + s; + quadruple += quadruple; + CHECK(ptr.to_string() == quadruple); + + // operator/ appends to a copy and leaves the operand unchanged + const json::json_pointer original(s); + CHECK((original / original) == expected); + CHECK(original.to_string() == s); + } + } + SECTION("equality comparison") { std::string ptr_string{"/foo/bar"}; diff --git a/tests/src/unit-json_view.cpp b/tests/src/unit-json_view.cpp index ee2300e3e..77f7aa4fa 100644 --- a/tests/src/unit-json_view.cpp +++ b/tests/src/unit-json_view.cpp @@ -749,6 +749,20 @@ std::string exception_of(F f) } return ""; } + +// the exception a call on a basic_json value throws, without the byte range of +// the value (JSON_DIAGNOSTIC_POSITIONS), which the exceptions of views lack +template +std::string json_exception_of(F f) +{ + std::string what = exception_of(f); + const std::size_t at = what.find("] (bytes "); + if (at != std::string::npos) + { + what.erase(at + 2, what.find(") ", at) - at); + } + return what; +} #endif // compares a view with the ordered_json value materialize() gives for it: @@ -957,24 +971,24 @@ TEST_CASE("json_view element access and iteration") #if !defined(JSON_NOEXCEPTION) if (!j.is_object()) { - CHECK(exception_of([&] { static_cast(v["a"]); }) == exception_of([&] { static_cast(j["a"]); })); + CHECK(exception_of([&] { static_cast(v["a"]); }) == json_exception_of([&] { static_cast(j["a"]); })); } if (!j.is_array()) { - CHECK(exception_of([&] { static_cast(v[0]); }) == exception_of([&] { static_cast(j[0]); })); + CHECK(exception_of([&] { static_cast(v[0]); }) == json_exception_of([&] { static_cast(j[0]); })); } - CHECK(exception_of([&] { static_cast(v.at("a")); }) == exception_of([&] { static_cast(j.at("a")); })); - CHECK(exception_of([&] { static_cast(v.at("missing")); }) == exception_of([&] { static_cast(j.at("missing")); })); - CHECK(exception_of([&] { static_cast(v.at(0)); }) == exception_of([&] { static_cast(j.at(0)); })); - CHECK(exception_of([&] { static_cast(v.at(5)); }) == exception_of([&] { static_cast(j.at(5)); })); + CHECK(exception_of([&] { static_cast(v.at("a")); }) == json_exception_of([&] { static_cast(j.at("a")); })); + CHECK(exception_of([&] { static_cast(v.at("missing")); }) == json_exception_of([&] { static_cast(j.at("missing")); })); + CHECK(exception_of([&] { static_cast(v.at(0)); }) == json_exception_of([&] { static_cast(j.at(0)); })); + CHECK(exception_of([&] { static_cast(v.at(5)); }) == json_exception_of([&] { static_cast(j.at(5)); })); if (!(j.is_object() && j.empty())) // (key() of an end iterator) { - CHECK(exception_of([&] { static_cast(v.begin().key()); }) == exception_of([&] { static_cast(j.begin().key()); })); + CHECK(exception_of([&] { static_cast(v.begin().key()); }) == json_exception_of([&] { static_cast(j.begin().key()); })); } if (!j.empty() || j.is_null()) { - CHECK(exception_of([&] { static_cast(v.front()); }) == exception_of([&] { static_cast(j.front()); })); - CHECK(exception_of([&] { static_cast(v.back()); }) == exception_of([&] { static_cast(j.back()); })); + CHECK(exception_of([&] { static_cast(v.front()); }) == json_exception_of([&] { static_cast(j.front()); })); + CHECK(exception_of([&] { static_cast(v.back()); }) == json_exception_of([&] { static_cast(j.back()); })); } #endif CHECK(v.contains("a") == j.contains("a")); diff --git a/tests/src/unit-json_view_edit_values.cpp b/tests/src/unit-json_view_edit_values.cpp index 39db8d0ba..881d957f8 100644 --- a/tests/src/unit-json_view_edit_values.cpp +++ b/tests/src/unit-json_view_edit_values.cpp @@ -54,14 +54,18 @@ int accepts_any(nlohmann::basic_json_view /*unused*/) return Mode == nlohmann::view_mode::editable ? 3 : 4; } -template -auto overload(View v) -> decltype(accepts_editable(v)) +// overloads that differ only in the mode; deduction ignores the conversion from +// read-only to editable views, so each view picks the overload of its own mode +// (two templates that differ only in a decltype return type are a redefinition +// for MSVC 2015) +template +int overload(nlohmann::basic_json_view v) { return accepts_editable(v); } -template -auto overload(View v) -> decltype(accepts_read_only(v)) +template +int overload(nlohmann::basic_json_view v) { return accepts_read_only(v); } diff --git a/tests/src/unit-json_view_from_json.cpp b/tests/src/unit-json_view_from_json.cpp index 4a776dd52..7e84a9081 100644 --- a/tests/src/unit-json_view_from_json.cpp +++ b/tests/src/unit-json_view_from_json.cpp @@ -69,9 +69,12 @@ struct point struct shape { // initialized for GCC -Weffc++ + // NOLINTNEXTLINE(readability-redundant-member-init) std::string name{}; + // NOLINTNEXTLINE(readability-redundant-member-init) std::vector points{}; point center{}; + // NOLINTNEXTLINE(readability-redundant-member-init) std::map named{}; NLOHMANN_DEFINE_TYPE_INTRUSIVE(shape, name, points, center, named) }; @@ -82,13 +85,17 @@ struct with_defaults std::string b = "default"; std::vector c = {1, 2}; }; +// NOLINTNEXTLINE(misc-use-internal-linkage): found by ADL, like the from_json() functions of users NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE_WITH_DEFAULT(with_defaults, a, b, c) struct with_names { int value = 0; - std::string text{}; // initialized for GCC -Weffc++ + // initialized for GCC -Weffc++ + // NOLINTNEXTLINE(readability-redundant-member-init) + std::string text{}; }; +// NOLINTNEXTLINE(misc-use-internal-linkage): found by ADL, like the from_json() functions of users NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE_WITH_NAMES(with_names, "Value", value, "Text", text) struct empty @@ -102,6 +109,7 @@ enum class color green, blue }; +// NOLINTNEXTLINE(misc-use-internal-linkage): found by ADL, like the from_json() functions of users NLOHMANN_JSON_SERIALIZE_ENUM(color, { {color::red, "red"}, @@ -114,6 +122,7 @@ enum class strict_color red, green }; +// NOLINTNEXTLINE(misc-use-internal-linkage): found by ADL, like the from_json() functions of users NLOHMANN_JSON_SERIALIZE_ENUM_STRICT(strict_color, { {strict_color::red, "red"}, @@ -154,6 +163,7 @@ struct derived_from_legacy : legacy_base { int extra = 0; }; +// NOLINTNEXTLINE(misc-use-internal-linkage): found by ADL, like the from_json() functions of users NLOHMANN_DEFINE_DERIVED_TYPE_NON_INTRUSIVE(derived_from_legacy, legacy_base, extra) // from_json() for json only: converted through materialize() @@ -301,18 +311,19 @@ TEST_CASE("json_view from_json") CHECK(same(v, j, "text")); CHECK(same(v, j, "nothing")); CHECK(same(v, j, "object")); - const std::valarray va = v["ints"].get>(); + const auto va = v["ints"].get>(); CHECK(va.size() == 4); CHECK(va[0] == 3); int c_array[2][2] = {{0, 0}, {0, 0}}; // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays): the conversion to C arrays v["nested"][0].get_to(c_array[0]); CHECK(c_array[0][1] == 2); #ifdef JSON_HAS_CPP_17 - CHECK(v["points"][0].get>()->y == 2); + const auto first = v["points"][0].get>(); + CHECK((first.has_value() && first->y == 2)); CHECK(!v["nothing"].get>().has_value()); #endif // containers of views keep views of the elements - const std::vector views = v["points"].get>(); + const auto views = v["points"].get>(); CHECK(views.size() == 2); CHECK(views[1]["x"].get() == 3); CHECK(v["object"].get>().at("a").get() == 2); @@ -326,7 +337,8 @@ TEST_CASE("json_view from_json") CHECK(d.root().get>().at("a") == 3); } -#if !defined(JSON_NOEXCEPTION) && !JSON_DIAGNOSTICS + // (the exceptions of views have neither the path nor the byte range of the value) +#if !defined(JSON_NOEXCEPTION) && !JSON_DIAGNOSTICS && !JSON_DIAGNOSTIC_POSITIONS SECTION("std types: the same errors as from the materialized value") { const json_document d = json_document::parse(R"([[1, "x"], {"a": 1}, "text", [1], 5, null])"); @@ -352,7 +364,7 @@ TEST_CASE("json_view from_json") { const json_document d = json_document::parse(R"({"name": "s", "points": [{"x": 1, "y": 2}, {"x": 3, "y": 4}], "center": {"x": 5, "y": 6}, "named": {"a": {"x": 7, "y": 8}}})"); - const shape s = d.root().get(); + const auto s = d.root().get(); CHECK(s.name == "s"); CHECK(s.points.size() == 2); CHECK(s.points[1].y == 4); @@ -370,13 +382,13 @@ TEST_CASE("json_view from_json") SECTION("defaults and names") { const json_document d = json_document::parse(R"({"a": 5, "Value": 3, "Text": "t"})"); - const with_defaults w = d.root().get(); + const auto w = d.root().get(); CHECK(w.a == 5); CHECK(w.b == "default"); CHECK(w.c == std::vector {1, 2}); const json_document null_document = json_document::parse("null"); CHECK(null_document.root().get().a == 1); - const with_names n = d.root().get(); + const auto n = d.root().get(); CHECK(n.value == 3); CHECK(n.text == "t"); } @@ -392,7 +404,7 @@ TEST_CASE("json_view from_json") #if !defined(JSON_NOEXCEPTION) const json j = d.root().materialize(); CHECK(exception_of_call([&] { static_cast(d.root()[2].get()); }) == "[json.exception.out_of_range.410] enum value out of range for strict_color: \"purple\""); -#if !JSON_DIAGNOSTICS +#if !JSON_DIAGNOSTICS && !JSON_DIAGNOSTIC_POSITIONS CHECK(exception_of_call([&] { static_cast(d.root()[2].get()); }) == exception_of_call([&] { static_cast(j[2].get()); })); #endif #endif @@ -401,13 +413,13 @@ TEST_CASE("json_view from_json") SECTION("derived types") { const json_document d = json_document::parse(R"({"id": 7, "c": "green", "extra": 3})"); - const derived x = d.root().get(); + const auto x = d.root().get(); CHECK(x.id == 7); CHECK(x.c == color::green); - const derived_with_defaults y = d.root().get(); + const auto y = d.root().get(); CHECK(y.id == 7); CHECK(y.label == "none"); - const derived_from_legacy z = d.root().get(); + const auto z = d.root().get(); CHECK(z.id == 7); CHECK(z.extra == 3); } @@ -440,10 +452,11 @@ TEST_CASE("json_view from_json") CHECK(o.root().get().center.y == 2); } -#if !defined(JSON_NOEXCEPTION) && !JSON_DIAGNOSTICS +#if !defined(JSON_NOEXCEPTION) && !JSON_DIAGNOSTICS && !JSON_DIAGNOSTIC_POSITIONS SECTION("errors are those of the same conversion of a json value") { - // (with JSON_DIAGNOSTICS, those of json name the path of the value) + // (with JSON_DIAGNOSTICS, those of json name the path of the value, + // and with JSON_DIAGNOSTIC_POSITIONS its bytes) const json_document d = json_document::parse(R"([{"x": 1}, {"x": "1", "y": 2}, [1, 2], {"id": "7", "c": "red"}])"); const json j = d.root().materialize(); for (std::size_t i = 0; i < 3; ++i) diff --git a/tests/src/unit-json_view_positions.cpp b/tests/src/unit-json_view_positions.cpp new file mode 100644 index 000000000..857b0cb8f --- /dev/null +++ b/tests/src/unit-json_view_positions.cpp @@ -0,0 +1,346 @@ +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ (supporting code) +// | | |__ | | | | | | version 3.12.0 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann +// SPDX-License-Identifier: MIT + +// source positions of materialize() (JSON_DIAGNOSTIC_POSITIONS, #5864) + +#include "doctest_compatibility.h" + +#ifndef JSON_DIAGNOSTICS + #define JSON_DIAGNOSTICS 1 +#endif +#define JSON_DIAGNOSTIC_POSITIONS 1 +#include +using nlohmann::json; +using nlohmann::ordered_json; +using nlohmann::json_document; +using nlohmann::json_editable_document; + +#include +#include +#include +#include +#include +#include + +#include "json_view_test_helpers.hpp" +#include + +namespace +{ +constexpr std::size_t npos = std::string::npos; + +// whether every value of a has the source positions of the value of b at the +// same place (a and b are equal; shallow test documents only) +template +bool same_positions(const BasicJsonType& a, const BasicJsonType& b) +{ + if (a.start_pos() != b.start_pos() || a.end_pos() != b.end_pos() || a.type() != b.type() || a.size() != b.size()) + { + return false; + } + if (a.is_object()) + { + for (auto it = a.begin(); it != a.end(); ++it) + { + if (!same_positions(it.value(), b.at(it.key()))) + { + return false; + } + } + } + else if (a.is_array()) + { + for (std::size_t i = 0; i < a.size(); ++i) + { + if (!same_positions(a[i], b[i])) + { + return false; + } + } + } + return true; +} + +// the materialized root of text has the positions of parse() +template +bool positions_as_parse(const std::string& text, bool comments = false, bool trailing_commas = false) +{ + const auto j = BasicJsonType::parse(text, nullptr, true, comments, trailing_commas); + const auto d = nlohmann::basic_json_document::parse(text, true, comments, trailing_commas); + const auto e = nlohmann::basic_json_document::parse(text, true, comments, trailing_commas); + return same_positions(d.root().materialize(), j) && same_positions(e.root().materialize(), j); +} + +std::string range(const json& j) +{ + return j.start_pos() == npos && j.end_pos() == npos ? "none" : std::to_string(j.start_pos()) + "-" + std::to_string(j.end_pos()); +} +} // namespace + +TEST_CASE("json_view materialize() source positions") +{ + SECTION("as parse()") + { + const std::vector texts = + { + "null", "true", "false", "0", "-12", "3.25e-2", "18446744073709551615", "-9223372036854775809", R"("a")", + R"("\n")", R"("\u00e9")", R"("\ud83d\ude00")", "\"\xf0\x9f\x98\x80\"", R"("a\"b\\")", R"("x\\")", + " [ ] ", "{}", "[[[]]]", R"({"a":{"b":[1,{"c":[]}]}})", + R"({"\u0041": "v\n", "k": "\"", "\t":[ "\/" ]})", + R"([1, 2.5, true, false, null, "s", "\u0000", -0])", + "\n\t[\r\n 1 ,\n {\n \"a\" : [ ]\n }\n]\n", + "\xef\xbb\xbf[1, {\"a\": \"b\"}]", "\xef\xbb\xbf\"\\n\"", + }; + for (const auto& text : texts) + { + CAPTURE(text) + CHECK(positions_as_parse(text)); + CHECK(positions_as_parse(text)); + } + } + + SECTION("comments and trailing commas") + { + const std::vector texts = + { + "[1 /* ] */]", "[1 // ]\n]", "{\"a\": 1 /* } */ , }", "[1, 2, ]", "[[1,],[2 , /**/ ] , ]", + "{\"a\": [] // }\n, \"b\": {} /* } ] */ }", "[ /* empty */ ]", "{ // }\n}", "/* a */ [1] // b", + R"(["\n" /* " */ , ])", + }; + for (const auto& text : texts) + { + CAPTURE(text) + CHECK(positions_as_parse(text, true, true)); + } + } + + SECTION("generated documents") + { + json_view_test::generator gen; + for (int i = 0; i < 300; ++i) + { + std::string text; + gen.value(text, 0); + CAPTURE(text) + CHECK(positions_as_parse(text)); + CHECK(positions_as_parse(text)); + } + } + + SECTION("test data") + { + for (const char* name : + { + "/json_testsuite/sample.json", "/nativejson-benchmark/canada.json", "/nativejson-benchmark/citm_catalog.json", + "/nativejson-benchmark/twitter.json", "/json_tests/pass1.json" + }) + { + CAPTURE(name) + const std::ifstream f(std::string(TEST_DATA_DIRECTORY) + name, std::ios::binary); + std::stringstream ss; + ss << f.rdbuf(); + CHECK(positions_as_parse(ss.str())); + } + } + + SECTION("subtrees") + { + const std::string text = R"({"a": [1, "\n", {"b": null}], "s": "x\ty", "n": -1.5})"; + const json j = json::parse(text); + const json_document d = json_document::parse(text); + CHECK(same_positions(d.root()["a"].materialize(), j["a"])); + CHECK(same_positions(d.root()["a"][2].materialize(), j["a"][2])); + CHECK(same_positions(d.root()["a"][1].materialize(), j["a"][1])); + CHECK(same_positions(d.root()["s"].materialize(), j["s"])); + CHECK(same_positions(d.root()["n"].materialize(), j["n"])); + CHECK(range(d.root()["a"][1].materialize()) == "10-14"); + CHECK(range(d.root()["s"].materialize()) == "35-41"); + } + +#if !defined(JSON_NOEXCEPTION) + SECTION("exception messages") + { + const std::string text = "{\n \"address\": {\"street\": \"Fake\\tStreet\", \"housenumber\": \"1\"},\n \"list\": [1, [true]]\n}"; + const json j = json::parse(text); + const json_document d = json_document::parse(text); + const json m = d.root().materialize(); + const std::string expected = json_view_test::exception_of_call([&] + { + static_cast(j.at("address").at("housenumber").get()); + }); + CHECK(expected.find("(bytes ") != std::string::npos); + CHECK(json_view_test::exception_of_call([&] + { + static_cast(m.at("address").at("housenumber").get()); + }) == expected); + CHECK(json_view_test::exception_of_call([&] + { + static_cast(m.at("address").at("street").get()); + }) == json_view_test::exception_of_call([&] + { + static_cast(j.at("address").at("street").get()); + })); + CHECK(json_view_test::exception_of_call([&] + { + static_cast(m.at("list").at(1).get()); + }) == json_view_test::exception_of_call([&] + { + static_cast(j.at("list").at(1).get()); + })); + } +#endif + + SECTION("edited documents") + { + const std::string text = R"({"a": [1, 2], "b": {"c": "\n", "d": true}, "e": [3, [4]], "f": "x"})"; + const json j = json::parse(text); + + SECTION("new scalars have none") + { + json_editable_document d = json_editable_document::parse(text); + d.set(d.root()["f"], 5); + d.set(d.root()["b"]["d"], "s"); + const json m = d.root().materialize(); + CHECK(range(m["f"]) == "none"); + CHECK(range(m["b"]["d"]) == "none"); + // the object whose last member is new has none; the others keep theirs + CHECK(range(m) == "none"); + CHECK(range(m["b"]) == "none"); + CHECK(same_positions(m["a"], j["a"])); + CHECK(same_positions(m["e"], j["e"])); + CHECK(same_positions(m["b"]["c"], j["b"]["c"])); + } + + SECTION("containers whose elements change have none") + { + json_editable_document d = json_editable_document::parse(text); + d.push_back(d.root()["a"], 7); + d.erase(d.root()["e"], 0); + const json m = d.root().materialize(); + CHECK(range(m["a"]) == "none"); + CHECK(range(m["a"][2]) == "none"); + CHECK(range(m["e"]) == "none"); + // the elements kept keep theirs + CHECK(range(m["a"][0]) == range(j["a"][0])); + CHECK(range(m["a"][1]) == range(j["a"][1])); + CHECK(same_positions(m["e"][0], j["e"][1])); + CHECK(same_positions(m["b"], j["b"])); + // the root's elements did not change + CHECK(range(m) == range(j)); + } + + SECTION("new containers have none") + { + json_editable_document d = json_editable_document::parse(text); + d.set(d.root()["b"], json::parse(R"({"x": [1]})")); + const json m = d.root().materialize(); + CHECK(range(m["b"]) == "none"); + CHECK(range(m["b"]["x"]) == "none"); + CHECK(range(m["b"]["x"][0]) == "none"); + CHECK(range(m) == range(j)); + } + + SECTION("copies of scalars of the document keep theirs") + { + // (a copy of an array or object is a new one; its scalars are the + // parsed ones) + json_editable_document d = json_editable_document::parse(text); + d.set(d.root()["f"], d.root()["e"]); + d.set(d.root()["a"], d.root()["b"]["c"]); + const json m = d.root().materialize(); + CHECK(range(m["a"]) == range(j["b"]["c"])); + CHECK(range(m["f"]) == "none"); + CHECK(range(m["f"][1]) == "none"); + CHECK(range(m["f"][0]) == range(j["e"][0])); + CHECK(range(m["f"][1][0]) == range(j["e"][1][0])); + } + } + +#if !(defined(__BYTE_ORDER__) && defined(__ORDER_BIG_ENDIAN__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__) + SECTION("images") + { + const std::string text = R"({"a\n": ["\u00e9", 1.5, {"b": [true, null]}], "c": "d"})"; + const json j = json::parse(text); + + SECTION("round trip") + { + const json_document d = json_document::load(json_document::parse(text).save()); + CHECK(same_positions(d.root().materialize(), j)); + } + + SECTION("edited") + { + json_editable_document e = json_editable_document::parse(text); + e.set(e.root()["c"], 1); + e.push_back(e.root()[std::string("a") + '\n'], false); + e.set(e.root()[std::string("a") + '\n'][2]["b"][0], "\""); + const json_document d = json_document::load(e.save()); + const json m = d.root().materialize(); + CHECK(range(m["c"]) == "none"); + CHECK(range(m) == "none"); + const json& a = m[std::string("a") + '\n']; + const json& ja = j[std::string("a") + '\n']; + CHECK(range(a) == "none"); + CHECK(range(a[3]) == "none"); + CHECK(range(a[0]) == range(ja[0])); + CHECK(range(a[1]) == range(ja[1])); + CHECK(range(a[2]["b"][0]) == "none"); + CHECK(range(a[2]["b"][1]) == range(ja[2]["b"][1])); + // the last element of b is still the parsed one + CHECK(range(a[2]["b"]) == range(ja[2]["b"])); + CHECK(range(a[2]) == range(ja[2])); + } + + SECTION("without positions") + { + // an image saved without JSON_DIAGNOSTIC_POSITIONS: next is 0 for + // the strings with escapes + std::vector image = json_document::parse(text).save(); + std::uint64_t count = 0; + std::memcpy(&count, image.data() + 8, sizeof(count)); + for (std::size_t i = 0; i < count; ++i) + { + nlohmann::detail::view::node n{}; + std::memcpy(&n, image.data() + 64 + (i * sizeof(n)), sizeof(n)); + if (n.kind == nlohmann::detail::view::node_kind::string) + { + n.next = 0; + } + std::memcpy(image.data() + 64 + (i * sizeof(n)), &n, sizeof(n)); + } + const json_document d = json_document::load(image); + const json m = d.root().materialize(); + CHECK(range(m[std::string("a") + '\n'][0]) == "none"); + CHECK(range(m["c"]) == range(j["c"])); + CHECK(same_positions(m[std::string("a") + '\n'][2], j[std::string("a") + '\n'][2])); + } + +#if !defined(JSON_NOEXCEPTION) + SECTION("a decoded string's position must be within the text") + { + std::vector image = json_document::parse(R"(["\n"])").save(); + nlohmann::detail::view::node n{}; + std::memcpy(&n, image.data() + 64 + sizeof(n), sizeof(n)); + REQUIRE(n.next == 2); + n.next = 7; + std::memcpy(image.data() + 64 + sizeof(n), &n, sizeof(n)); + const auto load_image = [&image] + { + // GCC ignores a cast to void for [[nodiscard]]-like results, so keep the document + const json_document d = json_document::load(image); + static_cast(d); + }; + CHECK_THROWS_AS(load_image(), json::parse_error&); + // without the check, the position is out of bounds and ignored + const json_document d = json_document::load(image, json_document::image_check::none); + CHECK(range(d.root()[0].materialize()) == "none"); + } +#endif + } +#endif +}