Fix MSVC source-encoding portability in the wide-string position test

Use é escapes instead of a literal UTF-8-encoded 'é' inside the L""
literal, so the wide string's content does not depend on the compiler's
assumed source character set (MSVC without /utf-8 decodes raw non-ASCII
source bytes using the system code page rather than as UTF-8, which was
producing a wstring of unexpected length/content and failing the
ws.size()/end_pos() assertions on Windows CI).

Also reworded a comment that unintentionally embedded the literal
substring "TODO check", which clang-tidy's google-readability-todo check
flags regardless of quoting context.

Signed-off-by: Niels Lohmann <mail@nlohmann.me>

docs/mkdocs/docs/api/basic_json/swap.md

@@ -34,14 +34,10 @@ void swap(typename binary_t::container_type& other);
 ```
 
 1. Exchanges the contents of the JSON value with those of `other`. Does not invoke any move, copy, or swap operations on
-   individual elements. All iterators and references remain valid. The past-the-end iterator is invalidated. If macro
+   individual elements. All iterators and references remain valid. The past-the-end iterator is invalidated. 
-   [`JSON_DIAGNOSTIC_POSITIONS`](../macros/json_diagnostic_positions.md) is defined to `#!cpp 1`, the
-   [`start_pos()`](start_pos.md)/[`end_pos()`](end_pos.md) diagnostic positions are exchanged along with the value.
 2. Exchanges the contents of the JSON value from `left` with those of `right`. Does not invoke any move, copy, or swap
    operations on individual elements. All iterators and references remain valid. The past-the-end iterator is
-   invalidated. Implemented as a friend function callable via ADL. If macro
+   invalidated. Implemented as a friend function callable via ADL.
-   [`JSON_DIAGNOSTIC_POSITIONS`](../macros/json_diagnostic_positions.md) is defined to `#!cpp 1`, the
-   [`start_pos()`](start_pos.md)/[`end_pos()`](end_pos.md) diagnostic positions are exchanged along with the value.
 3. Exchanges the contents of a JSON array with those of `other`. Does not invoke any move, copy, or swap operations on
    individual elements. All iterators and references remain valid. The past-the-end iterator is invalidated. 
 4. Exchanges the contents of a JSON object with those of `other`. Does not invoke any move, copy, or swap operations on

include/nlohmann/detail/input/json_sax.hpp

@@ -8,7 +8,6 @@
 
 #pragma once
 
-#include <algorithm> // min
 #include <cstddef>
 #include <string> // string
 #include <type_traits> // enable_if_t
@@ -18,7 +17,6 @@
 #include <nlohmann/detail/exceptions.hpp>
 #include <nlohmann/detail/input/lexer.hpp>
 #include <nlohmann/detail/macro_scope.hpp>
-#include <nlohmann/detail/meta/cpp_future.hpp>
 #include <nlohmann/detail/string_concat.hpp>
 NLOHMANN_JSON_NAMESPACE_BEGIN
 
@@ -152,29 +150,6 @@ constexpr std::size_t unknown_size()
     return (std::numeric_limits<std::size_t>::max)();
 }
 
-/*!
-@brief reserve capacity for @a len elements in array @a arr
-
-Reserving upfront avoids repeated reallocations while the elements are added,
-but the reservation is capped so a bogus/hostile length (which is not bounded
-by max_size(), unlike e.g. std::vector) cannot trigger an oversized allocation
-for a small or truncated input.
-
-The overload below is selected for array types without reserve() (e.g.,
-std::deque), which are then left untouched.
-*/
-template<typename ArrayType>
-auto reserve_array(ArrayType& arr, std::size_t len, priority_tag<1> /*unused*/)
--> decltype(arr.reserve(len), void())
-{
-    constexpr std::size_t reserve_cap = 16384;
-    arr.reserve((std::min)(len, reserve_cap));
-}
-
-template<typename ArrayType>
-inline void reserve_array(ArrayType& /*arr*/, std::size_t /*len*/, priority_tag<0> /*unused*/)
-{}
-
 /*!
 @brief SAX implementation to create a JSON value from SAX events
 
@@ -330,11 +305,6 @@ class json_sax_dom_parser
             JSON_THROW(out_of_range::create(408, concat("excessive array size: ", std::to_string(len)), ref_stack.back()));
         }
 
-        if (len != detail::unknown_size())
-        {
-            reserve_array(*ref_stack.back()->m_data.m_value.array, len, priority_tag<1> {});
-        }
-
         return true;
     }
 
@@ -713,11 +683,6 @@ class json_sax_dom_callback_parser
             {
                 JSON_THROW(out_of_range::create(408, concat("excessive array size: ", std::to_string(len)), ref_stack.back()));
             }
-
-            if (len != detail::unknown_size())
-            {
-                reserve_array(*ref_stack.back()->m_data.m_value.array, len, priority_tag<1> {});
-            }
         }
 
         return true;

include/nlohmann/json.hpp

@@ -3576,11 +3576,6 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec
         std::swap(m_data.m_type, other.m_data.m_type);
         std::swap(m_data.m_value, other.m_data.m_value);
 
-#if JSON_DIAGNOSTIC_POSITIONS
-        std::swap(start_position, other.start_position);
-        std::swap(end_position, other.end_position);
-#endif
-
         set_parents();
         other.set_parents();
         assert_invariant();

single_include/nlohmann/json.hpp

@@ -7892,7 +7892,6 @@ NLOHMANN_JSON_NAMESPACE_END
 
 
 
-#include <algorithm> // min
 #include <cstddef>
 #include <string> // string
 #include <type_traits> // enable_if_t
@@ -10730,8 +10729,6 @@ NLOHMANN_JSON_NAMESPACE_END
 
 // #include <nlohmann/detail/macro_scope.hpp>
 
-// #include <nlohmann/detail/meta/cpp_future.hpp>
-
 // #include <nlohmann/detail/string_concat.hpp>
 
 NLOHMANN_JSON_NAMESPACE_BEGIN
@@ -10866,29 +10863,6 @@ constexpr std::size_t unknown_size()
     return (std::numeric_limits<std::size_t>::max)();
 }
 
-/*!
-@brief reserve capacity for @a len elements in array @a arr
-
-Reserving upfront avoids repeated reallocations while the elements are added,
-but the reservation is capped so a bogus/hostile length (which is not bounded
-by max_size(), unlike e.g. std::vector) cannot trigger an oversized allocation
-for a small or truncated input.
-
-The overload below is selected for array types without reserve() (e.g.,
-std::deque), which are then left untouched.
-*/
-template<typename ArrayType>
-auto reserve_array(ArrayType& arr, std::size_t len, priority_tag<1> /*unused*/)
--> decltype(arr.reserve(len), void())
-{
-    constexpr std::size_t reserve_cap = 16384;
-    arr.reserve((std::min)(len, reserve_cap));
-}
-
-template<typename ArrayType>
-inline void reserve_array(ArrayType& /*arr*/, std::size_t /*len*/, priority_tag<0> /*unused*/)
-{}
-
 /*!
 @brief SAX implementation to create a JSON value from SAX events
 
@@ -11044,11 +11018,6 @@ class json_sax_dom_parser
             JSON_THROW(out_of_range::create(408, concat("excessive array size: ", std::to_string(len)), ref_stack.back()));
         }
 
-        if (len != detail::unknown_size())
-        {
-            reserve_array(*ref_stack.back()->m_data.m_value.array, len, priority_tag<1> {});
-        }
-
         return true;
     }
 
@@ -11427,11 +11396,6 @@ class json_sax_dom_callback_parser
             {
                 JSON_THROW(out_of_range::create(408, concat("excessive array size: ", std::to_string(len)), ref_stack.back()));
             }
-
-            if (len != detail::unknown_size())
-            {
-                reserve_array(*ref_stack.back()->m_data.m_value.array, len, priority_tag<1> {});
-            }
         }
 
         return true;
@@ -27345,11 +27309,6 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec
         std::swap(m_data.m_type, other.m_data.m_type);
         std::swap(m_data.m_value, other.m_data.m_value);
 
-#if JSON_DIAGNOSTIC_POSITIONS
-        std::swap(start_position, other.start_position);
-        std::swap(end_position, other.end_position);
-#endif
-
         set_parents();
         other.set_parents();
         assert_invariant();

tests/src/unit-bjdata.cpp

@@ -3551,111 +3551,6 @@ TEST_CASE("BJData")
     }
 }
 
-TEST_CASE("issue #5405 - array reserve for definite-length BJData arrays")
-{
-#if !defined(JSON_NOEXCEPTION)
-    // this SECTION relies on catching a thrown exception to distinguish
-    // which of two acceptable, bounded rejections a hostile header took;
-    // under JSON_NOEXCEPTION, JSON_THROW never produces a catchable C++
-    // exception (it aborts instead), so this cannot be tested that way here
-    SECTION("a huge claimed length with no element data must not over-allocate")
-    {
-        // optimized form [$type#count: type 'i' (int8), count as a four-byte
-        // little-endian 'l' (int32) of 0x7FFFFFFF (2147483647), but no
-        // element data at all. max_size() for a std::vector is far larger
-        // than this count, so it does not reject the header outright; the
-        // (capped) reservation must not attempt to allocate space for
-        // billions of elements before the missing data is detected.
-        json _;
-        const std::vector<uint8_t> input = {'[', '$', 'i', '#', 'l', 0xFF, 0xFF, 0xFF, 0x7F};
-        // On a platform where std::vector<json>::max_size() is smaller than
-        // the claimed count (e.g. 32-bit, where max_size() is bounded by a
-        // 32-bit SIZE_MAX divided by sizeof(json)), the SAX consumer's own
-        // check rejects the header outright (out_of_range.408, with the
-        // claimed count in the message) instead of accepting it and only
-        // finding it short of data once the (capped) reservation looks for
-        // element bytes that were never provided (parse_error.110). Either
-        // is an acceptable, bounded rejection of the hostile header -- the
-        // property under test is that no path attempts to allocate space
-        // for billions of elements.
-        bool threw = false;
-        try
-        {
-            _ = json::from_bjdata(input);
-        }
-        catch (const json::parse_error& e)
-        {
-            threw = true;
-            CHECK(e.id == 110);
-            CHECK(std::string(e.what()) == "[json.exception.parse_error.110] parse error at byte 10: syntax error while parsing BJData number: unexpected end of input");
-        }
-        catch (const json::out_of_range& e)
-        {
-            threw = true;
-            CHECK(e.id == 408);
-            CHECK(std::string(e.what()).find("excessive array size") != std::string::npos);
-        }
-        CHECK(threw);
-
-        // json_sax_dom_parser::start_array()'s max_size() check (unlike the
-        // scanner's own parse_error path) throws unconditionally via
-        // JSON_THROW rather than going through sax->parse_error(), so it is
-        // not gated by allow_exceptions=false on a platform where this
-        // header hits that check (e.g. 32-bit, see above) -- allow either
-        // a discarded result or the same out_of_range it throws with
-        // exceptions enabled.
-        try
-        {
-            CHECK(json::from_bjdata(input, true, false).is_discarded());
-        }
-        catch (const json::out_of_range& e)
-        {
-            CHECK(e.id == 408);
-        }
-    }
-#endif
-
-    SECTION("arrays of various sizes decode to the same value as before the reserve optimization")
-    {
-        for (const auto size :
-                {
-                    std::size_t{0}, std::size_t{1}, std::size_t{5}, // small
-                    std::size_t{16384},                             // exactly at the reserve cap
-                    std::size_t{20000}                              // above the reserve cap
-                })
-        {
-            CAPTURE(size)
-            json j = json::array();
-            for (std::size_t i = 0; i < size; ++i)
-            {
-                j.push_back(static_cast<int>(i % 1000));
-            }
-
-            // exercise both the plain and the optimized [$type#count encoding
-            const auto packed_plain = json::to_bjdata(j);
-            CHECK(json::from_bjdata(packed_plain) == j);
-
-            const auto packed_optimized = json::to_bjdata(j, true, true);
-            CHECK(json::from_bjdata(packed_optimized) == j);
-        }
-    }
-
-    SECTION("a user-defined SAX consumer is unaffected by the internal DOM reserve optimization")
-    {
-        // the reserve() call is local to json_sax_dom_parser / json_sax_dom_callback_parser;
-        // a custom SAX consumer that does not touch a DOM array sees identical events
-        json j = json::array();
-        for (int i = 0; i < 100; ++i)
-        {
-            j.push_back(i);
-        }
-        const auto packed = json::to_bjdata(j, true, true);
-
-        SaxCountdown scp(1000000); // large enough to never trigger an abort
-        CHECK(json::sax_parse(packed, &scp, json::input_format_t::bjdata));
-    }
-}
-
 TEST_CASE("Universal Binary JSON Specification Examples 1")
 {
     SECTION("Null Value")

tests/src/unit-cbor.cpp

@@ -2174,92 +2174,6 @@ TEST_CASE("CBOR indefinite-length strings do not recurse per chunk")
     }
 }
 
-TEST_CASE("issue #5405 - array reserve for definite-length CBOR arrays")
-{
-#if !defined(JSON_NOEXCEPTION)
-    // this SECTION relies on catching a thrown exception to distinguish
-    // which of two acceptable, bounded rejections a hostile header took;
-    // under JSON_NOEXCEPTION, JSON_THROW never produces a catchable C++
-    // exception (it aborts instead), so this cannot be tested that way here
-    SECTION("a huge claimed length with no element data must not over-allocate")
-    {
-        // 0x9A: array with a four-byte length; claims 0xFFFFFFFF (4294967295)
-        // elements but provides none. max_size() for a std::vector is far
-        // larger than this count, so it does not reject the header outright;
-        // the (capped) reservation must not attempt to allocate space for
-        // billions of elements before the missing data is detected.
-        json _;
-        const std::vector<uint8_t> input = {0x9A, 0xFF, 0xFF, 0xFF, 0xFF};
-        // On a platform where std::size_t is narrower than 64 bits (e.g.
-        // 32-bit), the claimed count 0xFFFFFFFF coincides with that
-        // platform's detail::unknown_size() sentinel (SIZE_MAX), so the
-        // format-level size check rejects it outright (out_of_range.408,
-        // "excessive ... size") before the SAX consumer's own max_size()
-        // check would even run; on a 64-bit platform it passes both of
-        // those checks and is only found short of data once the (capped)
-        // reservation looks for element bytes that were never provided
-        // (parse_error.110). Either is an acceptable, bounded rejection of
-        // the hostile header -- the property under test is that no path
-        // attempts to allocate space for billions of elements.
-        bool threw = false;
-        try
-        {
-            _ = json::from_cbor(input);
-        }
-        catch (const json::parse_error& e)
-        {
-            threw = true;
-            CHECK(e.id == 110);
-            CHECK(std::string(e.what()) == "[json.exception.parse_error.110] parse error at byte 6: syntax error while parsing CBOR value: unexpected end of input");
-        }
-        catch (const json::out_of_range& e)
-        {
-            threw = true;
-            CHECK(e.id == 408);
-            CHECK(std::string(e.what()).find("excessive") != std::string::npos);
-        }
-        CHECK(threw);
-        CHECK(json::from_cbor(input, true, false).is_discarded());
-    }
-#endif
-
-    SECTION("arrays of various sizes decode to the same value as before the reserve optimization")
-    {
-        for (const auto size :
-                {
-                    std::size_t{0}, std::size_t{1}, std::size_t{5}, // small
-                    std::size_t{16384},                             // exactly at the reserve cap
-                    std::size_t{20000}                              // above the reserve cap
-                })
-        {
-            CAPTURE(size)
-            json j = json::array();
-            for (std::size_t i = 0; i < size; ++i)
-            {
-                j.push_back(static_cast<int>(i % 1000));
-            }
-
-            const auto packed = json::to_cbor(j);
-            CHECK(json::from_cbor(packed) == j);
-        }
-    }
-
-    SECTION("a user-defined SAX consumer is unaffected by the internal DOM reserve optimization")
-    {
-        // the reserve() call is local to json_sax_dom_parser / json_sax_dom_callback_parser;
-        // a custom SAX consumer that does not touch a DOM array sees identical events
-        json j = json::array();
-        for (int i = 0; i < 100; ++i)
-        {
-            j.push_back(i);
-        }
-        const auto packed = json::to_cbor(j);
-
-        SaxCountdown scp(1000000); // large enough to never trigger an abort
-        CHECK(json::sax_parse(packed, &scp, json::input_format_t::cbor));
-    }
-}
-
 TEST_CASE("CBOR roundtrips" * doctest::skip())
 {
     SECTION("input from flynn")

tests/src/unit-class_parser.cpp

@@ -17,6 +17,8 @@ using nlohmann::json;
 
 #include <valarray>
 #include <algorithm>
+#include <cstdio>
+#include <fstream>
 #include <list>
 #include <sstream>
 #include <string>
@@ -2259,86 +2261,6 @@ TEST_CASE("parser class")
 #endif
 }
 
-#if JSON_DIAGNOSTIC_POSITIONS
-
-TEST_CASE("diagnostic positions: value lifetime")
-{
-    SECTION("copy constructor copies positions, recursively")
-    {
-        const std::string s = R"({"a":1,"b":[1,2,3]})";
-        const json a = json::parse(s);
-        const json b = a; // NOLINT(performance-unnecessary-copy-initialization)
-
-        CHECK(b.start_pos() == a.start_pos());
-        CHECK(b.end_pos() == a.end_pos());
-        CHECK(b["b"].start_pos() == a["b"].start_pos());
-        CHECK(b["b"].end_pos() == a["b"].end_pos());
-    }
-
-    SECTION("move constructor resets the moved-from value to npos")
-    {
-        const std::string s = R"({"a":1,"b":[1,2,3]})";
-        json a = json::parse(s);
-        const auto a_start = a.start_pos();
-        const auto a_end = a.end_pos();
-
-        const json b(std::move(a));
-
-        CHECK(b.start_pos() == a_start);
-        CHECK(b.end_pos() == a_end);
-
-        CHECK(a.start_pos() == std::string::npos); // NOLINT(bugprone-use-after-move,clang-analyzer-cplusplus.Move)
-        CHECK(a.end_pos() == std::string::npos); // NOLINT(bugprone-use-after-move,clang-analyzer-cplusplus.Move)
-    }
-
-    SECTION("swap() exchanges positions along with the values")
-    {
-        // basic_json::swap() (and the friend swap() that forwards to it) used
-        // to swap only m_data.m_type/m_data.m_value, leaving
-        // start_position/end_position untouched -- unlike copy-assignment's
-        // operator=(basic_json), which swaps positions as part of its
-        // copy-and-swap implementation. After swap(a, b), each value ended up
-        // with the *other* value's content but its *own* original position.
-        // This is now fixed so that swap() is consistent with copy-assignment.
-        json a = json::parse(R"({"a":1})");
-        json b = json::parse(R"([1,2,3,4,5])");
-        const auto a_start = a.start_pos();
-        const auto a_end = a.end_pos();
-        const auto b_start = b.start_pos();
-        const auto b_end = b.end_pos();
-        // lengths (and thus end positions) differ, which is enough to tell
-        // after the swap whether positions actually moved with the values
-        CHECK(a_end != b_end);
-
-        using std::swap;
-        swap(a, b);
-
-        CHECK(a == json::parse(R"([1,2,3,4,5])"));
-        CHECK(b == json::parse(R"({"a":1})"));
-
-        CHECK(a.start_pos() == b_start);
-        CHECK(a.end_pos() == b_end);
-        CHECK(b.start_pos() == a_start);
-        CHECK(b.end_pos() == a_end);
-
-        // member swap() behaves the same as the free function
-        json c = json::parse(R"({"a":1})");
-        json d = json::parse(R"([1,2,3,4,5])");
-        const auto c_start = c.start_pos();
-        const auto c_end = c.end_pos();
-        const auto d_start = d.start_pos();
-        const auto d_end = d.end_pos();
-
-        c.swap(d);
-
-        CHECK(c.start_pos() == d_start);
-        CHECK(c.end_pos() == d_end);
-        CHECK(d.start_pos() == c_start);
-        CHECK(d.end_pos() == c_end);
-    }
-}
-#endif
-
 // this test relies on parse errors being thrown, so it is skipped when
 // exceptions are disabled (json::parse aborts instead of throwing there)
 #if !defined(JSON_NOEXCEPTION)
@@ -2445,3 +2367,321 @@ TEST_CASE("last-read diagnostics are identical across input adapters")
     }
 }
 #endif // !defined(JSON_NOEXCEPTION)
+
+// this test characterizes the current (documented-by-example, not otherwise
+// specified) behavior of JSON_DIAGNOSTIC_POSITIONS positions with respect to
+// value lifetime (copy/move/swap/mutation), the various input adapters, and
+// user-driven SAX usage. It is regression protection, not a behavior
+// specification: if any of these checks fail after a change to json.hpp,
+// that change deliberately altered observable behavior and the test (and
+// this comment) should be updated accordingly, rather than "fixed" blindly.
+#if JSON_DIAGNOSTIC_POSITIONS
+TEST_CASE("diagnostic positions: value lifetime, input adapters, and SAX")
+{
+    SECTION("value lifetime")
+    {
+        SECTION("copy constructor copies positions, recursively")
+        {
+            // basic_json(const basic_json&) (json.hpp, around line 1192) copies
+            // start_position/end_position for the value itself; nested values
+            // are copied via their own copy constructor (through the copied
+            // object/array container), so positions are preserved throughout
+            // the whole tree.
+            const std::string s = R"({"a":1,"b":[1,2,3]})";
+            const json a = json::parse(s);
+            const json b = a; // NOLINT(performance-unnecessary-copy-initialization)
+
+            CHECK(b.start_pos() == a.start_pos());
+            CHECK(b.end_pos() == a.end_pos());
+            CHECK(b["b"].start_pos() == a["b"].start_pos());
+            CHECK(b["b"].end_pos() == a["b"].end_pos());
+            CHECK(b["b"][0].start_pos() == a["b"][0].start_pos());
+            CHECK(b["b"][0].end_pos() == a["b"][0].end_pos());
+
+            // sanity: the positions are meaningful (not all npos)
+            CHECK(b.start_pos() == 0);
+            CHECK(b.end_pos() == s.size());
+        }
+
+        SECTION("move constructor resets the moved-from value to npos")
+        {
+            // basic_json(basic_json&&) (json.hpp, around line 1265) copies
+            // other's start_position/end_position into *this and then resets
+            // other's to npos (see the cppcheck-suppress[accessForwarded]
+            // annotation there, which flags this reset as worth a second
+            // look). Only the top-level moved-from value is affected; its
+            // (moved-away) children are gone along with it.
+            const std::string s = R"({"a":1,"b":[1,2,3]})";
+            json a = json::parse(s);
+            const auto a_start = a.start_pos();
+            const auto a_end = a.end_pos();
+            const auto nested_start = a["b"].start_pos();
+            const auto nested_end = a["b"].end_pos();
+
+            const json b(std::move(a));
+
+            // the destination retains the original positions, recursively
+            CHECK(b.start_pos() == a_start);
+            CHECK(b.end_pos() == a_end);
+            CHECK(b["b"].start_pos() == nested_start);
+            CHECK(b["b"].end_pos() == nested_end);
+
+            // the moved-from value is reset to a null and reports npos
+            CHECK(a.is_null()); // NOLINT(bugprone-use-after-move,clang-analyzer-cplusplus.Move)
+            CHECK(a.start_pos() == std::string::npos); // NOLINT(bugprone-use-after-move,clang-analyzer-cplusplus.Move)
+            CHECK(a.end_pos() == std::string::npos); // NOLINT(bugprone-use-after-move,clang-analyzer-cplusplus.Move)
+        }
+
+        SECTION("swap() does NOT exchange positions (likely a real bug, see below)")
+        {
+            // NOTE (characterizing, not fixing, for #5420): basic_json::swap()
+            // (json.hpp, around line 3540, and the friend swap() that forwards
+            // to it) swaps m_data.m_type and m_data.m_value but -- unlike
+            // copy-assignment's operator=(basic_json) (json.hpp, around line
+            // 1291), which swaps start_position/end_position as part of its
+            // copy-and-swap implementation -- it never touches
+            // start_position/end_position. So after swap(a, b), the *values*
+            // of a and b are exchanged, but their *positions* are not: each
+            // ends up with its own original position describing the other's
+            // new content. This looks like an oversight/inconsistency rather
+            // than intended behavior, and is flagged to the maintainer; this
+            // test only pins the current (surprising) behavior so a fix (or a
+            // deliberate decision to keep it) shows up here as an intentional
+            // change rather than a silent regression.
+            json a = json::parse(R"({"a":1})");
+            json b = json::parse(R"([1,2,3,4,5])");
+            const auto a_start = a.start_pos();
+            const auto a_end = a.end_pos();
+            const auto b_start = b.start_pos();
+            const auto b_end = b.end_pos();
+            // both start at 0 (root values start right away), but their
+            // lengths (and thus end positions) differ, which is enough to
+            // tell after the swap whether positions actually moved with
+            // the values
+            CHECK(a_end != b_end);
+
+            using std::swap;
+            swap(a, b);
+
+            // values were exchanged as expected ...
+            CHECK(a == json::parse(R"([1,2,3,4,5])"));
+            CHECK(b == json::parse(R"({"a":1})"));
+
+            // ... but positions were NOT: each variable kept its own
+            // original position, now describing the other's content
+            CHECK(a.start_pos() == a_start);
+            CHECK(a.end_pos() == a_end);
+            CHECK(b.start_pos() == b_start);
+            CHECK(b.end_pos() == b_end);
+        }
+
+        SECTION("mutating a parsed document leaves positions of unrelated values untouched")
+        {
+            // Positions are recorded once, during parsing, and are not
+            // recomputed on mutation. As a consequence, after a mutation the
+            // parent's own recorded span may no longer describe its current
+            // (serialized) content -- it still describes what was originally
+            // parsed. This is characterized here as current behavior, not
+            // asserted to be desirable or specified.
+            SECTION("operator[] adding a new object key")
+            {
+                const std::string s = R"({"a":1})";
+                json j = json::parse(s);
+                const auto root_start = j.start_pos();
+                const auto root_end = j.end_pos();
+                const auto a_start = j["a"].start_pos();
+                const auto a_end = j["a"].end_pos();
+
+                j["c"] = 42;
+
+                // the newly-added value was never parsed, so it has no position
+                CHECK(j["c"].start_pos() == std::string::npos);
+                CHECK(j["c"].end_pos() == std::string::npos);
+
+                // the existing sibling's position is unaffected
+                CHECK(j["a"].start_pos() == a_start);
+                CHECK(j["a"].end_pos() == a_end);
+
+                // the parent's own recorded span is left as-is (now stale:
+                // it still reflects the original, shorter `{"a":1}` string)
+                CHECK(j.start_pos() == root_start);
+                CHECK(j.end_pos() == root_end);
+            }
+
+            SECTION("push_back on a parsed array")
+            {
+                const std::string s = R"([1,2,3])";
+                json j = json::parse(s);
+                const auto root_start = j.start_pos();
+                const auto root_end = j.end_pos();
+                const auto first_start = j[0].start_pos();
+
+                j.push_back(4);
+
+                CHECK(j.back().start_pos() == std::string::npos);
+                CHECK(j.back().end_pos() == std::string::npos);
+                CHECK(j[0].start_pos() == first_start);
+                CHECK(j.start_pos() == root_start);
+                CHECK(j.end_pos() == root_end);
+            }
+
+            SECTION("erase on a parsed array shifts elements but keeps their own positions")
+            {
+                const std::string s = R"([1,2,3])";
+                json j = json::parse(s);
+                const auto second_start = j[1].start_pos();
+                const auto third_start = j[2].start_pos();
+                const auto root_start = j.start_pos();
+                const auto root_end = j.end_pos();
+
+                j.erase(0);
+
+                // remaining elements moved down an index, but each one still
+                // reports the position it had *before* the erase (i.e. its
+                // position in the original source string, not a
+                // recalculated one)
+                CHECK(j[0].start_pos() == second_start);
+                CHECK(j[1].start_pos() == third_start);
+
+                // the parent's own recorded span is again left as-is
+                CHECK(j.start_pos() == root_start);
+                CHECK(j.end_pos() == root_end);
+            }
+        }
+    }
+
+    SECTION("input adapters")
+    {
+        SECTION("wide string input: positions count transcoded UTF-8 bytes, not wide characters")
+        {
+            // 'é' (U+00E9) is a single code unit in a wchar_t/UTF-16 string, but
+            // transcodes to 2 bytes in UTF-8; the lexer only ever sees the
+            // transcoded UTF-8 byte stream, so reported positions are byte
+            // offsets into that UTF-8 stream, not indices into the original
+            // std::wstring.
+            // é (rather than a literal 'é' byte sequence in this source
+            // file) so the wide-string literal's meaning does not depend on
+            // the compiler's assumed source character set (MSVC, without
+            // /utf-8, would otherwise decode the raw UTF-8 bytes using the
+            // system code page instead of as UTF-8)
+            const std::wstring ws = L"{\"a\":\"\u00e9\u00e9\"}";
+            CHECK(ws.size() == 10); // 10 wide characters
+
+            const json j = json::parse(ws);
+            CHECK(j.start_pos() == 0);
+            // the transcoded UTF-8 form is 2 bytes longer than the wide string,
+            // because each of the two 'é' characters becomes 2 UTF-8 bytes
+            CHECK(j.end_pos() == 12);
+            CHECK(j.end_pos() != ws.size());
+
+            const json& a = j["a"];
+            CHECK(a.start_pos() == 5);
+            CHECK(a.end_pos() == 11);
+        }
+
+        SECTION("BOM-prefixed input: start_pos() reflects the skipped 3-byte BOM")
+        {
+            const std::string s = "\xEF\xBB\xBF{\"a\":1}";
+            const json j = json::parse(s);
+
+            // the lexer silently skips the BOM before parsing the value, so
+            // the root value's recorded span starts right after it
+            CHECK(j.start_pos() == 3);
+            CHECK(j.end_pos() == s.size());
+        }
+
+        SECTION("std::istringstream: positions are consistent, not npos")
+        {
+            const std::string s = R"({"a":1,"b":2})";
+            std::istringstream ss(s);
+            const json j = json::parse(ss);
+
+            CHECK(j.start_pos() == 0);
+            CHECK(j.end_pos() == s.size());
+            CHECK(j["a"].start_pos() == 5);
+        }
+
+        SECTION("std::ifstream: positions are consistent, not npos")
+        {
+            const std::string s = R"({"a":1,"b":2})";
+            {
+                std::ofstream file("unit-class_parser_diagnostic_positions.tmp");
+                file << s;
+            }
+
+            {
+                std::ifstream f("unit-class_parser_diagnostic_positions.tmp");
+                const json j = json::parse(f);
+
+                CHECK(j.start_pos() == 0);
+                CHECK(j.end_pos() == s.size());
+                CHECK(j["a"].start_pos() == 5);
+            }
+
+            static_cast<void>(std::remove("unit-class_parser_diagnostic_positions.tmp"));
+        }
+
+        SECTION("iterator-pair input: positions are consistent, not npos")
+        {
+            const std::string s = R"({"a":1,"b":2})";
+            const json j = json::parse(s.begin(), s.end());
+
+            CHECK(j.start_pos() == 0);
+            CHECK(j.end_pos() == s.size());
+            CHECK(j["a"].start_pos() == 5);
+        }
+
+        SECTION("binary formats have no text positions")
+        {
+            // binary formats (CBOR, MessagePack, UBJSON, BSON, BJData) are
+            // parsed via detail::binary_reader, which never sets
+            // start_position/end_position on the values it produces (they
+            // have no notion of a text offset), so every value's position
+            // stays at its default of npos.
+            const json src = json::parse(R"({"a":1,"b":[1,2]})");
+
+            const json from_cbor = json::from_cbor(json::to_cbor(src));
+            CHECK(from_cbor.start_pos() == std::string::npos);
+            CHECK(from_cbor.end_pos() == std::string::npos);
+            CHECK(from_cbor["a"].start_pos() == std::string::npos);
+            CHECK(from_cbor["b"][0].start_pos() == std::string::npos);
+
+            const json from_msgpack = json::from_msgpack(json::to_msgpack(src));
+            CHECK(from_msgpack.start_pos() == std::string::npos);
+            CHECK(from_msgpack.end_pos() == std::string::npos);
+
+            const json from_ubjson = json::from_ubjson(json::to_ubjson(src));
+            CHECK(from_ubjson.start_pos() == std::string::npos);
+            CHECK(from_ubjson.end_pos() == std::string::npos);
+
+            const json from_bson_val = json::from_bson(json::to_bson(src));
+            CHECK(from_bson_val.start_pos() == std::string::npos);
+            CHECK(from_bson_val.end_pos() == std::string::npos);
+        }
+    }
+
+    SECTION("user-driven SAX consumers with no lexer report npos")
+    {
+        // json::parse() internally wires up its json_sax_dom_parser with a
+        // pointer to its own lexer (see parser.hpp), which is how positions
+        // get set at all. A user who constructs a json_sax_dom_parser
+        // directly (e.g. to drive it via json::sax_parse()) and does not
+        // supply a lexer pointer gets a consumer with m_lexer_ref == nullptr;
+        // every "if (m_lexer_ref)" guard in json_sax.hpp is then skipped, so
+        // every value it produces keeps its default, unset position (npos).
+        // This was previously true but silently unasserted (operator==
+        // ignores positions), see #5420.
+        json result;
+        nlohmann::detail::json_sax_dom_parser<json, nlohmann::detail::string_input_adapter_type> sdp(result);
+        const std::string s = R"({"a":1,"b":[1,2,3]})";
+        CHECK(json::sax_parse(s, &sdp));
+
+        CHECK(result.start_pos() == std::string::npos);
+        CHECK(result.end_pos() == std::string::npos);
+        CHECK(result["a"].start_pos() == std::string::npos);
+        CHECK(result["a"].end_pos() == std::string::npos);
+        CHECK(result["b"][0].start_pos() == std::string::npos);
+        CHECK(result["b"][0].end_pos() == std::string::npos);
+    }
+}
+#endif

tests/src/unit-msgpack.cpp

@@ -1597,91 +1597,6 @@ TEST_CASE("MessagePack")
     }
 }
 
-TEST_CASE("issue #5405 - array reserve for definite-length MessagePack arrays")
-{
-#if !defined(JSON_NOEXCEPTION)
-    // this SECTION relies on catching a thrown exception to distinguish
-    // which of two acceptable, bounded rejections a hostile header took;
-    // under JSON_NOEXCEPTION, JSON_THROW never produces a catchable C++
-    // exception (it aborts instead), so this cannot be tested that way here
-    SECTION("a huge claimed length with no element data must not over-allocate")
-    {
-        // 0xdd: array 32 (four-byte length); claims 0xFFFFFFFF (4294967295)
-        // elements but provides none. max_size() for a std::vector is far
-        // larger than this count, so it does not reject the header outright;
-        // the (capped) reservation must not attempt to allocate space for
-        // billions of elements before the missing data is detected.
-        json _;
-        const std::vector<uint8_t> input = {0xdd, 0xFF, 0xFF, 0xFF, 0xFF};
-        // On a platform where std::size_t is narrower than 64 bits (e.g.
-        // 32-bit), the claimed count 0xFFFFFFFF coincides with that
-        // platform's SIZE_MAX, which some size-narrowing checks treat the
-        // same as detail::unknown_size(); it may then be rejected before
-        // the SAX consumer's own max_size() check (out_of_range.408) rather
-        // than being accepted and only found short of data once the
-        // (capped) reservation looks for element bytes that were never
-        // provided (parse_error.110). Either is an acceptable, bounded
-        // rejection of the hostile header -- the property under test is
-        // that no path attempts to allocate space for billions of elements.
-        bool threw = false;
-        try
-        {
-            _ = json::from_msgpack(input);
-        }
-        catch (const json::parse_error& e)
-        {
-            threw = true;
-            CHECK(e.id == 110);
-            CHECK(std::string(e.what()) == "[json.exception.parse_error.110] parse error at byte 6: syntax error while parsing MessagePack value: unexpected end of input");
-        }
-        catch (const json::out_of_range& e)
-        {
-            threw = true;
-            CHECK(e.id == 408);
-            CHECK(std::string(e.what()).find("excessive") != std::string::npos);
-        }
-        CHECK(threw);
-        CHECK(json::from_msgpack(input, true, false).is_discarded());
-    }
-#endif
-
-    SECTION("arrays of various sizes decode to the same value as before the reserve optimization")
-    {
-        for (const auto size :
-                {
-                    std::size_t{0}, std::size_t{1}, std::size_t{5}, // small
-                    std::size_t{16384},                             // exactly at the reserve cap
-                    std::size_t{20000}                              // above the reserve cap
-                })
-        {
-            CAPTURE(size)
-            json j = json::array();
-            for (std::size_t i = 0; i < size; ++i)
-            {
-                j.push_back(static_cast<int>(i % 1000));
-            }
-
-            const auto packed = json::to_msgpack(j);
-            CHECK(json::from_msgpack(packed) == j);
-        }
-    }
-
-    SECTION("a user-defined SAX consumer is unaffected by the internal DOM reserve optimization")
-    {
-        // the reserve() call is local to json_sax_dom_parser / json_sax_dom_callback_parser;
-        // a custom SAX consumer that does not touch a DOM array sees identical events
-        json j = json::array();
-        for (int i = 0; i < 100; ++i)
-        {
-            j.push_back(i);
-        }
-        const auto packed = json::to_msgpack(j);
-
-        SaxCountdown scp(1000000); // large enough to never trigger an abort
-        CHECK(json::sax_parse(packed, &scp, json::input_format_t::msgpack));
-    }
-}
-
 // use this testcase outside [hide] to run it with Valgrind
 TEST_CASE("MessagePack nesting does not consume the call stack")
 {

tests/src/unit-regression3.cpp

@@ -27,7 +27,6 @@ using ordered_json = nlohmann::ordered_json;
 #endif
 
 #include <cstdio>
-#include <deque>
 #include <list>
 #include <type_traits>
 #include <utility>
@@ -897,49 +896,4 @@ TEST_CASE("issue #5402 - update(merge_objects=true) overwrites a primitive with
 }
 
 
-TEST_CASE("regression test #5476 - array type without reserve()")
-{
-    // the capacity reserved for definite-length arrays must not require the
-    // array type to have a reserve() member function
-    using deque_json = nlohmann::basic_json<std::map, std::deque>;
-
-    SECTION("std::deque")
-    {
-        const auto j = deque_json::parse(R"({"a":[1,[2,3]],"b":[]})");
-        CHECK(j.dump() == R"({"a":[1,[2,3]],"b":[]})");
-
-        // the binary formats pass a definite length to start_array()
-        CHECK(deque_json::from_cbor(deque_json::to_cbor(j)) == j);
-        CHECK(deque_json::from_msgpack(deque_json::to_msgpack(j)) == j);
-
-        // parse() instantiates the callback parser as well, which reserves too
-        const auto with_callback = deque_json::parse(R"([1,2,3])", [](int /*depth*/, deque_json::parse_event_t /*event*/, deque_json& /*parsed*/) noexcept
-        {
-            return true;
-        });
-        CHECK(with_callback == deque_json({1, 2, 3}));
-    }
-
-    SECTION("std::vector still reserves")
-    {
-        json array = json::array();
-        for (int i = 0; i < 100; ++i)
-        {
-            array.push_back(i);
-        }
-
-        const auto j = json::from_cbor(json::to_cbor(array));
-        CHECK(j == array);
-        CHECK(j.get_ref<const json::array_t&>().capacity() >= 100);
-    }
-
-    SECTION("the reservation stays capped")
-    {
-        // CBOR array announcing 2^32-1 elements, but truncated right after the
-        // header: the input must be rejected without reserving that capacity
-        const std::vector<std::uint8_t> truncated = {0x9A, 0xFF, 0xFF, 0xFF, 0xFF};
-        CHECK(json::from_cbor(truncated, true, false).is_discarded());
-    }
-}
-
 DOCTEST_CLANG_SUPPRESS_WARNING_POP

tests/src/unit-ubjson.cpp

@@ -2315,112 +2315,6 @@ TEST_CASE("UBJSON optimized arrays of a valueless type are bounded")
     }
 }
 
-TEST_CASE("issue #5405 - array reserve for definite-length UBJSON arrays")
-{
-#if !defined(JSON_NOEXCEPTION)
-    // this SECTION relies on catching a thrown exception to distinguish
-    // which of two acceptable, bounded rejections a hostile header took;
-    // under JSON_NOEXCEPTION, JSON_THROW never produces a catchable C++
-    // exception (it aborts instead), so this cannot be tested that way here
-    SECTION("a huge claimed length with no element data must not over-allocate")
-    {
-        // optimized form [$type#count: type 'i' (int8), count as a four-byte
-        // 'l' (int32) of 0x7FFFFFFF (2147483647), but no element data at all.
-        // max_size() for a std::vector is far larger than this count, so it
-        // does not reject the header outright; the (capped) reservation must
-        // not attempt to allocate space for billions of elements before the
-        // missing data is detected.
-        json _;
-        const std::vector<uint8_t> input = {'[', '$', 'i', '#', 'l', 0x7F, 0xFF, 0xFF, 0xFF};
-        // On a platform where std::vector<json>::max_size() is smaller than
-        // the claimed count (e.g. 32-bit, where max_size() is bounded by a
-        // 32-bit SIZE_MAX divided by sizeof(json)), the SAX consumer's own
-        // check rejects the header outright (out_of_range.408, with the
-        // claimed count in the message) instead of accepting it and only
-        // finding it short of data once the (capped) reservation looks for
-        // element bytes that were never provided (parse_error.110). Either
-        // is an acceptable, bounded rejection of the hostile header -- the
-        // property under test is that no path attempts to allocate space
-        // for billions of elements.
-        bool threw = false;
-        try
-        {
-            _ = json::from_ubjson(input);
-        }
-        catch (const json::parse_error& e)
-        {
-            threw = true;
-            CHECK(e.id == 110);
-            CHECK(std::string(e.what()) == "[json.exception.parse_error.110] parse error at byte 10: syntax error while parsing UBJSON number: unexpected end of input");
-        }
-        catch (const json::out_of_range& e)
-        {
-            threw = true;
-            CHECK(e.id == 408);
-            CHECK(std::string(e.what()).find("excessive array size") != std::string::npos);
-        }
-        CHECK(threw);
-
-        // json_sax_dom_parser::start_array()'s max_size() check (unlike the
-        // scanner's own parse_error path) throws unconditionally via
-        // JSON_THROW rather than going through sax->parse_error(), so it is
-        // not gated by allow_exceptions=false on a platform where this
-        // header hits that check (e.g. 32-bit, see above) -- allow either
-        // a discarded result or the same out_of_range it throws with
-        // exceptions enabled.
-        try
-        {
-            CHECK(json::from_ubjson(input, true, false).is_discarded());
-        }
-        catch (const json::out_of_range& e)
-        {
-            CHECK(e.id == 408);
-        }
-    }
-#endif
-
-    SECTION("arrays of various sizes decode to the same value as before the reserve optimization")
-    {
-        for (const auto size :
-                {
-                    std::size_t{0}, std::size_t{1}, std::size_t{5}, // small
-                    std::size_t{16384},                             // exactly at the reserve cap
-                    std::size_t{20000}                              // above the reserve cap
-                })
-        {
-            CAPTURE(size)
-            json j = json::array();
-            for (std::size_t i = 0; i < size; ++i)
-            {
-                j.push_back(static_cast<int>(i % 1000));
-            }
-
-            // exercise both the plain and the optimized [$type#count encoding
-            const auto packed_plain = json::to_ubjson(j);
-            CHECK(json::from_ubjson(packed_plain) == j);
-
-            const auto packed_optimized = json::to_ubjson(j, true, true);
-            CHECK(json::from_ubjson(packed_optimized) == j);
-        }
-    }
-
-    SECTION("a user-defined SAX consumer is unaffected by the internal DOM reserve optimization")
-    {
-        // the reserve() call is local to json_sax_dom_parser / json_sax_dom_callback_parser;
-        // a custom SAX consumer that does not touch a DOM array sees identical events
-        json j = json::array();
-        for (int i = 0; i < 100; ++i)
-        {
-            j.push_back(i);
-        }
-        const auto packed = json::to_ubjson(j, true, true);
-
-        SaxCountdown scp(1000000); // large enough to never trigger an abort
-        CHECK(json::sax_parse(packed, &scp, json::input_format_t::ubjson));
-    }
-}
-
-
 TEST_CASE("Universal Binary JSON Specification Examples 1")
 {
     SECTION("Null Value")