Make serializer's indent_string lazily allocated

The serializer constructor unconditionally allocated a 512-byte
indent_string, even though it is only ever read inside the
pretty_print branches of dump(). This wasted a heap allocation (and
its matching deallocation) on every compact (i.e. default, non-pretty)
dump() call.

indent_string is now default-constructed empty and lazily grown to
512 bytes, filled with indent_char, the first time a pretty-print
branch actually needs it. The existing doubling/growth logic for
larger indents is otherwise untouched, so output remains byte-identical
to before -- including in the pre-existing edge case where growth
beyond the initial buffer fills with ' ' instead of indent_char
(tracked separately by open PR #5186, which is left alone here).

The second, larger optimization mentioned in #5413 (removing the
shared_ptr-based output adapter) is intentionally out of scope, as it
overlaps open PR #5285.

Fixes #5413

Signed-off-by: Niels Lohmann <mail@nlohmann.me>

include/nlohmann/detail/input/json_sax.hpp

@@ -301,16 +301,6 @@ class json_sax_dom_parser
             JSON_THROW(out_of_range::create(408, concat("excessive array size: ", std::to_string(len)), ref_stack.back()));
         }
 
-        if (len != detail::unknown_size())
-        {
-            // reserve upfront to avoid repeated reallocations while adding elements,
-            // but cap the reservation so a bogus/hostile length (which is not bounded
-            // by max_size(), unlike e.g. std::vector) cannot trigger an oversized
-            // allocation for a small or truncated input
-            constexpr std::size_t reserve_cap = 16384;
-            ref_stack.back()->m_data.m_value.array->reserve(len < reserve_cap ? len : reserve_cap);
-        }
-
         return true;
     }
 
@@ -671,16 +661,6 @@ class json_sax_dom_callback_parser
             {
                 JSON_THROW(out_of_range::create(408, concat("excessive array size: ", std::to_string(len)), ref_stack.back()));
             }
-
-            if (len != detail::unknown_size())
-            {
-                // reserve upfront to avoid repeated reallocations while adding elements,
-                // but cap the reservation so a bogus/hostile length (which is not bounded
-                // by max_size(), unlike e.g. std::vector) cannot trigger an oversized
-                // allocation for a small or truncated input
-                constexpr std::size_t reserve_cap = 16384;
-                ref_stack.back()->m_data.m_value.array->reserve(len < reserve_cap ? len : reserve_cap);
-            }
         }
 
         return true;

include/nlohmann/detail/output/serializer.hpp

@@ -71,7 +71,7 @@ class serializer
         , thousands_sep(loc->thousands_sep == nullptr ? '\0' : std::char_traits<char>::to_char_type(* (loc->thousands_sep)))
         , decimal_point(loc->decimal_point == nullptr ? '\0' : std::char_traits<char>::to_char_type(* (loc->decimal_point)))
         , indent_char(ichar)
-        , indent_string(512, indent_char)
+        , indent_string()
         , error_handler(error_handler_)
     {}
 
@@ -126,6 +126,10 @@ class serializer
 
                     // variable to hold indentation for recursive calls
                     const auto new_indent = current_indent + indent_step;
+                    if (JSON_HEDLEY_UNLIKELY(indent_string.empty()))
+                    {
+                        indent_string.resize(512, indent_char);
+                    }
                     if (JSON_HEDLEY_UNLIKELY(indent_string.size() < new_indent))
                     {
                         indent_string.resize(indent_string.size() * 2, ' ');
@@ -199,6 +203,10 @@ class serializer
 
                     // variable to hold indentation for recursive calls
                     const auto new_indent = current_indent + indent_step;
+                    if (JSON_HEDLEY_UNLIKELY(indent_string.empty()))
+                    {
+                        indent_string.resize(512, indent_char);
+                    }
                     if (JSON_HEDLEY_UNLIKELY(indent_string.size() < new_indent))
                     {
                         indent_string.resize(indent_string.size() * 2, ' ');
@@ -260,6 +268,10 @@ class serializer
 
                     // variable to hold indentation for recursive calls
                     const auto new_indent = current_indent + indent_step;
+                    if (JSON_HEDLEY_UNLIKELY(indent_string.empty()))
+                    {
+                        indent_string.resize(512, indent_char);
+                    }
                     if (JSON_HEDLEY_UNLIKELY(indent_string.size() < new_indent))
                     {
                         indent_string.resize(indent_string.size() * 2, ' ');
@@ -1010,7 +1022,7 @@ class serializer
 
     /// the indentation character
     const char indent_char;
-    /// the indentation string
+    /// the indentation string (lazily allocated on first use by a pretty-print branch)
     string_t indent_string;
 
     /// error_handler how to react on decoding errors

single_include/nlohmann/json.hpp

@@ -9829,16 +9829,6 @@ class json_sax_dom_parser
             JSON_THROW(out_of_range::create(408, concat("excessive array size: ", std::to_string(len)), ref_stack.back()));
         }
 
-        if (len != detail::unknown_size())
-        {
-            // reserve upfront to avoid repeated reallocations while adding elements,
-            // but cap the reservation so a bogus/hostile length (which is not bounded
-            // by max_size(), unlike e.g. std::vector) cannot trigger an oversized
-            // allocation for a small or truncated input
-            constexpr std::size_t reserve_cap = 16384;
-            ref_stack.back()->m_data.m_value.array->reserve(len < reserve_cap ? len : reserve_cap);
-        }
-
         return true;
     }
 
@@ -10199,16 +10189,6 @@ class json_sax_dom_callback_parser
             {
                 JSON_THROW(out_of_range::create(408, concat("excessive array size: ", std::to_string(len)), ref_stack.back()));
             }
-
-            if (len != detail::unknown_size())
-            {
-                // reserve upfront to avoid repeated reallocations while adding elements,
-                // but cap the reservation so a bogus/hostile length (which is not bounded
-                // by max_size(), unlike e.g. std::vector) cannot trigger an oversized
-                // allocation for a small or truncated input
-                constexpr std::size_t reserve_cap = 16384;
-                ref_stack.back()->m_data.m_value.array->reserve(len < reserve_cap ? len : reserve_cap);
-            }
         }
 
         return true;
@@ -20170,7 +20150,7 @@ class serializer
         , thousands_sep(loc->thousands_sep == nullptr ? '\0' : std::char_traits<char>::to_char_type(* (loc->thousands_sep)))
         , decimal_point(loc->decimal_point == nullptr ? '\0' : std::char_traits<char>::to_char_type(* (loc->decimal_point)))
         , indent_char(ichar)
-        , indent_string(512, indent_char)
+        , indent_string()
         , error_handler(error_handler_)
     {}
 
@@ -20225,6 +20205,10 @@ class serializer
 
                     // variable to hold indentation for recursive calls
                     const auto new_indent = current_indent + indent_step;
+                    if (JSON_HEDLEY_UNLIKELY(indent_string.empty()))
+                    {
+                        indent_string.resize(512, indent_char);
+                    }
                     if (JSON_HEDLEY_UNLIKELY(indent_string.size() < new_indent))
                     {
                         indent_string.resize(indent_string.size() * 2, ' ');
@@ -20298,6 +20282,10 @@ class serializer
 
                     // variable to hold indentation for recursive calls
                     const auto new_indent = current_indent + indent_step;
+                    if (JSON_HEDLEY_UNLIKELY(indent_string.empty()))
+                    {
+                        indent_string.resize(512, indent_char);
+                    }
                     if (JSON_HEDLEY_UNLIKELY(indent_string.size() < new_indent))
                     {
                         indent_string.resize(indent_string.size() * 2, ' ');
@@ -20359,6 +20347,10 @@ class serializer
 
                     // variable to hold indentation for recursive calls
                     const auto new_indent = current_indent + indent_step;
+                    if (JSON_HEDLEY_UNLIKELY(indent_string.empty()))
+                    {
+                        indent_string.resize(512, indent_char);
+                    }
                     if (JSON_HEDLEY_UNLIKELY(indent_string.size() < new_indent))
                     {
                         indent_string.resize(indent_string.size() * 2, ' ');
@@ -21109,7 +21101,7 @@ class serializer
 
     /// the indentation character
     const char indent_char;
-    /// the indentation string
+    /// the indentation string (lazily allocated on first use by a pretty-print branch)
     string_t indent_string;
 
     /// error_handler how to react on decoding errors

tests/src/unit-bjdata.cpp

@@ -3489,111 +3489,6 @@ TEST_CASE("BJData")
     }
 }
 
-TEST_CASE("issue #5405 - array reserve for definite-length BJData arrays")
-{
-#if !defined(JSON_NOEXCEPTION)
-    // this SECTION relies on catching a thrown exception to distinguish
-    // which of two acceptable, bounded rejections a hostile header took;
-    // under JSON_NOEXCEPTION, JSON_THROW never produces a catchable C++
-    // exception (it aborts instead), so this cannot be tested that way here
-    SECTION("a huge claimed length with no element data must not over-allocate")
-    {
-        // optimized form [$type#count: type 'i' (int8), count as a four-byte
-        // little-endian 'l' (int32) of 0x7FFFFFFF (2147483647), but no
-        // element data at all. max_size() for a std::vector is far larger
-        // than this count, so it does not reject the header outright; the
-        // (capped) reservation must not attempt to allocate space for
-        // billions of elements before the missing data is detected.
-        json _;
-        const std::vector<uint8_t> input = {'[', '$', 'i', '#', 'l', 0xFF, 0xFF, 0xFF, 0x7F};
-        // On a platform where std::vector<json>::max_size() is smaller than
-        // the claimed count (e.g. 32-bit, where max_size() is bounded by a
-        // 32-bit SIZE_MAX divided by sizeof(json)), the SAX consumer's own
-        // check rejects the header outright (out_of_range.408, with the
-        // claimed count in the message) instead of accepting it and only
-        // finding it short of data once the (capped) reservation looks for
-        // element bytes that were never provided (parse_error.110). Either
-        // is an acceptable, bounded rejection of the hostile header -- the
-        // property under test is that no path attempts to allocate space
-        // for billions of elements.
-        bool threw = false;
-        try
-        {
-            _ = json::from_bjdata(input);
-        }
-        catch (const json::parse_error& e)
-        {
-            threw = true;
-            CHECK(e.id == 110);
-            CHECK(std::string(e.what()) == "[json.exception.parse_error.110] parse error at byte 10: syntax error while parsing BJData number: unexpected end of input");
-        }
-        catch (const json::out_of_range& e)
-        {
-            threw = true;
-            CHECK(e.id == 408);
-            CHECK(std::string(e.what()).find("excessive array size") != std::string::npos);
-        }
-        CHECK(threw);
-
-        // json_sax_dom_parser::start_array()'s max_size() check (unlike the
-        // scanner's own parse_error path) throws unconditionally via
-        // JSON_THROW rather than going through sax->parse_error(), so it is
-        // not gated by allow_exceptions=false on a platform where this
-        // header hits that check (e.g. 32-bit, see above) -- allow either
-        // a discarded result or the same out_of_range it throws with
-        // exceptions enabled.
-        try
-        {
-            CHECK(json::from_bjdata(input, true, false).is_discarded());
-        }
-        catch (const json::out_of_range& e)
-        {
-            CHECK(e.id == 408);
-        }
-    }
-#endif
-
-    SECTION("arrays of various sizes decode to the same value as before the reserve optimization")
-    {
-        for (const auto size :
-                {
-                    std::size_t{0}, std::size_t{1}, std::size_t{5}, // small
-                    std::size_t{16384},                             // exactly at the reserve cap
-                    std::size_t{20000}                              // above the reserve cap
-                })
-        {
-            CAPTURE(size)
-            json j = json::array();
-            for (std::size_t i = 0; i < size; ++i)
-            {
-                j.push_back(static_cast<int>(i % 1000));
-            }
-
-            // exercise both the plain and the optimized [$type#count encoding
-            const auto packed_plain = json::to_bjdata(j);
-            CHECK(json::from_bjdata(packed_plain) == j);
-
-            const auto packed_optimized = json::to_bjdata(j, true, true);
-            CHECK(json::from_bjdata(packed_optimized) == j);
-        }
-    }
-
-    SECTION("a user-defined SAX consumer is unaffected by the internal DOM reserve optimization")
-    {
-        // the reserve() call is local to json_sax_dom_parser / json_sax_dom_callback_parser;
-        // a custom SAX consumer that does not touch a DOM array sees identical events
-        json j = json::array();
-        for (int i = 0; i < 100; ++i)
-        {
-            j.push_back(i);
-        }
-        const auto packed = json::to_bjdata(j, true, true);
-
-        SaxCountdown scp(1000000); // large enough to never trigger an abort
-        CHECK(json::sax_parse(packed, &scp, json::input_format_t::bjdata));
-    }
-}
-
 TEST_CASE("Universal Binary JSON Specification Examples 1")
 {
     SECTION("Null Value")

tests/src/unit-cbor.cpp

@@ -2035,92 +2035,6 @@ TEST_CASE("CBOR definite length equal to the indefinite-length sentinel")
     }
 }
 
-TEST_CASE("issue #5405 - array reserve for definite-length CBOR arrays")
-{
-#if !defined(JSON_NOEXCEPTION)
-    // this SECTION relies on catching a thrown exception to distinguish
-    // which of two acceptable, bounded rejections a hostile header took;
-    // under JSON_NOEXCEPTION, JSON_THROW never produces a catchable C++
-    // exception (it aborts instead), so this cannot be tested that way here
-    SECTION("a huge claimed length with no element data must not over-allocate")
-    {
-        // 0x9A: array with a four-byte length; claims 0xFFFFFFFF (4294967295)
-        // elements but provides none. max_size() for a std::vector is far
-        // larger than this count, so it does not reject the header outright;
-        // the (capped) reservation must not attempt to allocate space for
-        // billions of elements before the missing data is detected.
-        json _;
-        const std::vector<uint8_t> input = {0x9A, 0xFF, 0xFF, 0xFF, 0xFF};
-        // On a platform where std::size_t is narrower than 64 bits (e.g.
-        // 32-bit), the claimed count 0xFFFFFFFF coincides with that
-        // platform's detail::unknown_size() sentinel (SIZE_MAX), so the
-        // format-level size check rejects it outright (out_of_range.408,
-        // "excessive ... size") before the SAX consumer's own max_size()
-        // check would even run; on a 64-bit platform it passes both of
-        // those checks and is only found short of data once the (capped)
-        // reservation looks for element bytes that were never provided
-        // (parse_error.110). Either is an acceptable, bounded rejection of
-        // the hostile header -- the property under test is that no path
-        // attempts to allocate space for billions of elements.
-        bool threw = false;
-        try
-        {
-            _ = json::from_cbor(input);
-        }
-        catch (const json::parse_error& e)
-        {
-            threw = true;
-            CHECK(e.id == 110);
-            CHECK(std::string(e.what()) == "[json.exception.parse_error.110] parse error at byte 6: syntax error while parsing CBOR value: unexpected end of input");
-        }
-        catch (const json::out_of_range& e)
-        {
-            threw = true;
-            CHECK(e.id == 408);
-            CHECK(std::string(e.what()).find("excessive") != std::string::npos);
-        }
-        CHECK(threw);
-        CHECK(json::from_cbor(input, true, false).is_discarded());
-    }
-#endif
-
-    SECTION("arrays of various sizes decode to the same value as before the reserve optimization")
-    {
-        for (const auto size :
-                {
-                    std::size_t{0}, std::size_t{1}, std::size_t{5}, // small
-                    std::size_t{16384},                             // exactly at the reserve cap
-                    std::size_t{20000}                              // above the reserve cap
-                })
-        {
-            CAPTURE(size)
-            json j = json::array();
-            for (std::size_t i = 0; i < size; ++i)
-            {
-                j.push_back(static_cast<int>(i % 1000));
-            }
-
-            const auto packed = json::to_cbor(j);
-            CHECK(json::from_cbor(packed) == j);
-        }
-    }
-
-    SECTION("a user-defined SAX consumer is unaffected by the internal DOM reserve optimization")
-    {
-        // the reserve() call is local to json_sax_dom_parser / json_sax_dom_callback_parser;
-        // a custom SAX consumer that does not touch a DOM array sees identical events
-        json j = json::array();
-        for (int i = 0; i < 100; ++i)
-        {
-            j.push_back(i);
-        }
-        const auto packed = json::to_cbor(j);
-
-        SaxCountdown scp(1000000); // large enough to never trigger an abort
-        CHECK(json::sax_parse(packed, &scp, json::input_format_t::cbor));
-    }
-}
-
 TEST_CASE("CBOR roundtrips" * doctest::skip())
 {
     SECTION("input from flynn")

tests/src/unit-msgpack.cpp

@@ -1597,91 +1597,6 @@ TEST_CASE("MessagePack")
     }
 }
 
-TEST_CASE("issue #5405 - array reserve for definite-length MessagePack arrays")
-{
-#if !defined(JSON_NOEXCEPTION)
-    // this SECTION relies on catching a thrown exception to distinguish
-    // which of two acceptable, bounded rejections a hostile header took;
-    // under JSON_NOEXCEPTION, JSON_THROW never produces a catchable C++
-    // exception (it aborts instead), so this cannot be tested that way here
-    SECTION("a huge claimed length with no element data must not over-allocate")
-    {
-        // 0xdd: array 32 (four-byte length); claims 0xFFFFFFFF (4294967295)
-        // elements but provides none. max_size() for a std::vector is far
-        // larger than this count, so it does not reject the header outright;
-        // the (capped) reservation must not attempt to allocate space for
-        // billions of elements before the missing data is detected.
-        json _;
-        const std::vector<uint8_t> input = {0xdd, 0xFF, 0xFF, 0xFF, 0xFF};
-        // On a platform where std::size_t is narrower than 64 bits (e.g.
-        // 32-bit), the claimed count 0xFFFFFFFF coincides with that
-        // platform's SIZE_MAX, which some size-narrowing checks treat the
-        // same as detail::unknown_size(); it may then be rejected before
-        // the SAX consumer's own max_size() check (out_of_range.408) rather
-        // than being accepted and only found short of data once the
-        // (capped) reservation looks for element bytes that were never
-        // provided (parse_error.110). Either is an acceptable, bounded
-        // rejection of the hostile header -- the property under test is
-        // that no path attempts to allocate space for billions of elements.
-        bool threw = false;
-        try
-        {
-            _ = json::from_msgpack(input);
-        }
-        catch (const json::parse_error& e)
-        {
-            threw = true;
-            CHECK(e.id == 110);
-            CHECK(std::string(e.what()) == "[json.exception.parse_error.110] parse error at byte 6: syntax error while parsing MessagePack value: unexpected end of input");
-        }
-        catch (const json::out_of_range& e)
-        {
-            threw = true;
-            CHECK(e.id == 408);
-            CHECK(std::string(e.what()).find("excessive") != std::string::npos);
-        }
-        CHECK(threw);
-        CHECK(json::from_msgpack(input, true, false).is_discarded());
-    }
-#endif
-
-    SECTION("arrays of various sizes decode to the same value as before the reserve optimization")
-    {
-        for (const auto size :
-                {
-                    std::size_t{0}, std::size_t{1}, std::size_t{5}, // small
-                    std::size_t{16384},                             // exactly at the reserve cap
-                    std::size_t{20000}                              // above the reserve cap
-                })
-        {
-            CAPTURE(size)
-            json j = json::array();
-            for (std::size_t i = 0; i < size; ++i)
-            {
-                j.push_back(static_cast<int>(i % 1000));
-            }
-
-            const auto packed = json::to_msgpack(j);
-            CHECK(json::from_msgpack(packed) == j);
-        }
-    }
-
-    SECTION("a user-defined SAX consumer is unaffected by the internal DOM reserve optimization")
-    {
-        // the reserve() call is local to json_sax_dom_parser / json_sax_dom_callback_parser;
-        // a custom SAX consumer that does not touch a DOM array sees identical events
-        json j = json::array();
-        for (int i = 0; i < 100; ++i)
-        {
-            j.push_back(i);
-        }
-        const auto packed = json::to_msgpack(j);
-
-        SaxCountdown scp(1000000); // large enough to never trigger an abort
-        CHECK(json::sax_parse(packed, &scp, json::input_format_t::msgpack));
-    }
-}
-
 // use this testcase outside [hide] to run it with Valgrind
 TEST_CASE("single MessagePack roundtrip")
 {

tests/src/unit-serialization.cpp

@@ -14,6 +14,40 @@ using nlohmann::json;
 #include <array>
 #include <sstream>
 #include <iomanip>
+#include <cstdlib>
+#include <new>
+
+namespace
+{
+// heap allocation counter used by the regression test for issue #5413
+// (https://github.com/nlohmann/json/issues/5413); disabled (and thus a
+// no-op besides the counting) unless explicitly toggled on
+bool count_heap_allocations = false; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables)
+std::size_t heap_allocations = 0; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables)
+} // namespace
+
+void* operator new (std::size_t size) // NOLINT(cppcoreguidelines-owning-memory,misc-new-delete-overloads)
+{
+    if (count_heap_allocations)
+    {
+        ++heap_allocations;
+    }
+    if (void* ptr = std::malloc(size)) // NOLINT(cppcoreguidelines-no-malloc,cppcoreguidelines-owning-memory)
+    {
+        return ptr;
+    }
+    throw std::bad_alloc(); // NOLINT(hicpp-exception-baseclass)
+}
+
+void operator delete (void* ptr) noexcept // NOLINT(cppcoreguidelines-owning-memory,misc-new-delete-overloads)
+{
+    std::free(ptr); // NOLINT(cppcoreguidelines-no-malloc,cppcoreguidelines-owning-memory)
+}
+
+void operator delete (void* ptr, std::size_t /*size*/) noexcept // NOLINT(cppcoreguidelines-owning-memory,misc-new-delete-overloads)
+{
+    std::free(ptr); // NOLINT(cppcoreguidelines-no-malloc,cppcoreguidelines-owning-memory)
+}
 
 TEST_CASE("serialization")
 {
@@ -382,3 +416,76 @@ TEST_CASE("dump for basic_json with long double number_float_t")
         check_same(100.0L,  100.0);
     }
 }
+
+TEST_CASE("regression test for issue #5413 - lazily allocated indent_string")
+{
+    // the serializer used to unconditionally allocate a 512-byte
+    // indent_string in its constructor, even though it is only ever read
+    // inside the pretty_print branches of dump(). This wasted a heap
+    // allocation (and its matching deallocation) on every single compact
+    // (i.e. non-pretty, the default) dump() call. indent_string is now
+    // allocated lazily, the first time a pretty-print branch actually
+    // needs it -- so a compact dump() must perform strictly fewer heap
+    // allocations than a pretty dump() of the same value.
+    const json j = {{"level", "info"}, {"msg", "hello world"}, {"id", 12345}};
+
+    // warm up anything unrelated to indentation (e.g., one-time locale
+    // lookups) that might otherwise allocate on first use regardless of
+    // pretty-printing, so it does not skew the counts measured below
+    const auto warmup = j.dump();
+    const auto warmup_pretty = j.dump(4);
+    CHECK(!warmup.empty());
+    CHECK(!warmup_pretty.empty());
+
+    SECTION("compact dump() has a stable, minimal allocation count")
+    {
+        count_heap_allocations = true;
+
+        heap_allocations = 0;
+        const auto compact1 = j.dump();
+        const auto allocs_compact1 = heap_allocations;
+
+        heap_allocations = 0;
+        const auto compact2 = j.dump(-1);
+        const auto allocs_compact2 = heap_allocations;
+
+        count_heap_allocations = false;
+
+        CHECK(compact1 == compact2);
+        // dump() and dump(-1) both take the compact code path and must
+        // never touch indent_string, so they allocate identically often
+        CHECK(allocs_compact1 == allocs_compact2);
+    }
+
+    SECTION("first pretty dump() allocates more than a compact dump()")
+    {
+        // use a tiny value whose compact ({"a":1}, 7 bytes) and pretty
+        // ({"a": 1} with 1-space indent, 11 bytes) serializations both stay
+        // well inside every common std::string small-string-optimization
+        // buffer (>= 15 bytes on libstdc++/MSVC STL, >= 22 on libc++), so
+        // building the result string itself causes no heap allocation
+        // either way -- isolating indent_string as the only thing that can
+        // possibly account for a difference in allocation count
+        const json tiny = {{"a", 1}};
+
+        count_heap_allocations = true;
+
+        heap_allocations = 0;
+        const auto compact = tiny.dump();
+        const auto allocs_compact = heap_allocations;
+
+        heap_allocations = 0;
+        const auto pretty = tiny.dump(1);
+        const auto allocs_pretty = heap_allocations;
+
+        count_heap_allocations = false;
+
+        CHECK(compact == "{\"a\":1}");
+        CHECK(pretty == "{\n \"a\": 1\n}");
+        // a fresh serializer is created per dump() call; the pretty branch
+        // lazily allocates indent_string on its first use, so it must
+        // allocate at least once more than the compact branch, which never
+        // touches indent_string at all
+        CHECK(allocs_pretty > allocs_compact);
+    }
+}

tests/src/unit-ubjson.cpp

@@ -2149,111 +2149,6 @@ TEST_CASE("UBJSON")
     }
 }
 
-TEST_CASE("issue #5405 - array reserve for definite-length UBJSON arrays")
-{
-#if !defined(JSON_NOEXCEPTION)
-    // this SECTION relies on catching a thrown exception to distinguish
-    // which of two acceptable, bounded rejections a hostile header took;
-    // under JSON_NOEXCEPTION, JSON_THROW never produces a catchable C++
-    // exception (it aborts instead), so this cannot be tested that way here
-    SECTION("a huge claimed length with no element data must not over-allocate")
-    {
-        // optimized form [$type#count: type 'i' (int8), count as a four-byte
-        // 'l' (int32) of 0x7FFFFFFF (2147483647), but no element data at all.
-        // max_size() for a std::vector is far larger than this count, so it
-        // does not reject the header outright; the (capped) reservation must
-        // not attempt to allocate space for billions of elements before the
-        // missing data is detected.
-        json _;
-        const std::vector<uint8_t> input = {'[', '$', 'i', '#', 'l', 0x7F, 0xFF, 0xFF, 0xFF};
-        // On a platform where std::vector<json>::max_size() is smaller than
-        // the claimed count (e.g. 32-bit, where max_size() is bounded by a
-        // 32-bit SIZE_MAX divided by sizeof(json)), the SAX consumer's own
-        // check rejects the header outright (out_of_range.408, with the
-        // claimed count in the message) instead of accepting it and only
-        // finding it short of data once the (capped) reservation looks for
-        // element bytes that were never provided (parse_error.110). Either
-        // is an acceptable, bounded rejection of the hostile header -- the
-        // property under test is that no path attempts to allocate space
-        // for billions of elements.
-        bool threw = false;
-        try
-        {
-            _ = json::from_ubjson(input);
-        }
-        catch (const json::parse_error& e)
-        {
-            threw = true;
-            CHECK(e.id == 110);
-            CHECK(std::string(e.what()) == "[json.exception.parse_error.110] parse error at byte 10: syntax error while parsing UBJSON number: unexpected end of input");
-        }
-        catch (const json::out_of_range& e)
-        {
-            threw = true;
-            CHECK(e.id == 408);
-            CHECK(std::string(e.what()).find("excessive array size") != std::string::npos);
-        }
-        CHECK(threw);
-
-        // json_sax_dom_parser::start_array()'s max_size() check (unlike the
-        // scanner's own parse_error path) throws unconditionally via
-        // JSON_THROW rather than going through sax->parse_error(), so it is
-        // not gated by allow_exceptions=false on a platform where this
-        // header hits that check (e.g. 32-bit, see above) -- allow either
-        // a discarded result or the same out_of_range it throws with
-        // exceptions enabled.
-        try
-        {
-            CHECK(json::from_ubjson(input, true, false).is_discarded());
-        }
-        catch (const json::out_of_range& e)
-        {
-            CHECK(e.id == 408);
-        }
-    }
-#endif
-
-    SECTION("arrays of various sizes decode to the same value as before the reserve optimization")
-    {
-        for (const auto size :
-                {
-                    std::size_t{0}, std::size_t{1}, std::size_t{5}, // small
-                    std::size_t{16384},                             // exactly at the reserve cap
-                    std::size_t{20000}                              // above the reserve cap
-                })
-        {
-            CAPTURE(size)
-            json j = json::array();
-            for (std::size_t i = 0; i < size; ++i)
-            {
-                j.push_back(static_cast<int>(i % 1000));
-            }
-
-            // exercise both the plain and the optimized [$type#count encoding
-            const auto packed_plain = json::to_ubjson(j);
-            CHECK(json::from_ubjson(packed_plain) == j);
-
-            const auto packed_optimized = json::to_ubjson(j, true, true);
-            CHECK(json::from_ubjson(packed_optimized) == j);
-        }
-    }
-
-    SECTION("a user-defined SAX consumer is unaffected by the internal DOM reserve optimization")
-    {
-        // the reserve() call is local to json_sax_dom_parser / json_sax_dom_callback_parser;
-        // a custom SAX consumer that does not touch a DOM array sees identical events
-        json j = json::array();
-        for (int i = 0; i < 100; ++i)
-        {
-            j.push_back(i);
-        }
-        const auto packed = json::to_ubjson(j, true, true);
-
-        SaxCountdown scp(1000000); // large enough to never trigger an abort
-        CHECK(json::sax_parse(packed, &scp, json::input_format_t::ubjson));
-    }
-}
-
 TEST_CASE("Universal Binary JSON Specification Examples 1")
 {
     SECTION("Null Value")