Docs: reading and writing files; patch_inplace() is not atomic (#6017)

output_vector_sink grew the vector by resizing it to its whole new
capacity. resize() zeroes the characters it adds, so every doubling
zeroed megabytes of a fresh allocation at once: that faults in all its
pages and evicts them from the cache before the writer reaches them,
and the writer's stores then miss again. The sink now reserves the same
capacity as before but resizes the vector only to 64 KiB
(vector_sink_window) ahead of the position, which is still cached when
the writer's stores come.

to_cbor(), to_msgpack(), to_ubjson(), to_bjdata() and to_bon8() of an
array of one million numbers into a fresh vector: 4-26 % faster on x86
(Skylake laptop with Apple clang 13, Haswell VPS with GCC 16 and
Clang 23), 1-2.5 % on Apple M1 Max; other documents unchanged (within
0.4 % in reruns). The capacity and growth of the vector are as before.

Signed-off-by: Niels Lohmann <mail@nlohmann.me>

.github/workflows/ubuntu.yml

@@ -25,7 +25,7 @@ jobs:
         with:
           persist-credentials: false
       - name: Get latest CMake and ninja
-        uses: lukka/get-cmake@f86dd60acae0d132056913e248bdabf0e0580b7a # v4.4.5
+        uses: lukka/get-cmake@a89e43e22a96abafcd7a62db652a65ecd33052c9 # v4.4.4
       - name: Run CMake
         run: cmake -S . -B build -DJSON_CI=On
       - name: Build
@@ -38,7 +38,7 @@ jobs:
         with:
           persist-credentials: false
       - name: Get latest CMake and ninja
-        uses: lukka/get-cmake@f86dd60acae0d132056913e248bdabf0e0580b7a # v4.4.5
+        uses: lukka/get-cmake@a89e43e22a96abafcd7a62db652a65ecd33052c9 # v4.4.4
       - name: Check that Meson and pkg-config offer the CMake options
         run: make check_build_options
       - name: Install Meson
@@ -92,7 +92,7 @@ jobs:
         with:
           persist-credentials: false
       - name: Get latest CMake and ninja
-        uses: lukka/get-cmake@f86dd60acae0d132056913e248bdabf0e0580b7a # v4.4.5
+        uses: lukka/get-cmake@a89e43e22a96abafcd7a62db652a65ecd33052c9 # v4.4.4
       - name: Run CMake
         run: cmake -S . -B build -DJSON_CI=On
       - name: Build
@@ -123,7 +123,7 @@ jobs:
         with:
           persist-credentials: false
       - name: Get latest CMake and ninja
-        uses: lukka/get-cmake@f86dd60acae0d132056913e248bdabf0e0580b7a # v4.4.5
+        uses: lukka/get-cmake@a89e43e22a96abafcd7a62db652a65ecd33052c9 # v4.4.4
       - name: Run CMake
         run: cmake -S . -B build -DJSON_CI=On
       - name: Build
@@ -142,7 +142,7 @@ jobs:
         with:
           persist-credentials: false
       - name: Get latest CMake and ninja
-        uses: lukka/get-cmake@f86dd60acae0d132056913e248bdabf0e0580b7a # v4.4.5
+        uses: lukka/get-cmake@a89e43e22a96abafcd7a62db652a65ecd33052c9 # v4.4.4
       - name: Run CMake
         run: cmake -S . -B build -DJSON_CI=On
       - name: Build
@@ -168,7 +168,7 @@ jobs:
         with:
           persist-credentials: false
       - name: Get latest CMake and ninja
-        uses: lukka/get-cmake@f86dd60acae0d132056913e248bdabf0e0580b7a # v4.4.5
+        uses: lukka/get-cmake@a89e43e22a96abafcd7a62db652a65ecd33052c9 # v4.4.4
       - name: Run CMake
         run: cmake -S . -B build -DJSON_CI=On
       - name: Build
@@ -244,7 +244,7 @@ jobs:
         with:
           persist-credentials: false
       - name: Get latest CMake and ninja
-        uses: lukka/get-cmake@f86dd60acae0d132056913e248bdabf0e0580b7a # v4.4.5
+        uses: lukka/get-cmake@a89e43e22a96abafcd7a62db652a65ecd33052c9 # v4.4.4
       - name: Run CMake
         run: CXX=g++-${{ matrix.compiler }} cmake -S . -B build -DJSON_CI=On
       - name: Build
@@ -262,7 +262,7 @@ jobs:
         with:
           persist-credentials: false
       - name: Get latest CMake and ninja
-        uses: lukka/get-cmake@f86dd60acae0d132056913e248bdabf0e0580b7a # v4.4.5
+        uses: lukka/get-cmake@a89e43e22a96abafcd7a62db652a65ecd33052c9 # v4.4.4
       - name: Run CMake
         run: cmake -S . -B build -DJSON_CI=On
       - name: Build
@@ -279,7 +279,7 @@ jobs:
         with:
           persist-credentials: false
       - name: Get latest CMake and ninja
-        uses: lukka/get-cmake@f86dd60acae0d132056913e248bdabf0e0580b7a # v4.4.5
+        uses: lukka/get-cmake@a89e43e22a96abafcd7a62db652a65ecd33052c9 # v4.4.4
       - name: Disable filesystem support (std::filesystem is unusable) for clang 7 / 8 / 9 / 10
         run: echo "JSON_FORCED_GLOBAL_COMPILE_OPTIONS=-DJSON_HAS_FILESYSTEM=0;-DJSON_HAS_EXPERIMENTAL_FILESYSTEM=0" >> "$GITHUB_ENV"
         if: ${{ matrix.compiler == '7' || matrix.compiler == '8' || matrix.compiler == '9' || matrix.compiler == '10' }}
@@ -299,7 +299,7 @@ jobs:
         with:
           persist-credentials: false
       - name: Get latest CMake and ninja
-        uses: lukka/get-cmake@f86dd60acae0d132056913e248bdabf0e0580b7a # v4.4.5
+        uses: lukka/get-cmake@a89e43e22a96abafcd7a62db652a65ecd33052c9 # v4.4.4
       - name: Run CMake
         run: cmake -S . -B build -DJSON_CI=On
       - name: Build
@@ -319,7 +319,7 @@ jobs:
         with:
           persist-credentials: false
       - name: Get latest CMake and ninja
-        uses: lukka/get-cmake@f86dd60acae0d132056913e248bdabf0e0580b7a # v4.4.5
+        uses: lukka/get-cmake@a89e43e22a96abafcd7a62db652a65ecd33052c9 # v4.4.4
       - name: Run CMake
         run: cmake -S . -B build -DJSON_CI=On
       - name: Build with libc++
@@ -346,7 +346,7 @@ jobs:
         with:
           persist-credentials: false
       - name: Get latest CMake and ninja
-        uses: lukka/get-cmake@f86dd60acae0d132056913e248bdabf0e0580b7a # v4.4.5
+        uses: lukka/get-cmake@a89e43e22a96abafcd7a62db652a65ecd33052c9 # v4.4.4
       - name: Run CMake
         run: cmake -S . -B build -DJSON_CI=On
       - name: Build
@@ -366,7 +366,7 @@ jobs:
       # import-std support. Its opt-in token is CMake-version-specific, so pin
       # CMake to the version whose token is set in tests/module_cpp20/CMakeLists.txt.
       - name: Get pinned CMake and ninja
-        uses: lukka/get-cmake@f86dd60acae0d132056913e248bdabf0e0580b7a # v4.4.5
+        uses: lukka/get-cmake@a89e43e22a96abafcd7a62db652a65ecd33052c9 # v4.4.4
         with:
           cmakeVersion: 4.3.4
       # Clang: the std library module is provided by libc++ (the image's libstdc++
@@ -392,7 +392,7 @@ jobs:
         with:
           persist-credentials: false
       - name: Get latest CMake and ninja
-        uses: lukka/get-cmake@f86dd60acae0d132056913e248bdabf0e0580b7a # v4.4.5
+        uses: lukka/get-cmake@a89e43e22a96abafcd7a62db652a65ecd33052c9 # v4.4.4
       - name: Run CMake
         run: cmake -S . -B build -DJSON_CI=On
       - name: Build
@@ -409,7 +409,7 @@ jobs:
         with:
           persist-credentials: false
       - name: Get latest CMake and ninja
-        uses: lukka/get-cmake@f86dd60acae0d132056913e248bdabf0e0580b7a # v4.4.5
+        uses: lukka/get-cmake@a89e43e22a96abafcd7a62db652a65ecd33052c9 # v4.4.4
       - name: Run CMake
         run: cmake -S . -B build -DJSON_CI=On
       - name: Build
@@ -423,7 +423,7 @@ jobs:
         with:
           persist-credentials: false
       - name: Get latest CMake and ninja
-        uses: lukka/get-cmake@f86dd60acae0d132056913e248bdabf0e0580b7a # v4.4.5
+        uses: lukka/get-cmake@a89e43e22a96abafcd7a62db652a65ecd33052c9 # v4.4.4
       - name: Run CMake
         run: cmake -S . -B build -DJSON_CI=On
       - name: Build
@@ -443,7 +443,7 @@ jobs:
         with:
           persist-credentials: false
       - name: Get latest CMake and ninja
-        uses: lukka/get-cmake@f86dd60acae0d132056913e248bdabf0e0580b7a # v4.4.5
+        uses: lukka/get-cmake@a89e43e22a96abafcd7a62db652a65ecd33052c9 # v4.4.4
       - name: Run CMake
         run: cmake -S . -B build -DCMAKE_TOOLCHAIN_FILE=$EMSDK/upstream/emscripten/cmake/Modules/Platform/Emscripten.cmake -GNinja
       - name: Build
@@ -484,7 +484,7 @@ jobs:
         with:
           persist-credentials: false
       - name: Get latest CMake and ninja
-        uses: lukka/get-cmake@f86dd60acae0d132056913e248bdabf0e0580b7a # v4.4.5
+        uses: lukka/get-cmake@a89e43e22a96abafcd7a62db652a65ecd33052c9 # v4.4.4
       - name: Check that the job runs natively on arm64
         run: uname -m && test "$(uname -m)" = aarch64
       - name: Run CMake

.github/workflows/windows.yml

@@ -90,7 +90,7 @@ jobs:
         with:
           persist-credentials: false
       - name: Get latest CMake and ninja
-        uses: lukka/get-cmake@f86dd60acae0d132056913e248bdabf0e0580b7a # v4.4.5
+        uses: lukka/get-cmake@a89e43e22a96abafcd7a62db652a65ecd33052c9 # v4.4.4
       - name: Set extra CXX_FLAGS for latest std_version
         # /wd5285 silences C5285 emitted by the bundled third-party doctest.h, which
         # specializes std::tuple (newly diagnosed by the VS2026 v145 toolset)
@@ -216,7 +216,7 @@ jobs:
       # import-std support. Its opt-in token is CMake-version-specific, so pin
       # CMake to the version whose token is set in tests/module_cpp20/CMakeLists.txt.
       - name: Get pinned CMake and ninja
-        uses: lukka/get-cmake@f86dd60acae0d132056913e248bdabf0e0580b7a # v4.4.5
+        uses: lukka/get-cmake@a89e43e22a96abafcd7a62db652a65ecd33052c9 # v4.4.4
         with:
           cmakeVersion: 4.3.4
       # CMAKE_CXX_FLAGS replaces CMake's MSVC defaults (which include /EHsc), and

docs/mkdocs/docs/api/basic_json/patch_inplace.md

@@ -15,7 +15,9 @@ the patch. This function applies a JSON patch in place and returns void.
 
 ## Exception safety
 
-No guarantees, value may be corrupted by an unsuccessful patch operation.
+No guarantees: if an operation fails, the operations before it stay applied, and a failing `move` operation may already
+have removed the value at its "from" location. Use [`patch`](patch.md) if the value must be unchanged after a failed
+patch.
 
 ## Exceptions
 
@@ -62,8 +64,19 @@ the JSON value, the complexity can usually be neglected.
 ## Notes
 
 Unlike [`patch`](patch.md), `patch_inplace` applies the operation "in place" and no copy of the JSON value is created.
-That makes it faster for large documents by avoiding the copy. However, the JSON value might be corrupted if the
-function throws an exception.
+That makes it faster for large documents by avoiding the copy.
+
+!!! warning "Not atomic"
+
+    `patch_inplace` applies the operations one after the other and does not undo them when one fails. After an
+    exception, the value holds the result of the operations before the failing one, and a failing `move` operation
+    (a "remove" followed by an "add") may already have removed the value at its "from" location; see the second example
+    below. [RFC 6902](https://tools.ietf.org/html/rfc6902#section-5) requires that a patch with a failing operation is
+    not deemed successful, but leaves open what happens to the document.
+
+    If the value must stay unchanged when the patch fails, use [`patch`](patch.md), which applies the patch to a copy:
+    `#!cpp j = j.patch(json_patch);`. Validating the patch beforehand is not enough, as "test" operations and missing
+    paths are only detected while the patch is applied.
 
 Duplicate member names inside an operation object are resolved by the parser before the patch is applied (the last one
 is kept, see [`object_t`](object_t.md#behavior)), so `#!json {"op": "add", "op": "remove", "path": "/a"}` is a

docs/mkdocs/docs/features/files.md

@@ -0,0 +1,130 @@
+# Reading and writing files
+
+The library has no functions that take a file name. It reads from and writes to streams, so files are handled with
+`#!cpp std::ifstream` and `#!cpp std::ofstream` (or a `#!cpp FILE*` when reading). This page collects the recipes for
+the common cases and the mistakes that show up again and again.
+
+## Reading a JSON file
+
+Pass the stream to [`parse`](../api/basic_json/parse.md) directly, and check that the file was opened:
+
+```cpp
+#include <fstream>
+#include <stdexcept>
+#include <string>
+#include <nlohmann/json.hpp>
+
+using json = nlohmann::json;
+
+json read_json_file(const std::string& path)
+{
+    std::ifstream f(path);
+    if (!f.is_open())
+    {
+        throw std::runtime_error("cannot open " + path);
+    }
+    return json::parse(f);
+}
+```
+
+- **Check that the file is open.** A stream that failed to open looks like an empty input, so `parse` reports
+  [`parse_error.101`](../home/exceptions.md#jsonexceptionparse_error101) ("unexpected end of input") at line 1,
+  column 1 instead of saying that the file is missing. A relative path is resolved against the working directory of the
+  running program, which is often not the directory of the source file (see the
+  [FAQ](../home/faq.md#parsing-a-file-fails-with-unexpected-end-of-input)).
+- **Don't read the file into a string first.** Parsing from a `#!cpp std::ifstream` is only about 10 to 20 % slower
+  than reading the whole file into a string with one `read()` and parsing that, and it needs no memory for the copy (see
+  [input sources and performance](parsing/index.md#input-sources-and-performance)). The common way to read a file with
+  `#!cpp std::istreambuf_iterator<char>` is so slow that it takes about twice as long as parsing from the stream. Read
+  the file into a string only if you need the text itself, for example for an error message. This advice is for
+  `#!cpp json::parse`. [`json_document`](json_view.md) reads a stream into a buffer that it owns anyway, so there is no
+  copy to save: reading the file into a `#!cpp std::string` yourself and passing that as an lvalue, which the document
+  borrows, was about 6 to 12 % faster than parsing from the stream, with the same single buffer.
+- `parse` requires the file to hold exactly one JSON value. Files with one value per line are read with
+  [JSON Lines](parsing/json_lines.md); files too large for memory are covered in
+  [Large, streamed, and partial input](parsing/large_and_partial_input.md).
+
+## Writing a JSON file
+
+[`dump`](../api/basic_json/dump.md) the value into a string first, then write the string:
+
+```cpp
+void write_json_file(const std::string& path, const json& j)
+{
+    const std::string text = j.dump(4) + '\n';  // may throw: nothing is written yet
+
+    std::ofstream f(path, std::ios::binary);
+    if (!f.is_open())
+    {
+        throw std::runtime_error("cannot open " + path);
+    }
+    f << text;
+    f.close();
+    if (f.fail())
+    {
+        throw std::runtime_error("cannot write " + path);
+    }
+}
+```
+
+- **Serialize before opening the file.** Opening a `#!cpp std::ofstream` truncates the file. `dump` and
+  [`operator<<`](../api/operator_ltlt.md) throw [`type_error.316`](../home/exceptions.md#jsonexceptiontype_error316)
+  when a string is not valid UTF-8 (see [handling invalid UTF-8](serialization.md#handling-invalid-utf-8)), and
+  `#!cpp f << std::setw(4) << j` may already have written part of the value when it throws. Serializing into a string
+  first leaves the old file untouched in that case.
+- **Check the stream after closing it.** A full disk or a missing permission sets the stream's error state; it does not
+  throw unless you enabled exceptions on the stream with `exceptions()`.
+- `std::ios::binary` writes the text as it is. Without it, Windows writes `\r\n` for every newline of a pretty-printed
+  value.
+
+## Replacing a file without corrupting it
+
+The library never writes a file itself, so this section is not about the library. It is general C++ advice for a
+problem users of the library keep running into.
+
+A program that crashes or is killed while it writes leaves a truncated file behind. To make sure readers see either the
+old or the new contents, write to a temporary file in the same directory and rename it over the original
+(C++17, `#!cpp <filesystem>`):
+
+```cpp
+#include <filesystem>
+
+void replace_json_file(const std::filesystem::path& path, const json& j)
+{
+    std::filesystem::path tmp = path;
+    tmp += ".tmp";  // use a unique name if several processes may write the file
+
+    write_json_file(tmp.string(), j);
+    std::filesystem::rename(tmp, path);  // replaces path
+}
+```
+
+The temporary file must be in the same directory (more precisely, on the same file system) as the target: a rename
+across file systems does not fall back to a copy, it fails and throws `#!cpp std::filesystem::filesystem_error`. On
+POSIX systems, the rename atomically replaces the file. Standard C++ has no way to flush a file to the disk (`fsync`),
+so this protects against a crashed program, but not necessarily against a power loss.
+
+## Binary formats
+
+The [binary formats](binary_formats/index.md) work the same way, with the streams opened in binary mode:
+
+```cpp
+// read
+std::ifstream in("data.cbor", std::ios::binary);
+json j = json::from_cbor(in);
+
+// write
+const std::vector<std::uint8_t> bytes = json::to_cbor(j);
+std::ofstream out("data.cbor", std::ios::binary);
+out.write(reinterpret_cast<const char*>(bytes.data()), static_cast<std::streamsize>(bytes.size()));
+```
+
+Without `std::ios::binary`, Windows turns the bytes `0x0D 0x0A` into `0x0A` and may stop at a byte `0x1A` while reading,
+and turns `0x0A` into `0x0D 0x0A` while writing, which corrupts the data.
+
+## See also
+
+- [Parsing](parsing/index.md) - the inputs `parse` accepts
+- [Serialization](serialization.md) - the options of `dump`
+- [`parse`](../api/basic_json/parse.md), [`dump`](../api/basic_json/dump.md)
+- [`operator>>`](../api/operator_gtgt.md), [`operator<<`](../api/operator_ltlt.md)

docs/mkdocs/docs/features/index.md

@@ -46,6 +46,8 @@ C++ types, and finally serialize it again.
   [BJData](binary_formats/bjdata.md), [BON8](binary_formats/bon8.md), [BSON](binary_formats/bson.md),
   [CBOR](binary_formats/cbor.md), [MessagePack](binary_formats/messagepack.md), or [UBJSON](binary_formats/ubjson.md).
 - [Binary values](binary_values.md) — store and exchange raw byte sequences.
+- [Reading and writing files](files.md) — read and write files safely, including replacing a file without
+  corrupting it.
 
 ## How values are stored and configured
 

docs/mkdocs/docs/features/json_patch.md

@@ -29,7 +29,8 @@ other three continue on its result.
 
     [`patch`](../api/basic_json/patch.md) leaves the original value unchanged and returns the patched result as a copy.
     If the document is large and the original value is no longer needed,
-    [`patch_inplace`](../api/basic_json/patch_inplace.md) applies the same operations in place instead.
+    [`patch_inplace`](../api/basic_json/patch_inplace.md) applies the same operations in place instead. Unlike
+    `patch`, it is not atomic: if an operation fails, the operations before it stay applied.
 
 !!! note "Duplicate member names"
 

docs/mkdocs/docs/features/json_view.md

@@ -160,7 +160,9 @@ document: `#!cpp auto v = json_document::parse(text).root();` does not compile.
   wide. The limit is 16 bytes below 2^32, because the parser reads in 16-byte steps, and a position one step past the
   end must still fit.
 - **A stream is always read to its end.** There is no partial/streaming read of an `#!cpp std::istream`: the stream
-  is read to its end into a buffer the document owns, and parsing starts after that.
+  is read to its end into a buffer the document owns, and parsing starts after that. For a file, reading it into a
+  `#!cpp std::string` yourself and parsing that (an lvalue is borrowed) is therefore not more expensive in memory, and
+  was about 6 to 12 % faster than parsing from an `#!cpp std::ifstream`.
 - **Fewer source positions after edits.** With
   [`JSON_DIAGNOSTIC_POSITIONS`](../api/macros/json_diagnostic_positions.md) enabled,
   [`materialize()`](../api/basic_json_view/materialize.md) sets the same positions as `#!cpp basic_json::parse()`.

docs/mkdocs/docs/features/parsing/index.md

@@ -35,6 +35,8 @@ std::ifstream f("example.json");
 json data = json::parse(f);
 ```
 
+See [Reading and writing files](../files.md) for checking that the file was opened and for writing files safely.
+
 The input must be encoded in UTF-8; other encodings are not supported. The exception is wide-character input, which
 is read as UTF-16 or UTF-32 — depending on the character type — and converted to UTF-8; see the
 [`parse`](../../api/basic_json/parse.md) page for an example. A single input may contain only one JSON value.
@@ -84,6 +86,9 @@ ahead: after parsing, it is positioned exactly as if every character had been re
 [`operator>>`](../../api/operator_gtgt.md) can still read several values from one stream (see also
 [JSON Lines](json_lines.md)).
 
+This does not hold for [`json_document`](../json_view.md): it reads a stream into a buffer it owns, so the stream saves
+no memory, and reading the file into a `#!cpp std::string` yourself and borrowing it is about 6 to 12 % faster.
+
 Two macros switch these optimizations off without changing any result:
 [`JSON_DISABLE_STREAM_BULK_SCAN`](../../api/macros/json_disable_stream_bulk_scan.md) for stream buffers that do not
 follow the usual `std::streambuf` contract, and

docs/mkdocs/docs/features/performance.md

@@ -39,7 +39,9 @@ iterator pair over non-contiguous but random-access storage (e.g. `#!cpp std::de
 optimization but not the second, since the byte-scanning fast path requires contiguous storage.
 
 Practically: a file can be passed directly to `parse`, which also avoids the memory for a copy of the file. Reading it
-into a string first is no longer necessary for speed:
+into a string first is no longer necessary for speed (this is for `#!cpp json::parse`; a
+[`json_document`](json_view.md) reads a stream into an owned buffer anyway, so for it reading the file into a string
+and borrowing that is the faster way):
 
 ```cpp
 // fast: the stream buffer's get area is scanned in bulk

docs/mkdocs/docs/features/serialization.md

@@ -17,6 +17,9 @@ To write a value directly to a stream (for example, a file or `#!cpp std::cout`)
 std::cout << j << std::endl;
 ```
 
+To write a file without leaving a truncated file behind when serialization fails, see
+[Reading and writing files](files.md#writing-a-json-file).
+
 !!! note "String, not raw value"
 
     `dump` always returns a **JSON text**. Serializing a JSON string therefore includes the surrounding quotes and

docs/mkdocs/docs/home/faq.md

@@ -110,7 +110,7 @@ in the source files that need it, instantiate `basic_json` once, and consider pr
 The parser did not see any character: the file was most likely not opened. A relative path is resolved against the
 working directory of the running program, which is often the build directory of your IDE, not the source directory.
 Check `is_open()` and try an absolute path. Other causes are an empty file and a stream that was already read. See
-[`parse_error.101`](exceptions.md#jsonexceptionparse_error101).
+[`parse_error.101`](exceptions.md#jsonexceptionparse_error101) and [Reading and writing files](../features/files.md).
 
 ### Input that arrives in chunks
 

docs/mkdocs/mkdocs.yml

@@ -130,6 +130,7 @@ nav:
           - features/element_access/unchecked_access.md
           - features/element_access/checked_access.md
           - features/element_access/default_value.md
+      - features/files.md
       - features/iterators.md
       - features/merge_patch.md
       - features/json_patch.md

include/nlohmann/detail/input/binary_reader.hpp

@@ -1024,7 +1024,7 @@ class binary_reader
             {
                 std::uint64_t len{};
                 std::size_t size{};
-                return get_cbor_argument(len) && get_cbor_container_size(len, size, "array") && enter_cbor_array(size);
+                return get_cbor_argument(len) && get_container_size(len, size, "array") && enter_cbor_array(size);
             }
 
             case 0x9F: // array (indefinite length)
@@ -1064,7 +1064,7 @@ class binary_reader
             {
                 std::uint64_t len{};
                 std::size_t size{};
-                return get_cbor_argument(len) && get_cbor_container_size(len, size, "map") && enter_object(size);
+                return get_cbor_argument(len) && get_container_size(len, size, "map") && enter_object(size);
             }
 
             case 0xBF: // map (indefinite length)
@@ -1700,7 +1700,7 @@ class binary_reader
     }
 
     /*!
-    @brief narrow a definite CBOR array/map length to std::size_t
+    @brief narrow a definite CBOR or MessagePack array/map length to std::size_t
 
     A definite length is rejected if it does not fit in std::size_t or if it
     equals detail::unknown_size(), which is reserved to mark an indefinite-
@@ -1713,7 +1713,7 @@ class binary_reader
     @param[in]  context  "array" or "map", for the error message
     @return whether the length is usable
     */
-    bool get_cbor_container_size(const std::uint64_t len, std::size_t& result, const char* context)
+    bool get_container_size(const std::uint64_t len, std::size_t& result, const char* context)
     {
         if (JSON_HEDLEY_UNLIKELY(!value_in_range_of<std::size_t>(len) || len == detail::unknown_size()))
         {
@@ -2309,7 +2309,8 @@ class binary_reader
             case 0xDD: // array 32
             {
                 std::uint32_t len{};
-                return get_number(len) && enter_msgpack_array(conditional_static_cast<std::size_t>(len));
+                std::size_t size{};
+                return get_number(len) && get_container_size(len, size, "array") && enter_msgpack_array(size);
             }
 
             case 0xDE: // map 16
@@ -2321,7 +2322,8 @@ class binary_reader
             case 0xDF: // map 32
             {
                 std::uint32_t len{};
-                return get_number(len) && enter_object(conditional_static_cast<std::size_t>(len));
+                std::size_t size{};
+                return get_number(len) && get_container_size(len, size, "map") && enter_object(size);
             }
 
             // negative fixint
@@ -3131,6 +3133,27 @@ class binary_reader
         return true;
     }
 
+    /*!
+    @brief store the count of an optimized container (or the length of a string)
+
+    A count equal to npos is rejected: npos marks a container whose count is
+    not known, so a count of that value would be read as "no count" (compare
+    get_container_size()). No container can hold that many elements.
+
+    @param[in]  count   the count read from the input
+    @param[out] result  set to @a count unless it is rejected
+    @return whether @a count is usable
+    */
+    bool store_ubjson_count(const std::size_t count, std::size_t& result)
+    {
+        if (JSON_HEDLEY_UNLIKELY(count == npos))
+        {
+            return out_of_range_error(exception_id::container_too_large, "integer value overflow", "size");
+        }
+        result = count;
+        return true;
+    }
+
     /*!
     @brief read a UBJSON/BJData optimized-container count of a signed marker
            type ('i', 'I', 'l', 'L') and narrow it to std::size_t
@@ -3161,8 +3184,7 @@ class binary_reader
         {
             return out_of_range_error(exception_id::container_too_large, "integer value overflow", "size");
         }
-        result = static_cast<std::size_t>(number); // NOLINT(bugprone-signed-char-misuse,cert-str34-c): number is not a char
-        return true;
+        return store_ubjson_count(static_cast<std::size_t>(number), result); // NOLINT(bugprone-signed-char-misuse,cert-str34-c): number is not a char
     }
 
     /*!
@@ -3239,8 +3261,7 @@ class binary_reader
                 {
                     return false;
                 }
-                result = conditional_static_cast<std::size_t>(number);
-                return true;
+                return store_ubjson_count(conditional_static_cast<std::size_t>(number), result);
             }
 
             case 'M':
@@ -3258,8 +3279,7 @@ class binary_reader
                 {
                     return out_of_range_error(exception_id::container_too_large, "integer value overflow", "size");
                 }
-                result = detail::conditional_static_cast<std::size_t>(number);
-                return true;
+                return store_ubjson_count(detail::conditional_static_cast<std::size_t>(number), result);
             }
 
             case '[':

include/nlohmann/detail/output/output_adapters.hpp

@@ -116,6 +116,10 @@ inline void copy_short(CharType* dst, const CharType* src, const std::size_t len
 /// the number of characters an empty vector is grown to by output_vector_sink
 constexpr std::size_t min_vector_sink_capacity = 64;
 
+/// the number of characters output_vector_sink makes writable ahead of its
+/// position at a time: 64 KiB of the vector's capacity, not all of it
+constexpr std::size_t vector_sink_window = 65536;
+
 /// @brief non-virtual output sink writing into a std::vector
 ///
 /// This sink is not part of the virtual output_adapter_protocol hierarchy: it is
@@ -129,7 +133,8 @@ constexpr std::size_t min_vector_sink_capacity = 64;
 /// encoding them: insert() is an out-of-line call into a routine that can insert
 /// anywhere, and resize() is one as well with libstdc++. The sink therefore keeps
 /// its own write position: it grows the vector geometrically ahead of the
-/// position and copies each write to it, so a write is a capacity check and a
+/// position (making @ref vector_sink_window characters of it writable at a
+/// time) and copies each write to it, so a write is a capacity check and a
 /// short copy. The destructor shrinks the vector to the bytes actually written,
 /// also when a write throws; shrinking a vector never allocates. The sink owns
 /// that trim, so it can be moved but not copied, and the vector must not be used
@@ -277,9 +282,17 @@ class output_vector_sink
         // reallocate for every few bytes
         const std::size_t written = position();
         const std::size_t needed = written + length;
-        v->resize((std::max)(min_vector_sink_capacity, needed <= v->capacity()
-                             ? v->capacity()
-                             : (std::max)(needed, v->capacity() * 2)));
+        const std::size_t capacity = (std::max)(min_vector_sink_capacity, needed <= v->capacity()
+                                                ? v->capacity()
+                                                : (std::max)(needed, v->capacity() * 2));
+        v->reserve(capacity);
+        // resize() zeroes the characters it adds. Zeroing all of a doubled
+        // capacity, megabytes of a fresh allocation, at once faults in its
+        // pages and evicts them from the cache before the writer gets there,
+        // so its stores miss again; a window ahead of the position is still
+        // cached when they come (the binary writers on a long array of
+        // numbers: 4-26 % faster on x86, 1-2.5 % on Apple M1)
+        v->resize((std::min)(capacity, (std::max)(needed, written + vector_sink_window)));
         cursor = v->data() + written;
         limit = v->data() + v->size();
     }

single_include/nlohmann/json.hpp

@@ -19039,7 +19039,7 @@ class binary_reader
             {
                 std::uint64_t len{};
                 std::size_t size{};
-                return get_cbor_argument(len) && get_cbor_container_size(len, size, "array") && enter_cbor_array(size);
+                return get_cbor_argument(len) && get_container_size(len, size, "array") && enter_cbor_array(size);
             }
 
             case 0x9F: // array (indefinite length)
@@ -19079,7 +19079,7 @@ class binary_reader
             {
                 std::uint64_t len{};
                 std::size_t size{};
-                return get_cbor_argument(len) && get_cbor_container_size(len, size, "map") && enter_object(size);
+                return get_cbor_argument(len) && get_container_size(len, size, "map") && enter_object(size);
             }
 
             case 0xBF: // map (indefinite length)
@@ -19715,7 +19715,7 @@ class binary_reader
     }
 
     /*!
-    @brief narrow a definite CBOR array/map length to std::size_t
+    @brief narrow a definite CBOR or MessagePack array/map length to std::size_t
 
     A definite length is rejected if it does not fit in std::size_t or if it
     equals detail::unknown_size(), which is reserved to mark an indefinite-
@@ -19728,7 +19728,7 @@ class binary_reader
     @param[in]  context  "array" or "map", for the error message
     @return whether the length is usable
     */
-    bool get_cbor_container_size(const std::uint64_t len, std::size_t& result, const char* context)
+    bool get_container_size(const std::uint64_t len, std::size_t& result, const char* context)
     {
         if (JSON_HEDLEY_UNLIKELY(!value_in_range_of<std::size_t>(len) || len == detail::unknown_size()))
         {
@@ -20324,7 +20324,8 @@ class binary_reader
             case 0xDD: // array 32
             {
                 std::uint32_t len{};
-                return get_number(len) && enter_msgpack_array(conditional_static_cast<std::size_t>(len));
+                std::size_t size{};
+                return get_number(len) && get_container_size(len, size, "array") && enter_msgpack_array(size);
             }
 
             case 0xDE: // map 16
@@ -20336,7 +20337,8 @@ class binary_reader
             case 0xDF: // map 32
             {
                 std::uint32_t len{};
-                return get_number(len) && enter_object(conditional_static_cast<std::size_t>(len));
+                std::size_t size{};
+                return get_number(len) && get_container_size(len, size, "map") && enter_object(size);
             }
 
             // negative fixint
@@ -21146,6 +21148,27 @@ class binary_reader
         return true;
     }
 
+    /*!
+    @brief store the count of an optimized container (or the length of a string)
+
+    A count equal to npos is rejected: npos marks a container whose count is
+    not known, so a count of that value would be read as "no count" (compare
+    get_container_size()). No container can hold that many elements.
+
+    @param[in]  count   the count read from the input
+    @param[out] result  set to @a count unless it is rejected
+    @return whether @a count is usable
+    */
+    bool store_ubjson_count(const std::size_t count, std::size_t& result)
+    {
+        if (JSON_HEDLEY_UNLIKELY(count == npos))
+        {
+            return out_of_range_error(exception_id::container_too_large, "integer value overflow", "size");
+        }
+        result = count;
+        return true;
+    }
+
     /*!
     @brief read a UBJSON/BJData optimized-container count of a signed marker
            type ('i', 'I', 'l', 'L') and narrow it to std::size_t
@@ -21176,8 +21199,7 @@ class binary_reader
         {
             return out_of_range_error(exception_id::container_too_large, "integer value overflow", "size");
         }
-        result = static_cast<std::size_t>(number); // NOLINT(bugprone-signed-char-misuse,cert-str34-c): number is not a char
-        return true;
+        return store_ubjson_count(static_cast<std::size_t>(number), result); // NOLINT(bugprone-signed-char-misuse,cert-str34-c): number is not a char
     }
 
     /*!
@@ -21254,8 +21276,7 @@ class binary_reader
                 {
                     return false;
                 }
-                result = conditional_static_cast<std::size_t>(number);
-                return true;
+                return store_ubjson_count(conditional_static_cast<std::size_t>(number), result);
             }
 
             case 'M':
@@ -21273,8 +21294,7 @@ class binary_reader
                 {
                     return out_of_range_error(exception_id::container_too_large, "integer value overflow", "size");
                 }
-                result = detail::conditional_static_cast<std::size_t>(number);
-                return true;
+                return store_ubjson_count(detail::conditional_static_cast<std::size_t>(number), result);
             }
 
             case '[':
@@ -27491,6 +27511,10 @@ inline void copy_short(CharType* dst, const CharType* src, const std::size_t len
 /// the number of characters an empty vector is grown to by output_vector_sink
 constexpr std::size_t min_vector_sink_capacity = 64;
 
+/// the number of characters output_vector_sink makes writable ahead of its
+/// position at a time: 64 KiB of the vector's capacity, not all of it
+constexpr std::size_t vector_sink_window = 65536;
+
 /// @brief non-virtual output sink writing into a std::vector
 ///
 /// This sink is not part of the virtual output_adapter_protocol hierarchy: it is
@@ -27504,7 +27528,8 @@ constexpr std::size_t min_vector_sink_capacity = 64;
 /// encoding them: insert() is an out-of-line call into a routine that can insert
 /// anywhere, and resize() is one as well with libstdc++. The sink therefore keeps
 /// its own write position: it grows the vector geometrically ahead of the
-/// position and copies each write to it, so a write is a capacity check and a
+/// position (making @ref vector_sink_window characters of it writable at a
+/// time) and copies each write to it, so a write is a capacity check and a
 /// short copy. The destructor shrinks the vector to the bytes actually written,
 /// also when a write throws; shrinking a vector never allocates. The sink owns
 /// that trim, so it can be moved but not copied, and the vector must not be used
@@ -27652,9 +27677,17 @@ class output_vector_sink
         // reallocate for every few bytes
         const std::size_t written = position();
         const std::size_t needed = written + length;
-        v->resize((std::max)(min_vector_sink_capacity, needed <= v->capacity()
-                             ? v->capacity()
-                             : (std::max)(needed, v->capacity() * 2)));
+        const std::size_t capacity = (std::max)(min_vector_sink_capacity, needed <= v->capacity()
+                                                ? v->capacity()
+                                                : (std::max)(needed, v->capacity() * 2));
+        v->reserve(capacity);
+        // resize() zeroes the characters it adds. Zeroing all of a doubled
+        // capacity, megabytes of a fresh allocation, at once faults in its
+        // pages and evicts them from the cache before the writer gets there,
+        // so its stores miss again; a window ahead of the position is still
+        // cached when they come (the binary writers on a long array of
+        // numbers: 4-26 % faster on x86, 1-2.5 % on Apple M1)
+        v->resize((std::min)(capacity, (std::max)(needed, written + vector_sink_window)));
         cursor = v->data() + written;
         limit = v->data() + v->size();
     }

tests/src/unit-32bit.cpp

@@ -97,3 +97,23 @@ TEST_CASE("CBOR")
         }
     }
 }
+
+TEST_CASE("MessagePack")
+{
+    SECTION("parse errors")
+    {
+        SECTION("array/map size equal to the unknown-size sentinel")
+        {
+            // on 32-bit platforms an array 32/map 32 length of 0xFFFFFFFF aliases unknown_size()
+            std::vector<uint8_t> const varr = {0xDD, 0xFF, 0xFF, 0xFF, 0xFF};
+            std::vector<uint8_t> const vmap = {0xDF, 0xFF, 0xFF, 0xFF, 0xFF};
+
+            json _;
+            CHECK_THROWS_WITH_AS(_ = json::from_msgpack(varr), "[json.exception.out_of_range.408] syntax error while parsing MessagePack size: excessive array size", json::out_of_range&);
+            CHECK(json::from_msgpack(varr, true, false).is_discarded());
+
+            CHECK_THROWS_WITH_AS(_ = json::from_msgpack(vmap), "[json.exception.out_of_range.408] syntax error while parsing MessagePack size: excessive map size", json::out_of_range&);
+            CHECK(json::from_msgpack(vmap, true, false).is_discarded());
+        }
+    }
+}

tests/src/unit-binary_writer_sinks.cpp

@@ -50,6 +50,14 @@ std::vector<json> test_values()
         large_payloads.push_back(json::binary(std::vector<std::uint8_t>(static_cast<std::size_t>(100 * i), static_cast<std::uint8_t>(i))));
     }
 
+    // several times vector_sink_window characters in every format: the vector
+    // sink makes the capacity it grows to writable a window at a time
+    json many_windows = json::array();
+    for (std::uint64_t i = 0; i < 30000; ++i)
+    {
+        many_windows.push_back((i * 0x9E3779B97F4A7C15u) >> 1u);
+    }
+
     return
     {
         json(nullptr), json(true), json(false),
@@ -61,7 +69,7 @@ std::vector<json> test_values()
         json::array(), json::object(),
         json::array({1, 2, 3}), json({{"a", 1}, {"b", nullptr}}),
         json({{"nested", {{"deep", json::array({1, "two", 3.0, nullptr})}}}}),
-        big_array, big_object, large_payloads,
+        big_array, big_object, large_payloads, many_windows,
         json::binary(std::vector<std::uint8_t>(5000, 0x42))
     };
 }

tests/src/unit-bjdata.cpp

@@ -3237,14 +3237,34 @@ TEST_CASE("BJData")
 #endif
                 CHECK(json::from_bjdata(vM, true, false).is_discarded());
 
-#if SIZE_MAX != 0xffffffff
-                CHECK_THROWS_WITH_AS(_ = json::from_bjdata(vMX), "[json.exception.out_of_range.408] syntax error while parsing BJData size: excessive ndarray size caused overflow", json::out_of_range&);
-#else
+                // the first dimension is 2^64-1, which is rejected as a count (see below)
                 CHECK_THROWS_WITH_AS(_ = json::from_bjdata(vMX), "[json.exception.out_of_range.408] syntax error while parsing BJData size: integer value overflow", json::out_of_range&);
-#endif
                 CHECK(json::from_bjdata(vMX, true, false).is_discarded());
             }
 
+            SECTION("optimized array: count equal to SIZE_MAX")
+            {
+                // the reader marks a container without a count by SIZE_MAX, so
+                // a count of that value must not be read as "no count"
+                // (it used to parse as [1])
+                json _;
+#if SIZE_MAX != 0xffffffff
+                std::vector<uint8_t> const vM = {'[', '#', 'M', 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 'i', 0x01, ']'};
+                CHECK_THROWS_WITH_AS(_ = json::from_bjdata(vM), "[json.exception.out_of_range.408] syntax error while parsing BJData size: integer value overflow", json::out_of_range&);
+                CHECK(json::from_bjdata(vM, true, false).is_discarded());
+                // one less is an ordinary excessive count
+                std::vector<uint8_t> const vM1 = {'[', '#', 'M', 0xFE, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 'i', 0x01, ']'};
+                CHECK_THROWS_WITH_AS(_ = json::from_bjdata(vM1), "[json.exception.out_of_range.408] excessive array size: 18446744073709551614", json::out_of_range&);
+#else
+                std::vector<uint8_t> const vm = {'[', '#', 'm', 0xFF, 0xFF, 0xFF, 0xFF, 'i', 0x01, ']'};
+                CHECK_THROWS_WITH_AS(_ = json::from_bjdata(vm), "[json.exception.out_of_range.408] syntax error while parsing BJData size: integer value overflow", json::out_of_range&);
+                CHECK(json::from_bjdata(vm, true, false).is_discarded());
+                std::vector<uint8_t> const vL = {'[', '#', 'L', 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, 'i', 0x01, ']'};
+                CHECK_THROWS_WITH_AS(_ = json::from_bjdata(vL), "[json.exception.out_of_range.408] syntax error while parsing BJData size: integer value overflow", json::out_of_range&);
+                CHECK(json::from_bjdata(vL, true, false).is_discarded());
+#endif
+            }
+
             SECTION("optimized array: integer value overflow")
             {
 #if SIZE_MAX == 0xffffffff

tests/src/unit-regression2.cpp

@@ -1009,7 +1009,7 @@ TEST_CASE("regression test - excessive binary container size honors allow_except
     // allow_exceptions=true (the default) must still throw exactly as before.
     // The exact message text is not checked here: on platforms where
     // std::size_t is 32-bit, the CBOR reader's own length-narrowing check
-    // (get_cbor_container_size(), unrelated to this fix) intercepts a
+    // (get_container_size(), unrelated to this fix) intercepts a
     // declared length of 2^63 before it ever reaches the check this test
     // targets, with different (but equally valid, and already correct)
     // wording -- see unit-cbor.cpp for coverage of that message.

tests/src/unit-ubjson.cpp

@@ -2385,6 +2385,18 @@ TEST_CASE("UBJSON optimized arrays of a valueless type are bounded")
         }
     }
 
+    SECTION("a count equal to SIZE_MAX is rejected")
+    {
+        // the reader marks a container without a count by SIZE_MAX, so a
+        // count of that value must not be read as "no count"; UBJSON's
+        // largest count type (int64) can only reach it where size_t is 32 bits
+#if SIZE_MAX == 0xffffffff
+        const std::vector<uint8_t> input = {'[', '#', 'L', 0x00, 0x00, 0x00, 0x00, 0xFF, 0xFF, 0xFF, 0xFF, 'i', 0x01, ']'};
+        CHECK_THROWS_WITH_AS(_ = json::from_ubjson(input), "[json.exception.out_of_range.408] syntax error while parsing UBJSON size: integer value overflow", json::out_of_range&);
+        CHECK(json::from_ubjson(input, true, false).is_discarded());
+#endif
+    }
+
     SECTION("ordinary counts are unaffected")
     {
         CHECK(json::from_ubjson(std::vector<uint8_t>({'[', '$', 'Z', '#', 'i', 3})) == json({nullptr, nullptr, nullptr}));