Report a missing 0xFF after a final BON8 string also without strict

A BON8 string ends at the first byte that cannot continue it. When a
string ends a value and is not terminated by 0xFF, that byte belongs to
whatever follows the value, but the reader has already taken it from the
input and keeps it in its push-back buffer. With strict = true the
end-of-input check reports it; with strict = false the buffer was
discarded, so a following value in a stream lost its first byte or bytes
without an error ({0x61, 0x90} read "a" and then dropped the 0 that
follows).

Report parse_error.110 "string at the end of the value is not terminated
by 0xFF" when the push-back buffer is not empty once a BON8 value is
complete and strict is false, as bon8.md requires the final 0xFF. Output
of to_bon8 always has it, so canonical input is not affected.

Fixes #6110.

Signed-off-by: Niels Lohmann <mail@nlohmann.me>

.github/workflows/ubuntu.yml

@@ -25,7 +25,7 @@ jobs:
         with:
           persist-credentials: false
       - name: Get latest CMake and ninja
-        uses: lukka/get-cmake@16e8153a8b4918849d92aad131befe67a3705daa # v4.5.0
+        uses: lukka/get-cmake@f86dd60acae0d132056913e248bdabf0e0580b7a # v4.4.5
       - name: Run CMake
         run: cmake -S . -B build -DJSON_CI=On
       - name: Build
@@ -38,7 +38,7 @@ jobs:
         with:
           persist-credentials: false
       - name: Get latest CMake and ninja
-        uses: lukka/get-cmake@16e8153a8b4918849d92aad131befe67a3705daa # v4.5.0
+        uses: lukka/get-cmake@f86dd60acae0d132056913e248bdabf0e0580b7a # v4.4.5
       - name: Check that Meson and pkg-config offer the CMake options
         run: make check_build_options
       - name: Install Meson
@@ -92,7 +92,7 @@ jobs:
         with:
           persist-credentials: false
       - name: Get latest CMake and ninja
-        uses: lukka/get-cmake@16e8153a8b4918849d92aad131befe67a3705daa # v4.5.0
+        uses: lukka/get-cmake@f86dd60acae0d132056913e248bdabf0e0580b7a # v4.4.5
       - name: Run CMake
         run: cmake -S . -B build -DJSON_CI=On
       - name: Build
@@ -123,7 +123,7 @@ jobs:
         with:
           persist-credentials: false
       - name: Get latest CMake and ninja
-        uses: lukka/get-cmake@16e8153a8b4918849d92aad131befe67a3705daa # v4.5.0
+        uses: lukka/get-cmake@f86dd60acae0d132056913e248bdabf0e0580b7a # v4.4.5
       - name: Run CMake
         run: cmake -S . -B build -DJSON_CI=On
       - name: Build
@@ -142,7 +142,7 @@ jobs:
         with:
           persist-credentials: false
       - name: Get latest CMake and ninja
-        uses: lukka/get-cmake@16e8153a8b4918849d92aad131befe67a3705daa # v4.5.0
+        uses: lukka/get-cmake@f86dd60acae0d132056913e248bdabf0e0580b7a # v4.4.5
       - name: Run CMake
         run: cmake -S . -B build -DJSON_CI=On
       - name: Build
@@ -168,7 +168,7 @@ jobs:
         with:
           persist-credentials: false
       - name: Get latest CMake and ninja
-        uses: lukka/get-cmake@16e8153a8b4918849d92aad131befe67a3705daa # v4.5.0
+        uses: lukka/get-cmake@f86dd60acae0d132056913e248bdabf0e0580b7a # v4.4.5
       - name: Run CMake
         run: cmake -S . -B build -DJSON_CI=On
       - name: Build
@@ -244,7 +244,7 @@ jobs:
         with:
           persist-credentials: false
       - name: Get latest CMake and ninja
-        uses: lukka/get-cmake@16e8153a8b4918849d92aad131befe67a3705daa # v4.5.0
+        uses: lukka/get-cmake@f86dd60acae0d132056913e248bdabf0e0580b7a # v4.4.5
       - name: Run CMake
         run: CXX=g++-${{ matrix.compiler }} cmake -S . -B build -DJSON_CI=On
       - name: Build
@@ -262,7 +262,7 @@ jobs:
         with:
           persist-credentials: false
       - name: Get latest CMake and ninja
-        uses: lukka/get-cmake@16e8153a8b4918849d92aad131befe67a3705daa # v4.5.0
+        uses: lukka/get-cmake@f86dd60acae0d132056913e248bdabf0e0580b7a # v4.4.5
       - name: Run CMake
         run: cmake -S . -B build -DJSON_CI=On
       - name: Build
@@ -279,7 +279,7 @@ jobs:
         with:
           persist-credentials: false
       - name: Get latest CMake and ninja
-        uses: lukka/get-cmake@16e8153a8b4918849d92aad131befe67a3705daa # v4.5.0
+        uses: lukka/get-cmake@f86dd60acae0d132056913e248bdabf0e0580b7a # v4.4.5
       - name: Disable filesystem support (std::filesystem is unusable) for clang 7 / 8 / 9 / 10
         run: echo "JSON_FORCED_GLOBAL_COMPILE_OPTIONS=-DJSON_HAS_FILESYSTEM=0;-DJSON_HAS_EXPERIMENTAL_FILESYSTEM=0" >> "$GITHUB_ENV"
         if: ${{ matrix.compiler == '7' || matrix.compiler == '8' || matrix.compiler == '9' || matrix.compiler == '10' }}
@@ -299,7 +299,7 @@ jobs:
         with:
           persist-credentials: false
       - name: Get latest CMake and ninja
-        uses: lukka/get-cmake@16e8153a8b4918849d92aad131befe67a3705daa # v4.5.0
+        uses: lukka/get-cmake@f86dd60acae0d132056913e248bdabf0e0580b7a # v4.4.5
       - name: Run CMake
         run: cmake -S . -B build -DJSON_CI=On
       - name: Build
@@ -319,7 +319,7 @@ jobs:
         with:
           persist-credentials: false
       - name: Get latest CMake and ninja
-        uses: lukka/get-cmake@16e8153a8b4918849d92aad131befe67a3705daa # v4.5.0
+        uses: lukka/get-cmake@f86dd60acae0d132056913e248bdabf0e0580b7a # v4.4.5
       - name: Run CMake
         run: cmake -S . -B build -DJSON_CI=On
       - name: Build with libc++
@@ -346,7 +346,7 @@ jobs:
         with:
           persist-credentials: false
       - name: Get latest CMake and ninja
-        uses: lukka/get-cmake@16e8153a8b4918849d92aad131befe67a3705daa # v4.5.0
+        uses: lukka/get-cmake@f86dd60acae0d132056913e248bdabf0e0580b7a # v4.4.5
       - name: Run CMake
         run: cmake -S . -B build -DJSON_CI=On
       - name: Build
@@ -362,11 +362,13 @@ jobs:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
         with:
           persist-credentials: false
-      # The module test uses `import std;`, which needs CMake 4.5 or newer.
-      - name: Get CMake 4.5+ and ninja
-        uses: lukka/get-cmake@16e8153a8b4918849d92aad131befe67a3705daa # v4.5.0
+      # The module test uses `import std;`, which needs CMake's experimental
+      # import-std support. Its opt-in token is CMake-version-specific, so pin
+      # CMake to the version whose token is set in tests/module_cpp20/CMakeLists.txt.
+      - name: Get pinned CMake and ninja
+        uses: lukka/get-cmake@f86dd60acae0d132056913e248bdabf0e0580b7a # v4.4.5
         with:
-          cmakeVersion: '^4.5.0'
+          cmakeVersion: 4.3.4
       # Clang: the std library module is provided by libc++ (the image's libstdc++
       # ships none), and the image's libc++ module manifest has a broken relative
       # path — repoint it at the real module sources.
@@ -390,7 +392,7 @@ jobs:
         with:
           persist-credentials: false
       - name: Get latest CMake and ninja
-        uses: lukka/get-cmake@16e8153a8b4918849d92aad131befe67a3705daa # v4.5.0
+        uses: lukka/get-cmake@f86dd60acae0d132056913e248bdabf0e0580b7a # v4.4.5
       - name: Run CMake
         run: cmake -S . -B build -DJSON_CI=On
       - name: Build
@@ -407,7 +409,7 @@ jobs:
         with:
           persist-credentials: false
       - name: Get latest CMake and ninja
-        uses: lukka/get-cmake@16e8153a8b4918849d92aad131befe67a3705daa # v4.5.0
+        uses: lukka/get-cmake@f86dd60acae0d132056913e248bdabf0e0580b7a # v4.4.5
       - name: Run CMake
         run: cmake -S . -B build -DJSON_CI=On
       - name: Build
@@ -421,7 +423,7 @@ jobs:
         with:
           persist-credentials: false
       - name: Get latest CMake and ninja
-        uses: lukka/get-cmake@16e8153a8b4918849d92aad131befe67a3705daa # v4.5.0
+        uses: lukka/get-cmake@f86dd60acae0d132056913e248bdabf0e0580b7a # v4.4.5
       - name: Run CMake
         run: cmake -S . -B build -DJSON_CI=On
       - name: Build
@@ -441,7 +443,7 @@ jobs:
         with:
           persist-credentials: false
       - name: Get latest CMake and ninja
-        uses: lukka/get-cmake@16e8153a8b4918849d92aad131befe67a3705daa # v4.5.0
+        uses: lukka/get-cmake@f86dd60acae0d132056913e248bdabf0e0580b7a # v4.4.5
       - name: Run CMake
         run: cmake -S . -B build -DCMAKE_TOOLCHAIN_FILE=$EMSDK/upstream/emscripten/cmake/Modules/Platform/Emscripten.cmake -GNinja
       - name: Build
@@ -482,7 +484,7 @@ jobs:
         with:
           persist-credentials: false
       - name: Get latest CMake and ninja
-        uses: lukka/get-cmake@16e8153a8b4918849d92aad131befe67a3705daa # v4.5.0
+        uses: lukka/get-cmake@f86dd60acae0d132056913e248bdabf0e0580b7a # v4.4.5
       - name: Check that the job runs natively on arm64
         run: uname -m && test "$(uname -m)" = aarch64
       - name: Run CMake

.github/workflows/windows.yml

@@ -90,7 +90,7 @@ jobs:
         with:
           persist-credentials: false
       - name: Get latest CMake and ninja
-        uses: lukka/get-cmake@16e8153a8b4918849d92aad131befe67a3705daa # v4.5.0
+        uses: lukka/get-cmake@f86dd60acae0d132056913e248bdabf0e0580b7a # v4.4.5
       - name: Set extra CXX_FLAGS for latest std_version
         # /wd5285 silences C5285 emitted by the bundled third-party doctest.h, which
         # specializes std::tuple (newly diagnosed by the VS2026 v145 toolset)
@@ -212,11 +212,13 @@ jobs:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
         with:
           persist-credentials: false
-      # The module test uses `import std;`, which needs CMake 4.5 or newer.
-      - name: Get CMake 4.5+ and ninja
-        uses: lukka/get-cmake@16e8153a8b4918849d92aad131befe67a3705daa # v4.5.0
+      # The module test uses `import std;`, which needs CMake's experimental
+      # import-std support. Its opt-in token is CMake-version-specific, so pin
+      # CMake to the version whose token is set in tests/module_cpp20/CMakeLists.txt.
+      - name: Get pinned CMake and ninja
+        uses: lukka/get-cmake@f86dd60acae0d132056913e248bdabf0e0580b7a # v4.4.5
         with:
-          cmakeVersion: '^4.5.0'
+          cmakeVersion: 4.3.4
       # CMAKE_CXX_FLAGS replaces CMake's MSVC defaults (which include /EHsc), and
       # ci_module_cpp20 forwards it to the module build, so list /EHsc explicitly
       - name: Run CMake (Debug)

README.md

@@ -165,8 +165,7 @@ std::ifstream f("example.json");
 json data = json::parse(f);
 ```
 
-If using modules (enabled with `NLOHMANN_JSON_BUILD_MODULES`; `import std;` needs CMake 4.5 or newer, see the
-[modules documentation](https://json.nlohmann.me/features/modules/)), this example becomes:
+If using modules (enabled with `NLOHMANN_JSON_BUILD_MODULES`), this example becomes:
 ```cpp
 import std;
 import nlohmann.json;

docs/mkdocs/docs/api/basic_json/from_bon8.md

@@ -69,8 +69,10 @@ Strong guarantee: if an exception is thrown, there are no changes in the JSON va
 
 ## Exceptions
 
-- Throws [parse_error.110](../../home/exceptions.md#jsonexceptionparse_error110) if the given input ends prematurely or
-  the end of the file was not reached when `strict` was set to true
+- Throws [parse_error.110](../../home/exceptions.md#jsonexceptionparse_error110) if the given input ends prematurely,
+  if the end of the file was not reached when `strict` was set to true, or if a string at the end of the value is not
+  terminated by 0xFF although a byte follows it (also when `strict` is false: the byte that ends the string is
+  taken from the input, so the next value would be lost)
 - Throws [parse_error.112](../../home/exceptions.md#jsonexceptionparse_error112) if a parse error occurs, for instance
   an invalid byte, a string that is not valid UTF-8, or an object key that is not a string
 

docs/mkdocs/docs/features/binary_formats/bon8.md

@@ -181,7 +181,10 @@ of the message are reported as [parse_error.110](../../home/exceptions.md#jsonex
     arrays and objects with up to four elements that are terminated by 0xFE, unsorted object keys, or a 0xFF after a
     string that would also end without it, are accepted. A second 0xFF is not a terminator but an empty string.
 
-    Strings must be valid UTF-8, and a string at the very end of a message must be terminated by 0xFF.
+    Strings must be valid UTF-8, and a string at the very end of a message must be terminated by 0xFF. This also
+    holds when `strict` is false and more data follows the value: the byte that would end the string belongs to the
+    next value, and reading it would remove it from the input, so the missing 0xFF is reported as
+    [parse_error.110](../../home/exceptions.md#jsonexceptionparse_error110).
 
     Duplicate object keys are accepted; the last value wins (`ordered_json` keeps the position of the first occurrence).
     The specification does not say how to treat them; the HikoGUI reference decoder keeps the first value.

docs/mkdocs/docs/features/modules.md

@@ -9,19 +9,7 @@ Importing the module instead of including the header reduces the compile time of
 ## Requirements
 The `nlohmann.json` module requires that the build system is configured to build and resolve modules when imported. Obviously, as modules were introduced in C++20, this feature can only be used in C++20 and subsequent versions.
 
-To enable building the `nlohmann.json` module (which is not done by default), set the CMake option
-[`NLOHMANN_JSON_BUILD_MODULES`](../integration/cmake.md#nlohmann_json_build_modules) and link the
-`nlohmann_json_modules` target.
-
-With CMake, building the module needs:
-
-- CMake 3.28 or newer, and
-- a generator that scans for modules: Ninja (1.11 or newer), Ninja Multi-Config, or Visual Studio 2022 or newer.
-
-The example below also uses `import std;`. CMake supports it without experimental opt-in since version 4.5, and then
-only with the Ninja generators and these toolchains: Clang 18.1.2 or newer (with libc++ or libstdc++), MSVC toolset 14.36
-(Visual Studio 17.6) or newer, and GCC 15 or newer (GCC 16 on macOS). With older CMake versions, `import std;` needs
-CMake's version-specific experimental opt-in.
+To enable building the `nlohmann.json` module (which is not done by default), the macro `NLOHMANN_JSON_BUILD_MODULES` must be passed to the build system.
 
 ## Example
 When using modules rather than headers, the previous example for creating a `json` object through a JSON file, would instead be:
@@ -79,7 +67,7 @@ and should not be used directly:
 
 ## Known issues
 
-C++20 modules support is exercised in CI with CMake 4.5 and `import std;` against current GCC and Clang on Ubuntu, and the default MSVC toolset on Windows Server 2022 — there is no documented minimum compiler version, unlike feature-test-macro-gated features such as [`JSON_HAS_RANGES`](../api/macros/json_has_ranges.md).
+C++20 modules support is exercised in CI against current GCC and Clang on Ubuntu, and the default MSVC toolset on Windows Server 2022 — there is no documented minimum compiler version, unlike feature-test-macro-gated features such as [`JSON_HAS_RANGES`](../api/macros/json_has_ranges.md).
 
 !!! info "Known compiler issues"
 

docs/mkdocs/docs/integration/cmake.md

@@ -269,9 +269,6 @@ target_link_libraries(myproject PRIVATE nlohmann_json_modules)
 target_compile_definitions(myproject PRIVATE NLOHMANN_JSON_BUILD_MODULES)
 ```
 
-If `main.cpp` also uses `import std;`, the project needs CMake 4.5 or newer and a Ninja generator; see
-[Modules](../features/modules.md#requirements) for the supported toolchains.
-
 ## See also
 
 - [Macros](../api/macros/index.md) - reference of all macros

include/nlohmann/detail/input/binary_reader.hpp

@@ -177,6 +177,17 @@ class binary_reader
                 JSON_ASSERT(false); // NOLINT(cert-dcl03-c,hicpp-static-assert,misc-static-assert) LCOV_EXCL_LINE
         }
 
+        // BON8: a string ends at the first byte that cannot continue it. If
+        // that byte follows the value, the reader has already taken it from
+        // the input and holds it in bon8_pushback. Without the end-of-input
+        // check below it would be dropped, and the next read would start
+        // behind it, so report the missing 0xFF instead.
+        if (result && !strict && input_format == input_format_t::bon8 && bon8_pushback_size != 0)
+        {
+            get_bon8();
+            return last_byte_error(exception_id::unexpected_end_of_input, "string at the end of the value is not terminated by 0xFF", "value", "");
+        }
+
         // strict mode: next byte must be EOF
         if (result && strict)
         {

single_include/nlohmann/json.hpp

@@ -18568,6 +18568,17 @@ class binary_reader
                 JSON_ASSERT(false); // NOLINT(cert-dcl03-c,hicpp-static-assert,misc-static-assert) LCOV_EXCL_LINE
         }
 
+        // BON8: a string ends at the first byte that cannot continue it. If
+        // that byte follows the value, the reader has already taken it from
+        // the input and holds it in bon8_pushback. Without the end-of-input
+        // check below it would be dropped, and the next read would start
+        // behind it, so report the missing 0xFF instead.
+        if (result && !strict && input_format == input_format_t::bon8 && bon8_pushback_size != 0)
+        {
+            get_bon8();
+            return last_byte_error(exception_id::unexpected_end_of_input, "string at the end of the value is not terminated by 0xFF", "value", "");
+        }
+
         // strict mode: next byte must be EOF
         if (result && strict)
         {

tests/module_cpp20/CMakeLists.txt

@@ -1,6 +1,11 @@
-# `import std;` (used by main.cpp) is supported without experimental opt-in
-# since CMake 4.5: it is available to every target that scans for C++ modules.
-cmake_minimum_required(VERSION 4.5)
+cmake_minimum_required(VERSION 3.30)
+
+# `import std;` (used by main.cpp) needs CMake's experimental C++ import-std
+# support. The accepted token is CMake-version-specific; this value matches the
+# CMake version pinned in the ci_module_cpp20 CI jobs (4.3.x). If that pin is
+# bumped, this token must be updated to match (configuration fails loudly
+# otherwise).
+set(CMAKE_EXPERIMENTAL_CXX_IMPORT_STD "451f2fe2-a8a2-47c3-bc32-94786d8fc91b")
 
 project(json_test CXX)
 
@@ -8,6 +13,7 @@ project(json_test CXX)
 set(CMAKE_CXX_STANDARD 23)
 set(CMAKE_CXX_STANDARD_REQUIRED ON)
 set(CMAKE_CXX_EXTENSIONS OFF)
+set(CMAKE_CXX_MODULE_STD ON)
 
 set(NLOHMANN_JSON_BUILD_MODULES ON CACHE BOOL "Enable nlohmann.json module support")
 

tests/module_cpp20/main.cpp

@@ -17,8 +17,8 @@ using namespace nlohmann::literals;
 // Standard-library facilities are pulled in via `import std;` rather than
 // textual `#include`s: mixing `import` with textual standard headers does not
 // compile under GCC's C++20 modules implementation, whereas `import std;` works
-// across GCC, Clang, and MSVC. This requires C++23 and CMake 4.5 or newer;
-// see CMakeLists.txt.
+// across GCC, Clang, and MSVC. This requires C++23 and CMake's (experimental)
+// import-std support; see CMakeLists.txt.
 int main()
 {
     // basic_json / json: parsing and value access

tests/src/unit-bon8.cpp

@@ -608,14 +608,26 @@ TEST_CASE("BON8")
             CHECK_THROWS_WITH_AS(_ = json::from_bon8(vec), "[json.exception.parse_error.110] parse error at byte 2: syntax error while parsing BON8 value: expected end of input; last byte: 0x90", json::parse_error&);
             CHECK(json::from_bon8(vec, true, false).is_discarded());
 
-            // the byte after a string that ends a container
+            // the byte after a string that ends a container: without strict, the
+            // missing 0xFF is reported, as the byte has been taken from the input
             const bytes vec2 = {0x81, 'a', 0x90};
-            CHECK(json::from_bon8(vec2, false) == json({"a"}));
+            CHECK_THROWS_WITH_AS(_ = json::from_bon8(vec2, false), "[json.exception.parse_error.110] parse error at byte 3: syntax error while parsing BON8 value: string at the end of the value is not terminated by 0xFF; last byte: 0x90", json::parse_error&);
+            CHECK(json::from_bon8(vec2, false, false).is_discarded());
             CHECK_THROWS_WITH_AS(_ = json::from_bon8(vec2), "[json.exception.parse_error.110] parse error at byte 3: syntax error while parsing BON8 value: expected end of input; last byte: 0x90", json::parse_error&);
 
             const bytes vec3 = {0x81, 'a', 0xC2, 0x00};
-            CHECK(json::from_bon8(vec3, false) == json({"a"}));
+            CHECK_THROWS_WITH_AS(_ = json::from_bon8(vec3, false), "[json.exception.parse_error.110] parse error at byte 3: syntax error while parsing BON8 value: string at the end of the value is not terminated by 0xFF; last byte: 0xC2", json::parse_error&);
             CHECK_THROWS_WITH_AS(_ = json::from_bon8(vec3), "[json.exception.parse_error.110] parse error at byte 3: syntax error while parsing BON8 value: expected end of input; last byte: 0xC2", json::parse_error&);
+
+            // with a terminated string, strict = false leaves the next value in the stream
+            std::istringstream terminated(std::string("\x61\xFF\x90", 3));
+            CHECK(json::from_bon8(terminated, false) == "a");
+            CHECK(terminated.tellg() == 2);
+            CHECK(json::from_bon8(terminated, false) == 0);
+
+            // without the terminator, the next value would be lost
+            std::istringstream unterminated(std::string("\x61\x90", 2));
+            CHECK_THROWS_WITH_AS(_ = json::from_bon8(unterminated, false), "[json.exception.parse_error.110] parse error at byte 2: syntax error while parsing BON8 value: string at the end of the value is not terminated by 0xFF; last byte: 0x90", json::parse_error&);
         }
     }