diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index b0f65230c..68f82c474 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -108,7 +108,9 @@ The tests are located in [`tests/src/unit-*.cpp`](https://github.com/nlohmann/js are structured along the features of the library or the nature of the tests. Usually, it should be clear from the context which existing file needs to be extended, and only very few cases require creating new test files. -When fixing a bug, edit `unit-regression2.cpp` and add a section referencing the fixed issue. +When fixing a bug, edit `unit-regression3.cpp` and add a section referencing the fixed issue. +`unit-regression2.cpp` holds the older tests; the two files exist because a single one grew large enough for the +MinGW linker to fail relocating it, so please keep adding to the smaller file rather than growing the larger one. #### Exceptions diff --git a/.github/workflows/check_amalgamation.yml b/.github/workflows/check_amalgamation.yml index baf4dbe31..f70ebfba0 100644 --- a/.github/workflows/check_amalgamation.yml +++ b/.github/workflows/check_amalgamation.yml @@ -11,7 +11,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Harden Runner - uses: step-security/harden-runner@05e31511f85b41b11d1cf0ef85d0992719546e2c # v2.21.0 + uses: step-security/harden-runner@e14015d583714f6e62063499dc959a02595150a1 # v2.21.1 with: egress-policy: audit @@ -34,7 +34,7 @@ jobs: steps: - name: Harden Runner - uses: step-security/harden-runner@05e31511f85b41b11d1cf0ef85d0992719546e2c # v2.21.0 + uses: step-security/harden-runner@e14015d583714f6e62063499dc959a02595150a1 # v2.21.1 with: egress-policy: audit diff --git a/.github/workflows/cifuzz.yml b/.github/workflows/cifuzz.yml index a64b11537..2e50fee9c 100644 --- a/.github/workflows/cifuzz.yml +++ b/.github/workflows/cifuzz.yml @@ -9,7 +9,7 @@ jobs: runs-on: ubuntu-22.04 steps: - name: Harden Runner - uses: step-security/harden-runner@05e31511f85b41b11d1cf0ef85d0992719546e2c # v2.21.0 + uses: step-security/harden-runner@e14015d583714f6e62063499dc959a02595150a1 # v2.21.1 with: egress-policy: audit diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 6f2c060c2..c5497ef9c 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -27,7 +27,7 @@ jobs: steps: - name: Harden Runner - uses: step-security/harden-runner@05e31511f85b41b11d1cf0ef85d0992719546e2c # v2.21.0 + uses: step-security/harden-runner@e14015d583714f6e62063499dc959a02595150a1 # v2.21.1 with: egress-policy: audit diff --git a/.github/workflows/comment_check_amalgamation.yml b/.github/workflows/comment_check_amalgamation.yml index 10ecb251c..788c1b8ce 100644 --- a/.github/workflows/comment_check_amalgamation.yml +++ b/.github/workflows/comment_check_amalgamation.yml @@ -19,7 +19,7 @@ jobs: pull-requests: write steps: - name: Harden Runner - uses: step-security/harden-runner@05e31511f85b41b11d1cf0ef85d0992719546e2c # v2.21.0 + uses: step-security/harden-runner@e14015d583714f6e62063499dc959a02595150a1 # v2.21.1 with: egress-policy: audit diff --git a/.github/workflows/dependency-review.yml b/.github/workflows/dependency-review.yml index a28624b77..aa09008a3 100644 --- a/.github/workflows/dependency-review.yml +++ b/.github/workflows/dependency-review.yml @@ -17,7 +17,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Harden Runner - uses: step-security/harden-runner@05e31511f85b41b11d1cf0ef85d0992719546e2c # v2.21.0 + uses: step-security/harden-runner@e14015d583714f6e62063499dc959a02595150a1 # v2.21.1 with: egress-policy: audit diff --git a/.github/workflows/flawfinder.yml b/.github/workflows/flawfinder.yml index 15f9f7825..7c4d22b3c 100644 --- a/.github/workflows/flawfinder.yml +++ b/.github/workflows/flawfinder.yml @@ -27,7 +27,7 @@ jobs: security-events: write steps: - name: Harden Runner - uses: step-security/harden-runner@05e31511f85b41b11d1cf0ef85d0992719546e2c # v2.21.0 + uses: step-security/harden-runner@e14015d583714f6e62063499dc959a02595150a1 # v2.21.1 with: egress-policy: audit diff --git a/.github/workflows/labeler.yml b/.github/workflows/labeler.yml index 3fb2af82a..3b4571b95 100644 --- a/.github/workflows/labeler.yml +++ b/.github/workflows/labeler.yml @@ -17,7 +17,7 @@ jobs: steps: - name: Harden Runner - uses: step-security/harden-runner@05e31511f85b41b11d1cf0ef85d0992719546e2c # v2.21.0 + uses: step-security/harden-runner@e14015d583714f6e62063499dc959a02595150a1 # v2.21.1 with: egress-policy: audit diff --git a/.github/workflows/publish_documentation.yml b/.github/workflows/publish_documentation.yml index 95790dcb9..189d95419 100644 --- a/.github/workflows/publish_documentation.yml +++ b/.github/workflows/publish_documentation.yml @@ -26,7 +26,7 @@ jobs: runs-on: ubuntu-22.04 steps: - name: Harden Runner - uses: step-security/harden-runner@05e31511f85b41b11d1cf0ef85d0992719546e2c # v2.21.0 + uses: step-security/harden-runner@e14015d583714f6e62063499dc959a02595150a1 # v2.21.1 with: egress-policy: audit diff --git a/.github/workflows/scorecards.yml b/.github/workflows/scorecards.yml index 0182fb062..de919b1ab 100644 --- a/.github/workflows/scorecards.yml +++ b/.github/workflows/scorecards.yml @@ -36,7 +36,7 @@ jobs: steps: - name: Harden Runner - uses: step-security/harden-runner@05e31511f85b41b11d1cf0ef85d0992719546e2c # v2.21.0 + uses: step-security/harden-runner@e14015d583714f6e62063499dc959a02595150a1 # v2.21.1 with: egress-policy: audit diff --git a/.github/workflows/semgrep.yml b/.github/workflows/semgrep.yml index d2afcb42e..38de00932 100644 --- a/.github/workflows/semgrep.yml +++ b/.github/workflows/semgrep.yml @@ -32,7 +32,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Harden Runner - uses: step-security/harden-runner@05e31511f85b41b11d1cf0ef85d0992719546e2c # v2.21.0 + uses: step-security/harden-runner@e14015d583714f6e62063499dc959a02595150a1 # v2.21.1 with: egress-policy: audit diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index 1cea13636..1e690ccdd 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -16,7 +16,7 @@ jobs: steps: - name: Harden Runner - uses: step-security/harden-runner@05e31511f85b41b11d1cf0ef85d0992719546e2c # v2.21.0 + uses: step-security/harden-runner@e14015d583714f6e62063499dc959a02595150a1 # v2.21.1 with: egress-policy: audit diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml index c5d04e1b2..69a3cbc45 100644 --- a/.github/workflows/ubuntu.yml +++ b/.github/workflows/ubuntu.yml @@ -35,7 +35,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Harden Runner - uses: step-security/harden-runner@05e31511f85b41b11d1cf0ef85d0992719546e2c # v2.21.0 + uses: step-security/harden-runner@e14015d583714f6e62063499dc959a02595150a1 # v2.21.1 with: egress-policy: audit @@ -60,7 +60,7 @@ jobs: target: [ci_test_amalgamation, ci_test_single_header, ci_cppcheck, ci_cpplint, ci_reproducible_tests, ci_non_git_tests, ci_offline_testdata, ci_reuse_compliance, ci_test_valgrind] steps: - name: Harden Runner - uses: step-security/harden-runner@05e31511f85b41b11d1cf0ef85d0992719546e2c # v2.21.0 + uses: step-security/harden-runner@e14015d583714f6e62063499dc959a02595150a1 # v2.21.1 with: egress-policy: audit @@ -100,7 +100,7 @@ jobs: container: ubuntu:focal strategy: matrix: - target: [ci_cmake_flags, ci_test_diagnostics, ci_test_diagnostic_positions, ci_test_noexceptions, ci_test_noimplicitconversions, ci_test_legacycomparison, ci_test_noglobaludls] + target: [ci_cmake_flags, ci_test_diagnostics, ci_test_diagnostic_positions, ci_test_noexceptions, ci_test_noimplicitconversions, ci_test_legacycomparison, ci_test_noglobaludls, ci_test_simdutf] steps: - name: Install build-essential run: apt-get update ; apt-get install -y build-essential unzip wget git libssl-dev @@ -118,7 +118,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Harden Runner - uses: step-security/harden-runner@05e31511f85b41b11d1cf0ef85d0992719546e2c # v2.21.0 + uses: step-security/harden-runner@e14015d583714f6e62063499dc959a02595150a1 # v2.21.1 with: egress-policy: audit @@ -369,7 +369,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Harden Runner - uses: step-security/harden-runner@05e31511f85b41b11d1cf0ef85d0992719546e2c # v2.21.0 + uses: step-security/harden-runner@e14015d583714f6e62063499dc959a02595150a1 # v2.21.1 with: egress-policy: audit @@ -392,7 +392,7 @@ jobs: target: [ci_test_examples, ci_test_build_documentation] steps: - name: Harden Runner - uses: step-security/harden-runner@05e31511f85b41b11d1cf0ef85d0992719546e2c # v2.21.0 + uses: step-security/harden-runner@e14015d583714f6e62063499dc959a02595150a1 # v2.21.1 with: egress-policy: audit diff --git a/Makefile b/Makefile index d99d6f5f3..e1a1d2b75 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: pretty clean ChangeLog.md release +.PHONY: pretty clean ChangeLog.md release update_hedley update_hedley_undef ########################################################################## # configuration @@ -41,6 +41,8 @@ all: @echo "fuzz_testing_ubjson - prepare fuzz testing of the UBJSON parser" @echo "pretty - beautify code with Artistic Style" @echo "run_benchmarks - build and run benchmarks" + @echo "update_hedley - download Hedley and regenerate hedley.hpp / hedley_undef.hpp" + @echo "update_hedley_undef - rebuild hedley_undef.hpp from the JSON_HEDLEY_* #define names in hedley.hpp" ########################################################################## @@ -241,11 +243,24 @@ update_hedley: rm -f include/nlohmann/thirdparty/hedley/hedley.hpp include/nlohmann/thirdparty/hedley/hedley_undef.hpp curl https://raw.githubusercontent.com/nemequ/hedley/master/hedley.h -o include/nlohmann/thirdparty/hedley/hedley.hpp $(SED) -i 's/HEDLEY_/JSON_HEDLEY_/g' include/nlohmann/thirdparty/hedley/hedley.hpp - grep "[[:blank:]]*#[[:blank:]]*undef" include/nlohmann/thirdparty/hedley/hedley.hpp | grep -v "__" | sort | uniq | $(SED) 's/ //g' | $(SED) 's/undef/undef /g' > include/nlohmann/thirdparty/hedley/hedley_undef.hpp $(SED) -i '1s/^/#pragma once\n\n/' include/nlohmann/thirdparty/hedley/hedley.hpp - $(SED) -i '1s/^/#pragma once\n\n/' include/nlohmann/thirdparty/hedley/hedley_undef.hpp + $(MAKE) update_hedley_undef $(MAKE) amalgamate +# Rebuild hedley_undef.hpp from every JSON_HEDLEY_* name that hedley.hpp +# #defines. Hedley does not #undef all of its public macros internally (see +# #5408), so grepping those #undef lines misses names such as +# JSON_HEDLEY_PRAGMA. cmake/scripts/gen_hedley_undef_check.cmake is the +# single source of truth for this extraction (tests/CMakeLists.txt uses the +# same script, in MODE=checks, to generate the matching leak-check test), so +# the vendored header, the generated #undef list, and the regression test +# cannot drift apart. +update_hedley_undef: + cmake -DHEDLEY_HPP=include/nlohmann/thirdparty/hedley/hedley.hpp \ + -DOUTPUT=include/nlohmann/thirdparty/hedley/hedley_undef.hpp \ + -DMODE=undef \ + -P cmake/scripts/gen_hedley_undef_check.cmake + ########################################################################## # serve_header.py ########################################################################## diff --git a/cmake/ci.cmake b/cmake/ci.cmake index 6b1d325d8..18fef2075 100644 --- a/cmake/ci.cmake +++ b/cmake/ci.cmake @@ -212,6 +212,24 @@ add_custom_target(ci_test_legacycomparison COMMENT "Compile and test with legacy discarded value comparison enabled" ) +############################################################################### +# Validate UTF-8 with simdutf. +############################################################################### + +add_custom_target(ci_test_simdutf + COMMAND ${CMAKE_COMMAND} + -DCMAKE_BUILD_TYPE=Debug -GNinja + -DJSON_BuildTests=ON -DJSON_TestSimdutf=ON + # simdutf needs C++17, so the library falls back to its scalar validator + # below that: build the suite at C++11 to cover the fallback with the macro + # defined, and at C++17 to run every test against simdutf itself + "-DJSON_TestStandards=11\;17" + -S${PROJECT_SOURCE_DIR} -B${PROJECT_BINARY_DIR}/build_simdutf + COMMAND ${CMAKE_COMMAND} --build ${PROJECT_BINARY_DIR}/build_simdutf + COMMAND cd ${PROJECT_BINARY_DIR}/build_simdutf && ${CMAKE_CTEST_COMMAND} --parallel ${N} --output-on-failure + COMMENT "Compile and test with simdutf UTF-8 validation enabled" +) + ############################################################################### # Enable brace-init copy semantics. ############################################################################### diff --git a/cmake/scripts/gen_hedley_undef_check.cmake b/cmake/scripts/gen_hedley_undef_check.cmake new file mode 100644 index 000000000..fc8cfec2c --- /dev/null +++ b/cmake/scripts/gen_hedley_undef_check.cmake @@ -0,0 +1,112 @@ +# Shared extractor for the JSON_HEDLEY_* macro names defined in hedley.hpp. +# +# Every macro that hedley.hpp #defines must be #undef-ed again once json.hpp +# has been fully processed (see include/nlohmann/detail/macro_unscope.hpp +# and https://github.com/nlohmann/json/issues/5408). Deriving the macro list +# straight from hedley.hpp here -- instead of hand-maintaining it in two +# places -- means hedley_undef.hpp and the regression test that checks for +# leaked macros can never drift apart, even after a future `make +# update_hedley` pulls in new macros from upstream Hedley. +# +# MODE=undef (default): write hedley_undef.hpp (SPDX header, #pragma once, +# one #undef per macro name) -- used by `make update_hedley_undef` +# MODE=checks: write one #ifdef/FAIL_CHECK/#endif per macro name, +# meant to be #include-d inside a TEST_CASE -- used by +# tests/CMakeLists.txt to (re)generate the include for +# tests/src/unit-no-macro-leak.cpp +# +# Required variables: +# HEDLEY_HPP path to include/nlohmann/thirdparty/hedley/hedley.hpp +# OUTPUT path of the file to (over)write +# Optional: +# MODE "undef" (default) or "checks" + +if(NOT DEFINED HEDLEY_HPP OR NOT DEFINED OUTPUT) + message(FATAL_ERROR "HEDLEY_HPP and OUTPUT must be set") +endif() + +if(NOT EXISTS "${HEDLEY_HPP}") + message(FATAL_ERROR "Hedley header not found: ${HEDLEY_HPP}") +endif() + +if(NOT DEFINED MODE) + set(MODE undef) +endif() + +if(NOT MODE STREQUAL "undef" AND NOT MODE STREQUAL "checks") + message(FATAL_ERROR "MODE must be undef or checks, got: ${MODE}") +endif() + +# Line-anchored, like `grep -oE "^[[:blank:]]*#[[:blank:]]*define[[:blank:]]+JSON_HEDLEY_[A-Za-z0-9_]+"`. +# Unanchored matching would also pick up JSON_HEDLEY_* mentions inside +# comments or string literals elsewhere in the file, which must not turn +# into #undef lines. +file(STRINGS "${HEDLEY_HPP}" hedley_lines) +set(macro_names) +foreach(line IN LISTS hedley_lines) + if("${line}" MATCHES "^[ \t]*#[ \t]*define[ \t]+(JSON_HEDLEY_[A-Za-z0-9_]+)") + list(APPEND macro_names "${CMAKE_MATCH_1}") + endif() +endforeach() + +if(NOT macro_names) + message(FATAL_ERROR "No JSON_HEDLEY_* macros found in ${HEDLEY_HPP}") +endif() + +list(REMOVE_DUPLICATES macro_names) +# Lexicographic, locale-independent (ASCII-only names) -- matches `LC_ALL=C sort`. +list(SORT macro_names COMPARE STRING) +list(LENGTH macro_names macro_count) + +set(generated "") +if(MODE STREQUAL "undef") + # Same banner `make update_hedley_undef` would stamp by hand, so the + # recipe is self-contained and its output is byte-stable across reruns. + # The embedded SPDX tags below are part of the *generated* file's + # content, not a REUSE header for this .cmake script itself (which is + # already covered by the blanket "Files: *" rule in .reuse/dep5) -- keep + # them wrapped in REUSE-IgnoreStart/End so `reuse lint` does not try to + # parse "MIT\n")" as this file's own SPDX-License-Identifier value. + # REUSE-IgnoreStart + string(APPEND generated "// __ _____ _____ _____\n") + string(APPEND generated "// __| | __| | | | JSON for Modern C++\n") + string(APPEND generated "// | | |__ | | | | | | version 3.12.0\n") + string(APPEND generated "// |_____|_____|_____|_|___| https://github.com/nlohmann/json\n") + string(APPEND generated "//\n") + string(APPEND generated "// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann \n") + string(APPEND generated "// SPDX-License-Identifier: MIT\n") + # REUSE-IgnoreEnd + string(APPEND generated "\n") + string(APPEND generated "#pragma once\n") + string(APPEND generated "\n") + foreach(name IN LISTS macro_names) + string(APPEND generated "#undef ${name}\n") + endforeach() +else() + string(APPEND generated "// This file is generated by cmake/scripts/gen_hedley_undef_check.cmake\n") + string(APPEND generated "// from include/nlohmann/thirdparty/hedley/hedley.hpp. Do not edit it by\n") + string(APPEND generated "// hand -- it is regenerated on every build. ${macro_count} macros checked.\n\n") + foreach(name IN LISTS macro_names) + string(APPEND generated "#ifdef ${name}\n") + string(APPEND generated " FAIL_CHECK(\"${name} leaked after including nlohmann/json.hpp\");\n") + string(APPEND generated "#endif\n") + endforeach() +endif() + +get_filename_component(output_dir "${OUTPUT}" DIRECTORY) +if(output_dir) + file(MAKE_DIRECTORY "${output_dir}") +endif() + +# Avoid rewriting the file (and busting downstream incremental rebuilds) +# when the content has not actually changed. +set(write_output TRUE) +if(EXISTS "${OUTPUT}") + file(READ "${OUTPUT}" existing_content) + if(existing_content STREQUAL generated) + set(write_output FALSE) + endif() +endif() +if(write_output) + file(WRITE "${OUTPUT}" "${generated}") +endif() diff --git a/docs/mkdocs/docs/api/basic_json/patch.md b/docs/mkdocs/docs/api/basic_json/patch.md index 432e7c128..fa25b2699 100644 --- a/docs/mkdocs/docs/api/basic_json/patch.md +++ b/docs/mkdocs/docs/api/basic_json/patch.md @@ -34,6 +34,10 @@ Strong guarantee: if an exception is thrown, there are no changes in the JSON va ("add", "remove", "move") - Throws [`out_of_range.411`](../../home/exceptions.md#jsonexceptionout_of_range411) if an "add" operation's target location has a parent that is neither an object nor an array. +- Throws [`out_of_range.413`](../../home/exceptions.md#jsonexceptionout_of_range413) if a "remove" operation's target + location has a parent that is neither an object nor an array. +- Throws [`out_of_range.414`](../../home/exceptions.md#jsonexceptionout_of_range414) if a "move" operation's "from" + location is a proper prefix of its "path" location. - Throws [`other_error.501`](../../home/exceptions.md#jsonexceptionother_error501) if "test" operation was unsuccessful. @@ -75,3 +79,7 @@ is thrown. In any case, the original value is not changed: the patch is applied - Added in version 2.0.0. - Added [`out_of_range.411`](../../home/exceptions.md#jsonexceptionout_of_range411) and stopped relying on an internal assertion when an "add" operation's target location has a non-object/non-array parent in version 3.13.0. +- Added [`out_of_range.413`](../../home/exceptions.md#jsonexceptionout_of_range413) and stopped silently ignoring a "remove" operation whose target + location has a non-object/non-array parent in version 3.13.0. +- Added [`out_of_range.414`](../../home/exceptions.md#jsonexceptionout_of_range414) and rejected a "move" operation whose "from" location is a proper + prefix of its "path" location instead of silently producing a corrupted result in version 3.13.0. diff --git a/docs/mkdocs/docs/api/basic_json/patch_inplace.md b/docs/mkdocs/docs/api/basic_json/patch_inplace.md index ae2b5e6a9..7ae85aaaa 100644 --- a/docs/mkdocs/docs/api/basic_json/patch_inplace.md +++ b/docs/mkdocs/docs/api/basic_json/patch_inplace.md @@ -30,6 +30,10 @@ No guarantees, value may be corrupted by an unsuccessful patch operation. ("add", "remove", "move") - Throws [`out_of_range.411`](../../home/exceptions.md#jsonexceptionout_of_range411) if an "add" operation's target location has a parent that is neither an object nor an array. +- Throws [`out_of_range.413`](../../home/exceptions.md#jsonexceptionout_of_range413) if a "remove" operation's target + location has a parent that is neither an object nor an array. +- Throws [`out_of_range.414`](../../home/exceptions.md#jsonexceptionout_of_range414) if a "move" operation's "from" + location is a proper prefix of its "path" location. - Throws [`other_error.501`](../../home/exceptions.md#jsonexceptionother_error501) if "test" operation was unsuccessful. @@ -72,3 +76,7 @@ function throws an exception. - Added in version 3.11.0. - Added [`out_of_range.411`](../../home/exceptions.md#jsonexceptionout_of_range411) and stopped relying on an internal assertion when an "add" operation's target location has a non-object/non-array parent in version 3.13.0. +- Added [`out_of_range.413`](../../home/exceptions.md#jsonexceptionout_of_range413) and stopped silently ignoring a "remove" operation whose target + location has a non-object/non-array parent in version 3.13.0. +- Added [`out_of_range.414`](../../home/exceptions.md#jsonexceptionout_of_range414) and rejected a "move" operation whose "from" location is a proper + prefix of its "path" location instead of silently producing a corrupted result in version 3.13.0. diff --git a/docs/mkdocs/docs/api/macros/index.md b/docs/mkdocs/docs/api/macros/index.md index 507c04932..e818f032a 100644 --- a/docs/mkdocs/docs/api/macros/index.md +++ b/docs/mkdocs/docs/api/macros/index.md @@ -24,6 +24,7 @@ header. See also the [macro overview page](../../features/macros.md). - [**JSON_NO_IO**](json_no_io.md) - switch off functions relying on certain C++ I/O headers - [**JSON_SKIP_UNSUPPORTED_COMPILER_CHECK**](json_skip_unsupported_compiler_check.md) - do not warn about unsupported compilers - [**JSON_USE_GLOBAL_UDLS**](json_use_global_udls.md) - place user-defined string literals (UDLs) into the global namespace +- [**JSON_USE_SIMDUTF**](json_use_simdutf.md) - use the simdutf library to accelerate UTF-8 validation ## Library version diff --git a/docs/mkdocs/docs/api/macros/json_use_simdutf.md b/docs/mkdocs/docs/api/macros/json_use_simdutf.md new file mode 100644 index 000000000..611c1e9f1 --- /dev/null +++ b/docs/mkdocs/docs/api/macros/json_use_simdutf.md @@ -0,0 +1,71 @@ +# JSON_USE_SIMDUTF + +```cpp +#define JSON_USE_SIMDUTF +``` + +When defined, the parser validates the UTF-8 content of JSON strings that come from a **contiguous byte input** +(`std::string`, `std::vector`/``, string literals, `const char*` ranges, …) using the +[simdutf](https://github.com/simdutf/simdutf) library instead of the built-in scalar validator. On text with many +non-ASCII characters (e.g. CJK or emoji) this can validate several times faster. + +This is an **opt-in external dependency**. The library itself remains header-only and its behavior is unchanged: the +same input is accepted or rejected either way, and every parse error is reported at the same position with the same +message (simdutf is only used to fast-path *valid* runs; anything it flags falls back to the scalar path so the exact +diagnostic is preserved). Streaming inputs (files, `std::istream`, wide strings, user-defined adapters) always use the +scalar path. + +When `JSON_USE_SIMDUTF` is defined you must make the `simdutf.h` header available on the include path and link the +simdutf library. When it is not defined, no simdutf header is included and there is no dependency. + +!!! note "Requires C++17" + + simdutf requires C++17 and its header rejects older standards with an `#!cpp #error`. The backend is therefore only + compiled in from C++17 on. In C++11 and C++14 the macro has no effect and the scalar validator is used, which + accepts and rejects exactly the same input -- only throughput differs. Setting the macro project-wide is therefore + safe even when some translation units are built with an older standard. + +!!! warning "Define consistently" + + The macro selects between two definitions of the same inline validation function. It must therefore be defined + identically for **every** translation unit that includes the library; mixing translation units that define it with + ones that do not is an ODR violation. Prefer setting it as a compile definition on the target rather than with + `#!cpp #define` in individual source files. + +## Default definition + +By default, `#!cpp JSON_USE_SIMDUTF` is not defined and the portable C++11 scalar validator is used. + +```cpp +#undef JSON_USE_SIMDUTF +``` + +## Examples + +??? example + + The code below enables the simdutf backend for UTF-8 validation. + + ```cpp + #define JSON_USE_SIMDUTF 1 + #include + + ... + ``` + + The project must also link against simdutf, e.g. with CMake: + + ```cmake + target_compile_definitions(your_target PRIVATE JSON_USE_SIMDUTF) + target_link_libraries(your_target PRIVATE simdutf::simdutf) + ``` + +!!! hint "Testing this configuration" + + The unit tests can be built against the simdutf backend with the CMake option `JSON_TestSimdutf` (`OFF` by + default), which fetches simdutf and defines `JSON_USE_SIMDUTF` for every test target. The `ci_test_simdutf` target + runs the whole test suite in that configuration. + +## Version history + +- Added in version 3.13.0. diff --git a/docs/mkdocs/docs/examples/get__ValueType_const.output b/docs/mkdocs/docs/examples/get__ValueType_const.output index 5cd9cd3aa..e7e9b5d59 100644 --- a/docs/mkdocs/docs/examples/get__ValueType_const.output +++ b/docs/mkdocs/docs/examples/get__ValueType_const.output @@ -4,8 +4,8 @@ Hello, world! 1 2 3 4 5 -string: "Hello, world!" number: {"floating-point":17.23,"integer":42} null: null +string: "Hello, world!" boolean: true array: [1,2,3,4,5] diff --git a/docs/mkdocs/docs/examples/get_to.output b/docs/mkdocs/docs/examples/get_to.output index 5cd9cd3aa..e7e9b5d59 100644 --- a/docs/mkdocs/docs/examples/get_to.output +++ b/docs/mkdocs/docs/examples/get_to.output @@ -4,8 +4,8 @@ Hello, world! 1 2 3 4 5 -string: "Hello, world!" number: {"floating-point":17.23,"integer":42} null: null +string: "Hello, world!" boolean: true array: [1,2,3,4,5] diff --git a/docs/mkdocs/docs/examples/operator__ValueType.output b/docs/mkdocs/docs/examples/operator__ValueType.output index a3bd9fff4..de471ec02 100644 --- a/docs/mkdocs/docs/examples/operator__ValueType.output +++ b/docs/mkdocs/docs/examples/operator__ValueType.output @@ -4,9 +4,9 @@ Hello, world! 1 2 3 4 5 -string: "Hello, world!" number: {"floating-point":17.23,"integer":42} null: null +string: "Hello, world!" boolean: true array: [1,2,3,4,5] [json.exception.type_error.302] type must be boolean, but is string diff --git a/docs/mkdocs/docs/features/binary_formats/ubjson.md b/docs/mkdocs/docs/features/binary_formats/ubjson.md index 76956d60a..be545b9fe 100644 --- a/docs/mkdocs/docs/features/binary_formats/ubjson.md +++ b/docs/mkdocs/docs/features/binary_formats/ubjson.md @@ -69,6 +69,13 @@ The library uses the following mapping from JSON values types to UBJSON types ac Note that `use_size = true` alone may result in larger representations - the benefit of this parameter is that the receiving side is immediately informed on the number of elements of the container. + An array whose type marker is `Z` (null), `T` (true) or `F` (false) stores no payload at all, because the marker + already is the value. Its declared count is therefore the only thing that decides how much memory the receiving side + allocates, and a handful of bytes can describe billions of elements. `from_ubjson` rejects such an array with + [`out_of_range.408`](../../home/exceptions.md#jsonexceptionout_of_range408) when the count exceeds 1,048,576 + (`1 << 20`), and `to_ubjson` writes longer arrays of these types without the annotation, so any value it produces + can be read back. + !!! info "Binary values" If the JSON data contains the binary type, the value stored is a list of integers, as suggested by the UBJSON diff --git a/docs/mkdocs/docs/features/macros.md b/docs/mkdocs/docs/features/macros.md index 1d169fdeb..c4602fa5a 100644 --- a/docs/mkdocs/docs/features/macros.md +++ b/docs/mkdocs/docs/features/macros.md @@ -137,6 +137,14 @@ behavior is deprecated and switched off (`0`) by default. See [full documentation of `JSON_USE_LEGACY_DISCARDED_VALUE_COMPARISON`](../api/macros/json_use_legacy_discarded_value_comparison.md). +## `JSON_USE_SIMDUTF` + +When defined, UTF-8 validation of JSON strings read from contiguous byte input is delegated to the +[simdutf](https://github.com/simdutf/simdutf) library instead of the built-in scalar validator. This is an opt-in +external dependency and is not defined by default. + +See [full documentation of `JSON_USE_SIMDUTF`](../api/macros/json_use_simdutf.md). + ## `NLOHMANN_DEFINE_TYPE_*(...)`, `NLOHMANN_DEFINE_DERIVED_TYPE_*(...)` The library defines 12 macros to simplify the serialization/deserialization of types. See the page on diff --git a/docs/mkdocs/docs/home/exceptions.md b/docs/mkdocs/docs/home/exceptions.md index 7c7d22e23..09cc8e178 100644 --- a/docs/mkdocs/docs/home/exceptions.md +++ b/docs/mkdocs/docs/home/exceptions.md @@ -868,6 +868,12 @@ The size of an array or object in a [binary format](../features/binary_formats/i the size following `#` for [UBJSON](../features/binary_formats/ubjson.md)/[BJData](../features/binary_formats/bjdata.md), or the encoded length for [CBOR](../features/binary_formats/cbor.md). +The exception is also thrown for a [UBJSON](../features/binary_formats/ubjson.md) array of a type that is encoded by its +marker alone (`Z`, `T` or `F`) whose declared count exceeds 1,048,576 (`1 << 20`). Such an array has no payload, so its +count alone decides how much memory is allocated, and a handful of bytes would otherwise describe billions of values. +[`to_ubjson`](../api/basic_json/to_ubjson.md) writes longer arrays of these types without the size and type annotation, +so any value it produces can still be read back. + !!! failure "Example messages" ``` @@ -879,6 +885,9 @@ or the encoded length for [CBOR](../features/binary_formats/cbor.md). ``` [json.exception.out_of_range.408] syntax error while parsing CBOR size: excessive map size ``` + ``` + [json.exception.out_of_range.408] syntax error while parsing UBJSON size: excessive array size + ``` ### json.exception.out_of_range.409 @@ -933,6 +942,34 @@ BSON stores the length of documents, arrays, strings, and binary values in a sig [`to_bson`](../api/basic_json/to_bson.md) produced documents with negative length prefixes that [`from_bson`](../api/basic_json/from_bson.md) rejected. +### json.exception.out_of_range.413 + +A JSON Patch `remove` operation cannot be applied because the target location's parent is neither an object nor an array. Per [RFC 6902](https://datatracker.ietf.org/doc/html/rfc6902), a `remove` target must reference a member of an existing object or an element of an existing array; a primitive value (string, number, boolean, etc.) or `null` has no members or elements to remove. + +!!! failure "Example message" + + ``` + cannot remove value: the JSON Patch 'remove' target's parent is of type number, but must be an object or array + ``` + +!!! note + + This exception was added in version 3.13.0. Before that, this situation was silently ignored (the `remove` operation had no effect). + +### json.exception.out_of_range.414 + +A JSON Patch `move` operation's `"from"` location is a proper prefix of its `"path"` location. Per [RFC 6902](https://datatracker.ietf.org/doc/html/rfc6902) (section 4.4), a location cannot be moved into one of its own children. + +!!! failure "Example message" + + ``` + cannot move value: 'from' path '/0' is a proper prefix of 'path' '/0/0' + ``` + +!!! note + + This exception was added in version 3.13.0. Before that, this situation could succeed with a corrupted result: for an array target, removing the "from" element before the "add" step shifted subsequent indices, so "path" silently re-resolved to a different element than intended. + ## Further exceptions This exception is thrown in case of errors that cannot be classified with the diff --git a/docs/mkdocs/mkdocs.yml b/docs/mkdocs/mkdocs.yml index 2e1337f47..ec3e462c1 100644 --- a/docs/mkdocs/mkdocs.yml +++ b/docs/mkdocs/mkdocs.yml @@ -296,6 +296,7 @@ nav: - 'JSON_USE_GLOBAL_UDLS': api/macros/json_use_global_udls.md - 'JSON_USE_IMPLICIT_CONVERSIONS': api/macros/json_use_implicit_conversions.md - 'JSON_USE_LEGACY_DISCARDED_VALUE_COMPARISON': api/macros/json_use_legacy_discarded_value_comparison.md + - 'JSON_USE_SIMDUTF': api/macros/json_use_simdutf.md - 'NLOHMANN_DEFINE_DERIVED_TYPE_INTRUSIVE, NLOHMANN_DEFINE_DERIVED_TYPE_INTRUSIVE_WITH_DEFAULT, NLOHMANN_DEFINE_DERIVED_TYPE_INTRUSIVE_ONLY_SERIALIZE, NLOHMANN_DEFINE_DERIVED_TYPE_NON_INTRUSIVE, NLOHMANN_DEFINE_DERIVED_TYPE_NON_INTRUSIVE_WITH_DEFAULT, NLOHMANN_DEFINE_DERIVED_TYPE_NON_INTRUSIVE_ONLY_SERIALIZE': api/macros/nlohmann_define_derived_type.md - 'NLOHMANN_DEFINE_TYPE_INTRUSIVE, NLOHMANN_DEFINE_TYPE_INTRUSIVE_WITH_DEFAULT, NLOHMANN_DEFINE_TYPE_INTRUSIVE_ONLY_SERIALIZE': api/macros/nlohmann_define_type_intrusive.md - 'NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE, NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE_WITH_DEFAULT, NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE_ONLY_SERIALIZE': api/macros/nlohmann_define_type_non_intrusive.md diff --git a/include/nlohmann/detail/conversions/from_json.hpp b/include/nlohmann/detail/conversions/from_json.hpp index 6856a0965..11e40f5f4 100644 --- a/include/nlohmann/detail/conversions/from_json.hpp +++ b/include/nlohmann/detail/conversions/from_json.hpp @@ -398,6 +398,17 @@ inline void from_json(const BasicJsonType& j, CompatibleArrayType& bin) } } +template +auto from_json_object_reserve(ConstructibleObjectType& obj, typename ConstructibleObjectType::size_type size, priority_tag<1> /*unused*/) +-> decltype(obj.reserve(size), void()) +{ + obj.reserve(size); +} + +template +inline void from_json_object_reserve(ConstructibleObjectType& /*obj*/, std::size_t /*size*/, priority_tag<0> /*unused*/) +{} + template::value, int> = 0> inline void from_json(const BasicJsonType& j, ConstructibleObjectType& obj) @@ -409,6 +420,7 @@ inline void from_json(const BasicJsonType& j, ConstructibleObjectType& obj) ConstructibleObjectType ret; const auto* inner_object = j.template get_ptr(); + from_json_object_reserve(ret, inner_object->size(), priority_tag<1> {}); for (const auto& p : *inner_object) { ret.emplace(p.first, p.second.template get()); diff --git a/include/nlohmann/detail/conversions/to_chars.hpp b/include/nlohmann/detail/conversions/to_chars.hpp index 70fb9b933..c0945ab9e 100644 --- a/include/nlohmann/detail/conversions/to_chars.hpp +++ b/include/nlohmann/detail/conversions/to_chars.hpp @@ -1075,8 +1075,8 @@ char* to_chars(char* first, const char* last, FloatType value) } #ifdef __GNUC__ -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wfloat-equal" + JSON_HEDLEY_DIAGNOSTIC_PUSH + JSON_HEDLEY_PRAGMA(GCC diagnostic ignored "-Wfloat-equal") #endif if (value == 0) // +-0 { @@ -1087,7 +1087,7 @@ char* to_chars(char* first, const char* last, FloatType value) return first; } #ifdef __GNUC__ -#pragma GCC diagnostic pop + JSON_HEDLEY_DIAGNOSTIC_POP #endif JSON_ASSERT(last - first >= std::numeric_limits::max_digits10); diff --git a/include/nlohmann/detail/exceptions.hpp b/include/nlohmann/detail/exceptions.hpp index cb87bb93e..3e5b45101 100644 --- a/include/nlohmann/detail/exceptions.hpp +++ b/include/nlohmann/detail/exceptions.hpp @@ -33,8 +33,8 @@ // code stumbling over this. See https://github.com/nlohmann/json/issues/4087 // for a discussion. #if defined(__clang__) - #pragma clang diagnostic push - #pragma clang diagnostic ignored "-Wweak-vtables" + JSON_HEDLEY_DIAGNOSTIC_PUSH + JSON_HEDLEY_PRAGMA(clang diagnostic ignored "-Wweak-vtables") #endif NLOHMANN_JSON_NAMESPACE_BEGIN @@ -287,5 +287,5 @@ class other_error : public exception NLOHMANN_JSON_NAMESPACE_END #if defined(__clang__) - #pragma clang diagnostic pop + JSON_HEDLEY_DIAGNOSTIC_POP #endif diff --git a/include/nlohmann/detail/input/binary_reader.hpp b/include/nlohmann/detail/input/binary_reader.hpp index 557d7669c..efa93070d 100644 --- a/include/nlohmann/detail/input/binary_reader.hpp +++ b/include/nlohmann/detail/input/binary_reader.hpp @@ -58,6 +58,26 @@ inline bool little_endianness(int num = 1) noexcept return *reinterpret_cast(&num) == 1; } +/*! +@brief largest element count accepted for a UBJSON container of a valueless type + +An element of type 'Z' (null), 'T' (true) or 'F' (false) is encoded by its +type marker alone, so an optimized container of one of those types has no +payload at all and its declared count is the only thing that decides how much +is allocated: `[$Z#L` followed by a large count turns some ten bytes of input +into that many values (see #2793, which reports 35 GB and 150 seconds). Every +other type costs at least one byte per element and is bounded by the end of +the input. + +This is a sanity bound rather than a security boundary, and it is far above +any container met in practice. @ref binary_writer falls back to the +unoptimized encoding for longer containers, so that a value serialized by +this library can always be read back. + +@sa https://github.com/nlohmann/json/issues/2793 +*/ +JSON_INLINE_VARIABLE constexpr std::size_t max_valueless_container_size = 1 << 20; + /////////////////// // binary reader // /////////////////// @@ -110,6 +130,7 @@ class binary_reader const cbor_tag_handler_t tag_handler = cbor_tag_handler_t::error) { sax = sax_; + container_stack.clear(); bool result = false; switch (format) @@ -159,6 +180,80 @@ class binary_reader } private: + //////////////////////// + // nested containers // + //////////////////////// + + /*! + @brief a container that has been opened and not closed yet + + The binary readers do not call themselves once per nesting level. Like + @ref parser::sax_parse_internal, which does the same for JSON text, they + keep the containers they are inside of on a heap-allocated stack, so that + the native call stack does not grow with the nesting depth of the input + and a deeply nested value is bounded by memory rather than by the stack + (see #5104). + + The members are ordered by decreasing alignment, which is the ordering that + keeps a struct from growing as members are added to it. + */ + struct container_frame + { + container_frame(const std::size_t remaining_, const bool is_object_, + const char_int_type type_marker_ = 0) noexcept + : remaining(remaining_), type_marker(type_marker_), is_object(is_object_) {} + + /// number of elements that have not been read yet, or npos when the + /// container is not sized and ends at a marker instead + std::size_t remaining; + /// BSON: value of chars_read before this document's size prefix, which + /// check_bson_document_size() needs once the document has been read + std::size_t start_position = 0; + /// UBJSON/BJData: the type marker of an optimized container, so that + /// its elements are read without one of their own; 0 otherwise + char_int_type type_marker; + /// BSON: the size this document declares, in bytes + std::int32_t declared_size = 0; + /// whether to close this container with end_object() or end_array() + bool is_object; + }; + + /*! + @brief open a nested array or object + + Emits the SAX start event and records the container. This is the only + place the binary readers start a container, so a check that rejects one + can be made here and is then guaranteed to run before the start event. + + @param[in] is_object whether an object (true) or an array (false) begins + @param[in] len number of elements the container declares + + @return whether the SAX parser accepted the start event + */ + bool enter_container(const bool is_object, const std::size_t len, + const char_int_type type_marker = 0) + { + if (JSON_HEDLEY_UNLIKELY(is_object ? !sax->start_object(len) : !sax->start_array(len))) + { + return false; + } + + container_stack.emplace_back(len, is_object, type_marker); + return true; + } + + /// @copydoc enter_container + bool enter_array(const std::size_t len, const char_int_type type_marker = 0) + { + return enter_container(/*is_object*/false, len, type_marker); + } + + /// @copydoc enter_container + bool enter_object(const std::size_t len, const char_int_type type_marker = 0) + { + return enter_container(/*is_object*/true, len, type_marker); + } + ////////// // BSON // ////////// @@ -193,8 +288,10 @@ class binary_reader @brief Reads in a BSON-object and passes it to the SAX-parser. @return whether a valid BSON-value was passed to the SAX parser */ - bool parse_bson_internal() + bool open_bson_document(const bool is_object) { + // recorded before the size prefix is read, because + // check_bson_document_size() measures the document from here const std::size_t document_start = chars_read; std::int32_t document_size{}; if (!get_number(input_format_t::bson, document_size)) @@ -202,22 +299,91 @@ class binary_reader return false; } - if (JSON_HEDLEY_UNLIKELY(!sax->start_object(detail::unknown_size()))) + if (JSON_HEDLEY_UNLIKELY(!enter_container(is_object, detail::unknown_size()))) { return false; } - if (JSON_HEDLEY_UNLIKELY(!parse_bson_element_list(/*is_array*/false))) + container_frame& frame = container_stack.back(); + frame.start_position = document_start; + frame.declared_size = document_size; + return true; + } + + /*! + @brief read a BSON document and everything nested inside it + + Reads elements until the document that was begun here is complete, + resuming the enclosing document each time an embedded one ends, so that + the nesting depth of the input costs heap rather than native stack + (see #5104). + + @return whether reading the document succeeded + */ + bool parse_bson_internal() + { + if (JSON_HEDLEY_UNLIKELY(!open_bson_document(/*is_object*/true))) { return false; } - if (JSON_HEDLEY_UNLIKELY(!check_bson_document_size(document_start, document_size))) - { - return false; - } + // the key currently being read; hoisted out of the loop so that its + // capacity is reused across elements and across nesting levels + string_t key; - return sax->end_object(); + while (true) + { + const auto element_type = get(); + + if (element_type == 0) // end of the innermost document + { + // a copy, not a reference: it must stay valid across the + // pop_back() below, which destroys the container_stack + // element it would otherwise alias + const container_frame top = container_stack.back(); + + if (JSON_HEDLEY_UNLIKELY(!check_bson_document_size(top.start_position, top.declared_size))) + { + return false; + } + + container_stack.pop_back(); + if (JSON_HEDLEY_UNLIKELY(top.is_object ? !sax->end_object() : !sax->end_array())) + { + return false; + } + // the document begun here is complete once it is not inside one + if (container_stack.empty()) + { + return true; + } + continue; + } + + if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format_t::bson, "element list"))) + { + return false; + } + + const std::size_t element_type_parse_position = chars_read; + key.clear(); + if (JSON_HEDLEY_UNLIKELY(!get_bson_cstr(key))) + { + return false; + } + + // an array's elements are named "0", "1", ... in the wire format, + // and those names are not passed on + if (container_stack.back().is_object && !sax->key(key)) + { + return false; + } + + if (JSON_HEDLEY_UNLIKELY(!parse_bson_element_internal(element_type, element_type_parse_position))) + { + return false; + } + } } /*! @@ -329,12 +495,12 @@ class binary_reader case 0x03: // object { - return parse_bson_internal(); + return open_bson_document(/*is_object*/true); } case 0x04: // array { - return parse_bson_array(); + return open_bson_document(/*is_object*/false); } case 0x05: // binary @@ -384,82 +550,7 @@ class binary_reader } } - /*! - @brief Read a BSON element list (as specified in the BSON-spec) - The same binary layout is used for objects and arrays, hence it must be - indicated with the argument @a is_array which one is expected - (true --> array, false --> object). - - @param[in] is_array Determines if the element list being read is to be - treated as an object (@a is_array == false), or as an - array (@a is_array == true). - @return whether a valid BSON-object/array was passed to the SAX parser - */ - bool parse_bson_element_list(const bool is_array) - { - string_t key; - - while (auto element_type = get()) - { - if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format_t::bson, "element list"))) - { - return false; - } - - const std::size_t element_type_parse_position = chars_read; - if (JSON_HEDLEY_UNLIKELY(!get_bson_cstr(key))) - { - return false; - } - - if (!is_array && !sax->key(key)) - { - return false; - } - - if (JSON_HEDLEY_UNLIKELY(!parse_bson_element_internal(element_type, element_type_parse_position))) - { - return false; - } - - // get_bson_cstr only appends - key.clear(); - } - - return true; - } - - /*! - @brief Reads an array from the BSON input and passes it to the SAX-parser. - @return whether a valid BSON-array was passed to the SAX parser - */ - bool parse_bson_array() - { - const std::size_t document_start = chars_read; - std::int32_t document_size{}; - if (!get_number(input_format_t::bson, document_size)) - { - return false; - } - - if (JSON_HEDLEY_UNLIKELY(!sax->start_array(detail::unknown_size()))) - { - return false; - } - - if (JSON_HEDLEY_UNLIKELY(!parse_bson_element_list(/*is_array*/true))) - { - return false; - } - - if (JSON_HEDLEY_UNLIKELY(!check_bson_document_size(document_start, document_size))) - { - return false; - } - - return sax->end_array(); - } ////////// // CBOR // @@ -491,9 +582,12 @@ class binary_reader @return whether a valid CBOR value was passed to the SAX parser */ - bool parse_cbor_internal(const bool get_char, - const cbor_tag_handler_t tag_handler) + bool parse_cbor_value(const bool get_char, + const cbor_tag_handler_t tag_handler, + bool& tag_pending) { + tag_pending = false; + switch (get_char ? get() : current) { // EOF @@ -685,37 +779,36 @@ class binary_reader case 0x95: case 0x96: case 0x97: - return get_cbor_array( - conditional_static_cast(static_cast(current) & 0x1Fu), tag_handler); + return enter_array(conditional_static_cast(static_cast(current) & 0x1Fu)); case 0x98: // array (one-byte uint8_t for n follows) { std::uint8_t len{}; - return get_number(input_format_t::cbor, len) && get_cbor_array(static_cast(len), tag_handler); + return get_number(input_format_t::cbor, len) && enter_array(static_cast(len)); } case 0x99: // array (two-byte uint16_t for n follow) { std::uint16_t len{}; - return get_number(input_format_t::cbor, len) && get_cbor_array(static_cast(len), tag_handler); + return get_number(input_format_t::cbor, len) && enter_array(static_cast(len)); } case 0x9A: // array (four-byte uint32_t for n follow) { std::uint32_t len{}; std::size_t size{}; - return get_number(input_format_t::cbor, len) && get_cbor_container_size(len, size, "array") && get_cbor_array(size, tag_handler); + return get_number(input_format_t::cbor, len) && get_cbor_container_size(len, size, "array") && enter_array(size); } case 0x9B: // array (eight-byte uint64_t for n follow) { std::uint64_t len{}; std::size_t size{}; - return get_number(input_format_t::cbor, len) && get_cbor_container_size(len, size, "array") && get_cbor_array(size, tag_handler); + return get_number(input_format_t::cbor, len) && get_cbor_container_size(len, size, "array") && enter_array(size); } case 0x9F: // array (indefinite length) - return get_cbor_array(detail::unknown_size(), tag_handler); + return enter_array(detail::unknown_size()); // map (0x00..0x17 pairs of data items follow) case 0xA0: @@ -742,36 +835,36 @@ class binary_reader case 0xB5: case 0xB6: case 0xB7: - return get_cbor_object(conditional_static_cast(static_cast(current) & 0x1Fu), tag_handler); + return enter_object(conditional_static_cast(static_cast(current) & 0x1Fu)); case 0xB8: // map (one-byte uint8_t for n follows) { std::uint8_t len{}; - return get_number(input_format_t::cbor, len) && get_cbor_object(static_cast(len), tag_handler); + return get_number(input_format_t::cbor, len) && enter_object(static_cast(len)); } case 0xB9: // map (two-byte uint16_t for n follow) { std::uint16_t len{}; - return get_number(input_format_t::cbor, len) && get_cbor_object(static_cast(len), tag_handler); + return get_number(input_format_t::cbor, len) && enter_object(static_cast(len)); } case 0xBA: // map (four-byte uint32_t for n follow) { std::uint32_t len{}; std::size_t size{}; - return get_number(input_format_t::cbor, len) && get_cbor_container_size(len, size, "map") && get_cbor_object(size, tag_handler); + return get_number(input_format_t::cbor, len) && get_cbor_container_size(len, size, "map") && enter_object(size); } case 0xBB: // map (eight-byte uint64_t for n follow) { std::uint64_t len{}; std::size_t size{}; - return get_number(input_format_t::cbor, len) && get_cbor_container_size(len, size, "map") && get_cbor_object(size, tag_handler); + return get_number(input_format_t::cbor, len) && get_cbor_container_size(len, size, "map") && enter_object(size); } case 0xBF: // map (indefinite length) - return get_cbor_object(detail::unknown_size(), tag_handler); + return enter_object(detail::unknown_size()); case 0xC0: // tagged item case 0xC1: @@ -855,7 +948,10 @@ class binary_reader default: break; } - return parse_cbor_internal(true, tag_handler); + // the tagged value follows; it is read by the loop in + // parse_cbor_internal() rather than by recursing here + tag_pending = true; + return true; } case cbor_tag_handler_t::store: @@ -905,7 +1001,11 @@ class binary_reader break; } default: - return parse_cbor_internal(true, tag_handler); + { + // as above, the tagged value is read by the caller + tag_pending = true; + return true; + } } get(); return get_cbor_binary(b) && sax->binary(b); @@ -996,23 +1096,21 @@ class binary_reader } /*! - @brief reads a CBOR string + @brief reads a definite-length CBOR string - This function first reads starting bytes to determine the expected - string length and then copies this number of bytes into a string. - Additionally, CBOR's strings with indefinite lengths are supported. + Reads everything @ref get_cbor_string accepts except the indefinite-length + form, which that function handles itself. The bytes are appended to @a + result, so consecutive chunks of an indefinite-length string can be read + into the same string. - @param[out] result created string + @param[out] result string the bytes are appended to @return whether string creation completed - */ - bool get_cbor_string(string_t& result) - { - if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format_t::cbor, "string"))) - { - return false; - } + @pre @a current is not EOF + */ + bool get_cbor_string_chunk(string_t& result) + { switch (current) { // UTF-8 string (0x00..0x17 bytes follow) @@ -1068,20 +1166,6 @@ class binary_reader return get_number(input_format_t::cbor, len) && get_string(input_format_t::cbor, len, result); } - case 0x7F: // UTF-8 string (indefinite length) - { - while (get() != 0xFF) - { - string_t chunk; - if (!get_cbor_string(chunk)) - { - return false; - } - result.append(chunk); - } - return true; - } - default: { auto last_token = get_token_string(); @@ -1092,23 +1176,82 @@ class binary_reader } /*! - @brief reads a CBOR byte array + @brief reads a CBOR string This function first reads starting bytes to determine the expected - byte array length and then copies this number of bytes into the byte array. - Additionally, CBOR's byte arrays with indefinite lengths are supported. + string length and then copies this number of bytes into a string. + Additionally, CBOR's strings with indefinite lengths are supported. - @param[out] result created byte array + @param[out] result created string + + @return whether string creation completed + */ + bool get_cbor_string(string_t& result) + { + // number of indefinite-length strings that have been opened and not + // closed yet. RFC 8949, Section 3.2.3 does not permit nesting them, + // but this reader has always accepted it, so the open levels are + // counted instead of recursed through, which overflowed the stack for + // an input of repeated 0x7F bytes (see #5104). Every chunk is appended + // to the same result, so no per-level state is needed. + std::size_t open = 0; + + while (true) + { + if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format_t::cbor, "string"))) + { + return false; + } + + if (current == 0x7F) // UTF-8 string (indefinite length) + { + ++open; + get(); + continue; + } + + // a break marker closes the innermost indefinite-length string; + // outside of one it is not a string and falls through to the error + if (open != 0 && current == 0xFF) + { + if (--open == 0) + { + return true; + } + get(); + continue; + } + + if (JSON_HEDLEY_UNLIKELY(!get_cbor_string_chunk(result))) + { + return false; + } + + if (open == 0) + { + return true; + } + + get(); + } + } + + /*! + @brief reads a definite-length CBOR byte array + + Reads everything @ref get_cbor_binary accepts except the indefinite-length + form, which that function handles itself. The bytes are appended to @a + result, so consecutive chunks of an indefinite-length byte array can be + read into the same byte array. + + @param[out] result byte array the bytes are appended to @return whether byte array creation completed - */ - bool get_cbor_binary(binary_t& result) - { - if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format_t::cbor, "binary"))) - { - return false; - } + @pre @a current is not EOF + */ + bool get_cbor_binary_chunk(binary_t& result) + { switch (current) { // Binary data (0x00..0x17 bytes follow) @@ -1168,20 +1311,6 @@ class binary_reader get_binary(input_format_t::cbor, len, result); } - case 0x5F: // Binary data (indefinite length) - { - while (get() != 0xFF) - { - binary_t chunk; - if (!get_cbor_binary(chunk)) - { - return false; - } - result.insert(result.end(), chunk.begin(), chunk.end()); - } - return true; - } - default: { auto last_token = get_token_string(); @@ -1191,6 +1320,63 @@ class binary_reader } } + /*! + @brief reads a CBOR byte array + + This function first reads starting bytes to determine the expected + byte array length and then copies this number of bytes into the byte array. + Additionally, CBOR's byte arrays with indefinite lengths are supported. + + @param[out] result created byte array + + @return whether byte array creation completed + */ + bool get_cbor_binary(binary_t& result) + { + // the open indefinite-length byte arrays are counted rather than + // recursed through, for the reason given in @ref get_cbor_string + std::size_t open = 0; + + while (true) + { + if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format_t::cbor, "binary"))) + { + return false; + } + + if (current == 0x5F) // Binary data (indefinite length) + { + ++open; + get(); + continue; + } + + // a break marker closes the innermost indefinite-length byte + // array; outside of one it falls through to the error below + if (open != 0 && current == 0xFF) + { + if (--open == 0) + { + return true; + } + get(); + continue; + } + + if (JSON_HEDLEY_UNLIKELY(!get_cbor_binary_chunk(result))) + { + return false; + } + + if (open == 0) + { + return true; + } + + get(); + } + } + /*! @brief narrow a definite CBOR array/map length to std::size_t @@ -1217,96 +1403,110 @@ class binary_reader } /*! - @param[in] len the length of the array or detail::unknown_size() for an - array of indefinite size + @brief read a CBOR value and everything nested inside it + + Reads values until the one that was begun here is complete, resuming the + enclosing container after each element, so that the nesting depth of the + input costs heap rather than native stack (see #5104). + + @param[in] get_char whether a new character should be retrieved from the + input (true) or whether the last read character + @a current should be considered instead @param[in] tag_handler how CBOR tags should be treated - @return whether array creation completed + + @return whether reading the value succeeded */ - bool get_cbor_array(const std::size_t len, - const cbor_tag_handler_t tag_handler) + bool parse_cbor_internal(const bool get_char, + const cbor_tag_handler_t tag_handler) { - if (JSON_HEDLEY_UNLIKELY(!sax->start_array(len))) - { - return false; - } + // whether the next value starts at a fresh byte or at the one already + // read into `current` + bool fetch = get_char; - if (len != detail::unknown_size()) + // the key currently being read; hoisted out of the loop so that its + // capacity is reused across elements and across nesting levels + string_t key; + + while (true) { - for (std::size_t i = 0; i < len; ++i) + if (!container_stack.empty()) { - if (JSON_HEDLEY_UNLIKELY(!parse_cbor_internal(true, tag_handler))) + // a copy, not a reference: it must stay valid across the + // pop_back() below, which destroys the container_stack element + // it would otherwise alias + container_frame top = container_stack.back(); + bool at_end = false; + + if (top.remaining != npos) { - return false; + // definite length: the container ends once its elements + // have been read + at_end = (top.remaining == 0); + if (!at_end) + { + // claim the element about to be read + --container_stack.back().remaining; + if (top.is_object) + { + get(); + } + } + fetch = true; } - } - } - else - { - while (get() != 0xFF) - { - if (JSON_HEDLEY_UNLIKELY(!parse_cbor_internal(false, tag_handler))) + else { - return false; + // indefinite length: the container ends at a break marker. + // Testing for it consumes a byte, which is the first byte + // of the next element when it is not one. + at_end = (get() == 0xFF); + fetch = top.is_object; } - } - } - return sax->end_array(); - } - - /*! - @param[in] len the length of the object or detail::unknown_size() for an - object of indefinite size - @param[in] tag_handler how CBOR tags should be treated - @return whether object creation completed - */ - bool get_cbor_object(const std::size_t len, - const cbor_tag_handler_t tag_handler) - { - if (JSON_HEDLEY_UNLIKELY(!sax->start_object(len))) - { - return false; - } - - if (len != 0) - { - string_t key; - if (len != detail::unknown_size()) - { - for (std::size_t i = 0; i < len; ++i) + if (at_end) { - get(); + container_stack.pop_back(); + if (JSON_HEDLEY_UNLIKELY(top.is_object ? !sax->end_object() : !sax->end_array())) + { + return false; + } + // the value begun here is complete once its container is + if (container_stack.empty()) + { + return true; + } + continue; + } + + if (top.is_object) + { + key.clear(); if (JSON_HEDLEY_UNLIKELY(!get_cbor_string(key) || !sax->key(key))) { return false; } - - if (JSON_HEDLEY_UNLIKELY(!parse_cbor_internal(true, tag_handler))) - { - return false; - } - key.clear(); + fetch = true; } } - else - { - while (get() != 0xFF) - { - if (JSON_HEDLEY_UNLIKELY(!get_cbor_string(key) || !sax->key(key))) - { - return false; - } - if (JSON_HEDLEY_UNLIKELY(!parse_cbor_internal(true, tag_handler))) - { - return false; - } - key.clear(); + // a tag is not a value of its own: read on until the tagged value + bool tag_pending = false; + do + { + if (JSON_HEDLEY_UNLIKELY(!parse_cbor_value(fetch, tag_handler, tag_pending))) + { + return false; } + fetch = true; + } + while (tag_pending); + + // a value that opened a container left it on the stack; one that + // did not, and that was not inside a container, was the whole value + if (container_stack.empty()) + { + return true; } } - - return sax->end_object(); } ///////////// @@ -1316,7 +1516,17 @@ class binary_reader /*! @return whether a valid MessagePack value was passed to the SAX parser */ - bool parse_msgpack_internal() + /*! + @brief read one MessagePack value + + Reads a single value and passes it to the SAX parser. A value that begins + a container is not read to its end: the container is opened with + @ref enter_container and its elements are read by + @ref parse_msgpack_internal, so that nesting does not consume native stack. + + @return whether reading the value succeeded + */ + bool parse_msgpack_value() { switch (get()) { @@ -1472,7 +1682,7 @@ class binary_reader case 0x8D: case 0x8E: case 0x8F: - return get_msgpack_object(conditional_static_cast(static_cast(current) & 0x0Fu)); + return enter_object(conditional_static_cast(static_cast(current) & 0x0Fu)); // fixarray case 0x90: @@ -1491,7 +1701,7 @@ class binary_reader case 0x9D: case 0x9E: case 0x9F: - return get_msgpack_array(conditional_static_cast(static_cast(current) & 0x0Fu)); + return enter_array(conditional_static_cast(static_cast(current) & 0x0Fu)); // fixstr case 0xA0: @@ -1622,25 +1832,25 @@ class binary_reader case 0xDC: // array 16 { std::uint16_t len{}; - return get_number(input_format_t::msgpack, len) && get_msgpack_array(static_cast(len)); + return get_number(input_format_t::msgpack, len) && enter_array(static_cast(len)); } case 0xDD: // array 32 { std::uint32_t len{}; - return get_number(input_format_t::msgpack, len) && get_msgpack_array(conditional_static_cast(len)); + return get_number(input_format_t::msgpack, len) && enter_array(conditional_static_cast(len)); } case 0xDE: // map 16 { std::uint16_t len{}; - return get_number(input_format_t::msgpack, len) && get_msgpack_object(static_cast(len)); + return get_number(input_format_t::msgpack, len) && enter_object(static_cast(len)); } case 0xDF: // map 32 { std::uint32_t len{}; - return get_number(input_format_t::msgpack, len) && get_msgpack_object(conditional_static_cast(len)); + return get_number(input_format_t::msgpack, len) && enter_object(conditional_static_cast(len)); } // negative fixint @@ -1888,55 +2098,69 @@ class binary_reader } /*! - @param[in] len the length of the array - @return whether array creation completed + @brief read a MessagePack value and everything nested inside it + + Reads values until the one that was begun here is complete, resuming the + enclosing container each time an element ends, so that the nesting depth + of the input costs heap rather than native stack (see #5104). + + @return whether reading the value succeeded */ - bool get_msgpack_array(const std::size_t len) + bool parse_msgpack_internal() { - if (JSON_HEDLEY_UNLIKELY(!sax->start_array(len))) - { - return false; - } - - for (std::size_t i = 0; i < len; ++i) - { - if (JSON_HEDLEY_UNLIKELY(!parse_msgpack_internal())) - { - return false; - } - } - - return sax->end_array(); - } - - /*! - @param[in] len the length of the object - @return whether object creation completed - */ - bool get_msgpack_object(const std::size_t len) - { - if (JSON_HEDLEY_UNLIKELY(!sax->start_object(len))) - { - return false; - } - + // the key currently being read; hoisted out of the loop so that its + // capacity is reused across elements and across nesting levels string_t key; - for (std::size_t i = 0; i < len; ++i) + + while (true) { - get(); - if (JSON_HEDLEY_UNLIKELY(!get_msgpack_string(key) || !sax->key(key))) + if (!container_stack.empty()) + { + // copied out before anything can push onto the stack and + // invalidate a reference into it + const bool is_object = container_stack.back().is_object; + + if (container_stack.back().remaining == 0) + { + container_stack.pop_back(); + if (JSON_HEDLEY_UNLIKELY(is_object ? !sax->end_object() : !sax->end_array())) + { + return false; + } + // the value begun here is complete once its container is + if (container_stack.empty()) + { + return true; + } + continue; + } + + // claim the element about to be read + --container_stack.back().remaining; + + if (is_object) + { + get(); + key.clear(); + if (JSON_HEDLEY_UNLIKELY(!get_msgpack_string(key) || !sax->key(key))) + { + return false; + } + } + } + + if (JSON_HEDLEY_UNLIKELY(!parse_msgpack_value())) { return false; } - if (JSON_HEDLEY_UNLIKELY(!parse_msgpack_internal())) + // a value that opened a container left it on the stack; one that + // did not, and that was not inside a container, was the whole value + if (container_stack.empty()) { - return false; + return true; } - key.clear(); } - - return sax->end_object(); } //////////// @@ -1952,7 +2176,103 @@ class binary_reader */ bool parse_ubjson_internal(const bool get_char = true) { - return get_ubjson_value(get_char ? get_ignore_noop() : current); + // the key currently being read; hoisted out of the loop so that its + // capacity is reused across elements and across nesting levels + string_t key; + + // the type marker of the value to read next + char_int_type prefix = get_char ? get_ignore_noop() : current; + + while (true) + { + const std::size_t depth = container_stack.size(); + + if (JSON_HEDLEY_UNLIKELY(!get_ubjson_value(prefix))) + { + return false; + } + + // the value begun here is complete once it is not inside anything + if (container_stack.empty()) + { + return true; + } + + // a value was completed rather than a container opened; a + // container that ends at a marker needs the next byte to test + if (container_stack.size() == depth && container_stack.back().remaining == npos) + { + get_ignore_noop(); + } + + // advance to the next element, closing the containers that ended. + // top is a copy, not a reference: it must stay valid across the + // pop_back() below, which destroys the container_stack element it + // would otherwise alias. + for (;;) + { + container_frame top = container_stack.back(); + + if (top.remaining != npos) + { + if (top.remaining != 0) + { + --container_stack.back().remaining; + if (top.is_object) + { + key.clear(); + if (JSON_HEDLEY_UNLIKELY(!get_ubjson_string(key) || !sax->key(key))) + { + return false; + } + } + // an optimized container gives its elements no marker + prefix = (top.type_marker != 0) ? top.type_marker : get_ignore_noop(); + break; + } + } + // the end marker is compared against a literal rather than + // against a conditional expression, because char_int_type is + // unsigned for some input adapters and MSVC then reports the + // comparison as a signed/unsigned mismatch + else if (top.is_object ? (current != '}') : (current != ']')) + { + // a container that ends at a marker is never optimized, so + // every element carries its own marker; for an object the + // byte tested above is the first byte of the key + if (top.is_object) + { + key.clear(); + if (JSON_HEDLEY_UNLIKELY(!get_ubjson_string(key, false) || !sax->key(key))) + { + return false; + } + prefix = get_ignore_noop(); + } + else + { + prefix = current; + } + break; + } + + container_stack.pop_back(); + if (JSON_HEDLEY_UNLIKELY(top.is_object ? !sax->end_object() : !sax->end_array())) + { + return false; + } + if (container_stack.empty()) + { + return true; + } + // the container that just ended was an element of the one + // below it, which may need the next byte for its own test + if (container_stack.back().remaining == npos) + { + get_ignore_noop(); + } + } + } } /*! @@ -2391,7 +2711,12 @@ class binary_reader { result.first = npos; // size result.second = 0; // type - bool is_ndarray = false; + // seed the flag with the caller's context: inside an ndarray dimension + // vector another ndarray is not allowed, and get_ubjson_size_value() + // rejects it up front instead of reading it and reporting afterwards. + // Seeding it with `false` made every '#' of a "[#[#[..." chain descend + // another level, which overflowed the stack (see #5104). + bool is_ndarray = inside_ndarray; get_ignore_noop(); @@ -2424,13 +2749,11 @@ class binary_reader } const bool is_error = get_ubjson_size_value(result.first, is_ndarray); - if (input_format == input_format_t::bjdata && is_ndarray) + // an ndarray was read here only if the flag flipped; when it was + // seeded true, get_ubjson_size_value() already rejected the nested + // dimension vector + if (input_format == input_format_t::bjdata && is_ndarray && !inside_ndarray) { - if (inside_ndarray) - { - return sax->parse_error(chars_read, get_token_string(), parse_error::create(112, chars_read, - exception_message(input_format, "ndarray can not be recursive", "size"), nullptr)); - } result.second |= (1 << 8); // use bit 8 to indicate ndarray, all UBJSON and BJData markers should be ASCII letters } return is_error; @@ -2439,7 +2762,7 @@ class binary_reader if (current == '#') { const bool is_error = get_ubjson_size_value(result.first, is_ndarray); - if (input_format == input_format_t::bjdata && is_ndarray) + if (input_format == input_format_t::bjdata && is_ndarray && !inside_ndarray) { return sax->parse_error(chars_read, get_token_string(), parse_error::create(112, chars_read, exception_message(input_format, "ndarray requires both type and size", "size"), nullptr)); @@ -2710,53 +3033,33 @@ class binary_reader if (size_and_type.first != npos) { - if (JSON_HEDLEY_UNLIKELY(!sax->start_array(size_and_type.first))) + // reading an element of a valueless type consumes no input, so the + // declared count alone decides how much is allocated; the check is + // made before the start event so that no container is opened that + // is then abandoned. See @ref max_valueless_container_size. + if (JSON_HEDLEY_UNLIKELY((size_and_type.second == 'Z' || size_and_type.second == 'T' || size_and_type.second == 'F') + && size_and_type.first > max_valueless_container_size)) + { + return sax->parse_error(chars_read, get_token_string(), out_of_range::create(408, + exception_message(input_format, "excessive array size", "size"), nullptr)); + } + + if (JSON_HEDLEY_UNLIKELY(!enter_array(size_and_type.first, size_and_type.second))) { return false; } - if (size_and_type.second != 0) + if (size_and_type.second == 'N') { - if (size_and_type.second != 'N') - { - for (std::size_t i = 0; i < size_and_type.first; ++i) - { - if (JSON_HEDLEY_UNLIKELY(!get_ubjson_value(size_and_type.second))) - { - return false; - } - } - } - } - else - { - for (std::size_t i = 0; i < size_and_type.first; ++i) - { - if (JSON_HEDLEY_UNLIKELY(!parse_ubjson_internal())) - { - return false; - } - } - } - } - else - { - if (JSON_HEDLEY_UNLIKELY(!sax->start_array(detail::unknown_size()))) - { - return false; + // a no-op is not a value, so a container of them holds none; + // the declared size has already been passed to the SAX parser + container_stack.back().remaining = 0; } - while (current != ']') - { - if (JSON_HEDLEY_UNLIKELY(!parse_ubjson_internal(false))) - { - return false; - } - get_ignore_noop(); - } + return true; } - return sax->end_array(); + return enter_array(detail::unknown_size()); } /*! @@ -2778,68 +3081,12 @@ class binary_reader exception_message(input_format, "BJData object does not support ND-array size in optimized format", "object"), nullptr)); } - string_t key; if (size_and_type.first != npos) { - if (JSON_HEDLEY_UNLIKELY(!sax->start_object(size_and_type.first))) - { - return false; - } - - if (size_and_type.second != 0) - { - for (std::size_t i = 0; i < size_and_type.first; ++i) - { - if (JSON_HEDLEY_UNLIKELY(!get_ubjson_string(key) || !sax->key(key))) - { - return false; - } - if (JSON_HEDLEY_UNLIKELY(!get_ubjson_value(size_and_type.second))) - { - return false; - } - key.clear(); - } - } - else - { - for (std::size_t i = 0; i < size_and_type.first; ++i) - { - if (JSON_HEDLEY_UNLIKELY(!get_ubjson_string(key) || !sax->key(key))) - { - return false; - } - if (JSON_HEDLEY_UNLIKELY(!parse_ubjson_internal())) - { - return false; - } - key.clear(); - } - } - } - else - { - if (JSON_HEDLEY_UNLIKELY(!sax->start_object(detail::unknown_size()))) - { - return false; - } - - while (current != '}') - { - if (JSON_HEDLEY_UNLIKELY(!get_ubjson_string(key, false) || !sax->key(key))) - { - return false; - } - if (JSON_HEDLEY_UNLIKELY(!parse_ubjson_internal())) - { - return false; - } - get_ignore_noop(); - key.clear(); - } + return enter_object(size_and_type.first, size_and_type.second); } - return sax->end_object(); + return enter_object(detail::unknown_size()); } // Note, no reader for UBJSON binary types is implemented because they do @@ -3227,6 +3474,9 @@ class binary_reader /// the SAX parser json_sax_t* sax = nullptr; + /// the containers that have been opened and not closed yet; see @ref container_frame + std::vector container_stack{}; + // excluded markers in bjdata optimized type #define JSON_BINARY_READER_MAKE_BJD_OPTIMIZED_TYPE_MARKERS_ \ make_array('F', 'H', 'N', 'S', 'T', 'Z', '[', '{') diff --git a/include/nlohmann/detail/input/input_adapters.hpp b/include/nlohmann/detail/input/input_adapters.hpp index ba8df07a6..bd19d32a8 100644 --- a/include/nlohmann/detail/input/input_adapters.hpp +++ b/include/nlohmann/detail/input/input_adapters.hpp @@ -155,11 +155,31 @@ class input_stream_adapter // General-purpose iterator-based adapter. It might not be as fast as // theoretically possible for some containers, but it is extremely versatile. -// SentinelType defaults to IteratorType for backward compatibility, but may -// be a different type (e.g., a C++20 sentinel or counted_iterator). +// SentinelType defaults to IteratorType for backward compatibility, but may be +// a different type, e.g. a C++20 sentinel such as std::default_sentinel_t when +// IteratorType is a std::counted_iterator. template class iterator_input_adapter { + // Whether the number of elements between two positions can be computed in + // O(1): either the iterator and the sentinel have the same type (plain + // std::distance) or, in C++20, the sentinel is a sized sentinel for the + // iterator (std::ranges::distance), e.g. std::default_sentinel_t paired + // with std::counted_iterator. + // + // JSON_HAS_RANGES gates the C++20 branch: on standard libraries with an + // incomplete (libstdc++ < 11, see #4440) evaluating + // std::contiguous_iterator on a std::counted_iterator is a hard error + // instead of yielding false, and these traits are instantiated for every + // adapter. Such toolchains fall back to the pointer-only test and simply + // use the byte-at-a-time scanner. + static constexpr bool sentinel_is_sized = +#if JSON_HAS_RANGES && defined(__cpp_lib_concepts) && defined(JSON_HAS_CPP_20) + std::is_same::value || std::sized_sentinel_for; +#else + std::is_same::value; +#endif + public: using char_type = typename std::iterator_traits::value_type; @@ -171,7 +191,7 @@ class iterator_input_adapter // in wide_string_input_adapter, which does not expose this). static constexpr bool supports_seek = std::is_same::iterator_category, std::random_access_iterator_tag>::value - && std::is_same::value + && sentinel_is_sized && sizeof(char_type) == 1; iterator_input_adapter(IteratorType first, SentinelType last) @@ -219,30 +239,60 @@ class iterator_input_adapter private: // whether IteratorType refers to a contiguous range and therefore supports // a std::memcpy fast path (pointers always do; in C++20 we can also detect - // library iterators such as those of std::vector and std::string). - // Computing the available element count needs either same-type iterators - // (plain std::distance) or, in C++20, a sized sentinel (std::ranges::distance), - // e.g. std::counted_iterator paired with std::default_sentinel_t. - static constexpr bool iterator_is_contiguous = -#if defined(__cpp_lib_concepts) && defined(JSON_HAS_CPP_20) - (std::is_same::value || std::sized_sentinel_for) - && (std::contiguous_iterator || std::is_pointer::value); + // library iterators such as those of std::vector and std::string). The + // available element count must also be computable in O(1), hence + // sentinel_is_sized. + static constexpr bool iterator_is_contiguous = sentinel_is_sized && +#if JSON_HAS_RANGES && defined(__cpp_lib_concepts) && defined(JSON_HAS_CPP_20) + (std::contiguous_iterator || std::is_pointer::value); #else - std::is_same::value && std::is_pointer::value; + std::is_pointer::value; #endif + // number of unread elements in [current, end) + std::size_t remaining_count() const + { +#if JSON_HAS_RANGES && defined(__cpp_lib_concepts) && defined(JSON_HAS_CPP_20) + // std::ranges::distance also supports sized sentinels of a different + // type (e.g. std::counted_iterator + std::default_sentinel_t) + return static_cast(std::ranges::distance(current, end)); +#else + return static_cast(std::distance(current, end)); +#endif + } + + public: + // Whether the remaining input is a single contiguous block of 1-byte + // elements that the lexer can inspect directly (used for the SWAR string + // fast path). + static constexpr bool supports_bulk_scan = + iterator_is_contiguous && sizeof(char_type) == 1; + + // Pointer to the next unread element; only valid when bulk_remaining() > 0. + const char_type* bulk_data() const + { + return &*current; + } + + // Number of unread elements available as one contiguous block. + std::size_t bulk_remaining() const + { + return remaining_count(); + } + + // Consume @a n elements previously inspected via bulk_data(). + void bulk_skip(std::size_t n) + { + std::advance(current, static_cast::difference_type>(n)); + } + + private: // contiguous fast path: bulk copy the remaining range with std::memcpy template std::size_t get_elements_impl(T* dest, std::size_t count, std::true_type /*contiguous*/) { const std::size_t wanted = count * sizeof(T); -#if defined(__cpp_lib_concepts) && defined(JSON_HAS_CPP_20) - // std::ranges::distance also supports sized sentinels of a different - // type (e.g. std::counted_iterator + std::default_sentinel_t) - const std::size_t available = static_cast(std::ranges::distance(current, end)) * sizeof(char_type); -#else - const std::size_t available = static_cast(std::distance(current, end)) * sizeof(char_type); -#endif + const std::size_t available = remaining_count() * sizeof(char_type); const std::size_t copied = (std::min)(wanted, available); if (JSON_HEDLEY_LIKELY(copied != 0)) { @@ -570,6 +620,46 @@ typename iterator_input_adapter_factory::adapter_typ return factory_type::create(first, last); } +// The element type a container's data() points at, cv-qualifiers removed. +// Ill-formed - and therefore SFINAE-friendly - for types without data(). +template +using container_data_t = typename std::remove_cv().data()) >::type >::type; + +// The container's own element type, cv-qualifiers removed. It is looked up on +// the bare type so it is also found when ContainerType is deduced as a +// reference by the forwarding-reference overload below. +template +using container_value_t = typename std::remove_cv < + typename std::remove_cv::type>::type::value_type >::type; + +// Detect a container that stores its elements contiguously as single bytes +// (std::string, std::vector, std::array, +// std::string_view, ...). Such inputs are wrapped in a pointer-based adapter so +// they benefit from the contiguous fast paths (bulk string scanning, memcpy for +// binary formats) in every C++ standard - not only in C++20, where the standard +// library iterators model std::contiguous_iterator and are detected directly. +// +// data() and size() on their own would be duck typing: they say nothing about +// size() counting the units data() points at, and reading [data(), data() + +// size()) as bytes would be wrong for a type where it does not. Requiring the +// container's own value_type to be that same single-byte element ties the two +// together; every contiguous standard container satisfies it. Anything else +// keeps the iterator-based adapter, which is always correct - only slower. +template +struct is_contiguous_byte_container : std::false_type {}; + +template +struct is_contiguous_byte_container < ContainerType, void_t < + container_data_t, + container_value_t, +decltype(std::declval().size()) >> + : std::integral_constant < bool, + std::is_pointer().data())>::value&& + std::is_integral>::value&& + sizeof(container_data_t) == 1 && + std::is_same, container_value_t>::value > {}; + // Convenience shorthand from container to iterator // Enables ADL on begin(container) and end(container) // Encloses the using declarations in namespace for not to leak them to outside scope @@ -597,12 +687,32 @@ struct container_input_adapter_factory< ContainerType, } // namespace container_input_adapter_factory_impl -template -typename container_input_adapter_factory_impl::container_input_adapter_factory::adapter_type input_adapter(ContainerType&& container) +// General container path (iterator-based). Contiguous single-byte containers +// are excluded here and routed through the pointer-based overload below. +template < typename ContainerType, + enable_if_t < !is_contiguous_byte_container::value, int > = 0 > +typename container_input_adapter_factory_impl::container_input_adapter_factory::adapter_type input_adapter(ContainerType && container) { return container_input_adapter_factory_impl::container_input_adapter_factory::create(std::forward(container)); } +// Contiguous single-byte containers (std::string, std::vector, ...) are +// wrapped in a pointer-based adapter so the contiguous fast paths apply in every +// standard. The pointer keeps the container's own element type (const char* for +// std::string, const std::uint8_t* for std::vector, ...), so the +// resulting char_type - and therefore the parsing behavior - is byte-for-byte +// identical to the iterator-based path; only the raw pointer additionally +// enables the bulk fast paths. The container outlives the adapter for the whole +// parse (temporaries live until the end of the full expression), exactly as the +// iterators it replaces did. +template < typename ContainerType, + enable_if_t < is_contiguous_byte_container::value, int > = 0 > +auto input_adapter(const ContainerType& container) +-> decltype(input_adapter(container.data(), container.data() + container.size())) +{ + return input_adapter(container.data(), container.data() + container.size()); +} + // specialization for std::string using string_input_adapter_type = decltype(input_adapter(std::declval())); diff --git a/include/nlohmann/detail/input/json_sax.hpp b/include/nlohmann/detail/input/json_sax.hpp index 63dd17e9f..60c468f30 100644 --- a/include/nlohmann/detail/input/json_sax.hpp +++ b/include/nlohmann/detail/input/json_sax.hpp @@ -223,12 +223,16 @@ class json_sax_dom_parser bool string(string_t& val) { - handle_value(val); + // json_sax documents that the passed value may be moved from, + // so hand the buffer over instead of copying it + handle_value(std::move(val)); return true; } bool binary(binary_t& val) { + // json_sax documents that the passed value may be moved from, + // so hand the buffer over instead of copying it handle_value(std::move(val)); return true; } @@ -543,12 +547,16 @@ class json_sax_dom_callback_parser bool string(string_t& val) { - handle_value(val); + // json_sax documents that the passed value may be moved from, + // so hand the buffer over instead of copying it + handle_value(std::move(val)); return true; } bool binary(binary_t& val) { + // json_sax documents that the passed value may be moved from, + // so hand the buffer over instead of copying it handle_value(std::move(val)); return true; } @@ -559,6 +567,11 @@ class json_sax_dom_callback_parser const bool keep = callback(static_cast(ref_stack.size()), parse_event_t::object_start, discarded); keep_stack.push_back(keep); + // the key this object will be stored under, read before handle_value() + // may consume it; kept in lockstep with ref_stack so end_object() can + // find the object in its parent again + container_key_stack.push_back(current_key()); + auto val = handle_value(BasicJsonType::value_t::object, true); ref_stack.push_back(val.second); @@ -592,6 +605,9 @@ class json_sax_dom_callback_parser // check callback for the key const bool keep = callback(static_cast(ref_stack.size()), parse_event_t::key, k); key_keep_stack.push_back(keep); + // remember the key so a rejected value can be erased without searching + // the object for it (kept in lockstep with key_keep_stack) + key_stack.push_back(val); // add discarded value at the given key and store the reference for later if (keep && ref_stack.back()) @@ -633,13 +649,16 @@ class json_sax_dom_callback_parser JSON_ASSERT(!ref_stack.empty()); JSON_ASSERT(!keep_stack.empty()); + JSON_ASSERT(!container_key_stack.empty()); ref_stack.pop_back(); keep_stack.pop_back(); + const string_t object_key = std::move(container_key_stack.back()); + container_key_stack.pop_back(); if (!ref_stack.empty() && ref_stack.back() && ref_stack.back()->is_structured()) { // remove discarded value - remove_discarded_value(*ref_stack.back()); + remove_discarded_value(*ref_stack.back(), object_key); } return true; @@ -650,6 +669,9 @@ class json_sax_dom_callback_parser const bool keep = callback(static_cast(ref_stack.size()), parse_event_t::array_start, discarded); keep_stack.push_back(keep); + // see start_object() + container_key_stack.push_back(current_key()); + auto val = handle_value(BasicJsonType::value_t::array, true); ref_stack.push_back(val.second); @@ -722,8 +744,11 @@ class json_sax_dom_callback_parser JSON_ASSERT(!ref_stack.empty()); JSON_ASSERT(!keep_stack.empty()); + JSON_ASSERT(!container_key_stack.empty()); ref_stack.pop_back(); keep_stack.pop_back(); + const string_t object_key = std::move(container_key_stack.back()); + container_key_stack.pop_back(); // remove discarded value if (!ref_stack.empty() && ref_stack.back()) @@ -737,7 +762,7 @@ class json_sax_dom_callback_parser // the array is either still stored under its key or was never // stored, leaving the placeholder key() wrote; both show up as // a discarded member of the parent object - remove_discarded_value(*ref_stack.back()); + remove_discarded_value(*ref_stack.back(), object_key); } } @@ -830,15 +855,56 @@ class json_sax_dom_callback_parser } #endif - /// remove the discarded value the callback rejected from its parent - static void remove_discarded_value(BasicJsonType& parent) + /*! + @brief the key the value now being handled will be stored under + + Empty unless the enclosing container is an object, in which case it is the + key of the pending key() event. Read before handle_value() consumes that + key, so it is also correct when the value never reaches its parent. + */ + string_t current_key() const { - for (auto it = parent.begin(); it != parent.end(); ++it) + if (!ref_stack.empty() && ref_stack.back() && ref_stack.back()->is_object() + && !key_stack.empty()) { - if (it->is_discarded()) + return key_stack.back(); + } + return string_t{}; + } + + /*! + @brief remove the discarded value the callback rejected from its parent + + A rejected value can only ever be the one most recently added to @a parent: + the last element of an array, or the placeholder key() stored under @a key + in an object. Looking there directly makes this O(1) resp. O(log n), where + searching @a parent for it made a filtering parse quadratic in the number of + members of a single container. + + Finding no discarded value there means none was stored in the first place - + the callback rejected the value before it reached its parent - so there is + nothing to remove. + + @param[in,out] parent the container to remove the rejected value from + @param[in] key the key the value was stored under; unused for arrays + */ + static void remove_discarded_value(BasicJsonType& parent, const string_t& key) + { + if (parent.is_array()) + { + auto& array = *parent.m_data.m_value.array; + if (!array.empty() && array.back().is_discarded()) { - parent.erase(it); - break; + array.pop_back(); + } + } + else if (parent.is_object()) + { + auto& object = *parent.m_data.m_value.object; + const auto it = object.find(key); + if (it != object.end() && it->second.is_discarded()) + { + object.erase(it); } } } @@ -888,11 +954,14 @@ class json_sax_dom_callback_parser if (!ref_stack.empty() && ref_stack.back() && ref_stack.back()->is_object()) { JSON_ASSERT(!key_keep_stack.empty()); + JSON_ASSERT(!key_stack.empty()); const bool placeholder_stored = key_keep_stack.back(); key_keep_stack.pop_back(); + const string_t key = std::move(key_stack.back()); + key_stack.pop_back(); if (placeholder_stored) { - remove_discarded_value(*ref_stack.back()); + remove_discarded_value(*ref_stack.back(), key); } } return {false, nullptr}; @@ -925,8 +994,10 @@ class json_sax_dom_callback_parser JSON_ASSERT(ref_stack.back()->is_object()); // check if we should store an element for the current key JSON_ASSERT(!key_keep_stack.empty()); + JSON_ASSERT(!key_stack.empty()); const bool store_element = key_keep_stack.back(); key_keep_stack.pop_back(); + key_stack.pop_back(); if (!store_element) { @@ -946,6 +1017,12 @@ class json_sax_dom_callback_parser std::vector keep_stack {}; // NOLINT(readability-redundant-member-init) /// stack to manage which object keys to keep std::vector key_keep_stack {}; // NOLINT(readability-redundant-member-init) + /// the keys key() stored a placeholder for, in lockstep with key_keep_stack + std::vector key_stack {}; // NOLINT(readability-redundant-member-init) + /// for each open container, the key it is stored under in its parent + /// object, in lockstep with ref_stack; unused where the parent is not an + /// object + std::vector container_key_stack {}; // NOLINT(readability-redundant-member-init) /// helper to hold the reference for the next object element BasicJsonType* object_element = nullptr; /// whether a syntax error occurred diff --git a/include/nlohmann/detail/input/lexer.hpp b/include/nlohmann/detail/input/lexer.hpp index 641d93fa4..bc31337f9 100644 --- a/include/nlohmann/detail/input/lexer.hpp +++ b/include/nlohmann/detail/input/lexer.hpp @@ -19,7 +19,9 @@ #include // vector #include +#include #include +#include #include #include @@ -125,6 +127,25 @@ constexpr bool input_adapter_supports_seek(std::false_type /*detected*/) return false; } +// Detect whether an input adapter exposes a contiguous byte block that the +// lexer can scan directly (see iterator_input_adapter::supports_bulk_scan). +// Adapters without the flag - file, stream, wide-string, user-defined - fall +// back to the character-at-a-time string scanner. +template +using detect_supports_bulk_scan = decltype(InputAdapterType::supports_bulk_scan); + +template +constexpr bool input_adapter_supports_bulk_scan(std::true_type /*detected*/) +{ + return InputAdapterType::supports_bulk_scan; +} + +template +constexpr bool input_adapter_supports_bulk_scan(std::false_type /*detected*/) +{ + return false; +} + /*! @brief lexical analysis @@ -146,13 +167,22 @@ class lexer : public lexer_base static constexpr bool lazy_token_string = input_adapter_supports_seek(is_detected {}); + /// whether string scanning may bulk-consume runs of ordinary characters + /// directly from a contiguous input buffer (SWAR fast path). This requires + /// the token to be reconstructible lazily (lazy_token_string), so bypassing + /// the per-character capture in get() cannot lose error diagnostics. + static constexpr bool bulk_scan = + lazy_token_string + && input_adapter_supports_bulk_scan(is_detected {}); + public: using token_type = typename lexer_base::token_type; - explicit lexer(InputAdapterType&& adapter, bool ignore_comments_ = false) noexcept + explicit lexer(InputAdapterType&& adapter, bool ignore_comments_ = false, bool discard_number_values_ = false) noexcept : ia(std::move(adapter)) , ignore_comments(ignore_comments_) , decimal_point_char(static_cast(get_decimal_point())) + , discard_number_values(discard_number_values_) {} // deleted because of pointer members @@ -265,6 +295,40 @@ class lexer : public lexer_base return true; } + /// contiguous input: bulk-append the run of ordinary characters and complete + /// well-formed UTF-8 sequences starting at the current read position, leaving + /// the first byte that needs individual handling (the closing quote, an + /// escape, a control character, or an ill-formed UTF-8 byte) for get() + void scan_string_bulk(std::true_type /*bulk*/) + { + // a pending unget must be consumed through the normal path first + if (next_unget) + { + return; + } + const std::size_t remaining = ia.bulk_remaining(); + if (remaining == 0) + { + return; + } + const auto* const data = reinterpret_cast(ia.bulk_data()); + + const std::size_t pos = string_bulk_run(data, remaining); + if (pos == 0) + { + return; + } + token_buffer.append(reinterpret_cast(data), pos); + ia.bulk_skip(pos); + // the run contains no newline (all bytes < 0x20 are treated as special), + // so only the flat character counters advance + position.chars_read_total += pos; + position.chars_read_current_line += pos; + } + + /// streaming input: no bulk fast path + void scan_string_bulk(std::false_type /*bulk*/) const noexcept {} + /*! @brief scan a string literal @@ -290,6 +354,10 @@ class lexer : public lexer_base while (true) { + // bulk-consume ordinary characters from contiguous input, then + // handle the next special byte through the switch below + scan_string_bulk(std::integral_constant {}); + // get the next character switch (get()) { @@ -1008,6 +1076,12 @@ class lexer : public lexer_base // changed if minus sign, decimal point, or exponent is read token_type number_type = token_type::value_unsigned; + // offset just past the last mantissa byte in token_buffer (i.e. the + // index of 'e'/'E', or the whole token when there is no exponent). + // convert_number() uses it to count significant digits; npos means + // "not seen an exponent yet" and is resolved at scan_number_done + std::size_t mantissa_end = std::string::npos; + // state (init): we just found out we need to scan a number switch (current) { @@ -1193,6 +1267,9 @@ scan_number_decimal2: scan_number_exponent: // we just parsed an exponent number_type = token_type::value_float; + // this label is reached only right after the 'e'/'E' was appended (from + // the zero, any1, and decimal2 states), so the mantissa ends before it + mantissa_end = token_buffer.size() - 1; switch (get()) { case '+': @@ -1279,45 +1356,199 @@ scan_number_done: // we are done scanning a number) unget(); - char* endptr = nullptr; // NOLINT(misc-const-correctness,cppcoreguidelines-pro-type-vararg,hicpp-vararg) - errno = 0; + // no exponent was scanned: the mantissa spans the whole token + if (mantissa_end == std::string::npos) + { + mantissa_end = token_buffer.size(); + } - // try to parse integers first and fall back to floats + return convert_number(number_type, mantissa_end); + } + + /*! + @brief convert an already-validated integer token to its value + + The digit sequence in [first, last) has been validated by the caller, so a + dedicated parser can avoid the locale/errno overhead of std::strtoull. + + @return the token type on success; token_type::uninitialized if @a + number_type is not an integer type or the value does not fit, in + which case the caller falls back to the floating-point conversion + (matching the previous std::strtoull/std::strtoll behavior) + */ + token_type convert_integer(token_type number_type, const char* first, const char* last) + { if (number_type == token_type::value_unsigned) { - const auto x = std::strtoull(token_buffer.data(), &endptr, 10); - - // we checked the number format before - JSON_ASSERT(endptr == token_buffer.data() + token_buffer.size()); - - if (errno != ERANGE) + if (parse_integer_unsigned(first, last, value_unsigned)) { - value_unsigned = static_cast(x); - if (value_unsigned == x) - { - return token_type::value_unsigned; - } + return token_type::value_unsigned; } } else if (number_type == token_type::value_integer) { - const auto x = std::strtoll(token_buffer.data(), &endptr, 10); - - // we checked the number format before - JSON_ASSERT(endptr == token_buffer.data() + token_buffer.size()); - - if (errno != ERANGE) + if (parse_integer_signed(first, last, value_integer)) { - value_integer = static_cast(x); - if (value_integer == x) - { - return token_type::value_integer; - } + return token_type::value_integer; + } + } + + return token_type::uninitialized; + } + + /*! + @brief check whether Clinger's fast path can still succeed for this token + + parse_float_fast() needs a significand below 2^53. A mantissa with 17 or + more significant digits is at least 10^16 and therefore always exceeds it, + so calling the fast path would walk the token one extra time only to + decline before strtod has to run anyway. + + Significant digits are the mantissa's digits from the first nonzero one on; + the sign, the decimal point, leading zeros, and the exponent do not count. + The answer is derived from indices - the digits are not scanned again - so + this stays off the hot path of the number scanners. + + @param[in] mantissa_end offset just past the last mantissa byte in + token_buffer + @return false if parse_float_fast() is guaranteed to decline + */ + bool mantissa_fits_clinger(std::size_t mantissa_end) const + { + // 10^16 already exceeds 2^53, so 17 digits can never fit + constexpr std::size_t limit = 17; + + const std::size_t neg = (!token_buffer.empty() && token_buffer[0] == '-') ? 1u : 0u; + const std::size_t has_dot = (decimal_point_position != std::string::npos) ? 1u : 0u; + // the JSON grammar restricts the integer part to "0" or [1-9][0-9]*, so + // a leading zero can only be a lone "0", which is not significant + const std::size_t lead_zero = (token_buffer[neg] == '0') ? 1u : 0u; + JSON_ASSERT(mantissa_end >= neg + has_dot + lead_zero); + std::size_t digits = mantissa_end - neg - has_dot - lead_zero; + + if (JSON_HEDLEY_LIKELY(digits < limit)) + { + return true; + } + + // Only a number below 1 can carry further insignificant zeros, and only + // while the count stays at the limit does removing them change the + // answer - so this loop is skipped for all but a few tokens. Note + // token_buffer holds the locale's decimal point, so the fraction is + // located through decimal_point_position rather than by searching '.'. + if (lead_zero != 0) + { + JSON_ASSERT(has_dot != 0); // an integer "0" cannot reach the limit + for (std::size_t i = decimal_point_position + 1; + digits >= limit && i < mantissa_end && token_buffer[i] == '0'; ++i) + { + --digits; + } + } + + return digits < limit; + } + + /*! + @brief convert the number text in token_buffer to its value and token type + + The digit sequence in token_buffer has already been validated (by the + scan_number() state machine or by the contiguous fast path) and holds the + locale decimal point in place of '.'. Integers are parsed first and fall + back to floating point on overflow. This is shared so both scanners produce + identical results. + + @param[in] mantissa_end offset just past the last mantissa byte in + token_buffer (the index of 'e'/'E', or + token_buffer.size() when there is no exponent); + used to skip Clinger's fast path when it cannot + possibly succeed - see mantissa_fits_clinger() + */ + token_type convert_number(token_type number_type, std::size_t mantissa_end) + { + // If the caller does not need the converted value (only whether the + // input is syntactically valid; see json_sax_acceptor/accept()), an + // unsigned/integer token can be reported without calling + // strtoull()/strtoll() at all, *provided* we can already tell from + // the digit count alone that the conversion cannot overflow 64 bits. + // Such tokens are always finite and are accepted unconditionally by + // the parser regardless of their actual value (parser::sax_parse_internal() + // never checks finiteness for value_unsigned/value_integer), so the + // classification below is all that is needed. + // + // A decimal number with up to 18 digits is always representable in + // both std::uint64_t and std::int64_t (18 nines is ~1e18, well below + // both UINT64_MAX ~1.8e19 and INT64_MAX ~9.2e18), so strtoull()/strtoll() + // could not have set errno to ERANGE for it. Numbers with more digits + // (rare in practice) fall through to the exact code below, unchanged, + // so their handling -- including reclassification to value_float when + // the value overflows 64 bits, and rejection when it is not even + // finite as a double -- is bit-for-bit identical to before this + // optimization. + // + // Note this reasons about std::uint64_t/std::int64_t, not about + // number_unsigned_t/number_integer_t (BasicJsonType's own, possibly + // narrower, template parameters -- e.g. std::uint32_t). That is fine + // *only* because discard_number_values is exclusively set by + // accept() (see json.hpp), and accept() always parses through the + // library's own json_sax_acceptor -- never a user-supplied SAX + // consumer -- whose number_unsigned()/number_integer()/number_float() + // callbacks unconditionally discard their argument and return true. + // So for every caller that can reach this branch, neither the token + // classification below nor the eventual (possibly narrowed, and on + // this fast path left stale/unset) value_unsigned/value_integer is + // ever consulted -- an unsigned/integer token is accepted outright, + // and even a >18-digit token that this fast path deliberately falls + // through for is, once reclassified to value_float, still finite + // (and thus accepted) for any digit count that fits in number_unsigned_t + // or number_integer_t regardless of that type's width. If this + // function is ever taught to run with discard_number_values true for + // a caller that *does* read the converted value, this reasoning (and + // the fast path below) would need to be revisited. + if (discard_number_values) + { + constexpr std::size_t safe_digit_count = 18; + if (number_type == token_type::value_unsigned && token_buffer.size() <= safe_digit_count) + { + return token_type::value_unsigned; + } + if (number_type == token_type::value_integer && token_buffer.size() - 1 <= safe_digit_count) + { + return token_type::value_integer; + } + } + + const char* const num_begin = token_buffer.data(); + const char* const num_end = num_begin + token_buffer.size(); + + if (number_type != token_type::value_float) + { + const token_type integer_result = convert_integer(number_type, num_begin, num_end); + if (integer_result != token_type::uninitialized) + { + return integer_result; } } // this code is reached if we parse a floating-point number or if an - // integer conversion above failed + // integer conversion above overflowed. Prefer std::from_chars + // (Eisel-Lemire, locale-independent, correctly rounded) when available; + // otherwise the exact Clinger fast path (double only); otherwise the + // locale-aware strtof/strtod. + if (parse_float_from_chars(num_begin, num_end, value_float)) + { + return token_type::value_float; + } + // Skipping a fast path that cannot succeed is lossless and saves a full + // extra pass over the token's bytes, which otherwise shows up on + // high-precision inputs such as canada.json + if (mantissa_fits_clinger(mantissa_end) + && parse_float_fast(num_begin, num_end, decimal_point_char, value_float)) + { + return token_type::value_float; + } + + char* endptr = nullptr; // NOLINT(misc-const-correctness,cppcoreguidelines-pro-type-vararg,hicpp-vararg) strtof(value_float, token_buffer.data(), &endptr); // we checked the number format before @@ -1326,6 +1557,158 @@ scan_number_done: return token_type::value_float; } + /*! + @brief contiguous fast path for scanning a number + + Parses the whole number token straight from the input buffer, avoiding the + per-character get()/add() of scan_number(). On success it fills token_buffer + (with the locale decimal point substituted, as scan_number() does) and + returns the token type. On anything it does not fully recognize as a + well-formed number it makes no state change and returns + token_type::uninitialized, so the caller falls back to scan_number(), which + then produces the exact diagnostic. @a current is the first digit or the + leading minus (already read); the remaining bytes are taken from the adapter. + */ + token_type scan_number_bulk_contiguous() + { + // a pending unget offsets the buffer position from current; fall back + if (next_unget) + { + return token_type::uninitialized; + } + const std::size_t rem = ia.bulk_remaining(); + if (rem == 0) + { + // the first digit is the last input byte; let scan_number() finish + return token_type::uninitialized; + } + // the byte before the next unread one is current (contiguous input) + const char* const data = reinterpret_cast(ia.bulk_data()) - 1; + const std::size_t avail = rem + 1; + + // validate + classify the number extent (mirrors scan_number()'s grammar) + std::size_t i = 0; + std::size_t dot_index = std::string::npos; + token_type number_type = token_type::value_unsigned; + if (data[0] == '-') + { + number_type = token_type::value_integer; + i = 1; + if (i >= avail) + { + return token_type::uninitialized; + } + } + if (data[i] == '0') + { + ++i; + } + else if (data[i] >= '1' && data[i] <= '9') + { + ++i; + while (i < avail && data[i] >= '0' && data[i] <= '9') + { + ++i; + } + } + else + { + return token_type::uninitialized; + } + if (i < avail && data[i] == '.') + { + number_type = token_type::value_float; + dot_index = i; + ++i; + if (i >= avail || !(data[i] >= '0' && data[i] <= '9')) + { + return token_type::uninitialized; + } + while (i < avail && data[i] >= '0' && data[i] <= '9') + { + ++i; + } + } + // the mantissa ends here, whether or not an exponent part follows + const std::size_t mantissa_end = i; + if (i < avail && (data[i] == 'e' || data[i] == 'E')) + { + number_type = token_type::value_float; + ++i; + if (i < avail && (data[i] == '+' || data[i] == '-')) + { + ++i; + } + if (i >= avail || !(data[i] >= '0' && data[i] <= '9')) + { + return token_type::uninitialized; + } + while (i < avail && data[i] >= '0' && data[i] <= '9') + { + ++i; + } + } + const std::size_t len = i; + + // reset() records where this token starts (for diagnostics), so it has + // to run before the input position advances below + reset(); + + // An integer token needs no token_buffer: the SAX callbacks for + // number_integer/number_unsigned take only the value, and the overflow + // diagnostic rebuilds the text from the input. Convert straight from the + // input buffer and leave token_buffer empty. (JSON_DIAGNOSTIC_POSITIONS + // derives a number's start position from get_string().size(), so there + // the token still has to be materialized.) +#if !JSON_DIAGNOSTIC_POSITIONS + if (number_type != token_type::value_float) + { + const token_type integer_result = convert_integer(number_type, data, data + len); + if (JSON_HEDLEY_LIKELY(integer_result != token_type::uninitialized)) + { + ia.bulk_skip(len - 1); + position.chars_read_total += (len - 1); + position.chars_read_current_line += (len - 1); + return integer_result; + } + // The value does not fit an integer, so this token converts as a + // float. Recording that here keeps convert_number() below from + // repeating the integer attempt that just failed. + number_type = token_type::value_float; + } +#endif + + // materialize the token exactly as scan_number() would, substituting the + // locale decimal point so convert_number()'s strtof fallback stays valid. + // reset() already cleared token_buffer, so append() fills it (assign() is + // avoided because custom string_t types need not provide it) + token_buffer.append(reinterpret_cast(data), len); + if (dot_index != std::string::npos) + { + token_buffer[dot_index] = static_cast(decimal_point_char); + decimal_point_position = dot_index; + } + + ia.bulk_skip(len - 1); + position.chars_read_total += (len - 1); + position.chars_read_current_line += (len - 1); + + return convert_number(number_type, mantissa_end); + } + + /// contiguous input: try the number fast path, else the byte-path scanner + token_type scan_number_dispatch(std::true_type /*bulk*/) + { + const token_type t = scan_number_bulk_contiguous(); + return (t != token_type::uninitialized) ? t : scan_number(); + } + + /// streaming input: always use the byte-path scanner + token_type scan_number_dispatch(std::false_type /*bulk*/) + { + return scan_number(); + } + /*! @param[in] literal_text the literal text to expect @param[in] length the length of the passed literal text @@ -1393,8 +1776,7 @@ scan_number_done: */ char_int_type get() { - ++position.chars_read_total; - ++position.chars_read_current_line; + advance_position(); if (next_unget) { @@ -1406,6 +1788,23 @@ scan_number_done: current = ia.get_character(); } + return track_after_read(); + } + + /// shared head of get() / get_ignoring_pending_unget(): bump the + /// per-character position counters (line-count-on-'\n' bookkeeping is + /// handled afterwards, in track_after_read(), once `current` is known) + void advance_position() noexcept + { + ++position.chars_read_total; + ++position.chars_read_current_line; + } + + /// shared tail of get() / get_ignoring_pending_unget(): capture the + /// character for error messages (if needed) and update line/column + /// bookkeeping for the character now in `current` + char_int_type track_after_read() + { // seekable adapters reconstruct the token lazily on error (see // get_token_string), so the eager per-character copy is skipped capture_char(std::integral_constant {}); @@ -1413,12 +1812,38 @@ scan_number_done: if (current == '\n') { ++position.lines_read; + // remember the column the newline was read at: chars_read_current_line + // is about to be cleared, and a matching unget() cannot reconstruct it + chars_read_before_newline = position.chars_read_current_line; position.chars_read_current_line = 0; } return current; } + /*! + @brief like get(), but for call sites that can prove no unget() is pending + + get() has to check the `next_unget` flag on every call, because a + previous token may have ended with unget() (e.g. scan_number() always + ungets the character that terminated the number, so the next call to + scan() can see it again). skip_whitespace() reads that first, + possibly-ungotten character via a plain get(), but every further + character it reads is guaranteed to be a fresh read: nothing between + those calls invokes unget(). This variant skips the (otherwise always + false) next_unget branch for those calls; it is not a general + replacement for get(). + */ + char_int_type get_ignoring_pending_unget() + { + JSON_ASSERT(!next_unget); + + advance_position(); + current = ia.get_character(); + + return track_after_read(); + } + /// seekable adapter: nothing to capture, the token is rebuilt on error void capture_char(std::true_type /*lazy*/) const noexcept {} @@ -1446,12 +1871,20 @@ scan_number_done: --position.chars_read_total; // in case we "unget" a newline, we have to also decrement the lines_read + // and restore the column that get() cleared when it saw the newline; + // chars_read_current_line == 0 can only mean the last get() read one if (position.chars_read_current_line == 0) { if (position.lines_read > 0) { --position.lines_read; } + + // chars_read_before_newline counts the newline itself, which is the + // character being ungotten, hence the -1 + position.chars_read_current_line = (chars_read_before_newline > 0) + ? chars_read_before_newline - 1 + : 0; } else { @@ -1612,13 +2045,37 @@ scan_number_done: return true; } + /// whether `current` is one of the four JSON whitespace characters + bool current_is_whitespace() const noexcept + { + return current == ' ' || current == '\t' || current == '\n' || current == '\r'; + } + void skip_whitespace() { + // the first character may be a pending unget() left over from the + // previous token (see get_ignoring_pending_unget()); every + // subsequent character read by this loop is guaranteed fresh, since + // nothing below calls unget() + get(); + + if (!current_is_whitespace()) + { + return; + } + + // this is written as an if-guarded do-while (rather than a plain + // while loop) because that shape is what lets both GCC and Clang + // keep the input adapter's read pointer in a register across + // iterations; the equivalent while-loop measurably defeated that + // optimization in testing, turning long whitespace runs (e.g. the + // indentation of pretty-printed JSON) from a register-only loop + // into one that reloads the pointer from memory every character do { - get(); + get_ignoring_pending_unget(); } - while (current == ' ' || current == '\t' || current == '\n' || current == '\r'); + while (current_is_whitespace()); } token_type scan() @@ -1694,7 +2151,7 @@ scan_number_done: case '7': case '8': case '9': - return scan_number(); + return scan_number_dispatch(std::integral_constant {}); // end of input (the null byte is needed when parsing from // string literals) @@ -1725,6 +2182,10 @@ scan_number_done: /// the start position of the current token position_t position {}; + /// the value chars_read_current_line had when the last newline was read, so + /// that unget() can restore the column instead of leaving it at 0 + std::size_t chars_read_before_newline = 0; + /// raw input token string for error messages; only populated for streaming /// adapters (seekable adapters reconstruct it lazily via token_string_start) std::vector token_string {}; @@ -1754,6 +2215,13 @@ scan_number_done: const char_int_type decimal_point_char = '.'; /// the position of the decimal point in the input std::size_t decimal_point_position = std::string::npos; + + /// whether the caller (e.g. accept()/json_sax_acceptor) only needs the + /// token classification and never looks at the converted numeric value; + /// when set, scan_number() may skip strtoull()/strtoll() for + /// value_unsigned/value_integer tokens whose digit count guarantees they + /// fit into 64 bits (see scan_number()) + const bool discard_number_values = false; }; } // namespace detail diff --git a/include/nlohmann/detail/input/number_parse.hpp b/include/nlohmann/detail/input/number_parse.hpp new file mode 100644 index 000000000..e50c3f67f --- /dev/null +++ b/include/nlohmann/detail/input/number_parse.hpp @@ -0,0 +1,302 @@ +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ +// | | |__ | | | | | | version 3.12.0 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann +// SPDX-License-Identifier: MIT + +#pragma once + +#include // array +#include // FLT_EVAL_METHOD +#include // size_t +#include // int64_t, uint64_t +#include // numeric_limits + +#include + +// std::from_chars lives in , but being in C++17 mode does not +// guarantee the header exists: GCC 7 sets __cplusplus to C++17 yet ships no +// (added in GCC 8; floating-point support in GCC 11). Guard the +// include with __has_include so such toolchains fall back to the scalar path. +#if defined(JSON_HAS_CPP_17) && defined(__has_include) + #if __has_include() + #include // from_chars (only used when __cpp_lib_to_chars is defined) + #include // errc + #endif +#endif + +// This file contains the value-conversion helpers used by the lexer to turn an +// already-validated number token into a value, without the locale/errno +// overhead of std::strtoull/std::strtod. They are free functions so the lexer +// stays focused on scanning; see lexer::convert_number(). + +NLOHMANN_JSON_NAMESPACE_BEGIN +namespace detail +{ + +/*! +@brief fast integer parser for an already-validated unsigned integer + +The number scanner has already checked that [first, last) is a valid JSON +integer, so this only needs to accumulate the digits and detect overflow. This +avoids the locale/errno machinery of std::strtoull, which dominates +integer-heavy inputs. + +@param[in] first pointer to the first character (a digit) +@param[in] last pointer past the last character +@param[out] value the parsed value on success +@return true if the value fit into @a NumberUnsignedType; false on overflow, in + which case the caller falls back to floating-point parsing (matching the + previous std::strtoull behavior) +*/ +template +bool parse_integer_unsigned(const char* first, const char* last, NumberUnsignedType& value) noexcept +{ + // accumulate in the widest unsigned type used by the previous strtoull + // path so the overflow behavior is unchanged for custom number types + std::uint64_t x = 0; + constexpr std::uint64_t cutoff = (std::numeric_limits::max)() / 10u; + constexpr std::uint64_t cutlim = (std::numeric_limits::max)() % 10u; + for (const char* p = first; p != last; ++p) + { + const auto digit = static_cast(static_cast(*p) - static_cast('0')); + if (JSON_HEDLEY_UNLIKELY(x > cutoff || (x == cutoff && digit > cutlim))) + { + return false; + } + x = (x * 10u) + digit; + } + value = static_cast(x); + // reject values that do not round-trip into a narrower NumberUnsignedType + return static_cast(value) == x; +} + +/*! +@brief fast integer parser for an already-validated negative integer + +@param[in] first pointer to the leading '-' +@param[in] last pointer past the last character +@param[out] value the parsed (negative) value on success +@return true on success; false on overflow (caller falls back to float) +*/ +template +bool parse_integer_signed(const char* first, const char* last, NumberIntegerType& value) noexcept +{ + // the state machine only reaches the signed path via a leading '-' + JSON_ASSERT(first != last && *first == '-'); + std::uint64_t magnitude = 0; + // |INT64_MIN| == INT64_MAX + 1; this is the largest admissible magnitude + constexpr std::uint64_t limit = static_cast((std::numeric_limits::max)()) + 1u; + for (const char* p = first + 1; p != last; ++p) + { + const auto digit = static_cast(static_cast(*p) - static_cast('0')); + if (JSON_HEDLEY_UNLIKELY(magnitude > (limit - digit) / 10u)) + { + return false; + } + magnitude = (magnitude * 10u) + digit; + } + const std::int64_t x = (magnitude == limit) + ? (std::numeric_limits::min)() + : -static_cast(magnitude); + value = static_cast(x); + // reject values that do not round-trip into a narrower NumberIntegerType + return static_cast(value) == x; +} + +/*! +@brief exact fast path for parsing a `double` (Clinger's algorithm) + +For the common case - at most 19 significant digits, a decimal exponent in +[-22, 22], and a significand below 2^53 - the value equals significand * +10^exp computed in IEEE-754 double arithmetic, which is exact under +round-to-nearest because both operands are exactly representable. This is the +same fast path used by fast_float/simdjson; the general cases are left to +std::strtod. The parser only activates for number_float_t == double; float and +long double keep the std::strtof/std::strtold paths (see the templated overload +below). + +@param[in] first pointer to the first character of the number +@param[in] last pointer past the last character +@param[in] decimal_point the (locale-dependent) decimal point character +@param[out] out the parsed value on success +@return true if the value was parsed exactly; false to fall back to strtod +*/ +template +bool parse_float_fast(const char* first, const char* last, DecimalPointType decimal_point, double& out) noexcept +{ +#if defined(FLT_EVAL_METHOD) && FLT_EVAL_METHOD != 0 + // Clinger's fast path is only exact when double operations are evaluated in + // true double precision. On platforms that keep intermediates in extended + // precision (e.g. the x87 FPU on 32-bit x86, where FLT_EVAL_METHOD == 2) the + // single significand * 10^scale step is double-rounded and can be 1 ULP off, + // so decline and let the caller fall back to the correctly-rounded + // std::from_chars / std::strtod path. + static_cast(first); + static_cast(last); + static_cast(decimal_point); + static_cast(out); + return false; +#else + static const std::array powers_of_ten = + { + { + 1e0, 1e1, 1e2, 1e3, 1e4, 1e5, 1e6, 1e7, 1e8, 1e9, 1e10, 1e11, + 1e12, 1e13, 1e14, 1e15, 1e16, 1e17, 1e18, 1e19, 1e20, 1e21, 1e22 + } + }; + + const char* p = first; + bool negative = false; + if (p != last && (*p == '-' || *p == '+')) + { + negative = (*p == '-'); + ++p; + } + + std::uint64_t significand = 0; + int num_digits = 0; + int fractional_digits = 0; + bool seen_dot = false; + bool any_digit = false; + for (; p != last; ++p) + { + const char c = *p; + if (c >= '0' && c <= '9') + { + any_digit = true; + if (JSON_HEDLEY_UNLIKELY(num_digits >= 19)) + { + return false; // significand may not fit into uint64_t + } + significand = (significand * 10u) + static_cast(c - '0'); + ++num_digits; + fractional_digits += static_cast(seen_dot); + } + else if (static_cast(c) == decimal_point) + { + if (JSON_HEDLEY_UNLIKELY(seen_dot)) + { + return false; + } + seen_dot = true; + } + else if (c == 'e' || c == 'E') + { + ++p; + break; + } + else + { + return false; + } + } + if (JSON_HEDLEY_UNLIKELY(!any_digit)) + { + return false; + } + + int exponent = 0; + if (p != last) // an exponent part remains + { + bool exp_negative = false; + if (p != last && (*p == '-' || *p == '+')) + { + exp_negative = (*p == '-'); + ++p; + } + bool any_exp_digit = false; + for (; p != last; ++p) + { + if (JSON_HEDLEY_UNLIKELY(*p < '0' || *p > '9')) + { + return false; + } + exponent = (exponent * 10) + (*p - '0'); + any_exp_digit = true; + if (JSON_HEDLEY_UNLIKELY(exponent > 9999)) + { + return false; + } + } + if (JSON_HEDLEY_UNLIKELY(!any_exp_digit)) + { + return false; + } + if (exp_negative) + { + exponent = -exponent; + } + } + + const int scale = exponent - fractional_digits; + if (JSON_HEDLEY_UNLIKELY(significand >= (static_cast(1) << 53))) + { + return false; // significand not exactly representable as double + } + + auto result = static_cast(significand); + if (scale >= 0) + { + if (JSON_HEDLEY_UNLIKELY(scale > 22)) + { + return false; + } + result *= powers_of_ten[static_cast(scale)]; + } + else + { + if (JSON_HEDLEY_UNLIKELY(-scale > 22)) + { + return false; + } + result /= powers_of_ten[static_cast(-scale)]; + } + out = negative ? -result : result; + return true; +#endif +} + +/// fast float path is only exact for `double`; decline for float/long double +template +bool parse_float_fast(const char* /*first*/, const char* /*last*/, DecimalPointType /*decimal_point*/, FloatType& /*out*/) noexcept +{ + return false; +} + +/*! +@brief parse a float with std::from_chars (Eisel-Lemire) when available + +std::from_chars is locale-independent, correctly rounded, and - via the +Eisel-Lemire algorithm in modern standard libraries - much faster than strtod +over the whole value range (not just the Clinger subset). It is used only when +__cpp_lib_to_chars indicates full floating-point support and only when it +consumes the entire token ([first, last)); a partial parse means the buffer +uses a non-'.' locale decimal point, in which case the caller falls back to the +locale-aware path. An under-/overflow (result_out_of_range) also declines, so +the caller's strtod fallback supplies the well-defined ±inf/0 result the parser +expects (side-stepping the P4168 divergence between implementations). + +@return true if the value was parsed exactly and fully; false to fall back +*/ +template +bool parse_float_from_chars(const char* first, const char* last, FloatType& out) noexcept +{ + // JSON_HAS_CPP_17 must gate the use as well as the include above: + // some standard libraries (e.g. libstdc++ 15) define __cpp_lib_to_chars even + // in C++14 mode, where is not included. +#if defined(JSON_HAS_CPP_17) && defined(__cpp_lib_to_chars) + const auto result = std::from_chars(first, last, out); + return result.ec == std::errc() && result.ptr == last; +#else + static_cast(first); + static_cast(last); + static_cast(out); + return false; +#endif +} + +} // namespace detail +NLOHMANN_JSON_NAMESPACE_END diff --git a/include/nlohmann/detail/input/parser.hpp b/include/nlohmann/detail/input/parser.hpp index ecc78eb72..a45ee4a0a 100644 --- a/include/nlohmann/detail/input/parser.hpp +++ b/include/nlohmann/detail/input/parser.hpp @@ -72,9 +72,10 @@ class parser parser_callback_t cb = nullptr, const bool allow_exceptions_ = true, const bool ignore_comments = false, - const bool ignore_trailing_commas_ = false) + const bool ignore_trailing_commas_ = false, + const bool discard_number_values_ = false) : callback(std::move(cb)) - , m_lexer(std::move(adapter), ignore_comments) + , m_lexer(std::move(adapter), ignore_comments, discard_number_values_) , allow_exceptions(allow_exceptions_) , ignore_trailing_commas(ignore_trailing_commas_) { diff --git a/include/nlohmann/detail/input/string_scan.hpp b/include/nlohmann/detail/input/string_scan.hpp new file mode 100644 index 000000000..6af0e6c5d --- /dev/null +++ b/include/nlohmann/detail/input/string_scan.hpp @@ -0,0 +1,287 @@ +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ +// | | |__ | | | | | | version 3.12.0 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann +// SPDX-License-Identifier: MIT + +#pragma once + +#include // size_t +#include // uint64_t +#include // memcpy + +#include + +// Optional SIMD backend for bulk UTF-8 validation. This is an opt-in external +// dependency: nlohmann/json itself stays header-only and the C++11 scalar +// validator below is always available; defining JSON_USE_SIMDUTF additionally +// requires the simdutf headers on the include path and linking the simdutf +// library. See string_bulk_run(). +// +// simdutf.h itself requires C++17 - it rejects older standards with an #error - +// so the backend is only compiled in from C++17 on. Below that the macro has no +// effect and the scalar validator is used; it accepts and rejects exactly the +// same input, so only throughput differs. macro_scope.hpp is included above to +// have JSON_HAS_CPP_17 available for this test. +#if defined(JSON_USE_SIMDUTF) && defined(JSON_HAS_CPP_17) + #include +#endif + +// This file contains the byte-level string-scanning helpers used by the lexer's +// contiguous fast path. They operate purely on raw bytes (no dependency on the +// lexer's template parameters) so they are free functions, keeping the lexer +// itself focused on the state machine; see lexer::scan_string_bulk(). + +NLOHMANN_JSON_NAMESPACE_BEGIN +namespace detail +{ + +// classify a single byte as needing individual string handling: the closing +// quote, an escape, a control character, or a non-ASCII (UTF-8) +// lead/continuation byte. Ordinary bytes (0x20..0x7F except '"' and '\\') are +// copied verbatim, which the bulk scanner does 8 bytes at a time. +inline bool is_string_special(unsigned char c) noexcept +{ + return c == '\"' || c == '\\' || c < 0x20u || c >= 0x80u; +} + +// SWAR helper: return a word whose high bit is set in every byte of @a v that +// is_string_special(); zero if the 8 bytes are all ordinary. +inline std::uint64_t swar_string_special(std::uint64_t v) noexcept +{ + constexpr std::uint64_t ones = 0x0101010101010101ull; + constexpr std::uint64_t high = 0x8080808080808080ull; + const std::uint64_t q = v ^ 0x2222222222222222ull; // '"' (0x22) + const std::uint64_t b = v ^ 0x5C5C5C5C5C5C5C5Cull; // '\\' (0x5C) + const std::uint64_t has_quote = (q - ones) & ~q & high; + const std::uint64_t has_backslash = (b - ones) & ~b & high; + const std::uint64_t has_control = (v - 0x2020202020202020ull) & ~v & high; // < 0x20 + const std::uint64_t has_non_ascii = v & high; // >= 0x80 + return has_quote | has_backslash | has_control | has_non_ascii; +} + +// return the index of the first is_string_special() byte in [data, data+n), or +// n if every byte is ordinary; scans 8 bytes at a time +inline std::size_t find_string_special(const unsigned char* data, std::size_t n) noexcept +{ + std::size_t i = 0; + for (; i + 8 <= n; i += 8) + { + std::uint64_t word = 0; + std::memcpy(&word, data + i, sizeof(word)); + if (swar_string_special(word) != 0) + { + // a special byte is in this word; locate it (endian-agnostic) + for (std::size_t j = 0; j < 8; ++j) + { + if (is_string_special(data[i + j])) + { + return i + j; + } + } + } + } + for (; i < n; ++i) + { + if (is_string_special(data[i])) + { + return i; + } + } + return n; +} + +// classify a byte as one the serializer must NOT copy verbatim when +// ensure_ascii is requested: the closing quote, an escape, a control character +// (< 0x20), DEL (0x7F), or any non-ASCII byte (>= 0x80). Everything else - +// printable ASCII except '"' and '\\' - is emitted unchanged. Note this differs +// from is_string_special() only in that 0x7F is also a stop (it is escaped as +// \u007f under ensure_ascii). +inline bool is_ascii_copyable(unsigned char c) noexcept +{ + return c >= 0x20u && c < 0x7Fu && c != '"' && c != '\\'; +} + +// return the index of the first byte in [data, data+n) that is NOT +// is_ascii_copyable(), or n if every byte can be copied verbatim; scans 8 bytes +// at a time. Used by the serializer's ensure_ascii fast path. +inline std::size_t find_ascii_copyable_run(const unsigned char* data, std::size_t n) noexcept +{ + constexpr std::uint64_t ones = 0x0101010101010101ull; + constexpr std::uint64_t high = 0x8080808080808080ull; + std::size_t i = 0; + for (; i + 8 <= n; i += 8) + { + std::uint64_t v = 0; + std::memcpy(&v, data + i, sizeof(v)); + const std::uint64_t q = v ^ 0x2222222222222222ull; // '"' (0x22) + const std::uint64_t b = v ^ 0x5C5C5C5C5C5C5C5Cull; // '\\' (0x5C) + const std::uint64_t d = v ^ 0x7F7F7F7F7F7F7F7Full; // DEL (0x7F) + const std::uint64_t stop = ((q - ones) & ~q & high) // == '"' + | ((b - ones) & ~b & high) // == '\\' + | ((d - ones) & ~d & high) // == 0x7F + | ((v - 0x2020202020202020ull) & ~v & high) // < 0x20 + | (v & high); // >= 0x80 + if (stop != 0) + { + break; + } + } + for (; i < n; ++i) + { + if (!is_ascii_copyable(data[i])) + { + return i; + } + } + return n; +} + +// Validate one UTF-8 sequence at the front of [data, data+avail). Returns its +// length (2..4) only when the bytes form a *well-formed* sequence using exactly +// the same ranges as scan_string()'s per-byte switch, so the bulk path accepts +// precisely what the byte path accepts. Returns 0 for anything that is invalid, +// incomplete, or that the byte path must diagnose (the caller then defers to +// that path, keeping error messages unchanged). Lead bytes < 0x80 are handled +// by the caller and never passed here. +inline std::size_t validate_one_utf8(const unsigned char* data, std::size_t avail) noexcept +{ + const unsigned char c0 = data[0]; + if (c0 >= 0xC2 && c0 <= 0xDF) // U+0080..U+07FF + { + if (avail >= 2 && data[1] >= 0x80 && data[1] <= 0xBF) + { + return 2; + } + } + else if (c0 == 0xE0) // U+0800..U+0FFF + { + if (avail >= 3 && data[1] >= 0xA0 && data[1] <= 0xBF && data[2] >= 0x80 && data[2] <= 0xBF) + { + return 3; + } + } + else if ((c0 >= 0xE1 && c0 <= 0xEC) || c0 == 0xEE || c0 == 0xEF) // U+1000..U+CFFF, U+E000..U+FFFF + { + if (avail >= 3 && data[1] >= 0x80 && data[1] <= 0xBF && data[2] >= 0x80 && data[2] <= 0xBF) + { + return 3; + } + } + else if (c0 == 0xED) // U+D000..U+D7FF (excludes surrogates) + { + if (avail >= 3 && data[1] >= 0x80 && data[1] <= 0x9F && data[2] >= 0x80 && data[2] <= 0xBF) + { + return 3; + } + } + else if (c0 == 0xF0) // U+10000..U+3FFFF + { + if (avail >= 4 && data[1] >= 0x90 && data[1] <= 0xBF && data[2] >= 0x80 && data[2] <= 0xBF && data[3] >= 0x80 && data[3] <= 0xBF) + { + return 4; + } + } + else if (c0 >= 0xF1 && c0 <= 0xF3) // U+40000..U+FFFFF + { + if (avail >= 4 && data[1] >= 0x80 && data[1] <= 0xBF && data[2] >= 0x80 && data[2] <= 0xBF && data[3] >= 0x80 && data[3] <= 0xBF) + { + return 4; + } + } + else if (c0 == 0xF4) // U+100000..U+10FFFF + { + if (avail >= 4 && data[1] >= 0x80 && data[1] <= 0x8F && data[2] >= 0x80 && data[2] <= 0xBF && data[3] >= 0x80 && data[3] <= 0xBF) + { + return 4; + } + } + return 0; // invalid, incomplete, or must be diagnosed by the byte path +} + +// Scalar (C++11) computation of the bulk run length: the number of leading +// bytes in [data, data+n) that are ordinary ASCII or complete well-formed UTF-8 +// sequences, stopping before the first byte that needs individual handling (the +// closing quote, an escape, a control character, or an ill-formed/truncated +// sequence). ASCII is skipped 8 bytes at a time. +inline std::size_t scalar_string_bulk_run(const unsigned char* data, std::size_t n) noexcept +{ + std::size_t pos = 0; + while (pos < n) + { + pos += find_string_special(data + pos, n - pos); + if (pos >= n || data[pos] < 0x80u) + { + break; // end of buffer, or a quote/escape/control byte + } + const std::size_t seq = validate_one_utf8(data + pos, n - pos); + if (seq == 0) + { + break; // ill-formed or truncated: let the byte path diagnose it + } + pos += seq; + } + return pos; +} + +#if defined(JSON_USE_SIMDUTF) && defined(JSON_HAS_CPP_17) +// Index of the first quote/escape/control byte in [data, data+n) (non-ASCII +// bytes are *not* stops here - the whole run is handed to simdutf), or n. +inline std::size_t find_string_delimiter(const unsigned char* data, std::size_t n) noexcept +{ + constexpr std::uint64_t ones = 0x0101010101010101ull; + constexpr std::uint64_t high = 0x8080808080808080ull; + std::size_t i = 0; + for (; i + 8 <= n; i += 8) + { + std::uint64_t v = 0; + std::memcpy(&v, data + i, sizeof(v)); + const std::uint64_t q = v ^ 0x2222222222222222ull; + const std::uint64_t b = v ^ 0x5C5C5C5C5C5C5C5Cull; + const std::uint64_t hit = ((q - ones) & ~q & high) + | ((b - ones) & ~b & high) + | ((v - 0x2020202020202020ull) & ~v & high); + if (hit != 0) + { + for (std::size_t j = 0; j < 8; ++j) + { + const unsigned char c = data[i + j]; + if (c == '\"' || c == '\\' || c < 0x20u) + { + return i + j; + } + } + } + } + for (; i < n; ++i) + { + const unsigned char c = data[i]; + if (c == '\"' || c == '\\' || c < 0x20u) + { + return i; + } + } + return n; +} +#endif + +// Backend-dispatched bulk run length. With JSON_USE_SIMDUTF the run up to the +// next delimiter is validated in one shot by simdutf; on the rare failure the +// scalar helper recomputes the exact valid prefix so the byte path still +// produces the precise diagnostic. Without it, the pure scalar path is used. +inline std::size_t string_bulk_run(const unsigned char* data, std::size_t n) noexcept +{ +#if defined(JSON_USE_SIMDUTF) && defined(JSON_HAS_CPP_17) + const std::size_t run = find_string_delimiter(data, n); + if (run != 0 && simdutf::validate_utf8(reinterpret_cast(data), run)) + { + return run; + } +#endif + return scalar_string_bulk_run(data, n); +} + +} // namespace detail +NLOHMANN_JSON_NAMESPACE_END diff --git a/include/nlohmann/detail/iterators/iteration_proxy.hpp b/include/nlohmann/detail/iterators/iteration_proxy.hpp index 99246d120..c8aa50dd2 100644 --- a/include/nlohmann/detail/iterators/iteration_proxy.hpp +++ b/include/nlohmann/detail/iterators/iteration_proxy.hpp @@ -18,6 +18,7 @@ #endif #include +#include #include #include #include @@ -206,10 +207,10 @@ NLOHMANN_JSON_NAMESPACE_END namespace std { +// Fix: https://github.com/nlohmann/json/issues/1401 #if defined(__clang__) - // Fix: https://github.com/nlohmann/json/issues/1401 - #pragma clang diagnostic push - #pragma clang diagnostic ignored "-Wmismatched-tags" + JSON_HEDLEY_DIAGNOSTIC_PUSH + JSON_HEDLEY_PRAGMA(clang diagnostic ignored "-Wmismatched-tags") #endif template class tuple_size<::nlohmann::detail::iteration_proxy_value> // NOLINT(cert-dcl58-cpp) @@ -224,7 +225,7 @@ class tuple_element> ::nlohmann::detail::iteration_proxy_value> ())); }; #if defined(__clang__) - #pragma clang diagnostic pop + JSON_HEDLEY_DIAGNOSTIC_POP #endif } // namespace std diff --git a/include/nlohmann/detail/json_pointer.hpp b/include/nlohmann/detail/json_pointer.hpp index 576ba62cc..247c5babb 100644 --- a/include/nlohmann/detail/json_pointer.hpp +++ b/include/nlohmann/detail/json_pointer.hpp @@ -748,6 +748,20 @@ class json_pointer } } + // the reference token consists only of digits at this point (cf. checks + // above); however, its numeric value might not be representable, in which + // case array_index() would throw out_of_range.404/410 -- contains() must + // not throw (see #5395), so such a reference token is treated as "not found" + errno = 0; // strtoull() does not reset errno on success + char* p_end = nullptr; // NOLINT(misc-const-correctness) + const unsigned long long magnitude = std::strtoull(reference_token.c_str(), &p_end, 10); // NOLINT(runtime/int) + if (JSON_HEDLEY_UNLIKELY(errno == ERANGE // the value exceeds ULLONG_MAX + || magnitude >= static_cast((std::numeric_limits::max)()))) // NOLINT(runtime/int) + { + // the array index cannot be represented as size_type + return false; + } + const auto idx = array_index(reference_token); if (idx >= ptr->size()) { diff --git a/include/nlohmann/detail/output/binary_writer.hpp b/include/nlohmann/detail/output/binary_writer.hpp index b8e9efa45..28290de3e 100644 --- a/include/nlohmann/detail/output/binary_writer.hpp +++ b/include/nlohmann/detail/output/binary_writer.hpp @@ -826,7 +826,17 @@ class binary_writer std::vector bjdx = {'[', '{', 'S', 'H', 'T', 'F', 'N', 'Z'}; // excluded markers in bjdata optimized type - if (same_prefix && !(use_bjdata && std::find(bjdx.begin(), bjdx.end(), first_prefix) != bjdx.end())) + // an optimized array of a valueless type carries no payload, so a + // reader has nothing but the declared count to bound the allocation + // by and refuses an excessive one. Write the unoptimized form for + // those, at one byte per element, so the result can be read back. + // Objects are not affected: every element is preceded by its key. + const bool valueless_type = (first_prefix == 'Z' || first_prefix == 'T' || first_prefix == 'F'); + const bool excessive_valueless = valueless_type + && j.m_data.m_value.array->size() > detail::max_valueless_container_size; + + if (same_prefix && !excessive_valueless + && !(use_bjdata && std::find(bjdx.begin(), bjdx.end(), first_prefix) != bjdx.end())) { prefix_required = false; oa->write_character(to_char_type('$')); @@ -1647,6 +1657,20 @@ class binary_writer return 'D'; // float 64 } + /*! + @brief checks whether a JSON number fits into @a TargetType + @param[in] el a JSON number of either the signed or unsigned integer kind + @return whether @a el's value can be represented by @a TargetType without + wrapping, regardless of which of the two kinds it is stored as + */ + template + static bool bjdata_ndarray_value_in_range(const BasicJsonType& el) + { + return el.is_number_unsigned() + ? value_in_range_of(el.template get()) + : value_in_range_of(el.template get()); + } + /*! @return false if the object is successfully converted to a bjdata ndarray, true if the type or size is invalid */ @@ -1731,6 +1755,60 @@ class binary_writer } } + // every element is cast to the (possibly narrower) C++ type matching + // dtype below; a value that does not fit that type would silently + // wrap (integers) or overflow to infinity (the "single" precision + // float) instead of being reported, so such an object falls back to + // a plain object encoding as well + for (const auto& el : value.at(key)) + { + bool in_range = true; + switch (dtype) + { + case 'U': + case 'C': + case 'B': + in_range = bjdata_ndarray_value_in_range(el); + break; + case 'i': + in_range = bjdata_ndarray_value_in_range(el); + break; + case 'u': + in_range = bjdata_ndarray_value_in_range(el); + break; + case 'I': + in_range = bjdata_ndarray_value_in_range(el); + break; + case 'm': + in_range = bjdata_ndarray_value_in_range(el); + break; + case 'l': + in_range = bjdata_ndarray_value_in_range(el); + break; + case 'M': + in_range = bjdata_ndarray_value_in_range(el); + break; + case 'L': + in_range = bjdata_ndarray_value_in_range(el); + break; + case 'd': + { + const auto dval = el.template get(); + in_range = !std::isfinite(dval) || + (dval >= static_cast(std::numeric_limits::lowest()) && + dval <= static_cast((std::numeric_limits::max)())); + break; + } + default: + // 'D' (double) already spans the full range of number_float_t + break; + } + if (!in_range) + { + return true; + } + } + oa->write_character('['); oa->write_character('$'); oa->write_character(dtype); @@ -1850,8 +1928,8 @@ class binary_writer void write_compact_float(const number_float_t n, detail::input_format_t format) { #ifdef __GNUC__ -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wfloat-equal" + JSON_HEDLEY_DIAGNOSTIC_PUSH + JSON_HEDLEY_PRAGMA(GCC diagnostic ignored "-Wfloat-equal") #endif if (!std::isfinite(n) || ((static_cast(n) >= static_cast(std::numeric_limits::lowest()) && static_cast(n) <= static_cast((std::numeric_limits::max)()) && @@ -1870,7 +1948,7 @@ class binary_writer write_number(n); } #ifdef __GNUC__ -#pragma GCC diagnostic pop + JSON_HEDLEY_DIAGNOSTIC_POP #endif } diff --git a/include/nlohmann/detail/output/serializer.hpp b/include/nlohmann/detail/output/serializer.hpp index 0b608f8e2..9560729ad 100644 --- a/include/nlohmann/detail/output/serializer.hpp +++ b/include/nlohmann/detail/output/serializer.hpp @@ -9,21 +9,23 @@ #pragma once -#include // reverse, remove, fill, find, none_of +#include // reverse, remove, fill, find, none_of, min #include // array #include // localeconv, lconv #include // labs, isfinite, isnan, signbit #include // size_t, ptrdiff_t #include // uint8_t #include // snprintf +#include // memcpy, memset #include // numeric_limits #include // string, char_traits -#include // setfill, setw #include // is_same #include // move +#include // vector #include #include +#include #include #include #include @@ -60,18 +62,32 @@ class serializer public: /*! - @param[in] s output stream to serialize to + @param[in] s output adapter to serialize to; not owned by the serializer, + so it must outlive it (it lives at the call site) @param[in] ichar indentation character to use + @param[in] pretty_print_ whether the output shall be pretty-printed + @param[in] ensure_ascii_ If @a ensure_ascii_ is true, all non-ASCII + characters in the output are escaped with `\uXXXX` sequences, and the + result consists of ASCII characters only. + @param[in] indent_step_ the indent level @param[in] error_handler_ how to react on decoding errors + + None of @a pretty_print_, @a ensure_ascii_ and @a indent_step_ change over + the life of the serializer, so they are captured once here instead of + being threaded through every call to @ref dump, @ref dump_internal and + @ref dump_iteratively. */ - serializer(output_adapter_t s, const char ichar, + serializer(output_adapter_protocol& s, const char ichar, + const bool pretty_print_ = false, + const bool ensure_ascii_ = false, + const std::size_t indent_step_ = 0, error_handler_t error_handler_ = error_handler_t::strict) - : o(std::move(s)) - , loc(std::localeconv()) - , thousands_sep(loc->thousands_sep == nullptr ? '\0' : std::char_traits::to_char_type(* (loc->thousands_sep))) - , decimal_point(loc->decimal_point == nullptr ? '\0' : std::char_traits::to_char_type(* (loc->decimal_point))) + : o(&s) + , locale(std::localeconv()) , indent_char(ichar) - , indent_string(512, indent_char) + , pretty_print(pretty_print_) + , ensure_ascii(ensure_ascii_) + , indent_step(indent_step_) , error_handler(error_handler_) {} @@ -87,8 +103,8 @@ class serializer This function is called by the public member function dump and organizes the serialization internally. The indentation level is propagated as - additional parameter. In case of arrays and objects, the function is - called recursively. + additional parameter. Arrays and objects are serialized without recursion, + however deeply they are nested. - strings and object keys are escaped using `escape_string()` - integer numbers are converted implicitly via `operator<<` @@ -97,89 +113,109 @@ class serializer byte array @param[in] val value to serialize - @param[in] pretty_print whether the output shall be pretty-printed - @param[in] ensure_ascii If @a ensure_ascii is true, all non-ASCII characters - in the output are escaped with `\uXXXX` sequences, and the result consists - of ASCII characters only. - @param[in] indent_step the indent level @param[in] current_indent the current indent level (only used internally) */ void dump(const BasicJsonType& val, - const bool pretty_print, - const bool ensure_ascii, - const unsigned int indent_step, - const unsigned int current_indent = 0) + const std::size_t current_indent = 0) + { + dump_internal(val, current_indent); + flush(); + } + + JSON_PRIVATE_UNLESS_TESTED: + /*! + @brief worker for @ref dump + + Identical in behavior to the historical @ref dump, but writes into the + serializer's internal @ref write_buffer instead of issuing a virtual call + per token. The public @ref dump wraps this and flushes the buffer once the + top-level value has been serialized. + + Serializing a container descends into its elements, so a value nested deeply + enough used to exhaust the call stack and terminate the process with no + exception to catch. The descent is bounded here: once @ref dump_depth_limit + levels have been entered, @ref dump_iteratively writes out what is left + without the call stack. A value nested less deeply than that - all but a + vanishing minority - is written by exactly the code that always wrote it. + + @sa https://github.com/nlohmann/json/issues/5387 + */ + void dump_internal(const BasicJsonType& val, + const std::size_t current_indent = 0, + const std::size_t depth = 0) { switch (val.m_data.m_type) { case value_t::object: { + if (JSON_HEDLEY_UNLIKELY(depth >= dump_depth_limit())) + { + dump_iteratively(val, current_indent); + return; + } + if (val.m_data.m_value.object->empty()) { - o->write_characters("{}", 2); + put_literal("{}"); return; } if (pretty_print) { - o->write_characters("{\n", 2); + put_literal("{\n"); // variable to hold indentation for recursive calls - const auto new_indent = current_indent + indent_step; - if (JSON_HEDLEY_UNLIKELY(indent_string.size() < new_indent)) - { - indent_string.resize(indent_string.size() * 2, ' '); - } + const auto new_indent = next_indent(current_indent, indent_step); // first n-1 elements auto i = val.m_data.m_value.object->cbegin(); for (std::size_t cnt = 0; cnt < val.m_data.m_value.object->size() - 1; ++cnt, ++i) { - o->write_characters(indent_string.c_str(), new_indent); - o->write_character('\"'); - dump_escaped(i->first, ensure_ascii); - o->write_characters("\": ", 3); - dump(i->second, true, ensure_ascii, indent_step, new_indent); - o->write_characters(",\n", 2); + put_indent(new_indent); + put_char('"'); + dump_escaped(i->first); + put_literal("\": "); + dump_internal(i->second, new_indent, depth + 1); + put_literal(",\n"); } // last element JSON_ASSERT(i != val.m_data.m_value.object->cend()); JSON_ASSERT(std::next(i) == val.m_data.m_value.object->cend()); - o->write_characters(indent_string.c_str(), new_indent); - o->write_character('\"'); - dump_escaped(i->first, ensure_ascii); - o->write_characters("\": ", 3); - dump(i->second, true, ensure_ascii, indent_step, new_indent); + put_indent(new_indent); + put_char('"'); + dump_escaped(i->first); + put_literal("\": "); + dump_internal(i->second, new_indent, depth + 1); - o->write_character('\n'); - o->write_characters(indent_string.c_str(), current_indent); - o->write_character('}'); + put_char('\n'); + put_indent(current_indent); + put_char('}'); } else { - o->write_character('{'); + put_char('{'); // first n-1 elements auto i = val.m_data.m_value.object->cbegin(); for (std::size_t cnt = 0; cnt < val.m_data.m_value.object->size() - 1; ++cnt, ++i) { - o->write_character('\"'); - dump_escaped(i->first, ensure_ascii); - o->write_characters("\":", 2); - dump(i->second, false, ensure_ascii, indent_step, current_indent); - o->write_character(','); + put_char('"'); + dump_escaped(i->first); + put_literal("\":"); + dump_internal(i->second, current_indent, depth + 1); + put_char(','); } // last element JSON_ASSERT(i != val.m_data.m_value.object->cend()); JSON_ASSERT(std::next(i) == val.m_data.m_value.object->cend()); - o->write_character('\"'); - dump_escaped(i->first, ensure_ascii); - o->write_characters("\":", 2); - dump(i->second, false, ensure_ascii, indent_step, current_indent); + put_char('"'); + dump_escaped(i->first); + put_literal("\":"); + dump_internal(i->second, current_indent, depth + 1); - o->write_character('}'); + put_char('}'); } return; @@ -187,58 +223,60 @@ class serializer case value_t::array: { + if (JSON_HEDLEY_UNLIKELY(depth >= dump_depth_limit())) + { + dump_iteratively(val, current_indent); + return; + } + if (val.m_data.m_value.array->empty()) { - o->write_characters("[]", 2); + put_literal("[]"); return; } if (pretty_print) { - o->write_characters("[\n", 2); + put_literal("[\n"); // variable to hold indentation for recursive calls - const auto new_indent = current_indent + indent_step; - if (JSON_HEDLEY_UNLIKELY(indent_string.size() < new_indent)) - { - indent_string.resize(indent_string.size() * 2, ' '); - } + const auto new_indent = next_indent(current_indent, indent_step); // first n-1 elements for (auto i = val.m_data.m_value.array->cbegin(); i != val.m_data.m_value.array->cend() - 1; ++i) { - o->write_characters(indent_string.c_str(), new_indent); - dump(*i, true, ensure_ascii, indent_step, new_indent); - o->write_characters(",\n", 2); + put_indent(new_indent); + dump_internal(*i, new_indent, depth + 1); + put_literal(",\n"); } // last element JSON_ASSERT(!val.m_data.m_value.array->empty()); - o->write_characters(indent_string.c_str(), new_indent); - dump(val.m_data.m_value.array->back(), true, ensure_ascii, indent_step, new_indent); + put_indent(new_indent); + dump_internal(val.m_data.m_value.array->back(), new_indent, depth + 1); - o->write_character('\n'); - o->write_characters(indent_string.c_str(), current_indent); - o->write_character(']'); + put_char('\n'); + put_indent(current_indent); + put_char(']'); } else { - o->write_character('['); + put_char('['); // first n-1 elements for (auto i = val.m_data.m_value.array->cbegin(); i != val.m_data.m_value.array->cend() - 1; ++i) { - dump(*i, false, ensure_ascii, indent_step, current_indent); - o->write_character(','); + dump_internal(*i, current_indent, depth + 1); + put_char(','); } // last element JSON_ASSERT(!val.m_data.m_value.array->empty()); - dump(val.m_data.m_value.array->back(), false, ensure_ascii, indent_step, current_indent); + dump_internal(val.m_data.m_value.array->back(), current_indent, depth + 1); - o->write_character(']'); + put_char(']'); } return; @@ -246,9 +284,9 @@ class serializer case value_t::string: { - o->write_character('\"'); - dump_escaped(*val.m_data.m_value.string, ensure_ascii); - o->write_character('\"'); + put_char('"'); + dump_escaped(*val.m_data.m_value.string); + put_char('"'); return; } @@ -256,70 +294,66 @@ class serializer { if (pretty_print) { - o->write_characters("{\n", 2); + put_literal("{\n"); // variable to hold indentation for recursive calls - const auto new_indent = current_indent + indent_step; - if (JSON_HEDLEY_UNLIKELY(indent_string.size() < new_indent)) - { - indent_string.resize(indent_string.size() * 2, ' '); - } + const auto new_indent = next_indent(current_indent, indent_step); - o->write_characters(indent_string.c_str(), new_indent); + put_indent(new_indent); - o->write_characters("\"bytes\": [", 10); + put_literal("\"bytes\": ["); if (!val.m_data.m_value.binary->empty()) { for (auto i = val.m_data.m_value.binary->cbegin(); i != val.m_data.m_value.binary->cend() - 1; ++i) { - dump_integer(*i); - o->write_characters(", ", 2); + dump_byte(*i); + put_literal(", "); } - dump_integer(val.m_data.m_value.binary->back()); + dump_byte(val.m_data.m_value.binary->back()); } - o->write_characters("],\n", 3); - o->write_characters(indent_string.c_str(), new_indent); + put_literal("],\n"); + put_indent(new_indent); - o->write_characters("\"subtype\": ", 11); + put_literal("\"subtype\": "); if (val.m_data.m_value.binary->has_subtype()) { dump_integer(val.m_data.m_value.binary->subtype()); } else { - o->write_characters("null", 4); + put_literal("null"); } - o->write_character('\n'); - o->write_characters(indent_string.c_str(), current_indent); - o->write_character('}'); + put_char('\n'); + put_indent(current_indent); + put_char('}'); } else { - o->write_characters("{\"bytes\":[", 10); + put_literal("{\"bytes\":["); if (!val.m_data.m_value.binary->empty()) { for (auto i = val.m_data.m_value.binary->cbegin(); i != val.m_data.m_value.binary->cend() - 1; ++i) { - dump_integer(*i); - o->write_character(','); + dump_byte(*i); + put_char(','); } - dump_integer(val.m_data.m_value.binary->back()); + dump_byte(val.m_data.m_value.binary->back()); } - o->write_characters("],\"subtype\":", 12); + put_literal("],\"subtype\":"); if (val.m_data.m_value.binary->has_subtype()) { dump_integer(val.m_data.m_value.binary->subtype()); - o->write_character('}'); + put_char('}'); } else { - o->write_characters("null}", 5); + put_literal("null}"); } } return; @@ -329,11 +363,11 @@ class serializer { if (val.m_data.m_value.boolean) { - o->write_characters("true", 4); + put_literal("true"); } else { - o->write_characters("false", 5); + put_literal("false"); } return; } @@ -358,13 +392,13 @@ class serializer case value_t::discarded: { - o->write_characters("", 11); + put_literal(""); return; } case value_t::null: { - o->write_characters("null", 4); + put_literal("null"); return; } @@ -373,6 +407,367 @@ class serializer } } + private: + /// the number of levels @ref dump_internal descends into before it hands + /// over to @ref dump_iteratively + static constexpr std::size_t dump_depth_limit() + { + return 128; + } + + /*! + @brief write out @a val and everything below it without the call stack + + Emits the same bytes as @ref dump_internal, keeping the containers it has + entered on an explicit stack instead of descending into them. Only reached + for values nested deeper than @ref dump_depth_limit, which is why it is not + written for speed: walking every value this way measured up to 20% slower on + object-heavy documents than letting the compiler drive the descent. + */ + void dump_iteratively(const BasicJsonType& val, + const std::size_t current_indent = 0) + { + // Scalars, empty containers and binary values are written by dump_value + // alone, so nothing is allocated for them: only a container with + // elements is ever pushed. + std::vector stack; + + dump_value(val, current_indent, stack); + + while (!stack.empty()) + { + dump_frame& frame = stack.back(); + + if (frame.value->m_data.m_type == value_t::object) + { + const auto* object = frame.value->m_data.m_value.object; + + if (frame.object_it == object->cend()) + { + if (pretty_print) + { + put_char('\n'); + put_indent(frame.current_indent); + } + + put_char('}'); + stack.pop_back(); + continue; + } + + // the separator goes in front of every element but the first, + // which puts exactly one between each pair and none at the end + if (frame.object_it != object->cbegin()) + { + if (pretty_print) + { + put_literal(",\n"); + } + else + { + put_char(','); + } + } + + if (pretty_print) + { + put_indent(frame.child_indent); + } + + put_char('"'); + dump_escaped(frame.object_it->first); + + if (pretty_print) + { + put_literal("\": "); + } + else + { + put_literal("\":"); + } + + const BasicJsonType& element = frame.object_it->second; + ++frame.object_it; + + // read everything needed from the frame before this: entering a + // container pushes another one and can move them all + const std::size_t element_indent = frame.child_indent; + dump_value(element, element_indent, stack); + } + else + { + const auto* array = frame.value->m_data.m_value.array; + + if (frame.array_it == array->cend()) + { + if (pretty_print) + { + put_char('\n'); + put_indent(frame.current_indent); + } + + put_char(']'); + stack.pop_back(); + continue; + } + + if (frame.array_it != array->cbegin()) + { + if (pretty_print) + { + put_literal(",\n"); + } + else + { + put_char(','); + } + } + + if (pretty_print) + { + put_indent(frame.child_indent); + } + + const BasicJsonType& element = *frame.array_it; + ++frame.array_it; + + // see above + const std::size_t element_indent = frame.child_indent; + dump_value(element, element_indent, stack); + } + } + } + + private: + /// @brief a container that has been opened but not closed yet + struct dump_frame + { + dump_frame(const BasicJsonType* value_, const std::size_t current_indent_, + const std::size_t child_indent_) noexcept + : value(value_) + , current_indent(current_indent_) + , child_indent(child_indent_) + {} + + /// the object or array being serialized + const BasicJsonType* value; + /// the element to serialize next; which of the two is live follows from + /// the type of @a value. They are kept side by side rather than in a + /// union, which would need its special members written out by hand, see + /// detail/iterators/internal_iterator.hpp + typename BasicJsonType::object_t::const_iterator object_it{}; + typename BasicJsonType::array_t::const_iterator array_it{}; + /// the indentation of the container itself, used by its closing bracket + std::size_t current_indent; + /// the indentation of the container's elements + std::size_t child_indent; + }; + + /*! + @brief serialize the value @a val, but not the elements of a container + + An object or array with elements is opened and pushed onto @a stack for + @ref dump_internal to walk; everything else - including a binary value, + which looks like an object but has no elements to descend into - is written + out here in full. + */ + void dump_value(const BasicJsonType& val, + const std::size_t current_indent, + std::vector& stack) + { + switch (val.m_data.m_type) + { + case value_t::object: + { + if (val.m_data.m_value.object->empty()) + { + put_literal("{}"); + return; + } + + std::size_t child_indent = current_indent; + + if (pretty_print) + { + put_literal("{\n"); + child_indent = next_indent(current_indent, indent_step); + } + else + { + put_char('{'); + } + + stack.emplace_back(&val, current_indent, child_indent); + stack.back().object_it = val.m_data.m_value.object->cbegin(); + return; + } + + case value_t::array: + { + if (val.m_data.m_value.array->empty()) + { + put_literal("[]"); + return; + } + + std::size_t child_indent = current_indent; + + if (pretty_print) + { + put_literal("[\n"); + child_indent = next_indent(current_indent, indent_step); + } + else + { + put_char('['); + } + + stack.emplace_back(&val, current_indent, child_indent); + stack.back().array_it = val.m_data.m_value.array->cbegin(); + return; + } + + case value_t::string: + { + put_char('"'); + dump_escaped(*val.m_data.m_value.string); + put_char('"'); + return; + } + + case value_t::binary: + { + if (pretty_print) + { + put_literal("{\n"); + + // variable to hold indentation for the bytes + const auto new_indent = next_indent(current_indent, indent_step); + + put_indent(new_indent); + + put_literal("\"bytes\": ["); + + if (!val.m_data.m_value.binary->empty()) + { + for (auto i = val.m_data.m_value.binary->cbegin(); + i != val.m_data.m_value.binary->cend() - 1; ++i) + { + dump_byte(*i); + put_literal(", "); + } + dump_byte(val.m_data.m_value.binary->back()); + } + + put_literal("],\n"); + put_indent(new_indent); + + put_literal("\"subtype\": "); + if (val.m_data.m_value.binary->has_subtype()) + { + dump_integer(val.m_data.m_value.binary->subtype()); + } + else + { + put_literal("null"); + } + put_char('\n'); + put_indent(current_indent); + put_char('}'); + } + else + { + put_literal("{\"bytes\":["); + + if (!val.m_data.m_value.binary->empty()) + { + for (auto i = val.m_data.m_value.binary->cbegin(); + i != val.m_data.m_value.binary->cend() - 1; ++i) + { + dump_byte(*i); + put_char(','); + } + dump_byte(val.m_data.m_value.binary->back()); + } + + put_literal("],\"subtype\":"); + if (val.m_data.m_value.binary->has_subtype()) + { + dump_integer(val.m_data.m_value.binary->subtype()); + put_char('}'); + } + else + { + put_literal("null}"); + } + } + return; + } + + case value_t::boolean: + { + if (val.m_data.m_value.boolean) + { + put_literal("true"); + } + else + { + put_literal("false"); + } + return; + } + + case value_t::number_integer: + { + dump_integer(val.m_data.m_value.number_integer); + return; + } + + case value_t::number_unsigned: + { + dump_integer(val.m_data.m_value.number_unsigned); + return; + } + + case value_t::number_float: + { + dump_float(val.m_data.m_value.number_float); + return; + } + + case value_t::discarded: + { + put_literal(""); + return; + } + + case value_t::null: + { + put_literal("null"); + return; + } + + default: // LCOV_EXCL_LINE + JSON_ASSERT(false); // NOLINT(cert-dcl03-c,hicpp-static-assert,misc-static-assert) LCOV_EXCL_LINE + } + } + + + /*! + @brief the indentation level to use for the children of the current value + + A very large @a indent_step can wrap the unsigned accumulation on deep + nesting, which would silently truncate the indentation. Far harder to reach + now that the accumulator is a std::size_t, but still reachable where that is + 32 bits wide. + */ + static std::size_t next_indent(const std::size_t current_indent, const std::size_t indent_step) + { + const std::size_t new_indent = current_indent + indent_step; + JSON_ASSERT(new_indent >= current_indent); + return new_indent; + } + JSON_PRIVATE_UNLESS_TESTED: /*! @brief dump escaped string @@ -383,12 +778,32 @@ class serializer representation. The escaped string is written to output stream @a o. @param[in] s the string to escape - @param[in] ensure_ascii whether to escape non-ASCII characters with - \uXXXX sequences @complexity Linear in the length of string @a s. */ - void dump_escaped(const string_t& s, const bool ensure_ascii) + void dump_escaped(const string_t& s) + { + // dispatch once here rather than test the flag inside the loop: it does + // not change while a string is written, and folding it lets each of the + // two scanners be inlined into a loop of its own + if (ensure_ascii) + { + dump_escaped_impl(s); + } + else + { + dump_escaped_impl(s); + } + } + + /*! + @brief worker for @ref dump_escaped + + @a ensure_ascii is a template parameter here so that the branch on it is + resolved once, outside the loop; see @ref dump_escaped. + */ + template + void dump_escaped_impl(const string_t& s) { std::uint32_t codepoint{}; std::uint8_t state = UTF8_ACCEPT; @@ -400,6 +815,56 @@ class serializer for (std::size_t i = 0; i < s.size(); ++i) { + // Fast path: at a character boundary (state == UTF8_ACCEPT), + // bulk-copy the longest run of bytes that need no escaping using a + // SWAR scanner shared with the lexer's contiguous path. The scanner + // stops exactly at the first byte dump_escaped would handle + // individually, so that byte is left to the byte-at-a-time path + // below, keeping escaping output and error diagnostics unchanged. + // + // - EnsureAscii == false: string_bulk_run() copies ordinary bytes + // and complete well-formed UTF-8, stopping at a quote, backslash, + // control character (< 0x20), or ill-formed/truncated sequence. + // - EnsureAscii == true: only printable ASCII may be copied + // verbatim; find_ascii_copyable_run() additionally stops at 0x7F + // and every non-ASCII byte (>= 0x80), which must be \u-escaped. + if (state == UTF8_ACCEPT) + { + const auto* const data = reinterpret_cast(s.data()); + // A run can only be non-empty when the very first byte is one + // the scanner may copy, so test that single byte before paying + // for the scan. Without it, text whose characters all have to be + // escaped - CJK under ensure_ascii, where every byte is >= 0x80 - + // runs the scanner once per character only to be told zero. + std::size_t run = 0; + if (!EnsureAscii) + { + run = string_bulk_run(data + i, s.size() - i); + } + else if (is_ascii_copyable(data[i])) + { + run = find_ascii_copyable_run(data + i, s.size() - i); + } + if (run != 0) + { + // emit any bytes still pending in string_buffer first to + // preserve output order, then write the run directly + if (bytes != 0) + { + put_buffer(string_buffer, bytes); + bytes = 0; + } + put_string(s, i, i + run); + bytes_after_last_accept = 0; + undumped_chars = 0; + i += run; + if (i >= s.size()) + { + break; + } + } + } + const auto byte = static_cast(s[i]); switch (decode(state, codepoint, byte)) @@ -446,7 +911,7 @@ class serializer case 0x22: // quotation mark { string_buffer[bytes++] = '\\'; - string_buffer[bytes++] = '\"'; + string_buffer[bytes++] = '"'; break; } @@ -460,8 +925,8 @@ class serializer default: { // escape control characters (0x00..0x1F) or, if - // ensure_ascii parameter is used, non-ASCII characters - if ((codepoint <= 0x1F) || (ensure_ascii && (codepoint >= 0x7F))) + // EnsureAscii parameter is used, non-ASCII characters + if ((codepoint <= 0x1F) || (EnsureAscii && (codepoint >= 0x7F))) { if (codepoint <= 0xFFFF) { @@ -488,7 +953,7 @@ class serializer // written ("\uxxxx\uxxxx\0") for one code point if (string_buffer.size() - bytes < 13) { - o->write_characters(string_buffer.data(), bytes); + put_buffer(string_buffer, bytes); bytes = 0; } @@ -526,7 +991,7 @@ class serializer if (error_handler == error_handler_t::replace) { // add a replacement character - if (ensure_ascii) + if (EnsureAscii) { string_buffer[bytes++] = '\\'; string_buffer[bytes++] = 'u'; @@ -547,7 +1012,7 @@ class serializer // written ("\uxxxx\uxxxx\0") for one code point if (string_buffer.size() - bytes < 13) { - o->write_characters(string_buffer.data(), bytes); + put_buffer(string_buffer, bytes); bytes = 0; } @@ -569,7 +1034,7 @@ class serializer default: // decode found yet incomplete multibyte code point { - if (!ensure_ascii) + if (!EnsureAscii) { // code point will not be escaped - copy byte to buffer string_buffer[bytes++] = s[i]; @@ -586,7 +1051,7 @@ class serializer // write buffer if (bytes > 0) { - o->write_characters(string_buffer.data(), bytes); + put_buffer(string_buffer, bytes); } } else @@ -602,22 +1067,22 @@ class serializer case error_handler_t::ignore: { // write all accepted bytes - o->write_characters(string_buffer.data(), bytes_after_last_accept); + put_buffer(string_buffer, bytes_after_last_accept); break; } case error_handler_t::replace: { // write all accepted bytes - o->write_characters(string_buffer.data(), bytes_after_last_accept); + put_buffer(string_buffer, bytes_after_last_accept); // add a replacement character - if (ensure_ascii) + if (EnsureAscii) { - o->write_characters("\\ufffd", 6); + put_literal("\\ufffd"); } else { - o->write_characters("\xEF\xBF\xBD", 3); + put_literal("\xEF\xBF\xBD"); } break; } @@ -628,6 +1093,160 @@ class serializer } } + private: + /*! + @brief append a single character to the write buffer + + Structural characters ('{', '"', ',', ...) previously went straight to the + output adapter, one virtual call each. Buffering them and flushing in bulk + turns those many indirect calls into a single memcpy plus an occasional + flush, which dominates the cost of serializing object/array-heavy values. + */ + void put_char(char c) + { + if (JSON_HEDLEY_UNLIKELY(write_buffer_pos == write_buffer.size())) + { + flush(); + } + write_buffer[write_buffer_pos++] = c; + } + + /*! + @brief append @a indent indentation characters to the write buffer + + Writes the indentation straight into the buffer instead of copying it out of + a pre-grown indentation string, so no auxiliary string has to be sized, + resized, or kept in sync with the deepest nesting level reached. + + An indentation wider than the buffer is emitted by filling the buffer with + the indentation character once and flushing that same content repeatedly: + flushing does not disturb what the buffer holds, so re-filling it between + flushes would be redundant work. + */ + void put_indent(std::size_t indent) + { + // closing braces at the outermost level ask for no indentation at all + if (indent == 0) + { + return; + } + + const std::size_t capacity = write_buffer.size(); + + // fill whatever room is left in the buffer; this is the whole job + // whenever the indentation is narrower than the buffer, which is the + // case for every sane indent_step + const std::size_t head = (std::min)(indent, capacity - write_buffer_pos); + std::memset(write_buffer.data() + write_buffer_pos, indent_char, head); + write_buffer_pos += head; + indent -= head; + + if (JSON_HEDLEY_LIKELY(indent == 0)) + { + return; + } + + // the buffer is full and the remainder spans whole buffer-fulls: flush + // what is pending, then fill the buffer with the indentation character + // exactly once and hand the same bytes to the adapter as often as needed + flush(); + std::memset(write_buffer.data(), indent_char, capacity); + + while (indent >= capacity) + { + write_buffer_pos = capacity; + flush(); + indent -= capacity; + } + + // the buffer still holds indentation characters throughout, so the tail + // only has to be claimed, not written again + write_buffer_pos = indent; + } + + /*! + @brief append a string literal to the write buffer + + The length comes from the array bound rather than a hand-written count, so + it cannot drift out of sync with the literal. A literal always fits into the + buffer (checked at compile time), so unlike @ref put_string this needs no + write-through path for oversized runs. + */ + template + void put_literal(const char (&s)[N]) // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays) + { + static_assert(N >= 2, "put_literal expects a non-empty string literal"); + // the array bound counts the terminating NUL, which is not written + constexpr std::size_t length = N - 1; + static_assert(length < write_buffer_size, "string literal must fit into the write buffer"); + + if (JSON_HEDLEY_UNLIKELY(write_buffer_pos + length > write_buffer.size())) + { + flush(); + } + std::memcpy(write_buffer.data() + write_buffer_pos, s, length); + write_buffer_pos += length; + } + + /*! + @brief append the characters of @a str in [@a start, @a end) + + The only way to append a run of characters: @a str carries its own bound, + so the range can be checked against it, which a bare pointer plus a count + could not do. Runs that do not fit the buffer are written straight through + the output adapter (after flushing what is pending), so large string and + number payloads are not copied an extra time. + */ + template + void put_string(const StringType& str, std::size_t start, std::size_t end) + { + JSON_ASSERT(start <= end); + JSON_ASSERT(end <= str.size()); + + const char* const s = str.data() + start; + const std::size_t length = end - start; + + if (JSON_HEDLEY_UNLIKELY(length >= write_buffer.size())) + { + flush(); + o->write_characters(s, length); + return; + } + if (JSON_HEDLEY_UNLIKELY(write_buffer_pos + length > write_buffer.size())) + { + flush(); + } + std::memcpy(write_buffer.data() + write_buffer_pos, s, length); + write_buffer_pos += length; + } + + /*! + @brief append the first @a length characters of a fixed-size buffer + */ + template + void put_buffer(const std::array& buffer, std::size_t length) + { + put_string(buffer, 0, length); + } + + JSON_PRIVATE_UNLESS_TESTED: + /*! + @brief flush the write buffer to the output adapter + + Writing zero characters is a well-defined no-op for every output adapter, so + the buffered length is passed through unconditionally (no empty-guard branch + to leave uncovered). + + @note dump_escaped() and dump_integer()/dump_float() write into the internal + write buffer; callers that invoke them directly (rather than through the + public dump()) must call flush() before inspecting the output. + */ + void flush() + { + o->write_characters(write_buffer.data(), write_buffer_pos); + write_buffer_pos = 0; + } + private: /*! @brief count digits @@ -716,6 +1335,62 @@ class serializer return false; } + /*! + @brief write the decimal representation of the byte @a value + + A binary value's bytes are always in [0, 255], so writing one needs neither + the digit counting nor the 64-bit arithmetic that @ref dump_integer does for + an arbitrary number, and the three digits it takes at most are written + straight into the write buffer. + + Any byte type that is not a plain unsigned byte is left to @ref dump_integer, + whose representation of it may differ. + */ + template + void dump_byte(const ByteType value) + { + dump_byte(value, std::integral_constant < bool, + std::is_unsigned::value && sizeof(ByteType) == 1 + && !std::is_same::value > {}); + } + + template + void dump_byte(const ByteType value, std::false_type /*is_plain_byte*/) + { + dump_integer(value); + } + + template + void dump_byte(const ByteType value, std::true_type /*is_plain_byte*/) + { + if (JSON_HEDLEY_UNLIKELY(write_buffer_pos + 3 > write_buffer.size())) + { + flush(); + } + + const auto byte = static_cast(value); + // Accumulate the offset in a local and store it back once. Writing + // through write_buffer[] is a char write, which may alias any object, + // so with the member updated in place the compiler has to reload and + // store it around every digit - measured 2.4x slower on a dump of a + // multi-megabyte binary value. + std::size_t pos = write_buffer_pos; + + if (byte >= 100) + { + write_buffer[pos++] = static_cast('0' + (byte / 100)); + write_buffer[pos++] = static_cast('0' + ((byte / 10) % 10)); + } + else if (byte >= 10) + { + write_buffer[pos++] = static_cast('0' + (byte / 10)); + } + + write_buffer[pos++] = static_cast('0' + (byte % 10)); + + write_buffer_pos = pos; + } + /*! @brief dump an integer @@ -752,7 +1427,7 @@ class serializer // special case for "0" if (x == 0) { - o->write_character('0'); + put_char('0'); return; } @@ -805,7 +1480,7 @@ class serializer *(--buffer_ptr) = static_cast('0' + abs_value); } - o->write_characters(number_buffer.data(), n_chars); + put_buffer(number_buffer, n_chars); } /*! @@ -821,7 +1496,7 @@ class serializer // NaN / inf if (!std::isfinite(x)) { - o->write_characters("null", 4); + put_literal("null"); return; } @@ -842,7 +1517,7 @@ class serializer auto* begin = number_buffer.data(); auto* end = ::nlohmann::detail::to_chars(begin, begin + number_buffer.size(), x); - o->write_characters(begin, static_cast(end - begin)); + put_buffer(number_buffer, static_cast(end - begin)); } JSON_HEDLEY_NON_NULL(1) @@ -873,27 +1548,27 @@ class serializer JSON_ASSERT(static_cast(len) < number_buffer.size()); // erase thousands separators - if (thousands_sep != '\0') + if (locale.thousands_sep != '\0') { // NOLINTNEXTLINE(readability-qualified-auto,llvm-qualified-auto): std::remove returns an iterator, see https://github.com/nlohmann/json/issues/3081 - const auto end = std::remove(number_buffer.begin(), number_buffer.begin() + len, thousands_sep); + const auto end = std::remove(number_buffer.begin(), number_buffer.begin() + len, locale.thousands_sep); std::fill(end, number_buffer.end(), '\0'); JSON_ASSERT((end - number_buffer.begin()) <= len); len = (end - number_buffer.begin()); } // convert decimal point to '.' - if (decimal_point != '\0' && decimal_point != '.') + if (locale.decimal_point != '\0' && locale.decimal_point != '.') { // NOLINTNEXTLINE(readability-qualified-auto,llvm-qualified-auto): std::find returns an iterator, see https://github.com/nlohmann/json/issues/3081 - const auto dec_pos = std::find(number_buffer.begin(), number_buffer.end(), decimal_point); + const auto dec_pos = std::find(number_buffer.begin(), number_buffer.end(), locale.decimal_point); if (dec_pos != number_buffer.end()) { *dec_pos = '.'; } } - o->write_characters(number_buffer.data(), static_cast(len)); + put_buffer(number_buffer, static_cast(len)); // determine if we need to append ".0" const bool value_is_int_like = @@ -905,7 +1580,7 @@ class serializer if (value_is_int_like) { - o->write_characters(".0", 2); + put_literal(".0"); } } @@ -992,29 +1667,53 @@ class serializer } private: - /// the output of the serializer - output_adapter_t o = nullptr; + /// the locale's thousand separator and decimal point characters + struct locale_chars + { + explicit locale_chars(const std::lconv* loc) noexcept + : thousands_sep(loc->thousands_sep == nullptr ? '\0' : std::char_traits::to_char_type(* (loc->thousands_sep))) + , decimal_point(loc->decimal_point == nullptr ? '\0' : std::char_traits::to_char_type(* (loc->decimal_point))) + {} + + const char thousands_sep; + const char decimal_point; + }; + + /// the output of the serializer (non-owning; the adapter lives at the call site) + output_adapter_protocol* o = nullptr; /// a (hopefully) large enough character buffer std::array number_buffer{{}}; - /// the locale - const std::lconv* loc = nullptr; - /// the locale's thousand separator character - const char thousands_sep = '\0'; - /// the locale's decimal point character - const char decimal_point = '\0'; + /// computed once from std::localeconv() at construction; @ref + /// locale_chars keeps std::localeconv()'s pointer from having to be held + /// past the constructor, while still letting these stay const + const locale_chars locale; /// string buffer std::array string_buffer{{}}; /// the indentation character const char indent_char; - /// the indentation string - string_t indent_string; + + /// whether to pretty-print the output + const bool pretty_print; + + /// whether to escape non-ASCII characters with \uXXXX sequences + const bool ensure_ascii; + + /// the indent level + const std::size_t indent_step; /// error_handler how to react on decoding errors const error_handler_t error_handler; + + /// buffer collecting output before it is flushed to the output adapter, so + /// that the many small structural writes become few bulk writes + static constexpr std::size_t write_buffer_size = 1024; + std::array write_buffer{{}}; + /// number of valid bytes currently held in @ref write_buffer + std::size_t write_buffer_pos = 0; }; } // namespace detail diff --git a/include/nlohmann/json.hpp b/include/nlohmann/json.hpp index be4ccb90f..016a4dd27 100644 --- a/include/nlohmann/json.hpp +++ b/include/nlohmann/json.hpp @@ -164,11 +164,12 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec detail::parser_callback_tcb = nullptr, const bool allow_exceptions = true, const bool ignore_comments = false, - const bool ignore_trailing_commas = false + const bool ignore_trailing_commas = false, + const bool discard_number_values = false ) { return ::nlohmann::detail::parser(std::move(adapter), - std::move(cb), allow_exceptions, ignore_comments, ignore_trailing_commas); + std::move(cb), allow_exceptions, ignore_comments, ignore_trailing_commas, discard_number_values); } private: @@ -1335,21 +1336,26 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec /// @brief serialization /// @sa https://json.nlohmann.me/api/basic_json/dump/ + JSON_HEDLEY_WARN_UNUSED_RESULT string_t dump(const int indent = -1, const char indent_char = ' ', const bool ensure_ascii = false, const error_handler_t error_handler = error_handler_t::strict) const { string_t result; - serializer s(detail::output_adapter(result), indent_char, error_handler); + detail::output_string_adapter string_adapter(result); if (indent >= 0) { - s.dump(*this, true, ensure_ascii, static_cast(indent)); + serializer s(string_adapter, indent_char, + true, ensure_ascii, static_cast(indent), error_handler); + s.dump(*this); } else { - s.dump(*this, false, ensure_ascii, 0); + serializer s(string_adapter, indent_char, + false, ensure_ascii, 0, error_handler); + s.dump(*this); } return result; @@ -1357,6 +1363,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec /// @brief return the type of the JSON value (explicit) /// @sa https://json.nlohmann.me/api/basic_json/type/ + JSON_HEDLEY_WARN_UNUSED_RESULT constexpr value_t type() const noexcept { return m_data.m_type; @@ -1364,6 +1371,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec /// @brief return whether type is primitive /// @sa https://json.nlohmann.me/api/basic_json/is_primitive/ + JSON_HEDLEY_WARN_UNUSED_RESULT constexpr bool is_primitive() const noexcept { return is_null() || is_string() || is_boolean() || is_number() || is_binary(); @@ -1371,6 +1379,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec /// @brief return whether type is structured /// @sa https://json.nlohmann.me/api/basic_json/is_structured/ + JSON_HEDLEY_WARN_UNUSED_RESULT constexpr bool is_structured() const noexcept { return is_array() || is_object(); @@ -1378,6 +1387,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec /// @brief return whether value is null /// @sa https://json.nlohmann.me/api/basic_json/is_null/ + JSON_HEDLEY_WARN_UNUSED_RESULT constexpr bool is_null() const noexcept { return m_data.m_type == value_t::null; @@ -1385,6 +1395,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec /// @brief return whether value is a boolean /// @sa https://json.nlohmann.me/api/basic_json/is_boolean/ + JSON_HEDLEY_WARN_UNUSED_RESULT constexpr bool is_boolean() const noexcept { return m_data.m_type == value_t::boolean; @@ -1392,6 +1403,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec /// @brief return whether value is a number /// @sa https://json.nlohmann.me/api/basic_json/is_number/ + JSON_HEDLEY_WARN_UNUSED_RESULT constexpr bool is_number() const noexcept { return is_number_integer() || is_number_float(); @@ -1399,6 +1411,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec /// @brief return whether value is an integer number /// @sa https://json.nlohmann.me/api/basic_json/is_number_integer/ + JSON_HEDLEY_WARN_UNUSED_RESULT constexpr bool is_number_integer() const noexcept { return m_data.m_type == value_t::number_integer || m_data.m_type == value_t::number_unsigned; @@ -1406,6 +1419,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec /// @brief return whether value is an unsigned integer number /// @sa https://json.nlohmann.me/api/basic_json/is_number_unsigned/ + JSON_HEDLEY_WARN_UNUSED_RESULT constexpr bool is_number_unsigned() const noexcept { return m_data.m_type == value_t::number_unsigned; @@ -1413,6 +1427,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec /// @brief return whether value is a floating-point number /// @sa https://json.nlohmann.me/api/basic_json/is_number_float/ + JSON_HEDLEY_WARN_UNUSED_RESULT constexpr bool is_number_float() const noexcept { return m_data.m_type == value_t::number_float; @@ -1420,6 +1435,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec /// @brief return whether value is an object /// @sa https://json.nlohmann.me/api/basic_json/is_object/ + JSON_HEDLEY_WARN_UNUSED_RESULT constexpr bool is_object() const noexcept { return m_data.m_type == value_t::object; @@ -1427,6 +1443,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec /// @brief return whether value is an array /// @sa https://json.nlohmann.me/api/basic_json/is_array/ + JSON_HEDLEY_WARN_UNUSED_RESULT constexpr bool is_array() const noexcept { return m_data.m_type == value_t::array; @@ -1434,6 +1451,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec /// @brief return whether value is a string /// @sa https://json.nlohmann.me/api/basic_json/is_string/ + JSON_HEDLEY_WARN_UNUSED_RESULT constexpr bool is_string() const noexcept { return m_data.m_type == value_t::string; @@ -1441,6 +1459,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec /// @brief return whether value is a binary array /// @sa https://json.nlohmann.me/api/basic_json/is_binary/ + JSON_HEDLEY_WARN_UNUSED_RESULT constexpr bool is_binary() const noexcept { return m_data.m_type == value_t::binary; @@ -1448,6 +1467,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec /// @brief return whether value is discarded /// @sa https://json.nlohmann.me/api/basic_json/is_discarded/ + JSON_HEDLEY_WARN_UNUSED_RESULT constexpr bool is_discarded() const noexcept { return m_data.m_type == value_t::discarded; @@ -2779,6 +2799,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec /// @brief returns the number of occurrences of a key in a JSON object /// @sa https://json.nlohmann.me/api/basic_json/count/ + JSON_HEDLEY_WARN_UNUSED_RESULT size_type count(const typename object_t::key_type& key) const { // return 0 for all nonobject types @@ -2789,6 +2810,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec /// @sa https://json.nlohmann.me/api/basic_json/count/ template::value, int> = 0> + JSON_HEDLEY_WARN_UNUSED_RESULT size_type count(KeyType && key) const { // return 0 for all nonobject types @@ -2797,6 +2819,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec /// @brief check the existence of an element in a JSON object /// @sa https://json.nlohmann.me/api/basic_json/contains/ + JSON_HEDLEY_WARN_UNUSED_RESULT bool contains(const typename object_t::key_type& key) const { return is_object() && m_data.m_value.object->find(key) != m_data.m_value.object->end(); @@ -2806,6 +2829,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec /// @sa https://json.nlohmann.me/api/basic_json/contains/ template::value, int> = 0> + JSON_HEDLEY_WARN_UNUSED_RESULT bool contains(KeyType && key) const { return is_object() && m_data.m_value.object->find(std::forward(key)) != m_data.m_value.object->end(); @@ -2813,12 +2837,14 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec /// @brief check the existence of an element in a JSON object given a JSON pointer /// @sa https://json.nlohmann.me/api/basic_json/contains/ + JSON_HEDLEY_WARN_UNUSED_RESULT bool contains(const json_pointer& ptr) const { return ptr.contains(this); } template::value, int> = 0> + JSON_HEDLEY_WARN_UNUSED_RESULT JSON_HEDLEY_DEPRECATED_FOR(3.11.0, basic_json::json_pointer or nlohmann::json_pointer) // NOLINT(readability/alt_tokens) bool contains(const typename ::nlohmann::json_pointer& ptr) const { @@ -2974,6 +3000,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec /// @brief checks whether the container is empty. /// @sa https://json.nlohmann.me/api/basic_json/empty/ + JSON_HEDLEY_WARN_UNUSED_RESULT bool empty() const noexcept { switch (m_data.m_type) @@ -3013,6 +3040,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec /// @brief returns the number of elements /// @sa https://json.nlohmann.me/api/basic_json/size/ + JSON_HEDLEY_WARN_UNUSED_RESULT size_type size() const noexcept { switch (m_data.m_type) @@ -3052,6 +3080,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec /// @brief returns the maximum possible number of elements /// @sa https://json.nlohmann.me/api/basic_json/max_size/ + JSON_HEDLEY_WARN_UNUSED_RESULT size_type max_size() const noexcept { switch (m_data.m_type) @@ -3770,13 +3799,13 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec bool operator==(const_reference rhs) const noexcept { #ifdef __GNUC__ -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wfloat-equal" + JSON_HEDLEY_DIAGNOSTIC_PUSH + JSON_HEDLEY_PRAGMA(GCC diagnostic ignored "-Wfloat-equal") #endif const_reference lhs = *this; JSON_IMPLEMENT_OPERATOR( ==, true, false, false) #ifdef __GNUC__ -#pragma GCC diagnostic pop + JSON_HEDLEY_DIAGNOSTIC_POP #endif } @@ -3863,12 +3892,12 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec friend bool operator==(const_reference lhs, const_reference rhs) noexcept { #ifdef __GNUC__ -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wfloat-equal" + JSON_HEDLEY_DIAGNOSTIC_PUSH + JSON_HEDLEY_PRAGMA(GCC diagnostic ignored "-Wfloat-equal") #endif JSON_IMPLEMENT_OPERATOR( ==, true, false, false) #ifdef __GNUC__ -#pragma GCC diagnostic pop + JSON_HEDLEY_DIAGNOSTIC_POP #endif } @@ -4055,8 +4084,10 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec o.width(0); // do the actual serialization - serializer s(detail::output_adapter(o), o.fill()); - s.dump(j, pretty_print, false, static_cast(indentation)); + detail::output_stream_adapter stream_adapter(o); + serializer s(stream_adapter, o.fill(), + pretty_print, false, static_cast(indentation)); + s.dump(j); return o; } @@ -4129,22 +4160,24 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec /// @brief check if the input is valid JSON /// @sa https://json.nlohmann.me/api/basic_json/accept/ template + JSON_HEDLEY_WARN_UNUSED_RESULT static bool accept(InputType&& i, const bool ignore_comments = false, const bool ignore_trailing_commas = false) { - return parser(detail::input_adapter(std::forward(i)), nullptr, false, ignore_comments, ignore_trailing_commas).accept(true); + return parser(detail::input_adapter(std::forward(i)), nullptr, false, ignore_comments, ignore_trailing_commas, true).accept(true); } /// @brief check if the input is valid JSON (iterator pair, or iterator+sentinel pair for C++20 ranges support) /// @sa https://json.nlohmann.me/api/basic_json/accept/ template::value, int> = 0> + JSON_HEDLEY_WARN_UNUSED_RESULT static bool accept(IteratorType first, SentinelType last, const bool ignore_comments = false, const bool ignore_trailing_commas = false) { - return parser(detail::input_adapter(std::move(first), std::move(last)), nullptr, false, ignore_comments, ignore_trailing_commas).accept(true); + return parser(detail::input_adapter(std::move(first), std::move(last)), nullptr, false, ignore_comments, ignore_trailing_commas, true).accept(true); } JSON_HEDLEY_WARN_UNUSED_RESULT @@ -4153,7 +4186,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec const bool ignore_comments = false, const bool ignore_trailing_commas = false) { - return parser(i.get(), nullptr, false, ignore_comments, ignore_trailing_commas).accept(true); + return parser(i.get(), nullptr, false, ignore_comments, ignore_trailing_commas, true).accept(true); } /// @brief generate SAX events @@ -4239,6 +4272,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec /// @brief return the type as string /// @sa https://json.nlohmann.me/api/basic_json/type_name/ + JSON_HEDLEY_WARN_UNUSED_RESULT JSON_HEDLEY_RETURNS_NON_NULL const char* type_name() const noexcept { @@ -4473,8 +4507,11 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec basic_json result; auto ia = detail::input_adapter(std::forward(i)); detail::json_sax_dom_parser sdp(result, allow_exceptions); - const bool res = binary_reader(std::move(ia), input_format_t::cbor).sax_parse(input_format_t::cbor, &sdp, strict, tag_handler); // cppcheck-suppress[accessMoved] - return res ? result : basic_json(value_t::discarded); + if (!binary_reader(std::move(ia), input_format_t::cbor).sax_parse(input_format_t::cbor, &sdp, strict, tag_handler)) // cppcheck-suppress[accessMoved] + { + result = value_t::discarded; + } + return result; } /// @brief create a JSON value from an input in CBOR format (iterator pair, or iterator+sentinel pair for C++20 ranges support) @@ -4490,8 +4527,11 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec basic_json result; auto ia = detail::input_adapter(std::move(first), std::move(last)); detail::json_sax_dom_parser sdp(result, allow_exceptions); - const bool res = binary_reader(std::move(ia), input_format_t::cbor).sax_parse(input_format_t::cbor, &sdp, strict, tag_handler); // cppcheck-suppress[accessMoved] - return res ? result : basic_json(value_t::discarded); + if (!binary_reader(std::move(ia), input_format_t::cbor).sax_parse(input_format_t::cbor, &sdp, strict, tag_handler)) // cppcheck-suppress[accessMoved] + { + result = value_t::discarded; + } + return result; } template @@ -4516,8 +4556,11 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec auto ia = i.get(); detail::json_sax_dom_parser sdp(result, allow_exceptions); // NOLINTNEXTLINE(hicpp-move-const-arg,performance-move-const-arg) - const bool res = binary_reader(std::move(ia), input_format_t::cbor).sax_parse(input_format_t::cbor, &sdp, strict, tag_handler); // cppcheck-suppress[accessMoved] - return res ? result : basic_json(value_t::discarded); + if (!binary_reader(std::move(ia), input_format_t::cbor).sax_parse(input_format_t::cbor, &sdp, strict, tag_handler)) // cppcheck-suppress[accessMoved] + { + result = value_t::discarded; + } + return result; } /// @brief create a JSON value from an input in MessagePack format @@ -4531,8 +4574,11 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec basic_json result; auto ia = detail::input_adapter(std::forward(i)); detail::json_sax_dom_parser sdp(result, allow_exceptions); - const bool res = binary_reader(std::move(ia), input_format_t::msgpack).sax_parse(input_format_t::msgpack, &sdp, strict); // cppcheck-suppress[accessMoved] - return res ? result : basic_json(value_t::discarded); + if (!binary_reader(std::move(ia), input_format_t::msgpack).sax_parse(input_format_t::msgpack, &sdp, strict)) // cppcheck-suppress[accessMoved] + { + result = value_t::discarded; + } + return result; } /// @brief create a JSON value from an input in MessagePack format (iterator pair, or iterator+sentinel pair for C++20 ranges support) @@ -4547,8 +4593,11 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec basic_json result; auto ia = detail::input_adapter(std::move(first), std::move(last)); detail::json_sax_dom_parser sdp(result, allow_exceptions); - const bool res = binary_reader(std::move(ia), input_format_t::msgpack).sax_parse(input_format_t::msgpack, &sdp, strict); // cppcheck-suppress[accessMoved] - return res ? result : basic_json(value_t::discarded); + if (!binary_reader(std::move(ia), input_format_t::msgpack).sax_parse(input_format_t::msgpack, &sdp, strict)) // cppcheck-suppress[accessMoved] + { + result = value_t::discarded; + } + return result; } template @@ -4571,8 +4620,11 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec auto ia = i.get(); detail::json_sax_dom_parser sdp(result, allow_exceptions); // NOLINTNEXTLINE(hicpp-move-const-arg,performance-move-const-arg) - const bool res = binary_reader(std::move(ia), input_format_t::msgpack).sax_parse(input_format_t::msgpack, &sdp, strict); // cppcheck-suppress[accessMoved] - return res ? result : basic_json(value_t::discarded); + if (!binary_reader(std::move(ia), input_format_t::msgpack).sax_parse(input_format_t::msgpack, &sdp, strict)) // cppcheck-suppress[accessMoved] + { + result = value_t::discarded; + } + return result; } /// @brief create a JSON value from an input in UBJSON format @@ -4586,8 +4638,11 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec basic_json result; auto ia = detail::input_adapter(std::forward(i)); detail::json_sax_dom_parser sdp(result, allow_exceptions); - const bool res = binary_reader(std::move(ia), input_format_t::ubjson).sax_parse(input_format_t::ubjson, &sdp, strict); // cppcheck-suppress[accessMoved] - return res ? result : basic_json(value_t::discarded); + if (!binary_reader(std::move(ia), input_format_t::ubjson).sax_parse(input_format_t::ubjson, &sdp, strict)) // cppcheck-suppress[accessMoved] + { + result = value_t::discarded; + } + return result; } /// @brief create a JSON value from an input in UBJSON format (iterator pair, or iterator+sentinel pair for C++20 ranges support) @@ -4602,8 +4657,11 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec basic_json result; auto ia = detail::input_adapter(std::move(first), std::move(last)); detail::json_sax_dom_parser sdp(result, allow_exceptions); - const bool res = binary_reader(std::move(ia), input_format_t::ubjson).sax_parse(input_format_t::ubjson, &sdp, strict); // cppcheck-suppress[accessMoved] - return res ? result : basic_json(value_t::discarded); + if (!binary_reader(std::move(ia), input_format_t::ubjson).sax_parse(input_format_t::ubjson, &sdp, strict)) // cppcheck-suppress[accessMoved] + { + result = value_t::discarded; + } + return result; } template @@ -4626,8 +4684,11 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec auto ia = i.get(); detail::json_sax_dom_parser sdp(result, allow_exceptions); // NOLINTNEXTLINE(hicpp-move-const-arg,performance-move-const-arg) - const bool res = binary_reader(std::move(ia), input_format_t::ubjson).sax_parse(input_format_t::ubjson, &sdp, strict); // cppcheck-suppress[accessMoved] - return res ? result : basic_json(value_t::discarded); + if (!binary_reader(std::move(ia), input_format_t::ubjson).sax_parse(input_format_t::ubjson, &sdp, strict)) // cppcheck-suppress[accessMoved] + { + result = value_t::discarded; + } + return result; } /// @brief create a JSON value from an input in BJData format @@ -4641,8 +4702,11 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec basic_json result; auto ia = detail::input_adapter(std::forward(i)); detail::json_sax_dom_parser sdp(result, allow_exceptions); - const bool res = binary_reader(std::move(ia), input_format_t::bjdata).sax_parse(input_format_t::bjdata, &sdp, strict); // cppcheck-suppress[accessMoved] - return res ? result : basic_json(value_t::discarded); + if (!binary_reader(std::move(ia), input_format_t::bjdata).sax_parse(input_format_t::bjdata, &sdp, strict)) // cppcheck-suppress[accessMoved] + { + result = value_t::discarded; + } + return result; } /// @brief create a JSON value from an input in BJData format (iterator pair, or iterator+sentinel pair for C++20 ranges support) @@ -4657,8 +4721,11 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec basic_json result; auto ia = detail::input_adapter(std::move(first), std::move(last)); detail::json_sax_dom_parser sdp(result, allow_exceptions); - const bool res = binary_reader(std::move(ia), input_format_t::bjdata).sax_parse(input_format_t::bjdata, &sdp, strict); // cppcheck-suppress[accessMoved] - return res ? result : basic_json(value_t::discarded); + if (!binary_reader(std::move(ia), input_format_t::bjdata).sax_parse(input_format_t::bjdata, &sdp, strict)) // cppcheck-suppress[accessMoved] + { + result = value_t::discarded; + } + return result; } /// @brief create a JSON value from an input in BSON format @@ -4672,8 +4739,11 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec basic_json result; auto ia = detail::input_adapter(std::forward(i)); detail::json_sax_dom_parser sdp(result, allow_exceptions); - const bool res = binary_reader(std::move(ia), input_format_t::bson).sax_parse(input_format_t::bson, &sdp, strict); // cppcheck-suppress[accessMoved] - return res ? result : basic_json(value_t::discarded); + if (!binary_reader(std::move(ia), input_format_t::bson).sax_parse(input_format_t::bson, &sdp, strict)) // cppcheck-suppress[accessMoved] + { + result = value_t::discarded; + } + return result; } /// @brief create a JSON value from an input in BSON format (iterator pair, or iterator+sentinel pair for C++20 ranges support) @@ -4688,8 +4758,11 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec basic_json result; auto ia = detail::input_adapter(std::move(first), std::move(last)); detail::json_sax_dom_parser sdp(result, allow_exceptions); - const bool res = binary_reader(std::move(ia), input_format_t::bson).sax_parse(input_format_t::bson, &sdp, strict); // cppcheck-suppress[accessMoved] - return res ? result : basic_json(value_t::discarded); + if (!binary_reader(std::move(ia), input_format_t::bson).sax_parse(input_format_t::bson, &sdp, strict)) // cppcheck-suppress[accessMoved] + { + result = value_t::discarded; + } + return result; } template @@ -4712,8 +4785,11 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec auto ia = i.get(); detail::json_sax_dom_parser sdp(result, allow_exceptions); // NOLINTNEXTLINE(hicpp-move-const-arg,performance-move-const-arg) - const bool res = binary_reader(std::move(ia), input_format_t::bson).sax_parse(input_format_t::bson, &sdp, strict); // cppcheck-suppress[accessMoved] - return res ? result : basic_json(value_t::discarded); + if (!binary_reader(std::move(ia), input_format_t::bson).sax_parse(input_format_t::bson, &sdp, strict)) // cppcheck-suppress[accessMoved] + { + result = value_t::discarded; + } + return result; } /// @} @@ -4939,6 +5015,36 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec // note erase performs range check parent.erase(json_pointer::template array_index(last_path)); } + else + { + // the parent of a "remove" target must be an object or array + // (see #5396) + JSON_THROW(out_of_range::create(413, detail::concat("cannot remove value: the JSON Patch 'remove' target's parent is of type ", parent.type_name(), ", but must be an object or array"), &parent)); + } + }; + + // RFC 6902 (section 4.4) forbids "from" from being a proper prefix + // of "path" for a "move" operation: a location cannot be moved into + // one of its own children. Compares reference tokens (already + // unescaped by json_pointer's parser) rather than the raw pointer + // strings, since a token may itself contain an escaped '/' or '~' + // that would defeat a naive string-prefix comparison. "from" equal + // to "path" is *not* a proper prefix and must return false. + const auto is_proper_prefix = [](const json_pointer & from, const json_pointer & to) + { + const auto from_size = from.reference_tokens.size(); + if (from_size >= to.reference_tokens.size()) + { + return false; + } + for (std::size_t i = 0; i < from_size; ++i) + { + if (!(from.reference_tokens[i] == to.reference_tokens[i])) + { + return false; + } + } + return true; }; // type check: top level value must be an array @@ -5016,6 +5122,11 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec const auto from_path = get_value("move", "from", true).template get(); json_pointer from_ptr(from_path); + if (JSON_HEDLEY_UNLIKELY(is_proper_prefix(from_ptr, ptr))) + { + JSON_THROW(out_of_range::create(414, detail::concat("cannot move value: 'from' path '", from_path, "' is a proper prefix of 'path' '", path, "'"), &result)); + } + // the "from" location must exist - use at() basic_json const v = result.at(from_ptr); diff --git a/include/nlohmann/thirdparty/hedley/hedley_undef.hpp b/include/nlohmann/thirdparty/hedley/hedley_undef.hpp index 1b8bd4338..e4d9838cb 100644 --- a/include/nlohmann/thirdparty/hedley/hedley_undef.hpp +++ b/include/nlohmann/thirdparty/hedley/hedley_undef.hpp @@ -17,7 +17,7 @@ #undef JSON_HEDLEY_CLANG_HAS_ATTRIBUTE #undef JSON_HEDLEY_CLANG_HAS_BUILTIN #undef JSON_HEDLEY_CLANG_HAS_CPP_ATTRIBUTE -#undef JSON_HEDLEY_CLANG_HAS_DECLSPEC_DECLSPEC_ATTRIBUTE +#undef JSON_HEDLEY_CLANG_HAS_DECLSPEC_ATTRIBUTE #undef JSON_HEDLEY_CLANG_HAS_EXTENSION #undef JSON_HEDLEY_CLANG_HAS_FEATURE #undef JSON_HEDLEY_CLANG_HAS_WARNING @@ -108,7 +108,10 @@ #undef JSON_HEDLEY_PELLES_VERSION_CHECK #undef JSON_HEDLEY_PGI_VERSION #undef JSON_HEDLEY_PGI_VERSION_CHECK +#undef JSON_HEDLEY_PRAGMA #undef JSON_HEDLEY_PREDICT +#undef JSON_HEDLEY_PREDICT_FALSE +#undef JSON_HEDLEY_PREDICT_TRUE #undef JSON_HEDLEY_PRINTF_FORMAT #undef JSON_HEDLEY_PRIVATE #undef JSON_HEDLEY_PUBLIC diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index 58d0aaa76..6bdd6563f 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -4945,8 +4945,8 @@ NLOHMANN_JSON_NAMESPACE_END // code stumbling over this. See https://github.com/nlohmann/json/issues/4087 // for a discussion. #if defined(__clang__) - #pragma clang diagnostic push - #pragma clang diagnostic ignored "-Wweak-vtables" + JSON_HEDLEY_DIAGNOSTIC_PUSH + JSON_HEDLEY_PRAGMA(clang diagnostic ignored "-Wweak-vtables") #endif NLOHMANN_JSON_NAMESPACE_BEGIN @@ -5199,7 +5199,7 @@ class other_error : public exception NLOHMANN_JSON_NAMESPACE_END #if defined(__clang__) - #pragma clang diagnostic pop + JSON_HEDLEY_DIAGNOSTIC_POP #endif // #include @@ -5693,6 +5693,17 @@ inline void from_json(const BasicJsonType& j, CompatibleArrayType& bin) } } +template +auto from_json_object_reserve(ConstructibleObjectType& obj, typename ConstructibleObjectType::size_type size, priority_tag<1> /*unused*/) +-> decltype(obj.reserve(size), void()) +{ + obj.reserve(size); +} + +template +inline void from_json_object_reserve(ConstructibleObjectType& /*obj*/, std::size_t /*size*/, priority_tag<0> /*unused*/) +{} + template::value, int> = 0> inline void from_json(const BasicJsonType& j, ConstructibleObjectType& obj) @@ -5704,6 +5715,7 @@ inline void from_json(const BasicJsonType& j, ConstructibleObjectType& obj) ConstructibleObjectType ret; const auto* inner_object = j.template get_ptr(); + from_json_object_reserve(ret, inner_object->size(), priority_tag<1> {}); for (const auto& p : *inner_object) { ret.emplace(p.first, p.second.template get()); @@ -5975,6 +5987,8 @@ NLOHMANN_JSON_NAMESPACE_END // #include +// #include + // #include // #include @@ -6204,10 +6218,10 @@ NLOHMANN_JSON_NAMESPACE_END namespace std { +// Fix: https://github.com/nlohmann/json/issues/1401 #if defined(__clang__) - // Fix: https://github.com/nlohmann/json/issues/1401 - #pragma clang diagnostic push - #pragma clang diagnostic ignored "-Wmismatched-tags" + JSON_HEDLEY_DIAGNOSTIC_PUSH + JSON_HEDLEY_PRAGMA(clang diagnostic ignored "-Wmismatched-tags") #endif template class tuple_size<::nlohmann::detail::iteration_proxy_value> // NOLINT(cert-dcl58-cpp) @@ -6222,7 +6236,7 @@ class tuple_element> ::nlohmann::detail::iteration_proxy_value> ())); }; #if defined(__clang__) - #pragma clang diagnostic pop + JSON_HEDLEY_DIAGNOSTIC_POP #endif } // namespace std @@ -7223,11 +7237,31 @@ class input_stream_adapter // General-purpose iterator-based adapter. It might not be as fast as // theoretically possible for some containers, but it is extremely versatile. -// SentinelType defaults to IteratorType for backward compatibility, but may -// be a different type (e.g., a C++20 sentinel or counted_iterator). +// SentinelType defaults to IteratorType for backward compatibility, but may be +// a different type, e.g. a C++20 sentinel such as std::default_sentinel_t when +// IteratorType is a std::counted_iterator. template class iterator_input_adapter { + // Whether the number of elements between two positions can be computed in + // O(1): either the iterator and the sentinel have the same type (plain + // std::distance) or, in C++20, the sentinel is a sized sentinel for the + // iterator (std::ranges::distance), e.g. std::default_sentinel_t paired + // with std::counted_iterator. + // + // JSON_HAS_RANGES gates the C++20 branch: on standard libraries with an + // incomplete (libstdc++ < 11, see #4440) evaluating + // std::contiguous_iterator on a std::counted_iterator is a hard error + // instead of yielding false, and these traits are instantiated for every + // adapter. Such toolchains fall back to the pointer-only test and simply + // use the byte-at-a-time scanner. + static constexpr bool sentinel_is_sized = +#if JSON_HAS_RANGES && defined(__cpp_lib_concepts) && defined(JSON_HAS_CPP_20) + std::is_same::value || std::sized_sentinel_for; +#else + std::is_same::value; +#endif + public: using char_type = typename std::iterator_traits::value_type; @@ -7239,7 +7273,7 @@ class iterator_input_adapter // in wide_string_input_adapter, which does not expose this). static constexpr bool supports_seek = std::is_same::iterator_category, std::random_access_iterator_tag>::value - && std::is_same::value + && sentinel_is_sized && sizeof(char_type) == 1; iterator_input_adapter(IteratorType first, SentinelType last) @@ -7287,30 +7321,60 @@ class iterator_input_adapter private: // whether IteratorType refers to a contiguous range and therefore supports // a std::memcpy fast path (pointers always do; in C++20 we can also detect - // library iterators such as those of std::vector and std::string). - // Computing the available element count needs either same-type iterators - // (plain std::distance) or, in C++20, a sized sentinel (std::ranges::distance), - // e.g. std::counted_iterator paired with std::default_sentinel_t. - static constexpr bool iterator_is_contiguous = -#if defined(__cpp_lib_concepts) && defined(JSON_HAS_CPP_20) - (std::is_same::value || std::sized_sentinel_for) - && (std::contiguous_iterator || std::is_pointer::value); + // library iterators such as those of std::vector and std::string). The + // available element count must also be computable in O(1), hence + // sentinel_is_sized. + static constexpr bool iterator_is_contiguous = sentinel_is_sized && +#if JSON_HAS_RANGES && defined(__cpp_lib_concepts) && defined(JSON_HAS_CPP_20) + (std::contiguous_iterator || std::is_pointer::value); #else - std::is_same::value && std::is_pointer::value; + std::is_pointer::value; #endif + // number of unread elements in [current, end) + std::size_t remaining_count() const + { +#if JSON_HAS_RANGES && defined(__cpp_lib_concepts) && defined(JSON_HAS_CPP_20) + // std::ranges::distance also supports sized sentinels of a different + // type (e.g. std::counted_iterator + std::default_sentinel_t) + return static_cast(std::ranges::distance(current, end)); +#else + return static_cast(std::distance(current, end)); +#endif + } + + public: + // Whether the remaining input is a single contiguous block of 1-byte + // elements that the lexer can inspect directly (used for the SWAR string + // fast path). + static constexpr bool supports_bulk_scan = + iterator_is_contiguous && sizeof(char_type) == 1; + + // Pointer to the next unread element; only valid when bulk_remaining() > 0. + const char_type* bulk_data() const + { + return &*current; + } + + // Number of unread elements available as one contiguous block. + std::size_t bulk_remaining() const + { + return remaining_count(); + } + + // Consume @a n elements previously inspected via bulk_data(). + void bulk_skip(std::size_t n) + { + std::advance(current, static_cast::difference_type>(n)); + } + + private: // contiguous fast path: bulk copy the remaining range with std::memcpy template std::size_t get_elements_impl(T* dest, std::size_t count, std::true_type /*contiguous*/) { const std::size_t wanted = count * sizeof(T); -#if defined(__cpp_lib_concepts) && defined(JSON_HAS_CPP_20) - // std::ranges::distance also supports sized sentinels of a different - // type (e.g. std::counted_iterator + std::default_sentinel_t) - const std::size_t available = static_cast(std::ranges::distance(current, end)) * sizeof(char_type); -#else - const std::size_t available = static_cast(std::distance(current, end)) * sizeof(char_type); -#endif + const std::size_t available = remaining_count() * sizeof(char_type); const std::size_t copied = (std::min)(wanted, available); if (JSON_HEDLEY_LIKELY(copied != 0)) { @@ -7638,6 +7702,46 @@ typename iterator_input_adapter_factory::adapter_typ return factory_type::create(first, last); } +// The element type a container's data() points at, cv-qualifiers removed. +// Ill-formed - and therefore SFINAE-friendly - for types without data(). +template +using container_data_t = typename std::remove_cv().data()) >::type >::type; + +// The container's own element type, cv-qualifiers removed. It is looked up on +// the bare type so it is also found when ContainerType is deduced as a +// reference by the forwarding-reference overload below. +template +using container_value_t = typename std::remove_cv < + typename std::remove_cv::type>::type::value_type >::type; + +// Detect a container that stores its elements contiguously as single bytes +// (std::string, std::vector, std::array, +// std::string_view, ...). Such inputs are wrapped in a pointer-based adapter so +// they benefit from the contiguous fast paths (bulk string scanning, memcpy for +// binary formats) in every C++ standard - not only in C++20, where the standard +// library iterators model std::contiguous_iterator and are detected directly. +// +// data() and size() on their own would be duck typing: they say nothing about +// size() counting the units data() points at, and reading [data(), data() + +// size()) as bytes would be wrong for a type where it does not. Requiring the +// container's own value_type to be that same single-byte element ties the two +// together; every contiguous standard container satisfies it. Anything else +// keeps the iterator-based adapter, which is always correct - only slower. +template +struct is_contiguous_byte_container : std::false_type {}; + +template +struct is_contiguous_byte_container < ContainerType, void_t < + container_data_t, + container_value_t, +decltype(std::declval().size()) >> + : std::integral_constant < bool, + std::is_pointer().data())>::value&& + std::is_integral>::value&& + sizeof(container_data_t) == 1 && + std::is_same, container_value_t>::value > {}; + // Convenience shorthand from container to iterator // Enables ADL on begin(container) and end(container) // Encloses the using declarations in namespace for not to leak them to outside scope @@ -7665,12 +7769,32 @@ struct container_input_adapter_factory< ContainerType, } // namespace container_input_adapter_factory_impl -template -typename container_input_adapter_factory_impl::container_input_adapter_factory::adapter_type input_adapter(ContainerType&& container) +// General container path (iterator-based). Contiguous single-byte containers +// are excluded here and routed through the pointer-based overload below. +template < typename ContainerType, + enable_if_t < !is_contiguous_byte_container::value, int > = 0 > +typename container_input_adapter_factory_impl::container_input_adapter_factory::adapter_type input_adapter(ContainerType && container) { return container_input_adapter_factory_impl::container_input_adapter_factory::create(std::forward(container)); } +// Contiguous single-byte containers (std::string, std::vector, ...) are +// wrapped in a pointer-based adapter so the contiguous fast paths apply in every +// standard. The pointer keeps the container's own element type (const char* for +// std::string, const std::uint8_t* for std::vector, ...), so the +// resulting char_type - and therefore the parsing behavior - is byte-for-byte +// identical to the iterator-based path; only the raw pointer additionally +// enables the bulk fast paths. The container outlives the adapter for the whole +// parse (temporaries live until the end of the full expression), exactly as the +// iterators it replaces did. +template < typename ContainerType, + enable_if_t < is_contiguous_byte_container::value, int > = 0 > +auto input_adapter(const ContainerType& container) +-> decltype(input_adapter(container.data(), container.data() + container.size())) +{ + return input_adapter(container.data(), container.data() + container.size()); +} + // specialization for std::string using string_input_adapter_type = decltype(input_adapter(std::declval())); @@ -7800,8 +7924,603 @@ NLOHMANN_JSON_NAMESPACE_END // #include +// #include +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ +// | | |__ | | | | | | version 3.12.0 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann +// SPDX-License-Identifier: MIT + + + +#include // array +#include // FLT_EVAL_METHOD +#include // size_t +#include // int64_t, uint64_t +#include // numeric_limits + +// #include + + +// std::from_chars lives in , but being in C++17 mode does not +// guarantee the header exists: GCC 7 sets __cplusplus to C++17 yet ships no +// (added in GCC 8; floating-point support in GCC 11). Guard the +// include with __has_include so such toolchains fall back to the scalar path. +#if defined(JSON_HAS_CPP_17) && defined(__has_include) + #if __has_include() + #include // from_chars (only used when __cpp_lib_to_chars is defined) + #include // errc + #endif +#endif + +// This file contains the value-conversion helpers used by the lexer to turn an +// already-validated number token into a value, without the locale/errno +// overhead of std::strtoull/std::strtod. They are free functions so the lexer +// stays focused on scanning; see lexer::convert_number(). + +NLOHMANN_JSON_NAMESPACE_BEGIN +namespace detail +{ + +/*! +@brief fast integer parser for an already-validated unsigned integer + +The number scanner has already checked that [first, last) is a valid JSON +integer, so this only needs to accumulate the digits and detect overflow. This +avoids the locale/errno machinery of std::strtoull, which dominates +integer-heavy inputs. + +@param[in] first pointer to the first character (a digit) +@param[in] last pointer past the last character +@param[out] value the parsed value on success +@return true if the value fit into @a NumberUnsignedType; false on overflow, in + which case the caller falls back to floating-point parsing (matching the + previous std::strtoull behavior) +*/ +template +bool parse_integer_unsigned(const char* first, const char* last, NumberUnsignedType& value) noexcept +{ + // accumulate in the widest unsigned type used by the previous strtoull + // path so the overflow behavior is unchanged for custom number types + std::uint64_t x = 0; + constexpr std::uint64_t cutoff = (std::numeric_limits::max)() / 10u; + constexpr std::uint64_t cutlim = (std::numeric_limits::max)() % 10u; + for (const char* p = first; p != last; ++p) + { + const auto digit = static_cast(static_cast(*p) - static_cast('0')); + if (JSON_HEDLEY_UNLIKELY(x > cutoff || (x == cutoff && digit > cutlim))) + { + return false; + } + x = (x * 10u) + digit; + } + value = static_cast(x); + // reject values that do not round-trip into a narrower NumberUnsignedType + return static_cast(value) == x; +} + +/*! +@brief fast integer parser for an already-validated negative integer + +@param[in] first pointer to the leading '-' +@param[in] last pointer past the last character +@param[out] value the parsed (negative) value on success +@return true on success; false on overflow (caller falls back to float) +*/ +template +bool parse_integer_signed(const char* first, const char* last, NumberIntegerType& value) noexcept +{ + // the state machine only reaches the signed path via a leading '-' + JSON_ASSERT(first != last && *first == '-'); + std::uint64_t magnitude = 0; + // |INT64_MIN| == INT64_MAX + 1; this is the largest admissible magnitude + constexpr std::uint64_t limit = static_cast((std::numeric_limits::max)()) + 1u; + for (const char* p = first + 1; p != last; ++p) + { + const auto digit = static_cast(static_cast(*p) - static_cast('0')); + if (JSON_HEDLEY_UNLIKELY(magnitude > (limit - digit) / 10u)) + { + return false; + } + magnitude = (magnitude * 10u) + digit; + } + const std::int64_t x = (magnitude == limit) + ? (std::numeric_limits::min)() + : -static_cast(magnitude); + value = static_cast(x); + // reject values that do not round-trip into a narrower NumberIntegerType + return static_cast(value) == x; +} + +/*! +@brief exact fast path for parsing a `double` (Clinger's algorithm) + +For the common case - at most 19 significant digits, a decimal exponent in +[-22, 22], and a significand below 2^53 - the value equals significand * +10^exp computed in IEEE-754 double arithmetic, which is exact under +round-to-nearest because both operands are exactly representable. This is the +same fast path used by fast_float/simdjson; the general cases are left to +std::strtod. The parser only activates for number_float_t == double; float and +long double keep the std::strtof/std::strtold paths (see the templated overload +below). + +@param[in] first pointer to the first character of the number +@param[in] last pointer past the last character +@param[in] decimal_point the (locale-dependent) decimal point character +@param[out] out the parsed value on success +@return true if the value was parsed exactly; false to fall back to strtod +*/ +template +bool parse_float_fast(const char* first, const char* last, DecimalPointType decimal_point, double& out) noexcept +{ +#if defined(FLT_EVAL_METHOD) && FLT_EVAL_METHOD != 0 + // Clinger's fast path is only exact when double operations are evaluated in + // true double precision. On platforms that keep intermediates in extended + // precision (e.g. the x87 FPU on 32-bit x86, where FLT_EVAL_METHOD == 2) the + // single significand * 10^scale step is double-rounded and can be 1 ULP off, + // so decline and let the caller fall back to the correctly-rounded + // std::from_chars / std::strtod path. + static_cast(first); + static_cast(last); + static_cast(decimal_point); + static_cast(out); + return false; +#else + static const std::array powers_of_ten = + { + { + 1e0, 1e1, 1e2, 1e3, 1e4, 1e5, 1e6, 1e7, 1e8, 1e9, 1e10, 1e11, + 1e12, 1e13, 1e14, 1e15, 1e16, 1e17, 1e18, 1e19, 1e20, 1e21, 1e22 + } + }; + + const char* p = first; + bool negative = false; + if (p != last && (*p == '-' || *p == '+')) + { + negative = (*p == '-'); + ++p; + } + + std::uint64_t significand = 0; + int num_digits = 0; + int fractional_digits = 0; + bool seen_dot = false; + bool any_digit = false; + for (; p != last; ++p) + { + const char c = *p; + if (c >= '0' && c <= '9') + { + any_digit = true; + if (JSON_HEDLEY_UNLIKELY(num_digits >= 19)) + { + return false; // significand may not fit into uint64_t + } + significand = (significand * 10u) + static_cast(c - '0'); + ++num_digits; + fractional_digits += static_cast(seen_dot); + } + else if (static_cast(c) == decimal_point) + { + if (JSON_HEDLEY_UNLIKELY(seen_dot)) + { + return false; + } + seen_dot = true; + } + else if (c == 'e' || c == 'E') + { + ++p; + break; + } + else + { + return false; + } + } + if (JSON_HEDLEY_UNLIKELY(!any_digit)) + { + return false; + } + + int exponent = 0; + if (p != last) // an exponent part remains + { + bool exp_negative = false; + if (p != last && (*p == '-' || *p == '+')) + { + exp_negative = (*p == '-'); + ++p; + } + bool any_exp_digit = false; + for (; p != last; ++p) + { + if (JSON_HEDLEY_UNLIKELY(*p < '0' || *p > '9')) + { + return false; + } + exponent = (exponent * 10) + (*p - '0'); + any_exp_digit = true; + if (JSON_HEDLEY_UNLIKELY(exponent > 9999)) + { + return false; + } + } + if (JSON_HEDLEY_UNLIKELY(!any_exp_digit)) + { + return false; + } + if (exp_negative) + { + exponent = -exponent; + } + } + + const int scale = exponent - fractional_digits; + if (JSON_HEDLEY_UNLIKELY(significand >= (static_cast(1) << 53))) + { + return false; // significand not exactly representable as double + } + + auto result = static_cast(significand); + if (scale >= 0) + { + if (JSON_HEDLEY_UNLIKELY(scale > 22)) + { + return false; + } + result *= powers_of_ten[static_cast(scale)]; + } + else + { + if (JSON_HEDLEY_UNLIKELY(-scale > 22)) + { + return false; + } + result /= powers_of_ten[static_cast(-scale)]; + } + out = negative ? -result : result; + return true; +#endif +} + +/// fast float path is only exact for `double`; decline for float/long double +template +bool parse_float_fast(const char* /*first*/, const char* /*last*/, DecimalPointType /*decimal_point*/, FloatType& /*out*/) noexcept +{ + return false; +} + +/*! +@brief parse a float with std::from_chars (Eisel-Lemire) when available + +std::from_chars is locale-independent, correctly rounded, and - via the +Eisel-Lemire algorithm in modern standard libraries - much faster than strtod +over the whole value range (not just the Clinger subset). It is used only when +__cpp_lib_to_chars indicates full floating-point support and only when it +consumes the entire token ([first, last)); a partial parse means the buffer +uses a non-'.' locale decimal point, in which case the caller falls back to the +locale-aware path. An under-/overflow (result_out_of_range) also declines, so +the caller's strtod fallback supplies the well-defined ±inf/0 result the parser +expects (side-stepping the P4168 divergence between implementations). + +@return true if the value was parsed exactly and fully; false to fall back +*/ +template +bool parse_float_from_chars(const char* first, const char* last, FloatType& out) noexcept +{ + // JSON_HAS_CPP_17 must gate the use as well as the include above: + // some standard libraries (e.g. libstdc++ 15) define __cpp_lib_to_chars even + // in C++14 mode, where is not included. +#if defined(JSON_HAS_CPP_17) && defined(__cpp_lib_to_chars) + const auto result = std::from_chars(first, last, out); + return result.ec == std::errc() && result.ptr == last; +#else + static_cast(first); + static_cast(last); + static_cast(out); + return false; +#endif +} + +} // namespace detail +NLOHMANN_JSON_NAMESPACE_END + // #include +// #include +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ +// | | |__ | | | | | | version 3.12.0 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann +// SPDX-License-Identifier: MIT + + + +#include // size_t +#include // uint64_t +#include // memcpy + +// #include + + +// Optional SIMD backend for bulk UTF-8 validation. This is an opt-in external +// dependency: nlohmann/json itself stays header-only and the C++11 scalar +// validator below is always available; defining JSON_USE_SIMDUTF additionally +// requires the simdutf headers on the include path and linking the simdutf +// library. See string_bulk_run(). +// +// simdutf.h itself requires C++17 - it rejects older standards with an #error - +// so the backend is only compiled in from C++17 on. Below that the macro has no +// effect and the scalar validator is used; it accepts and rejects exactly the +// same input, so only throughput differs. macro_scope.hpp is included above to +// have JSON_HAS_CPP_17 available for this test. +#if defined(JSON_USE_SIMDUTF) && defined(JSON_HAS_CPP_17) + #include +#endif + +// This file contains the byte-level string-scanning helpers used by the lexer's +// contiguous fast path. They operate purely on raw bytes (no dependency on the +// lexer's template parameters) so they are free functions, keeping the lexer +// itself focused on the state machine; see lexer::scan_string_bulk(). + +NLOHMANN_JSON_NAMESPACE_BEGIN +namespace detail +{ + +// classify a single byte as needing individual string handling: the closing +// quote, an escape, a control character, or a non-ASCII (UTF-8) +// lead/continuation byte. Ordinary bytes (0x20..0x7F except '"' and '\\') are +// copied verbatim, which the bulk scanner does 8 bytes at a time. +inline bool is_string_special(unsigned char c) noexcept +{ + return c == '\"' || c == '\\' || c < 0x20u || c >= 0x80u; +} + +// SWAR helper: return a word whose high bit is set in every byte of @a v that +// is_string_special(); zero if the 8 bytes are all ordinary. +inline std::uint64_t swar_string_special(std::uint64_t v) noexcept +{ + constexpr std::uint64_t ones = 0x0101010101010101ull; + constexpr std::uint64_t high = 0x8080808080808080ull; + const std::uint64_t q = v ^ 0x2222222222222222ull; // '"' (0x22) + const std::uint64_t b = v ^ 0x5C5C5C5C5C5C5C5Cull; // '\\' (0x5C) + const std::uint64_t has_quote = (q - ones) & ~q & high; + const std::uint64_t has_backslash = (b - ones) & ~b & high; + const std::uint64_t has_control = (v - 0x2020202020202020ull) & ~v & high; // < 0x20 + const std::uint64_t has_non_ascii = v & high; // >= 0x80 + return has_quote | has_backslash | has_control | has_non_ascii; +} + +// return the index of the first is_string_special() byte in [data, data+n), or +// n if every byte is ordinary; scans 8 bytes at a time +inline std::size_t find_string_special(const unsigned char* data, std::size_t n) noexcept +{ + std::size_t i = 0; + for (; i + 8 <= n; i += 8) + { + std::uint64_t word = 0; + std::memcpy(&word, data + i, sizeof(word)); + if (swar_string_special(word) != 0) + { + // a special byte is in this word; locate it (endian-agnostic) + for (std::size_t j = 0; j < 8; ++j) + { + if (is_string_special(data[i + j])) + { + return i + j; + } + } + } + } + for (; i < n; ++i) + { + if (is_string_special(data[i])) + { + return i; + } + } + return n; +} + +// classify a byte as one the serializer must NOT copy verbatim when +// ensure_ascii is requested: the closing quote, an escape, a control character +// (< 0x20), DEL (0x7F), or any non-ASCII byte (>= 0x80). Everything else - +// printable ASCII except '"' and '\\' - is emitted unchanged. Note this differs +// from is_string_special() only in that 0x7F is also a stop (it is escaped as +// \u007f under ensure_ascii). +inline bool is_ascii_copyable(unsigned char c) noexcept +{ + return c >= 0x20u && c < 0x7Fu && c != '"' && c != '\\'; +} + +// return the index of the first byte in [data, data+n) that is NOT +// is_ascii_copyable(), or n if every byte can be copied verbatim; scans 8 bytes +// at a time. Used by the serializer's ensure_ascii fast path. +inline std::size_t find_ascii_copyable_run(const unsigned char* data, std::size_t n) noexcept +{ + constexpr std::uint64_t ones = 0x0101010101010101ull; + constexpr std::uint64_t high = 0x8080808080808080ull; + std::size_t i = 0; + for (; i + 8 <= n; i += 8) + { + std::uint64_t v = 0; + std::memcpy(&v, data + i, sizeof(v)); + const std::uint64_t q = v ^ 0x2222222222222222ull; // '"' (0x22) + const std::uint64_t b = v ^ 0x5C5C5C5C5C5C5C5Cull; // '\\' (0x5C) + const std::uint64_t d = v ^ 0x7F7F7F7F7F7F7F7Full; // DEL (0x7F) + const std::uint64_t stop = ((q - ones) & ~q & high) // == '"' + | ((b - ones) & ~b & high) // == '\\' + | ((d - ones) & ~d & high) // == 0x7F + | ((v - 0x2020202020202020ull) & ~v & high) // < 0x20 + | (v & high); // >= 0x80 + if (stop != 0) + { + break; + } + } + for (; i < n; ++i) + { + if (!is_ascii_copyable(data[i])) + { + return i; + } + } + return n; +} + +// Validate one UTF-8 sequence at the front of [data, data+avail). Returns its +// length (2..4) only when the bytes form a *well-formed* sequence using exactly +// the same ranges as scan_string()'s per-byte switch, so the bulk path accepts +// precisely what the byte path accepts. Returns 0 for anything that is invalid, +// incomplete, or that the byte path must diagnose (the caller then defers to +// that path, keeping error messages unchanged). Lead bytes < 0x80 are handled +// by the caller and never passed here. +inline std::size_t validate_one_utf8(const unsigned char* data, std::size_t avail) noexcept +{ + const unsigned char c0 = data[0]; + if (c0 >= 0xC2 && c0 <= 0xDF) // U+0080..U+07FF + { + if (avail >= 2 && data[1] >= 0x80 && data[1] <= 0xBF) + { + return 2; + } + } + else if (c0 == 0xE0) // U+0800..U+0FFF + { + if (avail >= 3 && data[1] >= 0xA0 && data[1] <= 0xBF && data[2] >= 0x80 && data[2] <= 0xBF) + { + return 3; + } + } + else if ((c0 >= 0xE1 && c0 <= 0xEC) || c0 == 0xEE || c0 == 0xEF) // U+1000..U+CFFF, U+E000..U+FFFF + { + if (avail >= 3 && data[1] >= 0x80 && data[1] <= 0xBF && data[2] >= 0x80 && data[2] <= 0xBF) + { + return 3; + } + } + else if (c0 == 0xED) // U+D000..U+D7FF (excludes surrogates) + { + if (avail >= 3 && data[1] >= 0x80 && data[1] <= 0x9F && data[2] >= 0x80 && data[2] <= 0xBF) + { + return 3; + } + } + else if (c0 == 0xF0) // U+10000..U+3FFFF + { + if (avail >= 4 && data[1] >= 0x90 && data[1] <= 0xBF && data[2] >= 0x80 && data[2] <= 0xBF && data[3] >= 0x80 && data[3] <= 0xBF) + { + return 4; + } + } + else if (c0 >= 0xF1 && c0 <= 0xF3) // U+40000..U+FFFFF + { + if (avail >= 4 && data[1] >= 0x80 && data[1] <= 0xBF && data[2] >= 0x80 && data[2] <= 0xBF && data[3] >= 0x80 && data[3] <= 0xBF) + { + return 4; + } + } + else if (c0 == 0xF4) // U+100000..U+10FFFF + { + if (avail >= 4 && data[1] >= 0x80 && data[1] <= 0x8F && data[2] >= 0x80 && data[2] <= 0xBF && data[3] >= 0x80 && data[3] <= 0xBF) + { + return 4; + } + } + return 0; // invalid, incomplete, or must be diagnosed by the byte path +} + +// Scalar (C++11) computation of the bulk run length: the number of leading +// bytes in [data, data+n) that are ordinary ASCII or complete well-formed UTF-8 +// sequences, stopping before the first byte that needs individual handling (the +// closing quote, an escape, a control character, or an ill-formed/truncated +// sequence). ASCII is skipped 8 bytes at a time. +inline std::size_t scalar_string_bulk_run(const unsigned char* data, std::size_t n) noexcept +{ + std::size_t pos = 0; + while (pos < n) + { + pos += find_string_special(data + pos, n - pos); + if (pos >= n || data[pos] < 0x80u) + { + break; // end of buffer, or a quote/escape/control byte + } + const std::size_t seq = validate_one_utf8(data + pos, n - pos); + if (seq == 0) + { + break; // ill-formed or truncated: let the byte path diagnose it + } + pos += seq; + } + return pos; +} + +#if defined(JSON_USE_SIMDUTF) && defined(JSON_HAS_CPP_17) +// Index of the first quote/escape/control byte in [data, data+n) (non-ASCII +// bytes are *not* stops here - the whole run is handed to simdutf), or n. +inline std::size_t find_string_delimiter(const unsigned char* data, std::size_t n) noexcept +{ + constexpr std::uint64_t ones = 0x0101010101010101ull; + constexpr std::uint64_t high = 0x8080808080808080ull; + std::size_t i = 0; + for (; i + 8 <= n; i += 8) + { + std::uint64_t v = 0; + std::memcpy(&v, data + i, sizeof(v)); + const std::uint64_t q = v ^ 0x2222222222222222ull; + const std::uint64_t b = v ^ 0x5C5C5C5C5C5C5C5Cull; + const std::uint64_t hit = ((q - ones) & ~q & high) + | ((b - ones) & ~b & high) + | ((v - 0x2020202020202020ull) & ~v & high); + if (hit != 0) + { + for (std::size_t j = 0; j < 8; ++j) + { + const unsigned char c = data[i + j]; + if (c == '\"' || c == '\\' || c < 0x20u) + { + return i + j; + } + } + } + } + for (; i < n; ++i) + { + const unsigned char c = data[i]; + if (c == '\"' || c == '\\' || c < 0x20u) + { + return i; + } + } + return n; +} +#endif + +// Backend-dispatched bulk run length. With JSON_USE_SIMDUTF the run up to the +// next delimiter is validated in one shot by simdutf; on the rare failure the +// scalar helper recomputes the exact valid prefix so the byte path still +// produces the precise diagnostic. Without it, the pure scalar path is used. +inline std::size_t string_bulk_run(const unsigned char* data, std::size_t n) noexcept +{ +#if defined(JSON_USE_SIMDUTF) && defined(JSON_HAS_CPP_17) + const std::size_t run = find_string_delimiter(data, n); + if (run != 0 && simdutf::validate_utf8(reinterpret_cast(data), run)) + { + return run; + } +#endif + return scalar_string_bulk_run(data, n); +} + +} // namespace detail +NLOHMANN_JSON_NAMESPACE_END + // #include // #include @@ -7909,6 +8628,25 @@ constexpr bool input_adapter_supports_seek(std::false_type /*detected*/) return false; } +// Detect whether an input adapter exposes a contiguous byte block that the +// lexer can scan directly (see iterator_input_adapter::supports_bulk_scan). +// Adapters without the flag - file, stream, wide-string, user-defined - fall +// back to the character-at-a-time string scanner. +template +using detect_supports_bulk_scan = decltype(InputAdapterType::supports_bulk_scan); + +template +constexpr bool input_adapter_supports_bulk_scan(std::true_type /*detected*/) +{ + return InputAdapterType::supports_bulk_scan; +} + +template +constexpr bool input_adapter_supports_bulk_scan(std::false_type /*detected*/) +{ + return false; +} + /*! @brief lexical analysis @@ -7930,13 +8668,22 @@ class lexer : public lexer_base static constexpr bool lazy_token_string = input_adapter_supports_seek(is_detected {}); + /// whether string scanning may bulk-consume runs of ordinary characters + /// directly from a contiguous input buffer (SWAR fast path). This requires + /// the token to be reconstructible lazily (lazy_token_string), so bypassing + /// the per-character capture in get() cannot lose error diagnostics. + static constexpr bool bulk_scan = + lazy_token_string + && input_adapter_supports_bulk_scan(is_detected {}); + public: using token_type = typename lexer_base::token_type; - explicit lexer(InputAdapterType&& adapter, bool ignore_comments_ = false) noexcept + explicit lexer(InputAdapterType&& adapter, bool ignore_comments_ = false, bool discard_number_values_ = false) noexcept : ia(std::move(adapter)) , ignore_comments(ignore_comments_) , decimal_point_char(static_cast(get_decimal_point())) + , discard_number_values(discard_number_values_) {} // deleted because of pointer members @@ -8049,6 +8796,40 @@ class lexer : public lexer_base return true; } + /// contiguous input: bulk-append the run of ordinary characters and complete + /// well-formed UTF-8 sequences starting at the current read position, leaving + /// the first byte that needs individual handling (the closing quote, an + /// escape, a control character, or an ill-formed UTF-8 byte) for get() + void scan_string_bulk(std::true_type /*bulk*/) + { + // a pending unget must be consumed through the normal path first + if (next_unget) + { + return; + } + const std::size_t remaining = ia.bulk_remaining(); + if (remaining == 0) + { + return; + } + const auto* const data = reinterpret_cast(ia.bulk_data()); + + const std::size_t pos = string_bulk_run(data, remaining); + if (pos == 0) + { + return; + } + token_buffer.append(reinterpret_cast(data), pos); + ia.bulk_skip(pos); + // the run contains no newline (all bytes < 0x20 are treated as special), + // so only the flat character counters advance + position.chars_read_total += pos; + position.chars_read_current_line += pos; + } + + /// streaming input: no bulk fast path + void scan_string_bulk(std::false_type /*bulk*/) const noexcept {} + /*! @brief scan a string literal @@ -8074,6 +8855,10 @@ class lexer : public lexer_base while (true) { + // bulk-consume ordinary characters from contiguous input, then + // handle the next special byte through the switch below + scan_string_bulk(std::integral_constant {}); + // get the next character switch (get()) { @@ -8792,6 +9577,12 @@ class lexer : public lexer_base // changed if minus sign, decimal point, or exponent is read token_type number_type = token_type::value_unsigned; + // offset just past the last mantissa byte in token_buffer (i.e. the + // index of 'e'/'E', or the whole token when there is no exponent). + // convert_number() uses it to count significant digits; npos means + // "not seen an exponent yet" and is resolved at scan_number_done + std::size_t mantissa_end = std::string::npos; + // state (init): we just found out we need to scan a number switch (current) { @@ -8977,6 +9768,9 @@ scan_number_decimal2: scan_number_exponent: // we just parsed an exponent number_type = token_type::value_float; + // this label is reached only right after the 'e'/'E' was appended (from + // the zero, any1, and decimal2 states), so the mantissa ends before it + mantissa_end = token_buffer.size() - 1; switch (get()) { case '+': @@ -9063,45 +9857,199 @@ scan_number_done: // we are done scanning a number) unget(); - char* endptr = nullptr; // NOLINT(misc-const-correctness,cppcoreguidelines-pro-type-vararg,hicpp-vararg) - errno = 0; + // no exponent was scanned: the mantissa spans the whole token + if (mantissa_end == std::string::npos) + { + mantissa_end = token_buffer.size(); + } - // try to parse integers first and fall back to floats + return convert_number(number_type, mantissa_end); + } + + /*! + @brief convert an already-validated integer token to its value + + The digit sequence in [first, last) has been validated by the caller, so a + dedicated parser can avoid the locale/errno overhead of std::strtoull. + + @return the token type on success; token_type::uninitialized if @a + number_type is not an integer type or the value does not fit, in + which case the caller falls back to the floating-point conversion + (matching the previous std::strtoull/std::strtoll behavior) + */ + token_type convert_integer(token_type number_type, const char* first, const char* last) + { if (number_type == token_type::value_unsigned) { - const auto x = std::strtoull(token_buffer.data(), &endptr, 10); - - // we checked the number format before - JSON_ASSERT(endptr == token_buffer.data() + token_buffer.size()); - - if (errno != ERANGE) + if (parse_integer_unsigned(first, last, value_unsigned)) { - value_unsigned = static_cast(x); - if (value_unsigned == x) - { - return token_type::value_unsigned; - } + return token_type::value_unsigned; } } else if (number_type == token_type::value_integer) { - const auto x = std::strtoll(token_buffer.data(), &endptr, 10); - - // we checked the number format before - JSON_ASSERT(endptr == token_buffer.data() + token_buffer.size()); - - if (errno != ERANGE) + if (parse_integer_signed(first, last, value_integer)) { - value_integer = static_cast(x); - if (value_integer == x) - { - return token_type::value_integer; - } + return token_type::value_integer; + } + } + + return token_type::uninitialized; + } + + /*! + @brief check whether Clinger's fast path can still succeed for this token + + parse_float_fast() needs a significand below 2^53. A mantissa with 17 or + more significant digits is at least 10^16 and therefore always exceeds it, + so calling the fast path would walk the token one extra time only to + decline before strtod has to run anyway. + + Significant digits are the mantissa's digits from the first nonzero one on; + the sign, the decimal point, leading zeros, and the exponent do not count. + The answer is derived from indices - the digits are not scanned again - so + this stays off the hot path of the number scanners. + + @param[in] mantissa_end offset just past the last mantissa byte in + token_buffer + @return false if parse_float_fast() is guaranteed to decline + */ + bool mantissa_fits_clinger(std::size_t mantissa_end) const + { + // 10^16 already exceeds 2^53, so 17 digits can never fit + constexpr std::size_t limit = 17; + + const std::size_t neg = (!token_buffer.empty() && token_buffer[0] == '-') ? 1u : 0u; + const std::size_t has_dot = (decimal_point_position != std::string::npos) ? 1u : 0u; + // the JSON grammar restricts the integer part to "0" or [1-9][0-9]*, so + // a leading zero can only be a lone "0", which is not significant + const std::size_t lead_zero = (token_buffer[neg] == '0') ? 1u : 0u; + JSON_ASSERT(mantissa_end >= neg + has_dot + lead_zero); + std::size_t digits = mantissa_end - neg - has_dot - lead_zero; + + if (JSON_HEDLEY_LIKELY(digits < limit)) + { + return true; + } + + // Only a number below 1 can carry further insignificant zeros, and only + // while the count stays at the limit does removing them change the + // answer - so this loop is skipped for all but a few tokens. Note + // token_buffer holds the locale's decimal point, so the fraction is + // located through decimal_point_position rather than by searching '.'. + if (lead_zero != 0) + { + JSON_ASSERT(has_dot != 0); // an integer "0" cannot reach the limit + for (std::size_t i = decimal_point_position + 1; + digits >= limit && i < mantissa_end && token_buffer[i] == '0'; ++i) + { + --digits; + } + } + + return digits < limit; + } + + /*! + @brief convert the number text in token_buffer to its value and token type + + The digit sequence in token_buffer has already been validated (by the + scan_number() state machine or by the contiguous fast path) and holds the + locale decimal point in place of '.'. Integers are parsed first and fall + back to floating point on overflow. This is shared so both scanners produce + identical results. + + @param[in] mantissa_end offset just past the last mantissa byte in + token_buffer (the index of 'e'/'E', or + token_buffer.size() when there is no exponent); + used to skip Clinger's fast path when it cannot + possibly succeed - see mantissa_fits_clinger() + */ + token_type convert_number(token_type number_type, std::size_t mantissa_end) + { + // If the caller does not need the converted value (only whether the + // input is syntactically valid; see json_sax_acceptor/accept()), an + // unsigned/integer token can be reported without calling + // strtoull()/strtoll() at all, *provided* we can already tell from + // the digit count alone that the conversion cannot overflow 64 bits. + // Such tokens are always finite and are accepted unconditionally by + // the parser regardless of their actual value (parser::sax_parse_internal() + // never checks finiteness for value_unsigned/value_integer), so the + // classification below is all that is needed. + // + // A decimal number with up to 18 digits is always representable in + // both std::uint64_t and std::int64_t (18 nines is ~1e18, well below + // both UINT64_MAX ~1.8e19 and INT64_MAX ~9.2e18), so strtoull()/strtoll() + // could not have set errno to ERANGE for it. Numbers with more digits + // (rare in practice) fall through to the exact code below, unchanged, + // so their handling -- including reclassification to value_float when + // the value overflows 64 bits, and rejection when it is not even + // finite as a double -- is bit-for-bit identical to before this + // optimization. + // + // Note this reasons about std::uint64_t/std::int64_t, not about + // number_unsigned_t/number_integer_t (BasicJsonType's own, possibly + // narrower, template parameters -- e.g. std::uint32_t). That is fine + // *only* because discard_number_values is exclusively set by + // accept() (see json.hpp), and accept() always parses through the + // library's own json_sax_acceptor -- never a user-supplied SAX + // consumer -- whose number_unsigned()/number_integer()/number_float() + // callbacks unconditionally discard their argument and return true. + // So for every caller that can reach this branch, neither the token + // classification below nor the eventual (possibly narrowed, and on + // this fast path left stale/unset) value_unsigned/value_integer is + // ever consulted -- an unsigned/integer token is accepted outright, + // and even a >18-digit token that this fast path deliberately falls + // through for is, once reclassified to value_float, still finite + // (and thus accepted) for any digit count that fits in number_unsigned_t + // or number_integer_t regardless of that type's width. If this + // function is ever taught to run with discard_number_values true for + // a caller that *does* read the converted value, this reasoning (and + // the fast path below) would need to be revisited. + if (discard_number_values) + { + constexpr std::size_t safe_digit_count = 18; + if (number_type == token_type::value_unsigned && token_buffer.size() <= safe_digit_count) + { + return token_type::value_unsigned; + } + if (number_type == token_type::value_integer && token_buffer.size() - 1 <= safe_digit_count) + { + return token_type::value_integer; + } + } + + const char* const num_begin = token_buffer.data(); + const char* const num_end = num_begin + token_buffer.size(); + + if (number_type != token_type::value_float) + { + const token_type integer_result = convert_integer(number_type, num_begin, num_end); + if (integer_result != token_type::uninitialized) + { + return integer_result; } } // this code is reached if we parse a floating-point number or if an - // integer conversion above failed + // integer conversion above overflowed. Prefer std::from_chars + // (Eisel-Lemire, locale-independent, correctly rounded) when available; + // otherwise the exact Clinger fast path (double only); otherwise the + // locale-aware strtof/strtod. + if (parse_float_from_chars(num_begin, num_end, value_float)) + { + return token_type::value_float; + } + // Skipping a fast path that cannot succeed is lossless and saves a full + // extra pass over the token's bytes, which otherwise shows up on + // high-precision inputs such as canada.json + if (mantissa_fits_clinger(mantissa_end) + && parse_float_fast(num_begin, num_end, decimal_point_char, value_float)) + { + return token_type::value_float; + } + + char* endptr = nullptr; // NOLINT(misc-const-correctness,cppcoreguidelines-pro-type-vararg,hicpp-vararg) strtof(value_float, token_buffer.data(), &endptr); // we checked the number format before @@ -9110,6 +10058,158 @@ scan_number_done: return token_type::value_float; } + /*! + @brief contiguous fast path for scanning a number + + Parses the whole number token straight from the input buffer, avoiding the + per-character get()/add() of scan_number(). On success it fills token_buffer + (with the locale decimal point substituted, as scan_number() does) and + returns the token type. On anything it does not fully recognize as a + well-formed number it makes no state change and returns + token_type::uninitialized, so the caller falls back to scan_number(), which + then produces the exact diagnostic. @a current is the first digit or the + leading minus (already read); the remaining bytes are taken from the adapter. + */ + token_type scan_number_bulk_contiguous() + { + // a pending unget offsets the buffer position from current; fall back + if (next_unget) + { + return token_type::uninitialized; + } + const std::size_t rem = ia.bulk_remaining(); + if (rem == 0) + { + // the first digit is the last input byte; let scan_number() finish + return token_type::uninitialized; + } + // the byte before the next unread one is current (contiguous input) + const char* const data = reinterpret_cast(ia.bulk_data()) - 1; + const std::size_t avail = rem + 1; + + // validate + classify the number extent (mirrors scan_number()'s grammar) + std::size_t i = 0; + std::size_t dot_index = std::string::npos; + token_type number_type = token_type::value_unsigned; + if (data[0] == '-') + { + number_type = token_type::value_integer; + i = 1; + if (i >= avail) + { + return token_type::uninitialized; + } + } + if (data[i] == '0') + { + ++i; + } + else if (data[i] >= '1' && data[i] <= '9') + { + ++i; + while (i < avail && data[i] >= '0' && data[i] <= '9') + { + ++i; + } + } + else + { + return token_type::uninitialized; + } + if (i < avail && data[i] == '.') + { + number_type = token_type::value_float; + dot_index = i; + ++i; + if (i >= avail || !(data[i] >= '0' && data[i] <= '9')) + { + return token_type::uninitialized; + } + while (i < avail && data[i] >= '0' && data[i] <= '9') + { + ++i; + } + } + // the mantissa ends here, whether or not an exponent part follows + const std::size_t mantissa_end = i; + if (i < avail && (data[i] == 'e' || data[i] == 'E')) + { + number_type = token_type::value_float; + ++i; + if (i < avail && (data[i] == '+' || data[i] == '-')) + { + ++i; + } + if (i >= avail || !(data[i] >= '0' && data[i] <= '9')) + { + return token_type::uninitialized; + } + while (i < avail && data[i] >= '0' && data[i] <= '9') + { + ++i; + } + } + const std::size_t len = i; + + // reset() records where this token starts (for diagnostics), so it has + // to run before the input position advances below + reset(); + + // An integer token needs no token_buffer: the SAX callbacks for + // number_integer/number_unsigned take only the value, and the overflow + // diagnostic rebuilds the text from the input. Convert straight from the + // input buffer and leave token_buffer empty. (JSON_DIAGNOSTIC_POSITIONS + // derives a number's start position from get_string().size(), so there + // the token still has to be materialized.) +#if !JSON_DIAGNOSTIC_POSITIONS + if (number_type != token_type::value_float) + { + const token_type integer_result = convert_integer(number_type, data, data + len); + if (JSON_HEDLEY_LIKELY(integer_result != token_type::uninitialized)) + { + ia.bulk_skip(len - 1); + position.chars_read_total += (len - 1); + position.chars_read_current_line += (len - 1); + return integer_result; + } + // The value does not fit an integer, so this token converts as a + // float. Recording that here keeps convert_number() below from + // repeating the integer attempt that just failed. + number_type = token_type::value_float; + } +#endif + + // materialize the token exactly as scan_number() would, substituting the + // locale decimal point so convert_number()'s strtof fallback stays valid. + // reset() already cleared token_buffer, so append() fills it (assign() is + // avoided because custom string_t types need not provide it) + token_buffer.append(reinterpret_cast(data), len); + if (dot_index != std::string::npos) + { + token_buffer[dot_index] = static_cast(decimal_point_char); + decimal_point_position = dot_index; + } + + ia.bulk_skip(len - 1); + position.chars_read_total += (len - 1); + position.chars_read_current_line += (len - 1); + + return convert_number(number_type, mantissa_end); + } + + /// contiguous input: try the number fast path, else the byte-path scanner + token_type scan_number_dispatch(std::true_type /*bulk*/) + { + const token_type t = scan_number_bulk_contiguous(); + return (t != token_type::uninitialized) ? t : scan_number(); + } + + /// streaming input: always use the byte-path scanner + token_type scan_number_dispatch(std::false_type /*bulk*/) + { + return scan_number(); + } + /*! @param[in] literal_text the literal text to expect @param[in] length the length of the passed literal text @@ -9177,8 +10277,7 @@ scan_number_done: */ char_int_type get() { - ++position.chars_read_total; - ++position.chars_read_current_line; + advance_position(); if (next_unget) { @@ -9190,6 +10289,23 @@ scan_number_done: current = ia.get_character(); } + return track_after_read(); + } + + /// shared head of get() / get_ignoring_pending_unget(): bump the + /// per-character position counters (line-count-on-'\n' bookkeeping is + /// handled afterwards, in track_after_read(), once `current` is known) + void advance_position() noexcept + { + ++position.chars_read_total; + ++position.chars_read_current_line; + } + + /// shared tail of get() / get_ignoring_pending_unget(): capture the + /// character for error messages (if needed) and update line/column + /// bookkeeping for the character now in `current` + char_int_type track_after_read() + { // seekable adapters reconstruct the token lazily on error (see // get_token_string), so the eager per-character copy is skipped capture_char(std::integral_constant {}); @@ -9197,12 +10313,38 @@ scan_number_done: if (current == '\n') { ++position.lines_read; + // remember the column the newline was read at: chars_read_current_line + // is about to be cleared, and a matching unget() cannot reconstruct it + chars_read_before_newline = position.chars_read_current_line; position.chars_read_current_line = 0; } return current; } + /*! + @brief like get(), but for call sites that can prove no unget() is pending + + get() has to check the `next_unget` flag on every call, because a + previous token may have ended with unget() (e.g. scan_number() always + ungets the character that terminated the number, so the next call to + scan() can see it again). skip_whitespace() reads that first, + possibly-ungotten character via a plain get(), but every further + character it reads is guaranteed to be a fresh read: nothing between + those calls invokes unget(). This variant skips the (otherwise always + false) next_unget branch for those calls; it is not a general + replacement for get(). + */ + char_int_type get_ignoring_pending_unget() + { + JSON_ASSERT(!next_unget); + + advance_position(); + current = ia.get_character(); + + return track_after_read(); + } + /// seekable adapter: nothing to capture, the token is rebuilt on error void capture_char(std::true_type /*lazy*/) const noexcept {} @@ -9230,12 +10372,20 @@ scan_number_done: --position.chars_read_total; // in case we "unget" a newline, we have to also decrement the lines_read + // and restore the column that get() cleared when it saw the newline; + // chars_read_current_line == 0 can only mean the last get() read one if (position.chars_read_current_line == 0) { if (position.lines_read > 0) { --position.lines_read; } + + // chars_read_before_newline counts the newline itself, which is the + // character being ungotten, hence the -1 + position.chars_read_current_line = (chars_read_before_newline > 0) + ? chars_read_before_newline - 1 + : 0; } else { @@ -9396,13 +10546,37 @@ scan_number_done: return true; } + /// whether `current` is one of the four JSON whitespace characters + bool current_is_whitespace() const noexcept + { + return current == ' ' || current == '\t' || current == '\n' || current == '\r'; + } + void skip_whitespace() { + // the first character may be a pending unget() left over from the + // previous token (see get_ignoring_pending_unget()); every + // subsequent character read by this loop is guaranteed fresh, since + // nothing below calls unget() + get(); + + if (!current_is_whitespace()) + { + return; + } + + // this is written as an if-guarded do-while (rather than a plain + // while loop) because that shape is what lets both GCC and Clang + // keep the input adapter's read pointer in a register across + // iterations; the equivalent while-loop measurably defeated that + // optimization in testing, turning long whitespace runs (e.g. the + // indentation of pretty-printed JSON) from a register-only loop + // into one that reloads the pointer from memory every character do { - get(); + get_ignoring_pending_unget(); } - while (current == ' ' || current == '\t' || current == '\n' || current == '\r'); + while (current_is_whitespace()); } token_type scan() @@ -9478,7 +10652,7 @@ scan_number_done: case '7': case '8': case '9': - return scan_number(); + return scan_number_dispatch(std::integral_constant {}); // end of input (the null byte is needed when parsing from // string literals) @@ -9509,6 +10683,10 @@ scan_number_done: /// the start position of the current token position_t position {}; + /// the value chars_read_current_line had when the last newline was read, so + /// that unget() can restore the column instead of leaving it at 0 + std::size_t chars_read_before_newline = 0; + /// raw input token string for error messages; only populated for streaming /// adapters (seekable adapters reconstruct it lazily via token_string_start) std::vector token_string {}; @@ -9538,6 +10716,13 @@ scan_number_done: const char_int_type decimal_point_char = '.'; /// the position of the decimal point in the input std::size_t decimal_point_position = std::string::npos; + + /// whether the caller (e.g. accept()/json_sax_acceptor) only needs the + /// token classification and never looks at the converted numeric value; + /// when set, scan_number() may skip strtoull()/strtoll() for + /// value_unsigned/value_integer tokens whose digit count guarantees they + /// fit into 64 bits (see scan_number()) + const bool discard_number_values = false; }; } // namespace detail @@ -9751,12 +10936,16 @@ class json_sax_dom_parser bool string(string_t& val) { - handle_value(val); + // json_sax documents that the passed value may be moved from, + // so hand the buffer over instead of copying it + handle_value(std::move(val)); return true; } bool binary(binary_t& val) { + // json_sax documents that the passed value may be moved from, + // so hand the buffer over instead of copying it handle_value(std::move(val)); return true; } @@ -10071,12 +11260,16 @@ class json_sax_dom_callback_parser bool string(string_t& val) { - handle_value(val); + // json_sax documents that the passed value may be moved from, + // so hand the buffer over instead of copying it + handle_value(std::move(val)); return true; } bool binary(binary_t& val) { + // json_sax documents that the passed value may be moved from, + // so hand the buffer over instead of copying it handle_value(std::move(val)); return true; } @@ -10087,6 +11280,11 @@ class json_sax_dom_callback_parser const bool keep = callback(static_cast(ref_stack.size()), parse_event_t::object_start, discarded); keep_stack.push_back(keep); + // the key this object will be stored under, read before handle_value() + // may consume it; kept in lockstep with ref_stack so end_object() can + // find the object in its parent again + container_key_stack.push_back(current_key()); + auto val = handle_value(BasicJsonType::value_t::object, true); ref_stack.push_back(val.second); @@ -10120,6 +11318,9 @@ class json_sax_dom_callback_parser // check callback for the key const bool keep = callback(static_cast(ref_stack.size()), parse_event_t::key, k); key_keep_stack.push_back(keep); + // remember the key so a rejected value can be erased without searching + // the object for it (kept in lockstep with key_keep_stack) + key_stack.push_back(val); // add discarded value at the given key and store the reference for later if (keep && ref_stack.back()) @@ -10161,13 +11362,16 @@ class json_sax_dom_callback_parser JSON_ASSERT(!ref_stack.empty()); JSON_ASSERT(!keep_stack.empty()); + JSON_ASSERT(!container_key_stack.empty()); ref_stack.pop_back(); keep_stack.pop_back(); + const string_t object_key = std::move(container_key_stack.back()); + container_key_stack.pop_back(); if (!ref_stack.empty() && ref_stack.back() && ref_stack.back()->is_structured()) { // remove discarded value - remove_discarded_value(*ref_stack.back()); + remove_discarded_value(*ref_stack.back(), object_key); } return true; @@ -10178,6 +11382,9 @@ class json_sax_dom_callback_parser const bool keep = callback(static_cast(ref_stack.size()), parse_event_t::array_start, discarded); keep_stack.push_back(keep); + // see start_object() + container_key_stack.push_back(current_key()); + auto val = handle_value(BasicJsonType::value_t::array, true); ref_stack.push_back(val.second); @@ -10250,8 +11457,11 @@ class json_sax_dom_callback_parser JSON_ASSERT(!ref_stack.empty()); JSON_ASSERT(!keep_stack.empty()); + JSON_ASSERT(!container_key_stack.empty()); ref_stack.pop_back(); keep_stack.pop_back(); + const string_t object_key = std::move(container_key_stack.back()); + container_key_stack.pop_back(); // remove discarded value if (!ref_stack.empty() && ref_stack.back()) @@ -10265,7 +11475,7 @@ class json_sax_dom_callback_parser // the array is either still stored under its key or was never // stored, leaving the placeholder key() wrote; both show up as // a discarded member of the parent object - remove_discarded_value(*ref_stack.back()); + remove_discarded_value(*ref_stack.back(), object_key); } } @@ -10358,15 +11568,56 @@ class json_sax_dom_callback_parser } #endif - /// remove the discarded value the callback rejected from its parent - static void remove_discarded_value(BasicJsonType& parent) + /*! + @brief the key the value now being handled will be stored under + + Empty unless the enclosing container is an object, in which case it is the + key of the pending key() event. Read before handle_value() consumes that + key, so it is also correct when the value never reaches its parent. + */ + string_t current_key() const { - for (auto it = parent.begin(); it != parent.end(); ++it) + if (!ref_stack.empty() && ref_stack.back() && ref_stack.back()->is_object() + && !key_stack.empty()) { - if (it->is_discarded()) + return key_stack.back(); + } + return string_t{}; + } + + /*! + @brief remove the discarded value the callback rejected from its parent + + A rejected value can only ever be the one most recently added to @a parent: + the last element of an array, or the placeholder key() stored under @a key + in an object. Looking there directly makes this O(1) resp. O(log n), where + searching @a parent for it made a filtering parse quadratic in the number of + members of a single container. + + Finding no discarded value there means none was stored in the first place - + the callback rejected the value before it reached its parent - so there is + nothing to remove. + + @param[in,out] parent the container to remove the rejected value from + @param[in] key the key the value was stored under; unused for arrays + */ + static void remove_discarded_value(BasicJsonType& parent, const string_t& key) + { + if (parent.is_array()) + { + auto& array = *parent.m_data.m_value.array; + if (!array.empty() && array.back().is_discarded()) { - parent.erase(it); - break; + array.pop_back(); + } + } + else if (parent.is_object()) + { + auto& object = *parent.m_data.m_value.object; + const auto it = object.find(key); + if (it != object.end() && it->second.is_discarded()) + { + object.erase(it); } } } @@ -10416,11 +11667,14 @@ class json_sax_dom_callback_parser if (!ref_stack.empty() && ref_stack.back() && ref_stack.back()->is_object()) { JSON_ASSERT(!key_keep_stack.empty()); + JSON_ASSERT(!key_stack.empty()); const bool placeholder_stored = key_keep_stack.back(); key_keep_stack.pop_back(); + const string_t key = std::move(key_stack.back()); + key_stack.pop_back(); if (placeholder_stored) { - remove_discarded_value(*ref_stack.back()); + remove_discarded_value(*ref_stack.back(), key); } } return {false, nullptr}; @@ -10453,8 +11707,10 @@ class json_sax_dom_callback_parser JSON_ASSERT(ref_stack.back()->is_object()); // check if we should store an element for the current key JSON_ASSERT(!key_keep_stack.empty()); + JSON_ASSERT(!key_stack.empty()); const bool store_element = key_keep_stack.back(); key_keep_stack.pop_back(); + key_stack.pop_back(); if (!store_element) { @@ -10474,6 +11730,12 @@ class json_sax_dom_callback_parser std::vector keep_stack {}; // NOLINT(readability-redundant-member-init) /// stack to manage which object keys to keep std::vector key_keep_stack {}; // NOLINT(readability-redundant-member-init) + /// the keys key() stored a placeholder for, in lockstep with key_keep_stack + std::vector key_stack {}; // NOLINT(readability-redundant-member-init) + /// for each open container, the key it is stored under in its parent + /// object, in lockstep with ref_stack; unused where the parent is not an + /// object + std::vector container_key_stack {}; // NOLINT(readability-redundant-member-init) /// helper to hold the reference for the next object element BasicJsonType* object_element = nullptr; /// whether a syntax error occurred @@ -10766,6 +12028,26 @@ inline bool little_endianness(int num = 1) noexcept return *reinterpret_cast(&num) == 1; } +/*! +@brief largest element count accepted for a UBJSON container of a valueless type + +An element of type 'Z' (null), 'T' (true) or 'F' (false) is encoded by its +type marker alone, so an optimized container of one of those types has no +payload at all and its declared count is the only thing that decides how much +is allocated: `[$Z#L` followed by a large count turns some ten bytes of input +into that many values (see #2793, which reports 35 GB and 150 seconds). Every +other type costs at least one byte per element and is bounded by the end of +the input. + +This is a sanity bound rather than a security boundary, and it is far above +any container met in practice. @ref binary_writer falls back to the +unoptimized encoding for longer containers, so that a value serialized by +this library can always be read back. + +@sa https://github.com/nlohmann/json/issues/2793 +*/ +JSON_INLINE_VARIABLE constexpr std::size_t max_valueless_container_size = 1 << 20; + /////////////////// // binary reader // /////////////////// @@ -10818,6 +12100,7 @@ class binary_reader const cbor_tag_handler_t tag_handler = cbor_tag_handler_t::error) { sax = sax_; + container_stack.clear(); bool result = false; switch (format) @@ -10867,6 +12150,80 @@ class binary_reader } private: + //////////////////////// + // nested containers // + //////////////////////// + + /*! + @brief a container that has been opened and not closed yet + + The binary readers do not call themselves once per nesting level. Like + @ref parser::sax_parse_internal, which does the same for JSON text, they + keep the containers they are inside of on a heap-allocated stack, so that + the native call stack does not grow with the nesting depth of the input + and a deeply nested value is bounded by memory rather than by the stack + (see #5104). + + The members are ordered by decreasing alignment, which is the ordering that + keeps a struct from growing as members are added to it. + */ + struct container_frame + { + container_frame(const std::size_t remaining_, const bool is_object_, + const char_int_type type_marker_ = 0) noexcept + : remaining(remaining_), type_marker(type_marker_), is_object(is_object_) {} + + /// number of elements that have not been read yet, or npos when the + /// container is not sized and ends at a marker instead + std::size_t remaining; + /// BSON: value of chars_read before this document's size prefix, which + /// check_bson_document_size() needs once the document has been read + std::size_t start_position = 0; + /// UBJSON/BJData: the type marker of an optimized container, so that + /// its elements are read without one of their own; 0 otherwise + char_int_type type_marker; + /// BSON: the size this document declares, in bytes + std::int32_t declared_size = 0; + /// whether to close this container with end_object() or end_array() + bool is_object; + }; + + /*! + @brief open a nested array or object + + Emits the SAX start event and records the container. This is the only + place the binary readers start a container, so a check that rejects one + can be made here and is then guaranteed to run before the start event. + + @param[in] is_object whether an object (true) or an array (false) begins + @param[in] len number of elements the container declares + + @return whether the SAX parser accepted the start event + */ + bool enter_container(const bool is_object, const std::size_t len, + const char_int_type type_marker = 0) + { + if (JSON_HEDLEY_UNLIKELY(is_object ? !sax->start_object(len) : !sax->start_array(len))) + { + return false; + } + + container_stack.emplace_back(len, is_object, type_marker); + return true; + } + + /// @copydoc enter_container + bool enter_array(const std::size_t len, const char_int_type type_marker = 0) + { + return enter_container(/*is_object*/false, len, type_marker); + } + + /// @copydoc enter_container + bool enter_object(const std::size_t len, const char_int_type type_marker = 0) + { + return enter_container(/*is_object*/true, len, type_marker); + } + ////////// // BSON // ////////// @@ -10901,8 +12258,10 @@ class binary_reader @brief Reads in a BSON-object and passes it to the SAX-parser. @return whether a valid BSON-value was passed to the SAX parser */ - bool parse_bson_internal() + bool open_bson_document(const bool is_object) { + // recorded before the size prefix is read, because + // check_bson_document_size() measures the document from here const std::size_t document_start = chars_read; std::int32_t document_size{}; if (!get_number(input_format_t::bson, document_size)) @@ -10910,22 +12269,91 @@ class binary_reader return false; } - if (JSON_HEDLEY_UNLIKELY(!sax->start_object(detail::unknown_size()))) + if (JSON_HEDLEY_UNLIKELY(!enter_container(is_object, detail::unknown_size()))) { return false; } - if (JSON_HEDLEY_UNLIKELY(!parse_bson_element_list(/*is_array*/false))) + container_frame& frame = container_stack.back(); + frame.start_position = document_start; + frame.declared_size = document_size; + return true; + } + + /*! + @brief read a BSON document and everything nested inside it + + Reads elements until the document that was begun here is complete, + resuming the enclosing document each time an embedded one ends, so that + the nesting depth of the input costs heap rather than native stack + (see #5104). + + @return whether reading the document succeeded + */ + bool parse_bson_internal() + { + if (JSON_HEDLEY_UNLIKELY(!open_bson_document(/*is_object*/true))) { return false; } - if (JSON_HEDLEY_UNLIKELY(!check_bson_document_size(document_start, document_size))) - { - return false; - } + // the key currently being read; hoisted out of the loop so that its + // capacity is reused across elements and across nesting levels + string_t key; - return sax->end_object(); + while (true) + { + const auto element_type = get(); + + if (element_type == 0) // end of the innermost document + { + // a copy, not a reference: it must stay valid across the + // pop_back() below, which destroys the container_stack + // element it would otherwise alias + const container_frame top = container_stack.back(); + + if (JSON_HEDLEY_UNLIKELY(!check_bson_document_size(top.start_position, top.declared_size))) + { + return false; + } + + container_stack.pop_back(); + if (JSON_HEDLEY_UNLIKELY(top.is_object ? !sax->end_object() : !sax->end_array())) + { + return false; + } + // the document begun here is complete once it is not inside one + if (container_stack.empty()) + { + return true; + } + continue; + } + + if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format_t::bson, "element list"))) + { + return false; + } + + const std::size_t element_type_parse_position = chars_read; + key.clear(); + if (JSON_HEDLEY_UNLIKELY(!get_bson_cstr(key))) + { + return false; + } + + // an array's elements are named "0", "1", ... in the wire format, + // and those names are not passed on + if (container_stack.back().is_object && !sax->key(key)) + { + return false; + } + + if (JSON_HEDLEY_UNLIKELY(!parse_bson_element_internal(element_type, element_type_parse_position))) + { + return false; + } + } } /*! @@ -11037,12 +12465,12 @@ class binary_reader case 0x03: // object { - return parse_bson_internal(); + return open_bson_document(/*is_object*/true); } case 0x04: // array { - return parse_bson_array(); + return open_bson_document(/*is_object*/false); } case 0x05: // binary @@ -11092,82 +12520,7 @@ class binary_reader } } - /*! - @brief Read a BSON element list (as specified in the BSON-spec) - The same binary layout is used for objects and arrays, hence it must be - indicated with the argument @a is_array which one is expected - (true --> array, false --> object). - - @param[in] is_array Determines if the element list being read is to be - treated as an object (@a is_array == false), or as an - array (@a is_array == true). - @return whether a valid BSON-object/array was passed to the SAX parser - */ - bool parse_bson_element_list(const bool is_array) - { - string_t key; - - while (auto element_type = get()) - { - if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format_t::bson, "element list"))) - { - return false; - } - - const std::size_t element_type_parse_position = chars_read; - if (JSON_HEDLEY_UNLIKELY(!get_bson_cstr(key))) - { - return false; - } - - if (!is_array && !sax->key(key)) - { - return false; - } - - if (JSON_HEDLEY_UNLIKELY(!parse_bson_element_internal(element_type, element_type_parse_position))) - { - return false; - } - - // get_bson_cstr only appends - key.clear(); - } - - return true; - } - - /*! - @brief Reads an array from the BSON input and passes it to the SAX-parser. - @return whether a valid BSON-array was passed to the SAX parser - */ - bool parse_bson_array() - { - const std::size_t document_start = chars_read; - std::int32_t document_size{}; - if (!get_number(input_format_t::bson, document_size)) - { - return false; - } - - if (JSON_HEDLEY_UNLIKELY(!sax->start_array(detail::unknown_size()))) - { - return false; - } - - if (JSON_HEDLEY_UNLIKELY(!parse_bson_element_list(/*is_array*/true))) - { - return false; - } - - if (JSON_HEDLEY_UNLIKELY(!check_bson_document_size(document_start, document_size))) - { - return false; - } - - return sax->end_array(); - } ////////// // CBOR // @@ -11199,9 +12552,12 @@ class binary_reader @return whether a valid CBOR value was passed to the SAX parser */ - bool parse_cbor_internal(const bool get_char, - const cbor_tag_handler_t tag_handler) + bool parse_cbor_value(const bool get_char, + const cbor_tag_handler_t tag_handler, + bool& tag_pending) { + tag_pending = false; + switch (get_char ? get() : current) { // EOF @@ -11393,37 +12749,36 @@ class binary_reader case 0x95: case 0x96: case 0x97: - return get_cbor_array( - conditional_static_cast(static_cast(current) & 0x1Fu), tag_handler); + return enter_array(conditional_static_cast(static_cast(current) & 0x1Fu)); case 0x98: // array (one-byte uint8_t for n follows) { std::uint8_t len{}; - return get_number(input_format_t::cbor, len) && get_cbor_array(static_cast(len), tag_handler); + return get_number(input_format_t::cbor, len) && enter_array(static_cast(len)); } case 0x99: // array (two-byte uint16_t for n follow) { std::uint16_t len{}; - return get_number(input_format_t::cbor, len) && get_cbor_array(static_cast(len), tag_handler); + return get_number(input_format_t::cbor, len) && enter_array(static_cast(len)); } case 0x9A: // array (four-byte uint32_t for n follow) { std::uint32_t len{}; std::size_t size{}; - return get_number(input_format_t::cbor, len) && get_cbor_container_size(len, size, "array") && get_cbor_array(size, tag_handler); + return get_number(input_format_t::cbor, len) && get_cbor_container_size(len, size, "array") && enter_array(size); } case 0x9B: // array (eight-byte uint64_t for n follow) { std::uint64_t len{}; std::size_t size{}; - return get_number(input_format_t::cbor, len) && get_cbor_container_size(len, size, "array") && get_cbor_array(size, tag_handler); + return get_number(input_format_t::cbor, len) && get_cbor_container_size(len, size, "array") && enter_array(size); } case 0x9F: // array (indefinite length) - return get_cbor_array(detail::unknown_size(), tag_handler); + return enter_array(detail::unknown_size()); // map (0x00..0x17 pairs of data items follow) case 0xA0: @@ -11450,36 +12805,36 @@ class binary_reader case 0xB5: case 0xB6: case 0xB7: - return get_cbor_object(conditional_static_cast(static_cast(current) & 0x1Fu), tag_handler); + return enter_object(conditional_static_cast(static_cast(current) & 0x1Fu)); case 0xB8: // map (one-byte uint8_t for n follows) { std::uint8_t len{}; - return get_number(input_format_t::cbor, len) && get_cbor_object(static_cast(len), tag_handler); + return get_number(input_format_t::cbor, len) && enter_object(static_cast(len)); } case 0xB9: // map (two-byte uint16_t for n follow) { std::uint16_t len{}; - return get_number(input_format_t::cbor, len) && get_cbor_object(static_cast(len), tag_handler); + return get_number(input_format_t::cbor, len) && enter_object(static_cast(len)); } case 0xBA: // map (four-byte uint32_t for n follow) { std::uint32_t len{}; std::size_t size{}; - return get_number(input_format_t::cbor, len) && get_cbor_container_size(len, size, "map") && get_cbor_object(size, tag_handler); + return get_number(input_format_t::cbor, len) && get_cbor_container_size(len, size, "map") && enter_object(size); } case 0xBB: // map (eight-byte uint64_t for n follow) { std::uint64_t len{}; std::size_t size{}; - return get_number(input_format_t::cbor, len) && get_cbor_container_size(len, size, "map") && get_cbor_object(size, tag_handler); + return get_number(input_format_t::cbor, len) && get_cbor_container_size(len, size, "map") && enter_object(size); } case 0xBF: // map (indefinite length) - return get_cbor_object(detail::unknown_size(), tag_handler); + return enter_object(detail::unknown_size()); case 0xC0: // tagged item case 0xC1: @@ -11563,7 +12918,10 @@ class binary_reader default: break; } - return parse_cbor_internal(true, tag_handler); + // the tagged value follows; it is read by the loop in + // parse_cbor_internal() rather than by recursing here + tag_pending = true; + return true; } case cbor_tag_handler_t::store: @@ -11613,7 +12971,11 @@ class binary_reader break; } default: - return parse_cbor_internal(true, tag_handler); + { + // as above, the tagged value is read by the caller + tag_pending = true; + return true; + } } get(); return get_cbor_binary(b) && sax->binary(b); @@ -11704,23 +13066,21 @@ class binary_reader } /*! - @brief reads a CBOR string + @brief reads a definite-length CBOR string - This function first reads starting bytes to determine the expected - string length and then copies this number of bytes into a string. - Additionally, CBOR's strings with indefinite lengths are supported. + Reads everything @ref get_cbor_string accepts except the indefinite-length + form, which that function handles itself. The bytes are appended to @a + result, so consecutive chunks of an indefinite-length string can be read + into the same string. - @param[out] result created string + @param[out] result string the bytes are appended to @return whether string creation completed - */ - bool get_cbor_string(string_t& result) - { - if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format_t::cbor, "string"))) - { - return false; - } + @pre @a current is not EOF + */ + bool get_cbor_string_chunk(string_t& result) + { switch (current) { // UTF-8 string (0x00..0x17 bytes follow) @@ -11776,20 +13136,6 @@ class binary_reader return get_number(input_format_t::cbor, len) && get_string(input_format_t::cbor, len, result); } - case 0x7F: // UTF-8 string (indefinite length) - { - while (get() != 0xFF) - { - string_t chunk; - if (!get_cbor_string(chunk)) - { - return false; - } - result.append(chunk); - } - return true; - } - default: { auto last_token = get_token_string(); @@ -11800,23 +13146,82 @@ class binary_reader } /*! - @brief reads a CBOR byte array + @brief reads a CBOR string This function first reads starting bytes to determine the expected - byte array length and then copies this number of bytes into the byte array. - Additionally, CBOR's byte arrays with indefinite lengths are supported. + string length and then copies this number of bytes into a string. + Additionally, CBOR's strings with indefinite lengths are supported. - @param[out] result created byte array + @param[out] result created string + + @return whether string creation completed + */ + bool get_cbor_string(string_t& result) + { + // number of indefinite-length strings that have been opened and not + // closed yet. RFC 8949, Section 3.2.3 does not permit nesting them, + // but this reader has always accepted it, so the open levels are + // counted instead of recursed through, which overflowed the stack for + // an input of repeated 0x7F bytes (see #5104). Every chunk is appended + // to the same result, so no per-level state is needed. + std::size_t open = 0; + + while (true) + { + if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format_t::cbor, "string"))) + { + return false; + } + + if (current == 0x7F) // UTF-8 string (indefinite length) + { + ++open; + get(); + continue; + } + + // a break marker closes the innermost indefinite-length string; + // outside of one it is not a string and falls through to the error + if (open != 0 && current == 0xFF) + { + if (--open == 0) + { + return true; + } + get(); + continue; + } + + if (JSON_HEDLEY_UNLIKELY(!get_cbor_string_chunk(result))) + { + return false; + } + + if (open == 0) + { + return true; + } + + get(); + } + } + + /*! + @brief reads a definite-length CBOR byte array + + Reads everything @ref get_cbor_binary accepts except the indefinite-length + form, which that function handles itself. The bytes are appended to @a + result, so consecutive chunks of an indefinite-length byte array can be + read into the same byte array. + + @param[out] result byte array the bytes are appended to @return whether byte array creation completed - */ - bool get_cbor_binary(binary_t& result) - { - if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format_t::cbor, "binary"))) - { - return false; - } + @pre @a current is not EOF + */ + bool get_cbor_binary_chunk(binary_t& result) + { switch (current) { // Binary data (0x00..0x17 bytes follow) @@ -11876,20 +13281,6 @@ class binary_reader get_binary(input_format_t::cbor, len, result); } - case 0x5F: // Binary data (indefinite length) - { - while (get() != 0xFF) - { - binary_t chunk; - if (!get_cbor_binary(chunk)) - { - return false; - } - result.insert(result.end(), chunk.begin(), chunk.end()); - } - return true; - } - default: { auto last_token = get_token_string(); @@ -11899,6 +13290,63 @@ class binary_reader } } + /*! + @brief reads a CBOR byte array + + This function first reads starting bytes to determine the expected + byte array length and then copies this number of bytes into the byte array. + Additionally, CBOR's byte arrays with indefinite lengths are supported. + + @param[out] result created byte array + + @return whether byte array creation completed + */ + bool get_cbor_binary(binary_t& result) + { + // the open indefinite-length byte arrays are counted rather than + // recursed through, for the reason given in @ref get_cbor_string + std::size_t open = 0; + + while (true) + { + if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format_t::cbor, "binary"))) + { + return false; + } + + if (current == 0x5F) // Binary data (indefinite length) + { + ++open; + get(); + continue; + } + + // a break marker closes the innermost indefinite-length byte + // array; outside of one it falls through to the error below + if (open != 0 && current == 0xFF) + { + if (--open == 0) + { + return true; + } + get(); + continue; + } + + if (JSON_HEDLEY_UNLIKELY(!get_cbor_binary_chunk(result))) + { + return false; + } + + if (open == 0) + { + return true; + } + + get(); + } + } + /*! @brief narrow a definite CBOR array/map length to std::size_t @@ -11925,96 +13373,110 @@ class binary_reader } /*! - @param[in] len the length of the array or detail::unknown_size() for an - array of indefinite size + @brief read a CBOR value and everything nested inside it + + Reads values until the one that was begun here is complete, resuming the + enclosing container after each element, so that the nesting depth of the + input costs heap rather than native stack (see #5104). + + @param[in] get_char whether a new character should be retrieved from the + input (true) or whether the last read character + @a current should be considered instead @param[in] tag_handler how CBOR tags should be treated - @return whether array creation completed + + @return whether reading the value succeeded */ - bool get_cbor_array(const std::size_t len, - const cbor_tag_handler_t tag_handler) + bool parse_cbor_internal(const bool get_char, + const cbor_tag_handler_t tag_handler) { - if (JSON_HEDLEY_UNLIKELY(!sax->start_array(len))) - { - return false; - } + // whether the next value starts at a fresh byte or at the one already + // read into `current` + bool fetch = get_char; - if (len != detail::unknown_size()) + // the key currently being read; hoisted out of the loop so that its + // capacity is reused across elements and across nesting levels + string_t key; + + while (true) { - for (std::size_t i = 0; i < len; ++i) + if (!container_stack.empty()) { - if (JSON_HEDLEY_UNLIKELY(!parse_cbor_internal(true, tag_handler))) + // a copy, not a reference: it must stay valid across the + // pop_back() below, which destroys the container_stack element + // it would otherwise alias + container_frame top = container_stack.back(); + bool at_end = false; + + if (top.remaining != npos) { - return false; + // definite length: the container ends once its elements + // have been read + at_end = (top.remaining == 0); + if (!at_end) + { + // claim the element about to be read + --container_stack.back().remaining; + if (top.is_object) + { + get(); + } + } + fetch = true; } - } - } - else - { - while (get() != 0xFF) - { - if (JSON_HEDLEY_UNLIKELY(!parse_cbor_internal(false, tag_handler))) + else { - return false; + // indefinite length: the container ends at a break marker. + // Testing for it consumes a byte, which is the first byte + // of the next element when it is not one. + at_end = (get() == 0xFF); + fetch = top.is_object; } - } - } - return sax->end_array(); - } - - /*! - @param[in] len the length of the object or detail::unknown_size() for an - object of indefinite size - @param[in] tag_handler how CBOR tags should be treated - @return whether object creation completed - */ - bool get_cbor_object(const std::size_t len, - const cbor_tag_handler_t tag_handler) - { - if (JSON_HEDLEY_UNLIKELY(!sax->start_object(len))) - { - return false; - } - - if (len != 0) - { - string_t key; - if (len != detail::unknown_size()) - { - for (std::size_t i = 0; i < len; ++i) + if (at_end) { - get(); + container_stack.pop_back(); + if (JSON_HEDLEY_UNLIKELY(top.is_object ? !sax->end_object() : !sax->end_array())) + { + return false; + } + // the value begun here is complete once its container is + if (container_stack.empty()) + { + return true; + } + continue; + } + + if (top.is_object) + { + key.clear(); if (JSON_HEDLEY_UNLIKELY(!get_cbor_string(key) || !sax->key(key))) { return false; } - - if (JSON_HEDLEY_UNLIKELY(!parse_cbor_internal(true, tag_handler))) - { - return false; - } - key.clear(); + fetch = true; } } - else - { - while (get() != 0xFF) - { - if (JSON_HEDLEY_UNLIKELY(!get_cbor_string(key) || !sax->key(key))) - { - return false; - } - if (JSON_HEDLEY_UNLIKELY(!parse_cbor_internal(true, tag_handler))) - { - return false; - } - key.clear(); + // a tag is not a value of its own: read on until the tagged value + bool tag_pending = false; + do + { + if (JSON_HEDLEY_UNLIKELY(!parse_cbor_value(fetch, tag_handler, tag_pending))) + { + return false; } + fetch = true; + } + while (tag_pending); + + // a value that opened a container left it on the stack; one that + // did not, and that was not inside a container, was the whole value + if (container_stack.empty()) + { + return true; } } - - return sax->end_object(); } ///////////// @@ -12024,7 +13486,17 @@ class binary_reader /*! @return whether a valid MessagePack value was passed to the SAX parser */ - bool parse_msgpack_internal() + /*! + @brief read one MessagePack value + + Reads a single value and passes it to the SAX parser. A value that begins + a container is not read to its end: the container is opened with + @ref enter_container and its elements are read by + @ref parse_msgpack_internal, so that nesting does not consume native stack. + + @return whether reading the value succeeded + */ + bool parse_msgpack_value() { switch (get()) { @@ -12180,7 +13652,7 @@ class binary_reader case 0x8D: case 0x8E: case 0x8F: - return get_msgpack_object(conditional_static_cast(static_cast(current) & 0x0Fu)); + return enter_object(conditional_static_cast(static_cast(current) & 0x0Fu)); // fixarray case 0x90: @@ -12199,7 +13671,7 @@ class binary_reader case 0x9D: case 0x9E: case 0x9F: - return get_msgpack_array(conditional_static_cast(static_cast(current) & 0x0Fu)); + return enter_array(conditional_static_cast(static_cast(current) & 0x0Fu)); // fixstr case 0xA0: @@ -12330,25 +13802,25 @@ class binary_reader case 0xDC: // array 16 { std::uint16_t len{}; - return get_number(input_format_t::msgpack, len) && get_msgpack_array(static_cast(len)); + return get_number(input_format_t::msgpack, len) && enter_array(static_cast(len)); } case 0xDD: // array 32 { std::uint32_t len{}; - return get_number(input_format_t::msgpack, len) && get_msgpack_array(conditional_static_cast(len)); + return get_number(input_format_t::msgpack, len) && enter_array(conditional_static_cast(len)); } case 0xDE: // map 16 { std::uint16_t len{}; - return get_number(input_format_t::msgpack, len) && get_msgpack_object(static_cast(len)); + return get_number(input_format_t::msgpack, len) && enter_object(static_cast(len)); } case 0xDF: // map 32 { std::uint32_t len{}; - return get_number(input_format_t::msgpack, len) && get_msgpack_object(conditional_static_cast(len)); + return get_number(input_format_t::msgpack, len) && enter_object(conditional_static_cast(len)); } // negative fixint @@ -12596,55 +14068,69 @@ class binary_reader } /*! - @param[in] len the length of the array - @return whether array creation completed + @brief read a MessagePack value and everything nested inside it + + Reads values until the one that was begun here is complete, resuming the + enclosing container each time an element ends, so that the nesting depth + of the input costs heap rather than native stack (see #5104). + + @return whether reading the value succeeded */ - bool get_msgpack_array(const std::size_t len) + bool parse_msgpack_internal() { - if (JSON_HEDLEY_UNLIKELY(!sax->start_array(len))) - { - return false; - } - - for (std::size_t i = 0; i < len; ++i) - { - if (JSON_HEDLEY_UNLIKELY(!parse_msgpack_internal())) - { - return false; - } - } - - return sax->end_array(); - } - - /*! - @param[in] len the length of the object - @return whether object creation completed - */ - bool get_msgpack_object(const std::size_t len) - { - if (JSON_HEDLEY_UNLIKELY(!sax->start_object(len))) - { - return false; - } - + // the key currently being read; hoisted out of the loop so that its + // capacity is reused across elements and across nesting levels string_t key; - for (std::size_t i = 0; i < len; ++i) + + while (true) { - get(); - if (JSON_HEDLEY_UNLIKELY(!get_msgpack_string(key) || !sax->key(key))) + if (!container_stack.empty()) + { + // copied out before anything can push onto the stack and + // invalidate a reference into it + const bool is_object = container_stack.back().is_object; + + if (container_stack.back().remaining == 0) + { + container_stack.pop_back(); + if (JSON_HEDLEY_UNLIKELY(is_object ? !sax->end_object() : !sax->end_array())) + { + return false; + } + // the value begun here is complete once its container is + if (container_stack.empty()) + { + return true; + } + continue; + } + + // claim the element about to be read + --container_stack.back().remaining; + + if (is_object) + { + get(); + key.clear(); + if (JSON_HEDLEY_UNLIKELY(!get_msgpack_string(key) || !sax->key(key))) + { + return false; + } + } + } + + if (JSON_HEDLEY_UNLIKELY(!parse_msgpack_value())) { return false; } - if (JSON_HEDLEY_UNLIKELY(!parse_msgpack_internal())) + // a value that opened a container left it on the stack; one that + // did not, and that was not inside a container, was the whole value + if (container_stack.empty()) { - return false; + return true; } - key.clear(); } - - return sax->end_object(); } //////////// @@ -12660,7 +14146,103 @@ class binary_reader */ bool parse_ubjson_internal(const bool get_char = true) { - return get_ubjson_value(get_char ? get_ignore_noop() : current); + // the key currently being read; hoisted out of the loop so that its + // capacity is reused across elements and across nesting levels + string_t key; + + // the type marker of the value to read next + char_int_type prefix = get_char ? get_ignore_noop() : current; + + while (true) + { + const std::size_t depth = container_stack.size(); + + if (JSON_HEDLEY_UNLIKELY(!get_ubjson_value(prefix))) + { + return false; + } + + // the value begun here is complete once it is not inside anything + if (container_stack.empty()) + { + return true; + } + + // a value was completed rather than a container opened; a + // container that ends at a marker needs the next byte to test + if (container_stack.size() == depth && container_stack.back().remaining == npos) + { + get_ignore_noop(); + } + + // advance to the next element, closing the containers that ended. + // top is a copy, not a reference: it must stay valid across the + // pop_back() below, which destroys the container_stack element it + // would otherwise alias. + for (;;) + { + container_frame top = container_stack.back(); + + if (top.remaining != npos) + { + if (top.remaining != 0) + { + --container_stack.back().remaining; + if (top.is_object) + { + key.clear(); + if (JSON_HEDLEY_UNLIKELY(!get_ubjson_string(key) || !sax->key(key))) + { + return false; + } + } + // an optimized container gives its elements no marker + prefix = (top.type_marker != 0) ? top.type_marker : get_ignore_noop(); + break; + } + } + // the end marker is compared against a literal rather than + // against a conditional expression, because char_int_type is + // unsigned for some input adapters and MSVC then reports the + // comparison as a signed/unsigned mismatch + else if (top.is_object ? (current != '}') : (current != ']')) + { + // a container that ends at a marker is never optimized, so + // every element carries its own marker; for an object the + // byte tested above is the first byte of the key + if (top.is_object) + { + key.clear(); + if (JSON_HEDLEY_UNLIKELY(!get_ubjson_string(key, false) || !sax->key(key))) + { + return false; + } + prefix = get_ignore_noop(); + } + else + { + prefix = current; + } + break; + } + + container_stack.pop_back(); + if (JSON_HEDLEY_UNLIKELY(top.is_object ? !sax->end_object() : !sax->end_array())) + { + return false; + } + if (container_stack.empty()) + { + return true; + } + // the container that just ended was an element of the one + // below it, which may need the next byte for its own test + if (container_stack.back().remaining == npos) + { + get_ignore_noop(); + } + } + } } /*! @@ -13099,7 +14681,12 @@ class binary_reader { result.first = npos; // size result.second = 0; // type - bool is_ndarray = false; + // seed the flag with the caller's context: inside an ndarray dimension + // vector another ndarray is not allowed, and get_ubjson_size_value() + // rejects it up front instead of reading it and reporting afterwards. + // Seeding it with `false` made every '#' of a "[#[#[..." chain descend + // another level, which overflowed the stack (see #5104). + bool is_ndarray = inside_ndarray; get_ignore_noop(); @@ -13132,13 +14719,11 @@ class binary_reader } const bool is_error = get_ubjson_size_value(result.first, is_ndarray); - if (input_format == input_format_t::bjdata && is_ndarray) + // an ndarray was read here only if the flag flipped; when it was + // seeded true, get_ubjson_size_value() already rejected the nested + // dimension vector + if (input_format == input_format_t::bjdata && is_ndarray && !inside_ndarray) { - if (inside_ndarray) - { - return sax->parse_error(chars_read, get_token_string(), parse_error::create(112, chars_read, - exception_message(input_format, "ndarray can not be recursive", "size"), nullptr)); - } result.second |= (1 << 8); // use bit 8 to indicate ndarray, all UBJSON and BJData markers should be ASCII letters } return is_error; @@ -13147,7 +14732,7 @@ class binary_reader if (current == '#') { const bool is_error = get_ubjson_size_value(result.first, is_ndarray); - if (input_format == input_format_t::bjdata && is_ndarray) + if (input_format == input_format_t::bjdata && is_ndarray && !inside_ndarray) { return sax->parse_error(chars_read, get_token_string(), parse_error::create(112, chars_read, exception_message(input_format, "ndarray requires both type and size", "size"), nullptr)); @@ -13418,53 +15003,33 @@ class binary_reader if (size_and_type.first != npos) { - if (JSON_HEDLEY_UNLIKELY(!sax->start_array(size_and_type.first))) + // reading an element of a valueless type consumes no input, so the + // declared count alone decides how much is allocated; the check is + // made before the start event so that no container is opened that + // is then abandoned. See @ref max_valueless_container_size. + if (JSON_HEDLEY_UNLIKELY((size_and_type.second == 'Z' || size_and_type.second == 'T' || size_and_type.second == 'F') + && size_and_type.first > max_valueless_container_size)) + { + return sax->parse_error(chars_read, get_token_string(), out_of_range::create(408, + exception_message(input_format, "excessive array size", "size"), nullptr)); + } + + if (JSON_HEDLEY_UNLIKELY(!enter_array(size_and_type.first, size_and_type.second))) { return false; } - if (size_and_type.second != 0) + if (size_and_type.second == 'N') { - if (size_and_type.second != 'N') - { - for (std::size_t i = 0; i < size_and_type.first; ++i) - { - if (JSON_HEDLEY_UNLIKELY(!get_ubjson_value(size_and_type.second))) - { - return false; - } - } - } - } - else - { - for (std::size_t i = 0; i < size_and_type.first; ++i) - { - if (JSON_HEDLEY_UNLIKELY(!parse_ubjson_internal())) - { - return false; - } - } - } - } - else - { - if (JSON_HEDLEY_UNLIKELY(!sax->start_array(detail::unknown_size()))) - { - return false; + // a no-op is not a value, so a container of them holds none; + // the declared size has already been passed to the SAX parser + container_stack.back().remaining = 0; } - while (current != ']') - { - if (JSON_HEDLEY_UNLIKELY(!parse_ubjson_internal(false))) - { - return false; - } - get_ignore_noop(); - } + return true; } - return sax->end_array(); + return enter_array(detail::unknown_size()); } /*! @@ -13486,68 +15051,12 @@ class binary_reader exception_message(input_format, "BJData object does not support ND-array size in optimized format", "object"), nullptr)); } - string_t key; if (size_and_type.first != npos) { - if (JSON_HEDLEY_UNLIKELY(!sax->start_object(size_and_type.first))) - { - return false; - } - - if (size_and_type.second != 0) - { - for (std::size_t i = 0; i < size_and_type.first; ++i) - { - if (JSON_HEDLEY_UNLIKELY(!get_ubjson_string(key) || !sax->key(key))) - { - return false; - } - if (JSON_HEDLEY_UNLIKELY(!get_ubjson_value(size_and_type.second))) - { - return false; - } - key.clear(); - } - } - else - { - for (std::size_t i = 0; i < size_and_type.first; ++i) - { - if (JSON_HEDLEY_UNLIKELY(!get_ubjson_string(key) || !sax->key(key))) - { - return false; - } - if (JSON_HEDLEY_UNLIKELY(!parse_ubjson_internal())) - { - return false; - } - key.clear(); - } - } - } - else - { - if (JSON_HEDLEY_UNLIKELY(!sax->start_object(detail::unknown_size()))) - { - return false; - } - - while (current != '}') - { - if (JSON_HEDLEY_UNLIKELY(!get_ubjson_string(key, false) || !sax->key(key))) - { - return false; - } - if (JSON_HEDLEY_UNLIKELY(!parse_ubjson_internal())) - { - return false; - } - get_ignore_noop(); - key.clear(); - } + return enter_object(size_and_type.first, size_and_type.second); } - return sax->end_object(); + return enter_object(detail::unknown_size()); } // Note, no reader for UBJSON binary types is implemented because they do @@ -13935,6 +15444,9 @@ class binary_reader /// the SAX parser json_sax_t* sax = nullptr; + /// the containers that have been opened and not closed yet; see @ref container_frame + std::vector container_stack{}; + // excluded markers in bjdata optimized type #define JSON_BINARY_READER_MAKE_BJD_OPTIMIZED_TYPE_MARKERS_ \ make_array('F', 'H', 'N', 'S', 'T', 'Z', '[', '{') @@ -14064,9 +15576,10 @@ class parser parser_callback_t cb = nullptr, const bool allow_exceptions_ = true, const bool ignore_comments = false, - const bool ignore_trailing_commas_ = false) + const bool ignore_trailing_commas_ = false, + const bool discard_number_values_ = false) : callback(std::move(cb)) - , m_lexer(std::move(adapter), ignore_comments) + , m_lexer(std::move(adapter), ignore_comments, discard_number_values_) , allow_exceptions(allow_exceptions_) , ignore_trailing_commas(ignore_trailing_commas_) { @@ -16415,6 +17928,20 @@ class json_pointer } } + // the reference token consists only of digits at this point (cf. checks + // above); however, its numeric value might not be representable, in which + // case array_index() would throw out_of_range.404/410 -- contains() must + // not throw (see #5395), so such a reference token is treated as "not found" + errno = 0; // strtoull() does not reset errno on success + char* p_end = nullptr; // NOLINT(misc-const-correctness) + const unsigned long long magnitude = std::strtoull(reference_token.c_str(), &p_end, 10); // NOLINT(runtime/int) + if (JSON_HEDLEY_UNLIKELY(errno == ERANGE // the value exceeds ULLONG_MAX + || magnitude >= static_cast((std::numeric_limits::max)()))) // NOLINT(runtime/int) + { + // the array index cannot be represented as size_type + return false; + } + const auto idx = array_index(reference_token); if (idx >= ptr->size()) { @@ -17855,7 +19382,17 @@ class binary_writer std::vector bjdx = {'[', '{', 'S', 'H', 'T', 'F', 'N', 'Z'}; // excluded markers in bjdata optimized type - if (same_prefix && !(use_bjdata && std::find(bjdx.begin(), bjdx.end(), first_prefix) != bjdx.end())) + // an optimized array of a valueless type carries no payload, so a + // reader has nothing but the declared count to bound the allocation + // by and refuses an excessive one. Write the unoptimized form for + // those, at one byte per element, so the result can be read back. + // Objects are not affected: every element is preceded by its key. + const bool valueless_type = (first_prefix == 'Z' || first_prefix == 'T' || first_prefix == 'F'); + const bool excessive_valueless = valueless_type + && j.m_data.m_value.array->size() > detail::max_valueless_container_size; + + if (same_prefix && !excessive_valueless + && !(use_bjdata && std::find(bjdx.begin(), bjdx.end(), first_prefix) != bjdx.end())) { prefix_required = false; oa->write_character(to_char_type('$')); @@ -18676,6 +20213,20 @@ class binary_writer return 'D'; // float 64 } + /*! + @brief checks whether a JSON number fits into @a TargetType + @param[in] el a JSON number of either the signed or unsigned integer kind + @return whether @a el's value can be represented by @a TargetType without + wrapping, regardless of which of the two kinds it is stored as + */ + template + static bool bjdata_ndarray_value_in_range(const BasicJsonType& el) + { + return el.is_number_unsigned() + ? value_in_range_of(el.template get()) + : value_in_range_of(el.template get()); + } + /*! @return false if the object is successfully converted to a bjdata ndarray, true if the type or size is invalid */ @@ -18760,6 +20311,60 @@ class binary_writer } } + // every element is cast to the (possibly narrower) C++ type matching + // dtype below; a value that does not fit that type would silently + // wrap (integers) or overflow to infinity (the "single" precision + // float) instead of being reported, so such an object falls back to + // a plain object encoding as well + for (const auto& el : value.at(key)) + { + bool in_range = true; + switch (dtype) + { + case 'U': + case 'C': + case 'B': + in_range = bjdata_ndarray_value_in_range(el); + break; + case 'i': + in_range = bjdata_ndarray_value_in_range(el); + break; + case 'u': + in_range = bjdata_ndarray_value_in_range(el); + break; + case 'I': + in_range = bjdata_ndarray_value_in_range(el); + break; + case 'm': + in_range = bjdata_ndarray_value_in_range(el); + break; + case 'l': + in_range = bjdata_ndarray_value_in_range(el); + break; + case 'M': + in_range = bjdata_ndarray_value_in_range(el); + break; + case 'L': + in_range = bjdata_ndarray_value_in_range(el); + break; + case 'd': + { + const auto dval = el.template get(); + in_range = !std::isfinite(dval) || + (dval >= static_cast(std::numeric_limits::lowest()) && + dval <= static_cast((std::numeric_limits::max)())); + break; + } + default: + // 'D' (double) already spans the full range of number_float_t + break; + } + if (!in_range) + { + return true; + } + } + oa->write_character('['); oa->write_character('$'); oa->write_character(dtype); @@ -18879,8 +20484,8 @@ class binary_writer void write_compact_float(const number_float_t n, detail::input_format_t format) { #ifdef __GNUC__ -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wfloat-equal" + JSON_HEDLEY_DIAGNOSTIC_PUSH + JSON_HEDLEY_PRAGMA(GCC diagnostic ignored "-Wfloat-equal") #endif if (!std::isfinite(n) || ((static_cast(n) >= static_cast(std::numeric_limits::lowest()) && static_cast(n) <= static_cast((std::numeric_limits::max)()) && @@ -18899,7 +20504,7 @@ class binary_writer write_number(n); } #ifdef __GNUC__ -#pragma GCC diagnostic pop + JSON_HEDLEY_DIAGNOSTIC_POP #endif } @@ -18982,18 +20587,19 @@ NLOHMANN_JSON_NAMESPACE_END -#include // reverse, remove, fill, find, none_of +#include // reverse, remove, fill, find, none_of, min #include // array #include // localeconv, lconv #include // labs, isfinite, isnan, signbit #include // size_t, ptrdiff_t #include // uint8_t #include // snprintf +#include // memcpy, memset #include // numeric_limits #include // string, char_traits -#include // setfill, setw #include // is_same #include // move +#include // vector // #include // __ _____ _____ _____ @@ -20074,8 +21680,8 @@ char* to_chars(char* first, const char* last, FloatType value) } #ifdef __GNUC__ -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wfloat-equal" + JSON_HEDLEY_DIAGNOSTIC_PUSH + JSON_HEDLEY_PRAGMA(GCC diagnostic ignored "-Wfloat-equal") #endif if (value == 0) // +-0 { @@ -20086,7 +21692,7 @@ char* to_chars(char* first, const char* last, FloatType value) return first; } #ifdef __GNUC__ -#pragma GCC diagnostic pop + JSON_HEDLEY_DIAGNOSTIC_POP #endif JSON_ASSERT(last - first >= std::numeric_limits::max_digits10); @@ -20118,6 +21724,8 @@ NLOHMANN_JSON_NAMESPACE_END // #include +// #include + // #include // #include @@ -20160,18 +21768,32 @@ class serializer public: /*! - @param[in] s output stream to serialize to + @param[in] s output adapter to serialize to; not owned by the serializer, + so it must outlive it (it lives at the call site) @param[in] ichar indentation character to use + @param[in] pretty_print_ whether the output shall be pretty-printed + @param[in] ensure_ascii_ If @a ensure_ascii_ is true, all non-ASCII + characters in the output are escaped with `\uXXXX` sequences, and the + result consists of ASCII characters only. + @param[in] indent_step_ the indent level @param[in] error_handler_ how to react on decoding errors + + None of @a pretty_print_, @a ensure_ascii_ and @a indent_step_ change over + the life of the serializer, so they are captured once here instead of + being threaded through every call to @ref dump, @ref dump_internal and + @ref dump_iteratively. */ - serializer(output_adapter_t s, const char ichar, + serializer(output_adapter_protocol& s, const char ichar, + const bool pretty_print_ = false, + const bool ensure_ascii_ = false, + const std::size_t indent_step_ = 0, error_handler_t error_handler_ = error_handler_t::strict) - : o(std::move(s)) - , loc(std::localeconv()) - , thousands_sep(loc->thousands_sep == nullptr ? '\0' : std::char_traits::to_char_type(* (loc->thousands_sep))) - , decimal_point(loc->decimal_point == nullptr ? '\0' : std::char_traits::to_char_type(* (loc->decimal_point))) + : o(&s) + , locale(std::localeconv()) , indent_char(ichar) - , indent_string(512, indent_char) + , pretty_print(pretty_print_) + , ensure_ascii(ensure_ascii_) + , indent_step(indent_step_) , error_handler(error_handler_) {} @@ -20187,8 +21809,8 @@ class serializer This function is called by the public member function dump and organizes the serialization internally. The indentation level is propagated as - additional parameter. In case of arrays and objects, the function is - called recursively. + additional parameter. Arrays and objects are serialized without recursion, + however deeply they are nested. - strings and object keys are escaped using `escape_string()` - integer numbers are converted implicitly via `operator<<` @@ -20197,89 +21819,109 @@ class serializer byte array @param[in] val value to serialize - @param[in] pretty_print whether the output shall be pretty-printed - @param[in] ensure_ascii If @a ensure_ascii is true, all non-ASCII characters - in the output are escaped with `\uXXXX` sequences, and the result consists - of ASCII characters only. - @param[in] indent_step the indent level @param[in] current_indent the current indent level (only used internally) */ void dump(const BasicJsonType& val, - const bool pretty_print, - const bool ensure_ascii, - const unsigned int indent_step, - const unsigned int current_indent = 0) + const std::size_t current_indent = 0) + { + dump_internal(val, current_indent); + flush(); + } + + JSON_PRIVATE_UNLESS_TESTED: + /*! + @brief worker for @ref dump + + Identical in behavior to the historical @ref dump, but writes into the + serializer's internal @ref write_buffer instead of issuing a virtual call + per token. The public @ref dump wraps this and flushes the buffer once the + top-level value has been serialized. + + Serializing a container descends into its elements, so a value nested deeply + enough used to exhaust the call stack and terminate the process with no + exception to catch. The descent is bounded here: once @ref dump_depth_limit + levels have been entered, @ref dump_iteratively writes out what is left + without the call stack. A value nested less deeply than that - all but a + vanishing minority - is written by exactly the code that always wrote it. + + @sa https://github.com/nlohmann/json/issues/5387 + */ + void dump_internal(const BasicJsonType& val, + const std::size_t current_indent = 0, + const std::size_t depth = 0) { switch (val.m_data.m_type) { case value_t::object: { + if (JSON_HEDLEY_UNLIKELY(depth >= dump_depth_limit())) + { + dump_iteratively(val, current_indent); + return; + } + if (val.m_data.m_value.object->empty()) { - o->write_characters("{}", 2); + put_literal("{}"); return; } if (pretty_print) { - o->write_characters("{\n", 2); + put_literal("{\n"); // variable to hold indentation for recursive calls - const auto new_indent = current_indent + indent_step; - if (JSON_HEDLEY_UNLIKELY(indent_string.size() < new_indent)) - { - indent_string.resize(indent_string.size() * 2, ' '); - } + const auto new_indent = next_indent(current_indent, indent_step); // first n-1 elements auto i = val.m_data.m_value.object->cbegin(); for (std::size_t cnt = 0; cnt < val.m_data.m_value.object->size() - 1; ++cnt, ++i) { - o->write_characters(indent_string.c_str(), new_indent); - o->write_character('\"'); - dump_escaped(i->first, ensure_ascii); - o->write_characters("\": ", 3); - dump(i->second, true, ensure_ascii, indent_step, new_indent); - o->write_characters(",\n", 2); + put_indent(new_indent); + put_char('"'); + dump_escaped(i->first); + put_literal("\": "); + dump_internal(i->second, new_indent, depth + 1); + put_literal(",\n"); } // last element JSON_ASSERT(i != val.m_data.m_value.object->cend()); JSON_ASSERT(std::next(i) == val.m_data.m_value.object->cend()); - o->write_characters(indent_string.c_str(), new_indent); - o->write_character('\"'); - dump_escaped(i->first, ensure_ascii); - o->write_characters("\": ", 3); - dump(i->second, true, ensure_ascii, indent_step, new_indent); + put_indent(new_indent); + put_char('"'); + dump_escaped(i->first); + put_literal("\": "); + dump_internal(i->second, new_indent, depth + 1); - o->write_character('\n'); - o->write_characters(indent_string.c_str(), current_indent); - o->write_character('}'); + put_char('\n'); + put_indent(current_indent); + put_char('}'); } else { - o->write_character('{'); + put_char('{'); // first n-1 elements auto i = val.m_data.m_value.object->cbegin(); for (std::size_t cnt = 0; cnt < val.m_data.m_value.object->size() - 1; ++cnt, ++i) { - o->write_character('\"'); - dump_escaped(i->first, ensure_ascii); - o->write_characters("\":", 2); - dump(i->second, false, ensure_ascii, indent_step, current_indent); - o->write_character(','); + put_char('"'); + dump_escaped(i->first); + put_literal("\":"); + dump_internal(i->second, current_indent, depth + 1); + put_char(','); } // last element JSON_ASSERT(i != val.m_data.m_value.object->cend()); JSON_ASSERT(std::next(i) == val.m_data.m_value.object->cend()); - o->write_character('\"'); - dump_escaped(i->first, ensure_ascii); - o->write_characters("\":", 2); - dump(i->second, false, ensure_ascii, indent_step, current_indent); + put_char('"'); + dump_escaped(i->first); + put_literal("\":"); + dump_internal(i->second, current_indent, depth + 1); - o->write_character('}'); + put_char('}'); } return; @@ -20287,58 +21929,60 @@ class serializer case value_t::array: { + if (JSON_HEDLEY_UNLIKELY(depth >= dump_depth_limit())) + { + dump_iteratively(val, current_indent); + return; + } + if (val.m_data.m_value.array->empty()) { - o->write_characters("[]", 2); + put_literal("[]"); return; } if (pretty_print) { - o->write_characters("[\n", 2); + put_literal("[\n"); // variable to hold indentation for recursive calls - const auto new_indent = current_indent + indent_step; - if (JSON_HEDLEY_UNLIKELY(indent_string.size() < new_indent)) - { - indent_string.resize(indent_string.size() * 2, ' '); - } + const auto new_indent = next_indent(current_indent, indent_step); // first n-1 elements for (auto i = val.m_data.m_value.array->cbegin(); i != val.m_data.m_value.array->cend() - 1; ++i) { - o->write_characters(indent_string.c_str(), new_indent); - dump(*i, true, ensure_ascii, indent_step, new_indent); - o->write_characters(",\n", 2); + put_indent(new_indent); + dump_internal(*i, new_indent, depth + 1); + put_literal(",\n"); } // last element JSON_ASSERT(!val.m_data.m_value.array->empty()); - o->write_characters(indent_string.c_str(), new_indent); - dump(val.m_data.m_value.array->back(), true, ensure_ascii, indent_step, new_indent); + put_indent(new_indent); + dump_internal(val.m_data.m_value.array->back(), new_indent, depth + 1); - o->write_character('\n'); - o->write_characters(indent_string.c_str(), current_indent); - o->write_character(']'); + put_char('\n'); + put_indent(current_indent); + put_char(']'); } else { - o->write_character('['); + put_char('['); // first n-1 elements for (auto i = val.m_data.m_value.array->cbegin(); i != val.m_data.m_value.array->cend() - 1; ++i) { - dump(*i, false, ensure_ascii, indent_step, current_indent); - o->write_character(','); + dump_internal(*i, current_indent, depth + 1); + put_char(','); } // last element JSON_ASSERT(!val.m_data.m_value.array->empty()); - dump(val.m_data.m_value.array->back(), false, ensure_ascii, indent_step, current_indent); + dump_internal(val.m_data.m_value.array->back(), current_indent, depth + 1); - o->write_character(']'); + put_char(']'); } return; @@ -20346,9 +21990,9 @@ class serializer case value_t::string: { - o->write_character('\"'); - dump_escaped(*val.m_data.m_value.string, ensure_ascii); - o->write_character('\"'); + put_char('"'); + dump_escaped(*val.m_data.m_value.string); + put_char('"'); return; } @@ -20356,70 +22000,66 @@ class serializer { if (pretty_print) { - o->write_characters("{\n", 2); + put_literal("{\n"); // variable to hold indentation for recursive calls - const auto new_indent = current_indent + indent_step; - if (JSON_HEDLEY_UNLIKELY(indent_string.size() < new_indent)) - { - indent_string.resize(indent_string.size() * 2, ' '); - } + const auto new_indent = next_indent(current_indent, indent_step); - o->write_characters(indent_string.c_str(), new_indent); + put_indent(new_indent); - o->write_characters("\"bytes\": [", 10); + put_literal("\"bytes\": ["); if (!val.m_data.m_value.binary->empty()) { for (auto i = val.m_data.m_value.binary->cbegin(); i != val.m_data.m_value.binary->cend() - 1; ++i) { - dump_integer(*i); - o->write_characters(", ", 2); + dump_byte(*i); + put_literal(", "); } - dump_integer(val.m_data.m_value.binary->back()); + dump_byte(val.m_data.m_value.binary->back()); } - o->write_characters("],\n", 3); - o->write_characters(indent_string.c_str(), new_indent); + put_literal("],\n"); + put_indent(new_indent); - o->write_characters("\"subtype\": ", 11); + put_literal("\"subtype\": "); if (val.m_data.m_value.binary->has_subtype()) { dump_integer(val.m_data.m_value.binary->subtype()); } else { - o->write_characters("null", 4); + put_literal("null"); } - o->write_character('\n'); - o->write_characters(indent_string.c_str(), current_indent); - o->write_character('}'); + put_char('\n'); + put_indent(current_indent); + put_char('}'); } else { - o->write_characters("{\"bytes\":[", 10); + put_literal("{\"bytes\":["); if (!val.m_data.m_value.binary->empty()) { for (auto i = val.m_data.m_value.binary->cbegin(); i != val.m_data.m_value.binary->cend() - 1; ++i) { - dump_integer(*i); - o->write_character(','); + dump_byte(*i); + put_char(','); } - dump_integer(val.m_data.m_value.binary->back()); + dump_byte(val.m_data.m_value.binary->back()); } - o->write_characters("],\"subtype\":", 12); + put_literal("],\"subtype\":"); if (val.m_data.m_value.binary->has_subtype()) { dump_integer(val.m_data.m_value.binary->subtype()); - o->write_character('}'); + put_char('}'); } else { - o->write_characters("null}", 5); + put_literal("null}"); } } return; @@ -20429,11 +22069,11 @@ class serializer { if (val.m_data.m_value.boolean) { - o->write_characters("true", 4); + put_literal("true"); } else { - o->write_characters("false", 5); + put_literal("false"); } return; } @@ -20458,13 +22098,13 @@ class serializer case value_t::discarded: { - o->write_characters("", 11); + put_literal(""); return; } case value_t::null: { - o->write_characters("null", 4); + put_literal("null"); return; } @@ -20473,6 +22113,367 @@ class serializer } } + private: + /// the number of levels @ref dump_internal descends into before it hands + /// over to @ref dump_iteratively + static constexpr std::size_t dump_depth_limit() + { + return 128; + } + + /*! + @brief write out @a val and everything below it without the call stack + + Emits the same bytes as @ref dump_internal, keeping the containers it has + entered on an explicit stack instead of descending into them. Only reached + for values nested deeper than @ref dump_depth_limit, which is why it is not + written for speed: walking every value this way measured up to 20% slower on + object-heavy documents than letting the compiler drive the descent. + */ + void dump_iteratively(const BasicJsonType& val, + const std::size_t current_indent = 0) + { + // Scalars, empty containers and binary values are written by dump_value + // alone, so nothing is allocated for them: only a container with + // elements is ever pushed. + std::vector stack; + + dump_value(val, current_indent, stack); + + while (!stack.empty()) + { + dump_frame& frame = stack.back(); + + if (frame.value->m_data.m_type == value_t::object) + { + const auto* object = frame.value->m_data.m_value.object; + + if (frame.object_it == object->cend()) + { + if (pretty_print) + { + put_char('\n'); + put_indent(frame.current_indent); + } + + put_char('}'); + stack.pop_back(); + continue; + } + + // the separator goes in front of every element but the first, + // which puts exactly one between each pair and none at the end + if (frame.object_it != object->cbegin()) + { + if (pretty_print) + { + put_literal(",\n"); + } + else + { + put_char(','); + } + } + + if (pretty_print) + { + put_indent(frame.child_indent); + } + + put_char('"'); + dump_escaped(frame.object_it->first); + + if (pretty_print) + { + put_literal("\": "); + } + else + { + put_literal("\":"); + } + + const BasicJsonType& element = frame.object_it->second; + ++frame.object_it; + + // read everything needed from the frame before this: entering a + // container pushes another one and can move them all + const std::size_t element_indent = frame.child_indent; + dump_value(element, element_indent, stack); + } + else + { + const auto* array = frame.value->m_data.m_value.array; + + if (frame.array_it == array->cend()) + { + if (pretty_print) + { + put_char('\n'); + put_indent(frame.current_indent); + } + + put_char(']'); + stack.pop_back(); + continue; + } + + if (frame.array_it != array->cbegin()) + { + if (pretty_print) + { + put_literal(",\n"); + } + else + { + put_char(','); + } + } + + if (pretty_print) + { + put_indent(frame.child_indent); + } + + const BasicJsonType& element = *frame.array_it; + ++frame.array_it; + + // see above + const std::size_t element_indent = frame.child_indent; + dump_value(element, element_indent, stack); + } + } + } + + private: + /// @brief a container that has been opened but not closed yet + struct dump_frame + { + dump_frame(const BasicJsonType* value_, const std::size_t current_indent_, + const std::size_t child_indent_) noexcept + : value(value_) + , current_indent(current_indent_) + , child_indent(child_indent_) + {} + + /// the object or array being serialized + const BasicJsonType* value; + /// the element to serialize next; which of the two is live follows from + /// the type of @a value. They are kept side by side rather than in a + /// union, which would need its special members written out by hand, see + /// detail/iterators/internal_iterator.hpp + typename BasicJsonType::object_t::const_iterator object_it{}; + typename BasicJsonType::array_t::const_iterator array_it{}; + /// the indentation of the container itself, used by its closing bracket + std::size_t current_indent; + /// the indentation of the container's elements + std::size_t child_indent; + }; + + /*! + @brief serialize the value @a val, but not the elements of a container + + An object or array with elements is opened and pushed onto @a stack for + @ref dump_internal to walk; everything else - including a binary value, + which looks like an object but has no elements to descend into - is written + out here in full. + */ + void dump_value(const BasicJsonType& val, + const std::size_t current_indent, + std::vector& stack) + { + switch (val.m_data.m_type) + { + case value_t::object: + { + if (val.m_data.m_value.object->empty()) + { + put_literal("{}"); + return; + } + + std::size_t child_indent = current_indent; + + if (pretty_print) + { + put_literal("{\n"); + child_indent = next_indent(current_indent, indent_step); + } + else + { + put_char('{'); + } + + stack.emplace_back(&val, current_indent, child_indent); + stack.back().object_it = val.m_data.m_value.object->cbegin(); + return; + } + + case value_t::array: + { + if (val.m_data.m_value.array->empty()) + { + put_literal("[]"); + return; + } + + std::size_t child_indent = current_indent; + + if (pretty_print) + { + put_literal("[\n"); + child_indent = next_indent(current_indent, indent_step); + } + else + { + put_char('['); + } + + stack.emplace_back(&val, current_indent, child_indent); + stack.back().array_it = val.m_data.m_value.array->cbegin(); + return; + } + + case value_t::string: + { + put_char('"'); + dump_escaped(*val.m_data.m_value.string); + put_char('"'); + return; + } + + case value_t::binary: + { + if (pretty_print) + { + put_literal("{\n"); + + // variable to hold indentation for the bytes + const auto new_indent = next_indent(current_indent, indent_step); + + put_indent(new_indent); + + put_literal("\"bytes\": ["); + + if (!val.m_data.m_value.binary->empty()) + { + for (auto i = val.m_data.m_value.binary->cbegin(); + i != val.m_data.m_value.binary->cend() - 1; ++i) + { + dump_byte(*i); + put_literal(", "); + } + dump_byte(val.m_data.m_value.binary->back()); + } + + put_literal("],\n"); + put_indent(new_indent); + + put_literal("\"subtype\": "); + if (val.m_data.m_value.binary->has_subtype()) + { + dump_integer(val.m_data.m_value.binary->subtype()); + } + else + { + put_literal("null"); + } + put_char('\n'); + put_indent(current_indent); + put_char('}'); + } + else + { + put_literal("{\"bytes\":["); + + if (!val.m_data.m_value.binary->empty()) + { + for (auto i = val.m_data.m_value.binary->cbegin(); + i != val.m_data.m_value.binary->cend() - 1; ++i) + { + dump_byte(*i); + put_char(','); + } + dump_byte(val.m_data.m_value.binary->back()); + } + + put_literal("],\"subtype\":"); + if (val.m_data.m_value.binary->has_subtype()) + { + dump_integer(val.m_data.m_value.binary->subtype()); + put_char('}'); + } + else + { + put_literal("null}"); + } + } + return; + } + + case value_t::boolean: + { + if (val.m_data.m_value.boolean) + { + put_literal("true"); + } + else + { + put_literal("false"); + } + return; + } + + case value_t::number_integer: + { + dump_integer(val.m_data.m_value.number_integer); + return; + } + + case value_t::number_unsigned: + { + dump_integer(val.m_data.m_value.number_unsigned); + return; + } + + case value_t::number_float: + { + dump_float(val.m_data.m_value.number_float); + return; + } + + case value_t::discarded: + { + put_literal(""); + return; + } + + case value_t::null: + { + put_literal("null"); + return; + } + + default: // LCOV_EXCL_LINE + JSON_ASSERT(false); // NOLINT(cert-dcl03-c,hicpp-static-assert,misc-static-assert) LCOV_EXCL_LINE + } + } + + + /*! + @brief the indentation level to use for the children of the current value + + A very large @a indent_step can wrap the unsigned accumulation on deep + nesting, which would silently truncate the indentation. Far harder to reach + now that the accumulator is a std::size_t, but still reachable where that is + 32 bits wide. + */ + static std::size_t next_indent(const std::size_t current_indent, const std::size_t indent_step) + { + const std::size_t new_indent = current_indent + indent_step; + JSON_ASSERT(new_indent >= current_indent); + return new_indent; + } + JSON_PRIVATE_UNLESS_TESTED: /*! @brief dump escaped string @@ -20483,12 +22484,32 @@ class serializer representation. The escaped string is written to output stream @a o. @param[in] s the string to escape - @param[in] ensure_ascii whether to escape non-ASCII characters with - \uXXXX sequences @complexity Linear in the length of string @a s. */ - void dump_escaped(const string_t& s, const bool ensure_ascii) + void dump_escaped(const string_t& s) + { + // dispatch once here rather than test the flag inside the loop: it does + // not change while a string is written, and folding it lets each of the + // two scanners be inlined into a loop of its own + if (ensure_ascii) + { + dump_escaped_impl(s); + } + else + { + dump_escaped_impl(s); + } + } + + /*! + @brief worker for @ref dump_escaped + + @a ensure_ascii is a template parameter here so that the branch on it is + resolved once, outside the loop; see @ref dump_escaped. + */ + template + void dump_escaped_impl(const string_t& s) { std::uint32_t codepoint{}; std::uint8_t state = UTF8_ACCEPT; @@ -20500,6 +22521,56 @@ class serializer for (std::size_t i = 0; i < s.size(); ++i) { + // Fast path: at a character boundary (state == UTF8_ACCEPT), + // bulk-copy the longest run of bytes that need no escaping using a + // SWAR scanner shared with the lexer's contiguous path. The scanner + // stops exactly at the first byte dump_escaped would handle + // individually, so that byte is left to the byte-at-a-time path + // below, keeping escaping output and error diagnostics unchanged. + // + // - EnsureAscii == false: string_bulk_run() copies ordinary bytes + // and complete well-formed UTF-8, stopping at a quote, backslash, + // control character (< 0x20), or ill-formed/truncated sequence. + // - EnsureAscii == true: only printable ASCII may be copied + // verbatim; find_ascii_copyable_run() additionally stops at 0x7F + // and every non-ASCII byte (>= 0x80), which must be \u-escaped. + if (state == UTF8_ACCEPT) + { + const auto* const data = reinterpret_cast(s.data()); + // A run can only be non-empty when the very first byte is one + // the scanner may copy, so test that single byte before paying + // for the scan. Without it, text whose characters all have to be + // escaped - CJK under ensure_ascii, where every byte is >= 0x80 - + // runs the scanner once per character only to be told zero. + std::size_t run = 0; + if (!EnsureAscii) + { + run = string_bulk_run(data + i, s.size() - i); + } + else if (is_ascii_copyable(data[i])) + { + run = find_ascii_copyable_run(data + i, s.size() - i); + } + if (run != 0) + { + // emit any bytes still pending in string_buffer first to + // preserve output order, then write the run directly + if (bytes != 0) + { + put_buffer(string_buffer, bytes); + bytes = 0; + } + put_string(s, i, i + run); + bytes_after_last_accept = 0; + undumped_chars = 0; + i += run; + if (i >= s.size()) + { + break; + } + } + } + const auto byte = static_cast(s[i]); switch (decode(state, codepoint, byte)) @@ -20546,7 +22617,7 @@ class serializer case 0x22: // quotation mark { string_buffer[bytes++] = '\\'; - string_buffer[bytes++] = '\"'; + string_buffer[bytes++] = '"'; break; } @@ -20560,8 +22631,8 @@ class serializer default: { // escape control characters (0x00..0x1F) or, if - // ensure_ascii parameter is used, non-ASCII characters - if ((codepoint <= 0x1F) || (ensure_ascii && (codepoint >= 0x7F))) + // EnsureAscii parameter is used, non-ASCII characters + if ((codepoint <= 0x1F) || (EnsureAscii && (codepoint >= 0x7F))) { if (codepoint <= 0xFFFF) { @@ -20588,7 +22659,7 @@ class serializer // written ("\uxxxx\uxxxx\0") for one code point if (string_buffer.size() - bytes < 13) { - o->write_characters(string_buffer.data(), bytes); + put_buffer(string_buffer, bytes); bytes = 0; } @@ -20626,7 +22697,7 @@ class serializer if (error_handler == error_handler_t::replace) { // add a replacement character - if (ensure_ascii) + if (EnsureAscii) { string_buffer[bytes++] = '\\'; string_buffer[bytes++] = 'u'; @@ -20647,7 +22718,7 @@ class serializer // written ("\uxxxx\uxxxx\0") for one code point if (string_buffer.size() - bytes < 13) { - o->write_characters(string_buffer.data(), bytes); + put_buffer(string_buffer, bytes); bytes = 0; } @@ -20669,7 +22740,7 @@ class serializer default: // decode found yet incomplete multibyte code point { - if (!ensure_ascii) + if (!EnsureAscii) { // code point will not be escaped - copy byte to buffer string_buffer[bytes++] = s[i]; @@ -20686,7 +22757,7 @@ class serializer // write buffer if (bytes > 0) { - o->write_characters(string_buffer.data(), bytes); + put_buffer(string_buffer, bytes); } } else @@ -20702,22 +22773,22 @@ class serializer case error_handler_t::ignore: { // write all accepted bytes - o->write_characters(string_buffer.data(), bytes_after_last_accept); + put_buffer(string_buffer, bytes_after_last_accept); break; } case error_handler_t::replace: { // write all accepted bytes - o->write_characters(string_buffer.data(), bytes_after_last_accept); + put_buffer(string_buffer, bytes_after_last_accept); // add a replacement character - if (ensure_ascii) + if (EnsureAscii) { - o->write_characters("\\ufffd", 6); + put_literal("\\ufffd"); } else { - o->write_characters("\xEF\xBF\xBD", 3); + put_literal("\xEF\xBF\xBD"); } break; } @@ -20728,6 +22799,160 @@ class serializer } } + private: + /*! + @brief append a single character to the write buffer + + Structural characters ('{', '"', ',', ...) previously went straight to the + output adapter, one virtual call each. Buffering them and flushing in bulk + turns those many indirect calls into a single memcpy plus an occasional + flush, which dominates the cost of serializing object/array-heavy values. + */ + void put_char(char c) + { + if (JSON_HEDLEY_UNLIKELY(write_buffer_pos == write_buffer.size())) + { + flush(); + } + write_buffer[write_buffer_pos++] = c; + } + + /*! + @brief append @a indent indentation characters to the write buffer + + Writes the indentation straight into the buffer instead of copying it out of + a pre-grown indentation string, so no auxiliary string has to be sized, + resized, or kept in sync with the deepest nesting level reached. + + An indentation wider than the buffer is emitted by filling the buffer with + the indentation character once and flushing that same content repeatedly: + flushing does not disturb what the buffer holds, so re-filling it between + flushes would be redundant work. + */ + void put_indent(std::size_t indent) + { + // closing braces at the outermost level ask for no indentation at all + if (indent == 0) + { + return; + } + + const std::size_t capacity = write_buffer.size(); + + // fill whatever room is left in the buffer; this is the whole job + // whenever the indentation is narrower than the buffer, which is the + // case for every sane indent_step + const std::size_t head = (std::min)(indent, capacity - write_buffer_pos); + std::memset(write_buffer.data() + write_buffer_pos, indent_char, head); + write_buffer_pos += head; + indent -= head; + + if (JSON_HEDLEY_LIKELY(indent == 0)) + { + return; + } + + // the buffer is full and the remainder spans whole buffer-fulls: flush + // what is pending, then fill the buffer with the indentation character + // exactly once and hand the same bytes to the adapter as often as needed + flush(); + std::memset(write_buffer.data(), indent_char, capacity); + + while (indent >= capacity) + { + write_buffer_pos = capacity; + flush(); + indent -= capacity; + } + + // the buffer still holds indentation characters throughout, so the tail + // only has to be claimed, not written again + write_buffer_pos = indent; + } + + /*! + @brief append a string literal to the write buffer + + The length comes from the array bound rather than a hand-written count, so + it cannot drift out of sync with the literal. A literal always fits into the + buffer (checked at compile time), so unlike @ref put_string this needs no + write-through path for oversized runs. + */ + template + void put_literal(const char (&s)[N]) // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays) + { + static_assert(N >= 2, "put_literal expects a non-empty string literal"); + // the array bound counts the terminating NUL, which is not written + constexpr std::size_t length = N - 1; + static_assert(length < write_buffer_size, "string literal must fit into the write buffer"); + + if (JSON_HEDLEY_UNLIKELY(write_buffer_pos + length > write_buffer.size())) + { + flush(); + } + std::memcpy(write_buffer.data() + write_buffer_pos, s, length); + write_buffer_pos += length; + } + + /*! + @brief append the characters of @a str in [@a start, @a end) + + The only way to append a run of characters: @a str carries its own bound, + so the range can be checked against it, which a bare pointer plus a count + could not do. Runs that do not fit the buffer are written straight through + the output adapter (after flushing what is pending), so large string and + number payloads are not copied an extra time. + */ + template + void put_string(const StringType& str, std::size_t start, std::size_t end) + { + JSON_ASSERT(start <= end); + JSON_ASSERT(end <= str.size()); + + const char* const s = str.data() + start; + const std::size_t length = end - start; + + if (JSON_HEDLEY_UNLIKELY(length >= write_buffer.size())) + { + flush(); + o->write_characters(s, length); + return; + } + if (JSON_HEDLEY_UNLIKELY(write_buffer_pos + length > write_buffer.size())) + { + flush(); + } + std::memcpy(write_buffer.data() + write_buffer_pos, s, length); + write_buffer_pos += length; + } + + /*! + @brief append the first @a length characters of a fixed-size buffer + */ + template + void put_buffer(const std::array& buffer, std::size_t length) + { + put_string(buffer, 0, length); + } + + JSON_PRIVATE_UNLESS_TESTED: + /*! + @brief flush the write buffer to the output adapter + + Writing zero characters is a well-defined no-op for every output adapter, so + the buffered length is passed through unconditionally (no empty-guard branch + to leave uncovered). + + @note dump_escaped() and dump_integer()/dump_float() write into the internal + write buffer; callers that invoke them directly (rather than through the + public dump()) must call flush() before inspecting the output. + */ + void flush() + { + o->write_characters(write_buffer.data(), write_buffer_pos); + write_buffer_pos = 0; + } + private: /*! @brief count digits @@ -20816,6 +23041,62 @@ class serializer return false; } + /*! + @brief write the decimal representation of the byte @a value + + A binary value's bytes are always in [0, 255], so writing one needs neither + the digit counting nor the 64-bit arithmetic that @ref dump_integer does for + an arbitrary number, and the three digits it takes at most are written + straight into the write buffer. + + Any byte type that is not a plain unsigned byte is left to @ref dump_integer, + whose representation of it may differ. + */ + template + void dump_byte(const ByteType value) + { + dump_byte(value, std::integral_constant < bool, + std::is_unsigned::value && sizeof(ByteType) == 1 + && !std::is_same::value > {}); + } + + template + void dump_byte(const ByteType value, std::false_type /*is_plain_byte*/) + { + dump_integer(value); + } + + template + void dump_byte(const ByteType value, std::true_type /*is_plain_byte*/) + { + if (JSON_HEDLEY_UNLIKELY(write_buffer_pos + 3 > write_buffer.size())) + { + flush(); + } + + const auto byte = static_cast(value); + // Accumulate the offset in a local and store it back once. Writing + // through write_buffer[] is a char write, which may alias any object, + // so with the member updated in place the compiler has to reload and + // store it around every digit - measured 2.4x slower on a dump of a + // multi-megabyte binary value. + std::size_t pos = write_buffer_pos; + + if (byte >= 100) + { + write_buffer[pos++] = static_cast('0' + (byte / 100)); + write_buffer[pos++] = static_cast('0' + ((byte / 10) % 10)); + } + else if (byte >= 10) + { + write_buffer[pos++] = static_cast('0' + (byte / 10)); + } + + write_buffer[pos++] = static_cast('0' + (byte % 10)); + + write_buffer_pos = pos; + } + /*! @brief dump an integer @@ -20852,7 +23133,7 @@ class serializer // special case for "0" if (x == 0) { - o->write_character('0'); + put_char('0'); return; } @@ -20905,7 +23186,7 @@ class serializer *(--buffer_ptr) = static_cast('0' + abs_value); } - o->write_characters(number_buffer.data(), n_chars); + put_buffer(number_buffer, n_chars); } /*! @@ -20921,7 +23202,7 @@ class serializer // NaN / inf if (!std::isfinite(x)) { - o->write_characters("null", 4); + put_literal("null"); return; } @@ -20942,7 +23223,7 @@ class serializer auto* begin = number_buffer.data(); auto* end = ::nlohmann::detail::to_chars(begin, begin + number_buffer.size(), x); - o->write_characters(begin, static_cast(end - begin)); + put_buffer(number_buffer, static_cast(end - begin)); } JSON_HEDLEY_NON_NULL(1) @@ -20973,27 +23254,27 @@ class serializer JSON_ASSERT(static_cast(len) < number_buffer.size()); // erase thousands separators - if (thousands_sep != '\0') + if (locale.thousands_sep != '\0') { // NOLINTNEXTLINE(readability-qualified-auto,llvm-qualified-auto): std::remove returns an iterator, see https://github.com/nlohmann/json/issues/3081 - const auto end = std::remove(number_buffer.begin(), number_buffer.begin() + len, thousands_sep); + const auto end = std::remove(number_buffer.begin(), number_buffer.begin() + len, locale.thousands_sep); std::fill(end, number_buffer.end(), '\0'); JSON_ASSERT((end - number_buffer.begin()) <= len); len = (end - number_buffer.begin()); } // convert decimal point to '.' - if (decimal_point != '\0' && decimal_point != '.') + if (locale.decimal_point != '\0' && locale.decimal_point != '.') { // NOLINTNEXTLINE(readability-qualified-auto,llvm-qualified-auto): std::find returns an iterator, see https://github.com/nlohmann/json/issues/3081 - const auto dec_pos = std::find(number_buffer.begin(), number_buffer.end(), decimal_point); + const auto dec_pos = std::find(number_buffer.begin(), number_buffer.end(), locale.decimal_point); if (dec_pos != number_buffer.end()) { *dec_pos = '.'; } } - o->write_characters(number_buffer.data(), static_cast(len)); + put_buffer(number_buffer, static_cast(len)); // determine if we need to append ".0" const bool value_is_int_like = @@ -21005,7 +23286,7 @@ class serializer if (value_is_int_like) { - o->write_characters(".0", 2); + put_literal(".0"); } } @@ -21092,29 +23373,53 @@ class serializer } private: - /// the output of the serializer - output_adapter_t o = nullptr; + /// the locale's thousand separator and decimal point characters + struct locale_chars + { + explicit locale_chars(const std::lconv* loc) noexcept + : thousands_sep(loc->thousands_sep == nullptr ? '\0' : std::char_traits::to_char_type(* (loc->thousands_sep))) + , decimal_point(loc->decimal_point == nullptr ? '\0' : std::char_traits::to_char_type(* (loc->decimal_point))) + {} + + const char thousands_sep; + const char decimal_point; + }; + + /// the output of the serializer (non-owning; the adapter lives at the call site) + output_adapter_protocol* o = nullptr; /// a (hopefully) large enough character buffer std::array number_buffer{{}}; - /// the locale - const std::lconv* loc = nullptr; - /// the locale's thousand separator character - const char thousands_sep = '\0'; - /// the locale's decimal point character - const char decimal_point = '\0'; + /// computed once from std::localeconv() at construction; @ref + /// locale_chars keeps std::localeconv()'s pointer from having to be held + /// past the constructor, while still letting these stay const + const locale_chars locale; /// string buffer std::array string_buffer{{}}; /// the indentation character const char indent_char; - /// the indentation string - string_t indent_string; + + /// whether to pretty-print the output + const bool pretty_print; + + /// whether to escape non-ASCII characters with \uXXXX sequences + const bool ensure_ascii; + + /// the indent level + const std::size_t indent_step; /// error_handler how to react on decoding errors const error_handler_t error_handler; + + /// buffer collecting output before it is flushed to the output adapter, so + /// that the many small structural writes become few bulk writes + static constexpr std::size_t write_buffer_size = 1024; + std::array write_buffer{{}}; + /// number of valid bytes currently held in @ref write_buffer + std::size_t write_buffer_pos = 0; }; } // namespace detail @@ -21613,11 +23918,12 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec detail::parser_callback_tcb = nullptr, const bool allow_exceptions = true, const bool ignore_comments = false, - const bool ignore_trailing_commas = false + const bool ignore_trailing_commas = false, + const bool discard_number_values = false ) { return ::nlohmann::detail::parser(std::move(adapter), - std::move(cb), allow_exceptions, ignore_comments, ignore_trailing_commas); + std::move(cb), allow_exceptions, ignore_comments, ignore_trailing_commas, discard_number_values); } private: @@ -22784,21 +25090,26 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec /// @brief serialization /// @sa https://json.nlohmann.me/api/basic_json/dump/ + JSON_HEDLEY_WARN_UNUSED_RESULT string_t dump(const int indent = -1, const char indent_char = ' ', const bool ensure_ascii = false, const error_handler_t error_handler = error_handler_t::strict) const { string_t result; - serializer s(detail::output_adapter(result), indent_char, error_handler); + detail::output_string_adapter string_adapter(result); if (indent >= 0) { - s.dump(*this, true, ensure_ascii, static_cast(indent)); + serializer s(string_adapter, indent_char, + true, ensure_ascii, static_cast(indent), error_handler); + s.dump(*this); } else { - s.dump(*this, false, ensure_ascii, 0); + serializer s(string_adapter, indent_char, + false, ensure_ascii, 0, error_handler); + s.dump(*this); } return result; @@ -22806,6 +25117,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec /// @brief return the type of the JSON value (explicit) /// @sa https://json.nlohmann.me/api/basic_json/type/ + JSON_HEDLEY_WARN_UNUSED_RESULT constexpr value_t type() const noexcept { return m_data.m_type; @@ -22813,6 +25125,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec /// @brief return whether type is primitive /// @sa https://json.nlohmann.me/api/basic_json/is_primitive/ + JSON_HEDLEY_WARN_UNUSED_RESULT constexpr bool is_primitive() const noexcept { return is_null() || is_string() || is_boolean() || is_number() || is_binary(); @@ -22820,6 +25133,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec /// @brief return whether type is structured /// @sa https://json.nlohmann.me/api/basic_json/is_structured/ + JSON_HEDLEY_WARN_UNUSED_RESULT constexpr bool is_structured() const noexcept { return is_array() || is_object(); @@ -22827,6 +25141,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec /// @brief return whether value is null /// @sa https://json.nlohmann.me/api/basic_json/is_null/ + JSON_HEDLEY_WARN_UNUSED_RESULT constexpr bool is_null() const noexcept { return m_data.m_type == value_t::null; @@ -22834,6 +25149,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec /// @brief return whether value is a boolean /// @sa https://json.nlohmann.me/api/basic_json/is_boolean/ + JSON_HEDLEY_WARN_UNUSED_RESULT constexpr bool is_boolean() const noexcept { return m_data.m_type == value_t::boolean; @@ -22841,6 +25157,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec /// @brief return whether value is a number /// @sa https://json.nlohmann.me/api/basic_json/is_number/ + JSON_HEDLEY_WARN_UNUSED_RESULT constexpr bool is_number() const noexcept { return is_number_integer() || is_number_float(); @@ -22848,6 +25165,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec /// @brief return whether value is an integer number /// @sa https://json.nlohmann.me/api/basic_json/is_number_integer/ + JSON_HEDLEY_WARN_UNUSED_RESULT constexpr bool is_number_integer() const noexcept { return m_data.m_type == value_t::number_integer || m_data.m_type == value_t::number_unsigned; @@ -22855,6 +25173,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec /// @brief return whether value is an unsigned integer number /// @sa https://json.nlohmann.me/api/basic_json/is_number_unsigned/ + JSON_HEDLEY_WARN_UNUSED_RESULT constexpr bool is_number_unsigned() const noexcept { return m_data.m_type == value_t::number_unsigned; @@ -22862,6 +25181,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec /// @brief return whether value is a floating-point number /// @sa https://json.nlohmann.me/api/basic_json/is_number_float/ + JSON_HEDLEY_WARN_UNUSED_RESULT constexpr bool is_number_float() const noexcept { return m_data.m_type == value_t::number_float; @@ -22869,6 +25189,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec /// @brief return whether value is an object /// @sa https://json.nlohmann.me/api/basic_json/is_object/ + JSON_HEDLEY_WARN_UNUSED_RESULT constexpr bool is_object() const noexcept { return m_data.m_type == value_t::object; @@ -22876,6 +25197,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec /// @brief return whether value is an array /// @sa https://json.nlohmann.me/api/basic_json/is_array/ + JSON_HEDLEY_WARN_UNUSED_RESULT constexpr bool is_array() const noexcept { return m_data.m_type == value_t::array; @@ -22883,6 +25205,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec /// @brief return whether value is a string /// @sa https://json.nlohmann.me/api/basic_json/is_string/ + JSON_HEDLEY_WARN_UNUSED_RESULT constexpr bool is_string() const noexcept { return m_data.m_type == value_t::string; @@ -22890,6 +25213,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec /// @brief return whether value is a binary array /// @sa https://json.nlohmann.me/api/basic_json/is_binary/ + JSON_HEDLEY_WARN_UNUSED_RESULT constexpr bool is_binary() const noexcept { return m_data.m_type == value_t::binary; @@ -22897,6 +25221,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec /// @brief return whether value is discarded /// @sa https://json.nlohmann.me/api/basic_json/is_discarded/ + JSON_HEDLEY_WARN_UNUSED_RESULT constexpr bool is_discarded() const noexcept { return m_data.m_type == value_t::discarded; @@ -24228,6 +26553,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec /// @brief returns the number of occurrences of a key in a JSON object /// @sa https://json.nlohmann.me/api/basic_json/count/ + JSON_HEDLEY_WARN_UNUSED_RESULT size_type count(const typename object_t::key_type& key) const { // return 0 for all nonobject types @@ -24238,6 +26564,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec /// @sa https://json.nlohmann.me/api/basic_json/count/ template::value, int> = 0> + JSON_HEDLEY_WARN_UNUSED_RESULT size_type count(KeyType && key) const { // return 0 for all nonobject types @@ -24246,6 +26573,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec /// @brief check the existence of an element in a JSON object /// @sa https://json.nlohmann.me/api/basic_json/contains/ + JSON_HEDLEY_WARN_UNUSED_RESULT bool contains(const typename object_t::key_type& key) const { return is_object() && m_data.m_value.object->find(key) != m_data.m_value.object->end(); @@ -24255,6 +26583,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec /// @sa https://json.nlohmann.me/api/basic_json/contains/ template::value, int> = 0> + JSON_HEDLEY_WARN_UNUSED_RESULT bool contains(KeyType && key) const { return is_object() && m_data.m_value.object->find(std::forward(key)) != m_data.m_value.object->end(); @@ -24262,12 +26591,14 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec /// @brief check the existence of an element in a JSON object given a JSON pointer /// @sa https://json.nlohmann.me/api/basic_json/contains/ + JSON_HEDLEY_WARN_UNUSED_RESULT bool contains(const json_pointer& ptr) const { return ptr.contains(this); } template::value, int> = 0> + JSON_HEDLEY_WARN_UNUSED_RESULT JSON_HEDLEY_DEPRECATED_FOR(3.11.0, basic_json::json_pointer or nlohmann::json_pointer) // NOLINT(readability/alt_tokens) bool contains(const typename ::nlohmann::json_pointer& ptr) const { @@ -24423,6 +26754,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec /// @brief checks whether the container is empty. /// @sa https://json.nlohmann.me/api/basic_json/empty/ + JSON_HEDLEY_WARN_UNUSED_RESULT bool empty() const noexcept { switch (m_data.m_type) @@ -24462,6 +26794,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec /// @brief returns the number of elements /// @sa https://json.nlohmann.me/api/basic_json/size/ + JSON_HEDLEY_WARN_UNUSED_RESULT size_type size() const noexcept { switch (m_data.m_type) @@ -24501,6 +26834,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec /// @brief returns the maximum possible number of elements /// @sa https://json.nlohmann.me/api/basic_json/max_size/ + JSON_HEDLEY_WARN_UNUSED_RESULT size_type max_size() const noexcept { switch (m_data.m_type) @@ -25219,13 +27553,13 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec bool operator==(const_reference rhs) const noexcept { #ifdef __GNUC__ -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wfloat-equal" + JSON_HEDLEY_DIAGNOSTIC_PUSH + JSON_HEDLEY_PRAGMA(GCC diagnostic ignored "-Wfloat-equal") #endif const_reference lhs = *this; JSON_IMPLEMENT_OPERATOR( ==, true, false, false) #ifdef __GNUC__ -#pragma GCC diagnostic pop + JSON_HEDLEY_DIAGNOSTIC_POP #endif } @@ -25312,12 +27646,12 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec friend bool operator==(const_reference lhs, const_reference rhs) noexcept { #ifdef __GNUC__ -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wfloat-equal" + JSON_HEDLEY_DIAGNOSTIC_PUSH + JSON_HEDLEY_PRAGMA(GCC diagnostic ignored "-Wfloat-equal") #endif JSON_IMPLEMENT_OPERATOR( ==, true, false, false) #ifdef __GNUC__ -#pragma GCC diagnostic pop + JSON_HEDLEY_DIAGNOSTIC_POP #endif } @@ -25504,8 +27838,10 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec o.width(0); // do the actual serialization - serializer s(detail::output_adapter(o), o.fill()); - s.dump(j, pretty_print, false, static_cast(indentation)); + detail::output_stream_adapter stream_adapter(o); + serializer s(stream_adapter, o.fill(), + pretty_print, false, static_cast(indentation)); + s.dump(j); return o; } @@ -25578,22 +27914,24 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec /// @brief check if the input is valid JSON /// @sa https://json.nlohmann.me/api/basic_json/accept/ template + JSON_HEDLEY_WARN_UNUSED_RESULT static bool accept(InputType&& i, const bool ignore_comments = false, const bool ignore_trailing_commas = false) { - return parser(detail::input_adapter(std::forward(i)), nullptr, false, ignore_comments, ignore_trailing_commas).accept(true); + return parser(detail::input_adapter(std::forward(i)), nullptr, false, ignore_comments, ignore_trailing_commas, true).accept(true); } /// @brief check if the input is valid JSON (iterator pair, or iterator+sentinel pair for C++20 ranges support) /// @sa https://json.nlohmann.me/api/basic_json/accept/ template::value, int> = 0> + JSON_HEDLEY_WARN_UNUSED_RESULT static bool accept(IteratorType first, SentinelType last, const bool ignore_comments = false, const bool ignore_trailing_commas = false) { - return parser(detail::input_adapter(std::move(first), std::move(last)), nullptr, false, ignore_comments, ignore_trailing_commas).accept(true); + return parser(detail::input_adapter(std::move(first), std::move(last)), nullptr, false, ignore_comments, ignore_trailing_commas, true).accept(true); } JSON_HEDLEY_WARN_UNUSED_RESULT @@ -25602,7 +27940,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec const bool ignore_comments = false, const bool ignore_trailing_commas = false) { - return parser(i.get(), nullptr, false, ignore_comments, ignore_trailing_commas).accept(true); + return parser(i.get(), nullptr, false, ignore_comments, ignore_trailing_commas, true).accept(true); } /// @brief generate SAX events @@ -25688,6 +28026,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec /// @brief return the type as string /// @sa https://json.nlohmann.me/api/basic_json/type_name/ + JSON_HEDLEY_WARN_UNUSED_RESULT JSON_HEDLEY_RETURNS_NON_NULL const char* type_name() const noexcept { @@ -25922,8 +28261,11 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec basic_json result; auto ia = detail::input_adapter(std::forward(i)); detail::json_sax_dom_parser sdp(result, allow_exceptions); - const bool res = binary_reader(std::move(ia), input_format_t::cbor).sax_parse(input_format_t::cbor, &sdp, strict, tag_handler); // cppcheck-suppress[accessMoved] - return res ? result : basic_json(value_t::discarded); + if (!binary_reader(std::move(ia), input_format_t::cbor).sax_parse(input_format_t::cbor, &sdp, strict, tag_handler)) // cppcheck-suppress[accessMoved] + { + result = value_t::discarded; + } + return result; } /// @brief create a JSON value from an input in CBOR format (iterator pair, or iterator+sentinel pair for C++20 ranges support) @@ -25939,8 +28281,11 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec basic_json result; auto ia = detail::input_adapter(std::move(first), std::move(last)); detail::json_sax_dom_parser sdp(result, allow_exceptions); - const bool res = binary_reader(std::move(ia), input_format_t::cbor).sax_parse(input_format_t::cbor, &sdp, strict, tag_handler); // cppcheck-suppress[accessMoved] - return res ? result : basic_json(value_t::discarded); + if (!binary_reader(std::move(ia), input_format_t::cbor).sax_parse(input_format_t::cbor, &sdp, strict, tag_handler)) // cppcheck-suppress[accessMoved] + { + result = value_t::discarded; + } + return result; } template @@ -25965,8 +28310,11 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec auto ia = i.get(); detail::json_sax_dom_parser sdp(result, allow_exceptions); // NOLINTNEXTLINE(hicpp-move-const-arg,performance-move-const-arg) - const bool res = binary_reader(std::move(ia), input_format_t::cbor).sax_parse(input_format_t::cbor, &sdp, strict, tag_handler); // cppcheck-suppress[accessMoved] - return res ? result : basic_json(value_t::discarded); + if (!binary_reader(std::move(ia), input_format_t::cbor).sax_parse(input_format_t::cbor, &sdp, strict, tag_handler)) // cppcheck-suppress[accessMoved] + { + result = value_t::discarded; + } + return result; } /// @brief create a JSON value from an input in MessagePack format @@ -25980,8 +28328,11 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec basic_json result; auto ia = detail::input_adapter(std::forward(i)); detail::json_sax_dom_parser sdp(result, allow_exceptions); - const bool res = binary_reader(std::move(ia), input_format_t::msgpack).sax_parse(input_format_t::msgpack, &sdp, strict); // cppcheck-suppress[accessMoved] - return res ? result : basic_json(value_t::discarded); + if (!binary_reader(std::move(ia), input_format_t::msgpack).sax_parse(input_format_t::msgpack, &sdp, strict)) // cppcheck-suppress[accessMoved] + { + result = value_t::discarded; + } + return result; } /// @brief create a JSON value from an input in MessagePack format (iterator pair, or iterator+sentinel pair for C++20 ranges support) @@ -25996,8 +28347,11 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec basic_json result; auto ia = detail::input_adapter(std::move(first), std::move(last)); detail::json_sax_dom_parser sdp(result, allow_exceptions); - const bool res = binary_reader(std::move(ia), input_format_t::msgpack).sax_parse(input_format_t::msgpack, &sdp, strict); // cppcheck-suppress[accessMoved] - return res ? result : basic_json(value_t::discarded); + if (!binary_reader(std::move(ia), input_format_t::msgpack).sax_parse(input_format_t::msgpack, &sdp, strict)) // cppcheck-suppress[accessMoved] + { + result = value_t::discarded; + } + return result; } template @@ -26020,8 +28374,11 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec auto ia = i.get(); detail::json_sax_dom_parser sdp(result, allow_exceptions); // NOLINTNEXTLINE(hicpp-move-const-arg,performance-move-const-arg) - const bool res = binary_reader(std::move(ia), input_format_t::msgpack).sax_parse(input_format_t::msgpack, &sdp, strict); // cppcheck-suppress[accessMoved] - return res ? result : basic_json(value_t::discarded); + if (!binary_reader(std::move(ia), input_format_t::msgpack).sax_parse(input_format_t::msgpack, &sdp, strict)) // cppcheck-suppress[accessMoved] + { + result = value_t::discarded; + } + return result; } /// @brief create a JSON value from an input in UBJSON format @@ -26035,8 +28392,11 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec basic_json result; auto ia = detail::input_adapter(std::forward(i)); detail::json_sax_dom_parser sdp(result, allow_exceptions); - const bool res = binary_reader(std::move(ia), input_format_t::ubjson).sax_parse(input_format_t::ubjson, &sdp, strict); // cppcheck-suppress[accessMoved] - return res ? result : basic_json(value_t::discarded); + if (!binary_reader(std::move(ia), input_format_t::ubjson).sax_parse(input_format_t::ubjson, &sdp, strict)) // cppcheck-suppress[accessMoved] + { + result = value_t::discarded; + } + return result; } /// @brief create a JSON value from an input in UBJSON format (iterator pair, or iterator+sentinel pair for C++20 ranges support) @@ -26051,8 +28411,11 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec basic_json result; auto ia = detail::input_adapter(std::move(first), std::move(last)); detail::json_sax_dom_parser sdp(result, allow_exceptions); - const bool res = binary_reader(std::move(ia), input_format_t::ubjson).sax_parse(input_format_t::ubjson, &sdp, strict); // cppcheck-suppress[accessMoved] - return res ? result : basic_json(value_t::discarded); + if (!binary_reader(std::move(ia), input_format_t::ubjson).sax_parse(input_format_t::ubjson, &sdp, strict)) // cppcheck-suppress[accessMoved] + { + result = value_t::discarded; + } + return result; } template @@ -26075,8 +28438,11 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec auto ia = i.get(); detail::json_sax_dom_parser sdp(result, allow_exceptions); // NOLINTNEXTLINE(hicpp-move-const-arg,performance-move-const-arg) - const bool res = binary_reader(std::move(ia), input_format_t::ubjson).sax_parse(input_format_t::ubjson, &sdp, strict); // cppcheck-suppress[accessMoved] - return res ? result : basic_json(value_t::discarded); + if (!binary_reader(std::move(ia), input_format_t::ubjson).sax_parse(input_format_t::ubjson, &sdp, strict)) // cppcheck-suppress[accessMoved] + { + result = value_t::discarded; + } + return result; } /// @brief create a JSON value from an input in BJData format @@ -26090,8 +28456,11 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec basic_json result; auto ia = detail::input_adapter(std::forward(i)); detail::json_sax_dom_parser sdp(result, allow_exceptions); - const bool res = binary_reader(std::move(ia), input_format_t::bjdata).sax_parse(input_format_t::bjdata, &sdp, strict); // cppcheck-suppress[accessMoved] - return res ? result : basic_json(value_t::discarded); + if (!binary_reader(std::move(ia), input_format_t::bjdata).sax_parse(input_format_t::bjdata, &sdp, strict)) // cppcheck-suppress[accessMoved] + { + result = value_t::discarded; + } + return result; } /// @brief create a JSON value from an input in BJData format (iterator pair, or iterator+sentinel pair for C++20 ranges support) @@ -26106,8 +28475,11 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec basic_json result; auto ia = detail::input_adapter(std::move(first), std::move(last)); detail::json_sax_dom_parser sdp(result, allow_exceptions); - const bool res = binary_reader(std::move(ia), input_format_t::bjdata).sax_parse(input_format_t::bjdata, &sdp, strict); // cppcheck-suppress[accessMoved] - return res ? result : basic_json(value_t::discarded); + if (!binary_reader(std::move(ia), input_format_t::bjdata).sax_parse(input_format_t::bjdata, &sdp, strict)) // cppcheck-suppress[accessMoved] + { + result = value_t::discarded; + } + return result; } /// @brief create a JSON value from an input in BSON format @@ -26121,8 +28493,11 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec basic_json result; auto ia = detail::input_adapter(std::forward(i)); detail::json_sax_dom_parser sdp(result, allow_exceptions); - const bool res = binary_reader(std::move(ia), input_format_t::bson).sax_parse(input_format_t::bson, &sdp, strict); // cppcheck-suppress[accessMoved] - return res ? result : basic_json(value_t::discarded); + if (!binary_reader(std::move(ia), input_format_t::bson).sax_parse(input_format_t::bson, &sdp, strict)) // cppcheck-suppress[accessMoved] + { + result = value_t::discarded; + } + return result; } /// @brief create a JSON value from an input in BSON format (iterator pair, or iterator+sentinel pair for C++20 ranges support) @@ -26137,8 +28512,11 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec basic_json result; auto ia = detail::input_adapter(std::move(first), std::move(last)); detail::json_sax_dom_parser sdp(result, allow_exceptions); - const bool res = binary_reader(std::move(ia), input_format_t::bson).sax_parse(input_format_t::bson, &sdp, strict); // cppcheck-suppress[accessMoved] - return res ? result : basic_json(value_t::discarded); + if (!binary_reader(std::move(ia), input_format_t::bson).sax_parse(input_format_t::bson, &sdp, strict)) // cppcheck-suppress[accessMoved] + { + result = value_t::discarded; + } + return result; } template @@ -26161,8 +28539,11 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec auto ia = i.get(); detail::json_sax_dom_parser sdp(result, allow_exceptions); // NOLINTNEXTLINE(hicpp-move-const-arg,performance-move-const-arg) - const bool res = binary_reader(std::move(ia), input_format_t::bson).sax_parse(input_format_t::bson, &sdp, strict); // cppcheck-suppress[accessMoved] - return res ? result : basic_json(value_t::discarded); + if (!binary_reader(std::move(ia), input_format_t::bson).sax_parse(input_format_t::bson, &sdp, strict)) // cppcheck-suppress[accessMoved] + { + result = value_t::discarded; + } + return result; } /// @} @@ -26388,6 +28769,36 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec // note erase performs range check parent.erase(json_pointer::template array_index(last_path)); } + else + { + // the parent of a "remove" target must be an object or array + // (see #5396) + JSON_THROW(out_of_range::create(413, detail::concat("cannot remove value: the JSON Patch 'remove' target's parent is of type ", parent.type_name(), ", but must be an object or array"), &parent)); + } + }; + + // RFC 6902 (section 4.4) forbids "from" from being a proper prefix + // of "path" for a "move" operation: a location cannot be moved into + // one of its own children. Compares reference tokens (already + // unescaped by json_pointer's parser) rather than the raw pointer + // strings, since a token may itself contain an escaped '/' or '~' + // that would defeat a naive string-prefix comparison. "from" equal + // to "path" is *not* a proper prefix and must return false. + const auto is_proper_prefix = [](const json_pointer & from, const json_pointer & to) + { + const auto from_size = from.reference_tokens.size(); + if (from_size >= to.reference_tokens.size()) + { + return false; + } + for (std::size_t i = 0; i < from_size; ++i) + { + if (!(from.reference_tokens[i] == to.reference_tokens[i])) + { + return false; + } + } + return true; }; // type check: top level value must be an array @@ -26465,6 +28876,11 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec const auto from_path = get_value("move", "from", true).template get(); json_pointer from_ptr(from_path); + if (JSON_HEDLEY_UNLIKELY(is_proper_prefix(from_ptr, ptr))) + { + JSON_THROW(out_of_range::create(414, detail::concat("cannot move value: 'from' path '", from_path, "' is a proper prefix of 'path' '", path, "'"), &result)); + } + // the "from" location must exist - use at() basic_json const v = result.at(from_ptr); @@ -26986,7 +29402,7 @@ struct formatter // NOLINT(cert-dcl58-c #undef JSON_HEDLEY_CLANG_HAS_ATTRIBUTE #undef JSON_HEDLEY_CLANG_HAS_BUILTIN #undef JSON_HEDLEY_CLANG_HAS_CPP_ATTRIBUTE -#undef JSON_HEDLEY_CLANG_HAS_DECLSPEC_DECLSPEC_ATTRIBUTE +#undef JSON_HEDLEY_CLANG_HAS_DECLSPEC_ATTRIBUTE #undef JSON_HEDLEY_CLANG_HAS_EXTENSION #undef JSON_HEDLEY_CLANG_HAS_FEATURE #undef JSON_HEDLEY_CLANG_HAS_WARNING @@ -27077,7 +29493,10 @@ struct formatter // NOLINT(cert-dcl58-c #undef JSON_HEDLEY_PELLES_VERSION_CHECK #undef JSON_HEDLEY_PGI_VERSION #undef JSON_HEDLEY_PGI_VERSION_CHECK +#undef JSON_HEDLEY_PRAGMA #undef JSON_HEDLEY_PREDICT +#undef JSON_HEDLEY_PREDICT_FALSE +#undef JSON_HEDLEY_PREDICT_TRUE #undef JSON_HEDLEY_PRINTF_FORMAT #undef JSON_HEDLEY_PRIVATE #undef JSON_HEDLEY_PUBLIC diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 4383b582c..86b4825d7 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -2,6 +2,9 @@ cmake_minimum_required(VERSION 3.13...4.0) option(JSON_Valgrind "Execute test suite with Valgrind." OFF) option(JSON_FastTests "Skip expensive/slow tests." OFF) +option(JSON_TestSimdutf "Build the unit tests against the simdutf UTF-8 validation backend." OFF) + +set(JSON_SIMDUTF_VERSION 9.1.0 CACHE STRING "The simdutf version used by JSON_TestSimdutf.") set(JSON_32bitTest AUTO CACHE STRING "Enable the 32bit unit test (ON/OFF/AUTO/ONLY).") set(JSON_TestStandards "" CACHE STRING "The list of standards to test explicitly.") @@ -125,6 +128,51 @@ json_test_set_test_options(test-unicode4 TEST_PROPERTIES TIMEOUT 3000) # add unit tests ############################################################################# +# Generate the leak checks for every JSON_HEDLEY_* macro defined in +# hedley.hpp; tests/src/unit-no-macro-leak.cpp #include-s the result after +# nlohmann/json.hpp (see issue #5408). Using the shared +# cmake/scripts/gen_hedley_undef_check.cmake script (also used by `make +# update_hedley_undef`) instead of a hand-maintained list of macro names +# means this test can never go stale after a future `make update_hedley`. +set(hedley_hpp "${PROJECT_SOURCE_DIR}/include/nlohmann/thirdparty/hedley/hedley.hpp") +set(hedley_undef_check_script "${PROJECT_SOURCE_DIR}/cmake/scripts/gen_hedley_undef_check.cmake") +set(hedley_undef_checks "${PROJECT_BINARY_DIR}/include/hedley_undef_checks.inc") + +# Reconfigure whenever the vendored header or the generator script changes, +# so a `cmake --build` after `make update_hedley` does not silently keep a +# stale generated file around. +set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS + "${hedley_hpp}" + "${hedley_undef_check_script}") + +# Generate once at configure time, so the very first build (before any +# custom-command build step has run) already has an up-to-date file. +execute_process( + COMMAND ${CMAKE_COMMAND} + "-DHEDLEY_HPP=${hedley_hpp}" + "-DOUTPUT=${hedley_undef_checks}" + -DMODE=checks + -P "${hedley_undef_check_script}" + RESULT_VARIABLE hedley_undef_check_result +) +if(NOT hedley_undef_check_result EQUAL 0) + message(FATAL_ERROR "Failed to generate ${hedley_undef_checks}") +endif() + +# Also (re)generate as a build step, so an incremental build after editing +# hedley.hpp without a full reconfigure still picks up the change. +add_custom_command( + OUTPUT "${hedley_undef_checks}" + COMMAND ${CMAKE_COMMAND} + "-DHEDLEY_HPP=${hedley_hpp}" + "-DOUTPUT=${hedley_undef_checks}" + -DMODE=checks + -P "${hedley_undef_check_script}" + DEPENDS "${hedley_hpp}" "${hedley_undef_check_script}" + COMMENT "Generating Hedley undef leak checks" + VERBATIM) +add_custom_target(generate_hedley_undef_checks DEPENDS "${hedley_undef_checks}") + if("${JSON_TestStandards}" STREQUAL "") set(test_cxx_standards 11 14 17 20 23) unset(test_force) @@ -149,6 +197,71 @@ if(test_force) endif() message(STATUS "${msg}") +############################################################################# +# optionally validate UTF-8 with simdutf (JSON_USE_SIMDUTF) +############################################################################# + +# The simdutf backend is opt-in and not vendored, so it is fetched here rather +# than being a checked-in dependency. Everything below hangs off test_main, +# whose usage requirements every test target inherits; the library target and +# the installed CMake package are deliberately left untouched. +if (JSON_TestSimdutf) + # simdutf requires C++17, both to compile itself and to be reachable from + # the library, which keeps its scalar validator below that. Find a tested + # standard that satisfies it. + set(simdutf_standard "") + foreach(cxx_standard ${test_cxx_standards}) + if(NOT cxx_standard LESS 17 AND compiler_supports_cpp_${cxx_standard}) + set(simdutf_standard ${cxx_standard}) + break() + endif() + endforeach() + + if("${simdutf_standard}" STREQUAL "") + # Building simdutf would fail outright without a C++17 compiler, and + # even with one it would go unused if no C++17-or-later standard is + # tested. Say so and fall back to the scalar validator rather than + # failing the build. + if(NOT compiler_supports_cpp_17) + set(simdutf_reason "the compiler does not support C++17") + else() + set(simdutf_reason "no tested standard is C++17 or later (testing ${msg_standards})") + endif() + message(WARNING + "JSON_TestSimdutf is enabled, but ${simdutf_reason}. simdutf requires C++17, so it " + "is not fetched and JSON_USE_SIMDUTF is not defined: the tests run against the " + "built-in scalar UTF-8 validator instead. Set JSON_TestStandards to include 17 or " + "later, or build with a compiler that supports C++17.") + else() + if (CMAKE_VERSION VERSION_LESS 3.18) + message(FATAL_ERROR "JSON_TestSimdutf requires CMake 3.18 or later (simdutf's minimum).") + endif() + + include(FetchContent) + + # simdutf builds its tests and tools by default, and its tests pull + # further dependencies of their own; only the library is needed here + set(SIMDUTF_TESTS OFF CACHE BOOL "" FORCE) + set(SIMDUTF_TOOLS OFF CACHE BOOL "" FORCE) + set(SIMDUTF_BENCHMARKS OFF CACHE BOOL "" FORCE) + set(SIMDUTF_ICONV OFF CACHE BOOL "" FORCE) + + FetchContent_Declare(simdutf + URL https://github.com/simdutf/simdutf/archive/refs/tags/v${JSON_SIMDUTF_VERSION}.tar.gz + DOWNLOAD_EXTRACT_TIMESTAMP TRUE + ) + FetchContent_MakeAvailable(simdutf) + + target_compile_definitions(test_main PUBLIC JSON_USE_SIMDUTF) + target_link_libraries(test_main PUBLIC simdutf::simdutf) + + # simdutf.h requires C++17; below that the library keeps its scalar + # validator, so any C++11/14 test targets exercise the fallback and the + # C++17-and-later ones exercise simdutf. Both must agree. + message(STATUS "UTF-8 validation delegated to simdutf ${JSON_SIMDUTF_VERSION} for C++17 and later (JSON_USE_SIMDUTF)") + endif() +endif() + # *DO* use json_test_set_test_options() above this line json_test_should_build_32bit_test(json_32bit_test json_32bit_test_only "${JSON_32bitTest}") @@ -163,6 +276,14 @@ foreach(file ${files}) json_test_add_test_for(${file} MAIN test_main CXX_STANDARDS ${test_cxx_standards} ${test_force}) endforeach() +# tests/src/unit-no-macro-leak.cpp #include-s the generated leak-check file, +# so its test targets must be built after generate_hedley_undef_checks. +foreach(cxx_standard ${test_cxx_standards}) + if(TARGET test-no-macro-leak_cpp${cxx_standard}) + add_dependencies(test-no-macro-leak_cpp${cxx_standard} generate_hedley_undef_checks) + endif() +endforeach() + if(json_32bit_test_only) # Skip all other tests in this file return() @@ -177,6 +298,24 @@ json_test_add_test_for(src/unit-comparison.cpp MAIN test_main CXX_STANDARDS ${test_cxx_standards} ${test_force} ) +# test the parser again with JSON_DIAGNOSTIC_POSITIONS enabled +json_test_set_test_options(test-class_parser_diagnostic_positions + COMPILE_DEFINITIONS JSON_DIAGNOSTIC_POSITIONS=1 +) +json_test_add_test_for(src/unit-class_parser.cpp + NAME test-class_parser_diagnostic_positions + MAIN test_main CXX_STANDARDS ${test_cxx_standards} ${test_force} +) + +# test diagnostic positions again without regular diagnostics (JSON pointer paths) +json_test_set_test_options(test-diagnostic-positions_only + COMPILE_DEFINITIONS JSON_DIAGNOSTICS=0 +) +json_test_add_test_for(src/unit-diagnostic-positions.cpp + NAME test-diagnostic-positions_only + MAIN test_main CXX_STANDARDS ${test_cxx_standards} ${test_force} +) + # *DO NOT* use json_test_set_test_options() below this line ############################################################################# diff --git a/tests/benchmarks/src/benchmarks.cpp b/tests/benchmarks/src/benchmarks.cpp index 4de238272..9522df5a4 100644 --- a/tests/benchmarks/src/benchmarks.cpp +++ b/tests/benchmarks/src/benchmarks.cpp @@ -81,6 +81,44 @@ BENCHMARK_CAPTURE(ParseString, signed_ints, TEST_DATA_DIRECTORY "/regressi BENCHMARK_CAPTURE(ParseString, unsigned_ints, TEST_DATA_DIRECTORY "/regression/unsigned_ints.json"); BENCHMARK_CAPTURE(ParseString, small_signed_ints, TEST_DATA_DIRECTORY "/regression/small_signed_ints.json"); +////////////////////////////////////////////////////////////////////////////// +// parse pretty-printed JSON from string +// +// Every file in the corpus above is minified or only lightly spaced, so none of +// them exercise the lexer's whitespace handling. Real-world JSON is frequently +// indented - configuration files, pretty-printed API responses, anything kept +// under version control - where insignificant whitespace can outweigh the data. +// Re-serializing a document with an indentation and parsing that keeps the +// content identical to the ParseString row above, so the pair isolates the cost +// of the whitespace alone. +////////////////////////////////////////////////////////////////////////////// + +static void ParseIndented(benchmark::State& state, const char* filename, int indent) +{ + std::ifstream f(filename); + std::string str((std::istreambuf_iterator(f)), std::istreambuf_iterator()); + const std::string indented = json::parse(str).dump(indent); + + while (state.KeepRunning()) + { + state.PauseTiming(); + auto* j = new json(); + state.ResumeTiming(); + + *j = json::parse(indented); + + state.PauseTiming(); + delete j; + state.ResumeTiming(); + } + + state.SetBytesProcessed(state.iterations() * indented.size()); +} +BENCHMARK_CAPTURE(ParseIndented, jeopardy / 4, TEST_DATA_DIRECTORY "/jeopardy/jeopardy.json", 4); +BENCHMARK_CAPTURE(ParseIndented, canada / 4, TEST_DATA_DIRECTORY "/nativejson-benchmark/canada.json", 4); +BENCHMARK_CAPTURE(ParseIndented, citm_catalog / 4, TEST_DATA_DIRECTORY "/nativejson-benchmark/citm_catalog.json", 4); +BENCHMARK_CAPTURE(ParseIndented, twitter / 4, TEST_DATA_DIRECTORY "/nativejson-benchmark/twitter.json", 4); + ////////////////////////////////////////////////////////////////////////////// // serialize JSON ////////////////////////////////////////////////////////////////////////////// diff --git a/tests/src/test_utils.hpp b/tests/src/test_utils.hpp index baa802f71..4c81a8ef4 100644 --- a/tests/src/test_utils.hpp +++ b/tests/src/test_utils.hpp @@ -15,6 +15,15 @@ namespace utils { +// Some tests intentionally discard the [[nodiscard]]/JSON_HEDLEY_WARN_UNUSED_RESULT +// return value of a call they only make to exercise its side effects (e.g. checking +// that it does not throw). A plain (void) cast on the call expression does not +// suppress GCC's warning for functions using the GNU __attribute__((warn_unused_result)) +// form (as opposed to the C++17 [[nodiscard]] attribute) -- passing the value into an +// ordinary function call does. +template +inline void ignore_return_value(T&& /*unused*/) noexcept {} + inline std::vector read_binary_file(const std::string& filename) { std::ifstream file(filename, std::ios::binary); diff --git a/tests/src/unit-bjdata.cpp b/tests/src/unit-bjdata.cpp index 629fc9d07..9338e663e 100644 --- a/tests/src/unit-bjdata.cpp +++ b/tests/src/unit-bjdata.cpp @@ -2776,6 +2776,53 @@ TEST_CASE("BJData") CHECK(out_num.at(0) == '{'); CHECK(json::from_bjdata(out_num) == j_num); } + + SECTION("ndarray with out-of-range _ArrayData_ elements stays as object") + { + // each element is cast to the (possibly narrower) C++ type + // named by _ArrayType_ before being written; a value that + // does not fit that type would silently wrap instead of + // being reported, so such an object falls back to a plain + // object encoding that still round-trips (see GitHub issue #5403) + + // an unsigned element that does not fit uint8 + json const j_uint8 = json({{"_ArrayType_", "uint8"}, {"_ArraySize_", {2}}, {"_ArrayData_", {1, 256}}}); + const auto out_uint8 = json::to_bjdata(j_uint8); + CHECK(out_uint8.at(0) == '{'); + CHECK(json::from_bjdata(out_uint8) == j_uint8); + + // a signed element that does not fit int8 + json const j_int8 = json({{"_ArrayType_", "int8"}, {"_ArraySize_", {2}}, {"_ArrayData_", {1, 200}}}); + const auto out_int8 = json::to_bjdata(j_int8); + CHECK(out_int8.at(0) == '{'); + CHECK(json::from_bjdata(out_int8) == j_int8); + + // a negative element is likewise out of range for an + // unsigned _ArrayType_ + json const j_uint16_neg = json({{"_ArrayType_", "uint16"}, {"_ArraySize_", {2}}, {"_ArrayData_", {1, -1}}}); + const auto out_uint16_neg = json::to_bjdata(j_uint16_neg); + CHECK(out_uint16_neg.at(0) == '{'); + CHECK(json::from_bjdata(out_uint16_neg) == j_uint16_neg); + + // a double element that overflows to infinity when narrowed + // to the "single" (float) precision named by _ArrayType_ + json const j_single = json({{"_ArrayType_", "single"}, {"_ArraySize_", {2}}, {"_ArrayData_", {1.5, 1e40}}}); + const auto out_single = json::to_bjdata(j_single); + CHECK(out_single.at(0) == '{'); + CHECK(json::from_bjdata(out_single) == j_single); + + // in-range boundary values still use the compact ndarray encoding + json const j_uint8_ok = json({{"_ArrayType_", "uint8"}, {"_ArraySize_", {2}}, {"_ArrayData_", {0, 255}}}); + CHECK(json::to_bjdata(j_uint8_ok) == std::vector({'[', '$', 'U', '#', '[', 'i', 2, ']', 0, 255})); + + json const j_int8_ok = json({{"_ArrayType_", "int8"}, {"_ArraySize_", {2}}, {"_ArrayData_", {-128, 127}}}); + CHECK(json::to_bjdata(j_int8_ok) == std::vector({'[', '$', 'i', '#', '[', 'i', 2, ']', 0x80, 0x7F})); + + json const j_single_ok = json({{"_ArrayType_", "single"}, {"_ArraySize_", {1}}, {"_ArrayData_", {1.5}}}); + const auto out_single_ok = json::to_bjdata(j_single_ok); + CHECK(out_single_ok.at(0) == '['); + CHECK(json::from_bjdata(out_single_ok) == json({1.5f})); + } } } @@ -3288,8 +3335,10 @@ TEST_CASE("BJData") CHECK_THROWS_WITH_AS(_ = json::from_bjdata(vR1), "[json.exception.parse_error.113] parse error at byte 6: syntax error while parsing BJData size: ndarray dimensional vector is not allowed", json::parse_error&); CHECK(json::from_bjdata(vR1, true, false).is_discarded()); + // a dimension vector that opens another one is rejected where the + // nested '[' is read, rather than after it has been descended into std::vector const vR2 = {'[', '$', 'i', '#', '[', '#', '[', 'i', 1, ']', ']', 1}; - CHECK_THROWS_WITH_AS(_ = json::from_bjdata(vR2), "[json.exception.parse_error.113] parse error at byte 11: syntax error while parsing BJData size: expected length type specification (U, i, u, I, m, l, M, L) after '#'; last byte: 0x5D", json::parse_error&); + CHECK_THROWS_WITH_AS(_ = json::from_bjdata(vR2), "[json.exception.parse_error.113] parse error at byte 7: syntax error while parsing BJData size: ndarray dimensional vector is not allowed", json::parse_error&); CHECK(json::from_bjdata(vR2, true, false).is_discarded()); std::vector const vR3 = {'[', '#', '[', 'i', '2', 'i', 2, ']'}; @@ -3297,7 +3346,7 @@ TEST_CASE("BJData") CHECK(json::from_bjdata(vR3, true, false).is_discarded()); std::vector const vR4 = {'[', '$', 'i', '#', '[', '$', 'i', '#', '[', 'i', 1, ']', 1}; - CHECK_THROWS_WITH_AS(_ = json::from_bjdata(vR4), "[json.exception.parse_error.110] parse error at byte 14: syntax error while parsing BJData number: unexpected end of input", json::parse_error&); + CHECK_THROWS_WITH_AS(_ = json::from_bjdata(vR4), "[json.exception.parse_error.113] parse error at byte 9: syntax error while parsing BJData size: ndarray dimensional vector is not allowed", json::parse_error&); CHECK(json::from_bjdata(vR4, true, false).is_discarded()); std::vector const vR5 = {'[', '$', 'i', '#', '[', '[', '[', ']', ']', ']'}; @@ -3305,12 +3354,25 @@ TEST_CASE("BJData") CHECK(json::from_bjdata(vR5, true, false).is_discarded()); std::vector const vR6 = {'[', '$', 'i', '#', '[', '$', 'i', '#', '[', 'i', '2', 'i', 2, ']'}; - CHECK_THROWS_WITH_AS(_ = json::from_bjdata(vR6), "[json.exception.parse_error.112] parse error at byte 14: syntax error while parsing BJData size: ndarray can not be recursive", json::parse_error&); + CHECK_THROWS_WITH_AS(_ = json::from_bjdata(vR6), "[json.exception.parse_error.113] parse error at byte 9: syntax error while parsing BJData size: ndarray dimensional vector is not allowed", json::parse_error&); CHECK(json::from_bjdata(vR6, true, false).is_discarded()); std::vector const vH = {'[', 'H', '[', '#', '[', '$', 'i', '#', '[', 'i', '2', 'i', 2, ']'}; CHECK_THROWS_WITH_AS(_ = json::from_bjdata(vH), "[json.exception.parse_error.113] parse error at byte 3: syntax error while parsing BJData size: ndarray dimensional vector is not allowed", json::parse_error&); CHECK(json::from_bjdata(vH, true, false).is_discarded()); + + // Every "#[" of this chain used to open another dimension vector + // and cost several stack frames before anything was rejected, so a + // long enough chain crashed the process (see #5104). The nested + // vector is refused where it is read, so the length is irrelevant. + std::vector vRdeep = {'['}; + for (std::size_t i = 0; i < 100000; ++i) + { + vRdeep.push_back('#'); + vRdeep.push_back('['); + } + CHECK_THROWS_WITH_AS(_ = json::from_bjdata(vRdeep), "[json.exception.parse_error.113] parse error at byte 5: syntax error while parsing BJData size: ndarray dimensional vector is not allowed", json::parse_error&); + CHECK(json::from_bjdata(vRdeep, true, false).is_discarded()); } SECTION("objects") diff --git a/tests/src/unit-bson.cpp b/tests/src/unit-bson.cpp index 7896b9f18..153e12d30 100644 --- a/tests/src/unit-bson.cpp +++ b/tests/src/unit-bson.cpp @@ -38,6 +38,54 @@ class huge_binary_t : public std::vector using huge_binary_json = nlohmann::basic_json < std::map, std::vector, std::string, bool, std::int64_t, std::uint64_t, double, std::allocator, nlohmann::adl_serializer, huge_binary_t, void >; + +// a string type that can be made to report a size beyond INT32_MAX without +// allocating that much memory, so BSON length overflow can be tested for +// strings and (embedded) documents as well, following the same idea as +// huge_binary_t. +// +// Unlike huge_binary_t (which is only ever used as the BSON *value* type), +// this type doubles as basic_json's StringType and is therefore also used +// for *object keys* (e.g. "s" or "nested" below). Only the designated test +// value is meant to lie about its size - if every huge_string_t (including +// keys) reported a huge size, the running totals computed while walking the +// BSON document (see calc_bson_object_size & friends in binary_writer.hpp) +// would need more than 32 bits, and on platforms where std::size_t is only +// 32 bits wide that arithmetic would silently wrap around, producing wrong +// (or even unguarded) lengths. The fake size is therefore opt-in via +// as_huge(), and plain strings - in particular object keys - keep reporting +// their real, small size. +class huge_string_t : public std::string +{ + public: + using std::string::string; + huge_string_t(const std::string& s) : std::string(s) {} // NOLINT(google-explicit-constructor,hicpp-explicit-conversions) + + // returns a copy of @a s whose size() pretends to be huge + static huge_string_t as_huge(const std::string& s) + { + huge_string_t result(s); + result.pretend_huge = true; + return result; + } + + size_type size() const noexcept + { + if (pretend_huge) + { + // one byte more than the BSON length field can represent + return static_cast((std::numeric_limits::max)()) + 1; + } + return std::string::size(); + } + + private: + bool pretend_huge = false; +}; + +using huge_string_json = nlohmann::basic_json < + std::map, std::vector, huge_string_t, bool, std::int64_t, std::uint64_t, + double, std::allocator, nlohmann::adl_serializer, std::vector, void >; } // namespace TEST_CASE("BSON") @@ -105,10 +153,36 @@ TEST_CASE("BSON") SECTION("lengths exceeding INT32_MAX cannot be serialized to BSON") { - huge_binary_json j; - j["b"] = huge_binary_json::binary(huge_binary_t{}); + // out_of_range.412 is thrown from a single shared helper + // (to_bson_length) that guards the BSON length fields of binary + // values, strings, and (embedded) documents alike + SECTION("binary") + { + huge_binary_json j; + j["b"] = huge_binary_json::binary(huge_binary_t{}); - CHECK_THROWS_WITH_AS(huge_binary_json::to_bson(j), "[json.exception.out_of_range.412] BSON length 2147483661 exceeds maximum of 2147483647", huge_binary_json::out_of_range&); + CHECK_THROWS_WITH_AS(huge_binary_json::to_bson(j), "[json.exception.out_of_range.412] BSON length 2147483661 exceeds maximum of 2147483647", huge_binary_json::out_of_range&); + } + + SECTION("string") + { + huge_string_json j; + j["s"] = huge_string_t::as_huge("value"); + + CHECK_THROWS_WITH_AS(huge_string_json::to_bson(j), "[json.exception.out_of_range.412] BSON length 2147483661 exceeds maximum of 2147483647", huge_string_json::out_of_range&); + } + + SECTION("document") + { + // an oversized string nested one level deep makes the + // *embedded* document's own length exceed INT32_MAX as well + huge_string_json nested; + nested["s"] = huge_string_t::as_huge("value"); + huge_string_json j; + j["nested"] = nested; + + CHECK_THROWS_WITH_AS(huge_string_json::to_bson(j), "[json.exception.out_of_range.412] BSON length 2147483674 exceeds maximum of 2147483647", huge_string_json::out_of_range&); + } } SECTION("string length must be at least 1") @@ -193,6 +267,23 @@ TEST_CASE("BSON") CHECK(json::from_bson(result, true, false) == j); } + SECTION("non-empty object with bool from a non-0/1 byte (lenient parsing)") + { + // documented lenient behavior (see gh-5333): any non-zero byte + // is accepted as `true`, not just 0x01 + std::vector const input = + { + 0x0D, 0x00, 0x00, 0x00, // size (little endian) + 0x08, // entry: boolean + 'e', 'n', 't', 'r', 'y', '\x00', + 0x02, // value = 0x02 (neither 0x00 nor 0x01) + 0x00 // end marker + }; + + const json expected = { { "entry", true } }; + CHECK(json::from_bson(input) == expected); + } + SECTION("non-empty object with double") { json const j = @@ -499,6 +590,29 @@ TEST_CASE("BSON") CHECK(json::from_bson(result, true, false) == j); } + SECTION("array elements with non-conforming keys (lenient parsing)") + { + // documented lenient behavior (see gh-5333): BSON array element + // keys are not checked against the required decimal sequence + // "0", "1", "2", ... - elements are taken in encoded order + std::vector const input = + { + 0x26, 0x00, 0x00, 0x00, // size (little endian) + 0x04, 'e', 'n', 't', 'r', 'y', '\x00', // entry: embedded array + + 0x1A, 0x00, 0x00, 0x00, // size (little endian) + 0x10, '5', 0x00, 0x0A, 0x00, 0x00, 0x00, // key "5" (bogus) -> 10 + 0x10, 'x', 0x00, 0x14, 0x00, 0x00, 0x00, // key "x" (non-numeric) -> 20 + 0x10, '1', 0x00, 0x1E, 0x00, 0x00, 0x00, // key "1" (out of order) -> 30 + 0x00, // end marker (embedded array) + + 0x00 // end marker + }; + + const json expected = { { "entry", json::array({10, 20, 30}) } }; + CHECK(json::from_bson(input) == expected); + } + SECTION("non-empty object with binary member") { const size_t N = 10; @@ -594,6 +708,31 @@ TEST_CASE("BSON") CHECK(json::from_bson(result, true, false) == j); } + SECTION("binary member with subtype 0x02 (old binary) keeps its inner length prefix (lenient parsing)") + { + // documented lenient behavior (see gh-5333): the payload for + // binary subtype 0x02 ("old binary") is returned as-is, + // including its own inner 4-byte length prefix; it is not + // stripped or reinterpreted + std::vector const input = + { + 0x17, 0x00, 0x00, 0x00, // size (little endian) + 0x05, 'e', 'n', 't', 'r', 'y', '\x00', // entry: binary + + 0x06, 0x00, 0x00, 0x00, // size of binary (little endian) + 0x02, // "old binary" subtype + 0x02, 0x00, 0x00, 0x00, // inner length prefix (part of the old-binary payload) + 0x68, 0x69, // payload ('h', 'i') + + 0x00 // end marker + }; + + // the inner length prefix is part of the (unmodified) payload + const std::vector expected_payload = {0x02, 0x00, 0x00, 0x00, 0x68, 0x69}; + const json expected = { { "entry", json::binary(expected_payload, 0x02) } }; + CHECK(json::from_bson(input) == expected); + } + SECTION("Some more complex document") { json const j = @@ -1011,6 +1150,91 @@ TEST_CASE("BSON document size mismatch") } } +TEST_CASE("BSON nesting does not consume the call stack") +{ + // An embedded document or array used to be read by calling back into the + // document reader, so the native call stack grew with the nesting depth of + // the input (#5104). The open documents are kept on a heap stack now. + // + // Deeply nested values must not be compared, copied or dumped here: those + // operations are still recursive and would reintroduce the crash. + + // A document nested deeply enough to have crashed. The bytes are built + // here rather than with to_bson(), because the writer still recurses once + // per level and would overflow the stack before the reader is ever + // reached. Every level is + // 0x03 'a' 0x00 0x00 + // so a level is eight bytes larger than the one it holds, and the sizes + // can be filled in from the outside in. + const std::size_t depth = 30000; + std::vector input; + input.reserve(5 + (8 * depth)); + for (std::size_t i = 0; i < depth; ++i) + { + const auto size = static_cast(5 + (8 * (depth - i))); + input.push_back(static_cast(size & 0xFF)); + input.push_back(static_cast((size >> 8) & 0xFF)); + input.push_back(static_cast((size >> 16) & 0xFF)); + input.push_back(static_cast((size >> 24) & 0xFF)); + input.push_back(0x03); // embedded document + input.push_back('a'); + input.push_back(0x00); + } + // the innermost document is empty, then one terminator closes each level + input.insert(input.end(), {0x05, 0x00, 0x00, 0x00, 0x00}); + input.insert(input.end(), depth, 0x00); + + SECTION("a well-formed deep document is read through the SAX interface") + { + SaxCountdown accept_all(1000000); + CHECK(json::sax_parse(input, &accept_all, json::input_format_t::bson)); + } + + SECTION("a well-formed deep document is read into a value") + { + json j = json::from_bson(input); + + // walked rather than compared: comparing, copying or dumping a value + // this deep is still recursive + std::size_t measured = 0; + const json* q = &j; + while (q->is_object() && !q->empty()) + { + q = &q->begin().value(); + ++measured; + } + CHECK(measured == depth); + } + + SECTION("embedded documents and arrays are still read the same way") + { + const json values = {{"a", {{"b", {{"c", 1}}}}}}; + CHECK(json::from_bson(json::to_bson(values)) == values); + + const json array = {{"a", {1, 2, 3}}}; + CHECK(json::from_bson(json::to_bson(array)) == array); + + const json mixed = {{"a", {json{{"x", 1}}, json{{"y", 2}}}}}; + CHECK(json::from_bson(json::to_bson(mixed)) == mixed); + + CHECK(json::from_bson(json::to_bson(json::object())) == json::object()); + } + + SECTION("a size that does not match is still reported per document") + { + // the embedded document claims one byte too many + std::vector const bad = + { + 0x15, 0x00, 0x00, 0x00, 0x03, 'a', 0x00, + 0x0D, 0x00, 0x00, 0x00, 0x08, 'b', 0x00, 0x01, 0x00, + 0x00 + }; + json _; + CHECK_THROWS_AS(_ = json::from_bson(bad), json::parse_error&); + CHECK(json::from_bson(bad, true, false).is_discarded()); + } +} + TEST_CASE("BSON numerical data") { SECTION("number") diff --git a/tests/src/unit-cbor.cpp b/tests/src/unit-cbor.cpp index 90437ac12..774f89bcb 100644 --- a/tests/src/unit-cbor.cpp +++ b/tests/src/unit-cbor.cpp @@ -2121,6 +2121,145 @@ TEST_CASE("issue #5405 - array reserve for definite-length CBOR arrays") } } +TEST_CASE("CBOR nesting does not consume the call stack") +{ + // Containers used to be read by calling back into the value reader once + // per element, and a tag by calling it for the tagged value, so the native + // call stack grew with the nesting depth of the input. Each of the three + // costs a single byte to encode -- 0x9F, 0x81 and 0xC2 -- so a payload of + // repeated bytes crashed the process (#5104). The containers are kept on a + // heap stack now, and a tag is read in a loop. + // + // Deeply nested values must not be compared, copied or dumped here: those + // operations are still recursive and would reintroduce the crash. + json _; + + SECTION("indefinite-length containers") + { + const std::vector input(500000, 0x9F); + CHECK_THROWS_WITH_AS(_ = json::from_cbor(input), "[json.exception.parse_error.110] parse error at byte 500001: syntax error while parsing CBOR value: unexpected end of input", json::parse_error&); + CHECK(json::from_cbor(input, true, false).is_discarded()); + } + + SECTION("definite-length containers") + { + const std::vector input(500000, 0x81); + CHECK_THROWS_WITH_AS(_ = json::from_cbor(input), "[json.exception.parse_error.110] parse error at byte 500001: syntax error while parsing CBOR value: unexpected end of input", json::parse_error&); + CHECK(json::from_cbor(input, true, false).is_discarded()); + } + + SECTION("tags") + { + // a tag is not a value of its own, so a chain of them used to recurse + const std::vector input(500000, 0xC2); + CHECK_THROWS_WITH_AS(_ = json::from_cbor(input, true, true, json::cbor_tag_handler_t::ignore), "[json.exception.parse_error.110] parse error at byte 500001: syntax error while parsing CBOR value: unexpected end of input", json::parse_error&); + CHECK(json::from_cbor(input, true, false, json::cbor_tag_handler_t::ignore).is_discarded()); + } + + SECTION("a well-formed deep value is read through the SAX interface") + { + std::vector input(200000, 0x9F); + input.insert(input.end(), 200000, 0xFF); + + SaxCountdown accept_all(1000000); + CHECK(json::sax_parse(input, &accept_all, json::input_format_t::cbor)); + } + + SECTION("a well-formed deep value is read into a value") + { + const std::size_t depth = 10000; + std::vector input(depth, 0x81); + input.push_back(0x00); + + json j = json::from_cbor(input); + + std::size_t measured = 0; + const json* p = &j; + while (p->is_array() && !p->empty()) + { + p = &p->front(); + ++measured; + } + CHECK(measured == depth); + CHECK(p->is_number()); + } + + SECTION("containers are still read the same way") + { + CHECK(json::from_cbor(std::vector({0x80})) == json::array()); + CHECK(json::from_cbor(std::vector({0xA0})) == json::object()); + CHECK(json::from_cbor(std::vector({0x9F, 0xFF})) == json::array()); + CHECK(json::from_cbor(std::vector({0xBF, 0xFF})) == json::object()); + CHECK(json::from_cbor(std::vector({0x9F, 0x01, 0x02, 0xFF})) == json({1, 2})); + CHECK(json::from_cbor(std::vector({0xBF, 0x61, 'a', 0x01, 0xFF})) == json({{"a", 1}})); + // definite and indefinite forms nested inside each other + CHECK(json::from_cbor(std::vector({0x9F, 0x82, 0x01, 0x02, 0xA1, 0x61, 'k', 0xBF, 0xFF, 0xFF})) == json({{1, 2}, {{"k", json::object()}}})); + } + + SECTION("tagged values are still read the same way") + { + const auto ignore = json::cbor_tag_handler_t::ignore; + CHECK(json::from_cbor(std::vector({0xC2, 0x01}), true, true, ignore) == json(1)); + // a chain of tags resolves to the value that follows it + CHECK(json::from_cbor(std::vector({0xC2, 0xC2, 0xC2, 0x01}), true, true, ignore) == json(1)); + // a tag inside a container, and one in front of a container + CHECK(json::from_cbor(std::vector({0x82, 0xC2, 0x01, 0x02}), true, true, ignore) == json({1, 2})); + CHECK(json::from_cbor(std::vector({0xC2, 0x82, 0x01, 0x02}), true, true, ignore) == json({1, 2})); + } +} + +TEST_CASE("CBOR indefinite-length strings do not recurse per chunk") +{ + // Reading an indefinite-length string or byte array used to call itself + // once per chunk, so a payload of repeated 0x7F (or 0x5F) bytes exhausted + // the call stack before any of the input was rejected. The open levels are + // counted now, and the levels below prove the reader still reads the same + // values and reports the same errors at the same byte offsets. + json _; + + SECTION("many open levels are reported, not crashed on") + { + const std::vector input(200000, 0x7F); + CHECK_THROWS_WITH_AS(_ = json::from_cbor(input), "[json.exception.parse_error.110] parse error at byte 200001: syntax error while parsing CBOR string: unexpected end of input", json::parse_error&); + CHECK(json::from_cbor(input, true, false).is_discarded()); + } + + SECTION("many open levels are reported, not crashed on (binary)") + { + const std::vector input(200000, 0x5F); + CHECK_THROWS_WITH_AS(_ = json::from_cbor(input), "[json.exception.parse_error.110] parse error at byte 200001: syntax error while parsing CBOR binary: unexpected end of input", json::parse_error&); + CHECK(json::from_cbor(input, true, false).is_discarded()); + } + + SECTION("chunks are still concatenated") + { + CHECK(json::from_cbor(std::vector({0x7F, 0xFF})) == json("")); + CHECK(json::from_cbor(std::vector({0x7F, 0x61, 0x61, 0xFF})) == json("a")); + // nested indefinite-length strings are concatenated across levels + CHECK(json::from_cbor(std::vector({0x7F, 0x7F, 0x61, 0x61, 0xFF, 0x61, 0x62, 0xFF})) == json("ab")); + CHECK(json::from_cbor(std::vector({0x7F, 0x7F, 0x7F, 0x61, 0x7A, 0xFF, 0xFF, 0xFF})) == json("z")); + CHECK(json::from_cbor(std::vector({0xA1, 0x7F, 0x61, 0x61, 0xFF, 0x01})) == json({{"a", 1}})); + } + + SECTION("chunks are still concatenated (binary)") + { + CHECK(json::from_cbor(std::vector({0x5F, 0x41, 0x61, 0xFF})) == json::binary({0x61})); + CHECK(json::from_cbor(std::vector({0x5F, 0x5F, 0x41, 0x61, 0xFF, 0x41, 0x62, 0xFF})) == json::binary({0x61, 0x62})); + } + + SECTION("a chunk that is not a string is still rejected") + { + CHECK_THROWS_WITH_AS(_ = json::from_cbor(std::vector({0x7F, 0x7F, 0x00})), "[json.exception.parse_error.113] parse error at byte 3: syntax error while parsing CBOR string: expected length specification (0x60-0x7B) or indefinite string type (0x7F); last byte: 0x00", json::parse_error&); + CHECK_THROWS_WITH_AS(_ = json::from_cbor(std::vector({0x5F, 0x5F, 0x00})), "[json.exception.parse_error.113] parse error at byte 3: syntax error while parsing CBOR binary: expected length specification (0x40-0x5B) or indefinite binary array type (0x5F); last byte: 0x00", json::parse_error&); + } + + SECTION("a break marker outside an indefinite-length string is not a string") + { + // 0xFF only closes a string that was opened; on its own it is not one + CHECK_THROWS_WITH_AS(_ = json::from_cbor(std::vector({0xA1, 0xFF, 0x01})), "[json.exception.parse_error.113] parse error at byte 2: syntax error while parsing CBOR string: expected length specification (0x60-0x7B) or indefinite string type (0x7F); last byte: 0xFF", json::parse_error&); + } +} + TEST_CASE("CBOR roundtrips" * doctest::skip()) { SECTION("input from flynn") diff --git a/tests/src/unit-class_lexer.cpp b/tests/src/unit-class_lexer.cpp index 64baf3da6..e89497738 100644 --- a/tests/src/unit-class_lexer.cpp +++ b/tests/src/unit-class_lexer.cpp @@ -12,6 +12,11 @@ #include using nlohmann::json; +#include // strtod +#include // stringstream +#include // string +#include // vector + namespace { // shortcut to scan a string literal @@ -224,3 +229,431 @@ TEST_CASE("lexer class") CHECK((scan_string("/**//**//**/", true) == json::lexer::token_type::end_of_input)); } } + +TEST_CASE("lexer number fast path") +{ + // The contiguous fast path (used for pointer/string input) must agree with + // the streaming byte path (used for std::istream) on token type, numeric + // value, and round-trip text for every well-formed number, and reject the + // same malformed numbers with the same message. + SECTION("contiguous vs streaming parity") + { + const std::vector numbers = + { + "0", "-0", "1", "-1", "42", "-42", "10", "100", "1234567890", + "0.0", "-0.0", "3.14", "-3.14", "0.5", "-0.001", "123.456789", + "1e0", "1E0", "1e10", "1e-10", "1e+10", "1.5e3", "-2.5E-4", + "9223372036854775807", // INT64_MAX -> unsigned + "9223372036854775808", // INT64_MAX + 1 -> unsigned + "18446744073709551615", // UINT64_MAX -> unsigned + "18446744073709551616", // UINT64_MAX + 1 -> float + "-9223372036854775808", // INT64_MIN -> integer + "-9223372036854775809", // INT64_MIN - 1 -> float + "123456789012345678901234567890", // huge -> float + "0.30000000000000004", "2.2250738585072014e-308", "1e308", + // high-precision / wide-exponent values that exercise the + // std::from_chars (Eisel-Lemire) path beyond the Clinger subset + "1.7976931348623157e308", "1.2345678901234567e-250", + "9007199254740993", "5e-324", "1e-320" + }; + + for (const auto& n : numbers) + { + const std::string doc = "[" + n + "]"; + + // contiguous fast path + const json a = json::parse(doc); + // streaming byte path + std::stringstream ss(doc); + const json b = json::parse(ss); + + CAPTURE(n); + CHECK(a == b); + CHECK(a.dump() == b.dump()); + CHECK(a[0].type() == b[0].type()); + } + } + + SECTION("significant-digit gate for the Clinger fast path") + { + // Clinger's fast path needs a significand below 2^53, so it cannot + // succeed once the mantissa has 17 or more significant digits (the + // significand would be at least 10^16). The lexer skips the attempt + // there. That is only allowed to save work: every value must still come + // out bit-exactly, and both scanners must agree. In particular the gate + // must not fire for tokens whose leading zeros merely look like extra + // digits - "0.1234567890123456" has 16 significant digits, not 17. + const std::vector numbers = + { + "1234567890123456", // 16 significant digits + "12345678901234567", // 17 -> attempt skipped + "123456789012345678", // 18 -> attempt skipped + "0.1234567890123456", // 16: the leading "0" is not significant + "0.12345678901234567", // 17 + "0.00000000000000001", // 1, in a long token + "0.000000000000000012345678901234", // 14, in a long token + "-0.0000000000000000000001", // 1, negative + "1.0000000000000000", // 17: trailing zeros are significant here + "10000000000000000", // 17 + "9007199254740992", // 2^53 + "9007199254740993", // 2^53 + 1 + "-65.613616999999977", // canada.json shape + "1.2345678901234567e-250", // 17 with an exponent + "1.234567890123456e-250", // 16 with an exponent + "1e10", "0.0", "-0.0", "0e0", "0.000123" + }; + + for (const auto& n : numbers) + { + CAPTURE(n); + const std::string doc = "[" + n + "]"; + + const json a = json::parse(doc); // contiguous fast path + std::stringstream ss(doc); + const json b = json::parse(ss); // streaming byte path + + CHECK(a[0].type() == b[0].type()); + CHECK(a == b); + + if (a[0].is_number_float()) + { + const double expected = std::strtod(n.c_str(), nullptr); + CHECK(a[0].get() == expected); + CHECK(b[0].get() == expected); + } + } + } + + SECTION("token type classification") + { + CHECK((scan_string("0") == json::lexer::token_type::value_unsigned)); + CHECK((scan_string("-1") == json::lexer::token_type::value_integer)); + CHECK((scan_string("1.5") == json::lexer::token_type::value_float)); + CHECK((scan_string("1e5") == json::lexer::token_type::value_float)); + CHECK((scan_string("18446744073709551615") == json::lexer::token_type::value_unsigned)); + CHECK((scan_string("18446744073709551616") == json::lexer::token_type::value_float)); + CHECK((scan_string("-9223372036854775808") == json::lexer::token_type::value_integer)); + CHECK((scan_string("-9223372036854775809") == json::lexer::token_type::value_float)); + } + + SECTION("malformed numbers are rejected identically") + { + for (const char* bad : + {"-", "1.", "1e", "1e+", "1.2e", "01", "-01", "1..2", "1.2.3" + }) + { + CAPTURE(bad); + // the contiguous fast path must decline and let the byte path report + const std::string doc = std::string("[") + bad + "]"; + CHECK_FALSE(json::accept(doc)); + std::stringstream ss(doc); + CHECK_FALSE(json::accept(ss)); + } + } + +#if !defined(JSON_NOEXCEPTION) + // these sections parse invalid input, which aborts when exceptions are off + SECTION("exhaustive grammar parity with the streaming path") + { + // The JSON number grammar is encoded twice: once as the scan_number() + // state machine and once as the contiguous fast path. Enumerate every + // short string over the number alphabet and require the two encodings to + // agree exactly - on acceptance, on the reported error, and on the parsed + // value - so they cannot drift apart. + const std::string alphabet = "01.eE+-"; + + // full outcome of parsing @a doc, so a mismatch in type, value, or error + // message is caught, not just a mismatch in acceptance + const auto outcome = [](const std::string & doc, bool streaming) -> std::string + { + try + { + if (streaming) + { + std::stringstream ss(doc); + const json j = json::parse(ss); + return std::string(j[0].type_name()) + '|' + j.dump(); + } + const json j = json::parse(doc); + return std::string(j[0].type_name()) + '|' + j.dump(); + } + catch (const json::parse_error& e) + { + return {e.what()}; + } + }; + + std::vector mismatches; + std::vector tokens{""}; + for (std::size_t length = 1; length <= 4; ++length) + { + std::vector next; + next.reserve(tokens.size() * alphabet.size()); + for (const auto& prefix : tokens) + { + for (const char c : alphabet) + { + next.push_back(prefix + c); + } + } + tokens = next; + + for (const auto& token : tokens) + { + const std::string doc = "[" + token + "]"; + if (outcome(doc, false) != outcome(doc, true)) + { + mismatches.push_back(doc); + } + } + } + + // 7 + 49 + 343 + 2401 tokens + CHECK(tokens.size() == 2401); + CAPTURE(mismatches); + CHECK(mismatches.empty()); + } + + SECTION("error positions match the streaming path") + { + // Rejecting identically is not enough: the fast path must also report the + // error at the same position as the byte path. A number directly followed + // by a newline is the interesting case, because the byte path reaches the + // newline (which resets the column) and then ungets it. + // returns the parse_error message, or "" if the document parsed + const auto contiguous_error = [](const std::string & doc) -> std::string + { + try + { + const json j = json::parse(doc); + static_cast(j); + } + catch (const json::parse_error& e) + { + return {e.what()}; + } + return {}; + }; + const auto streaming_error = [](const std::string & doc) -> std::string + { + try + { + std::stringstream ss(doc); + const json j = json::parse(ss); + static_cast(j); + } + catch (const json::parse_error& e) + { + return {e.what()}; + } + return {}; + }; + + for (const char* bad : + {"[01\n]", "[00\n]", "[-01\n]", "{1\n}", "[1\n2]", "[1.2.3\n]", + "[1 \n2]", "[\n1\n2]", "1\n2", "[01\r\n]", "[1e\n]", "[-\n]" + }) + { + CAPTURE(bad); + const std::string doc = bad; + const std::string contiguous_what = contiguous_error(doc); + + CHECK_FALSE(contiguous_what.empty()); + CHECK(contiguous_what == streaming_error(doc)); + } + + // A number terminated by a newline must report the same position as the + // same number terminated by anything else: scan_number() reads the + // terminator and ungets it, so the reported column is the one reached + // after the number's last character - not the 0 that an unget() across + // the newline used to leave behind. + CHECK(contiguous_error("[01\n]") == contiguous_error("[01 ]")); + CHECK(contiguous_error("[01\n]") == + "[json.exception.parse_error.101] parse error at line 1, column 3: " + "syntax error while parsing array - unexpected number literal; expected ']'"); + + // the same for a multi-character token, where the column of the last + // character (the '3' of "-2.5e3") differs from the column it starts at + CHECK(contiguous_error("null -2.5e3\nfalse") == contiguous_error("null -2.5e3 false")); + CHECK(contiguous_error("null -2.5e3\nfalse") == + "[json.exception.parse_error.101] parse error at line 1, column 11: " + "syntax error while parsing value - unexpected number literal; expected end of input"); + } +#endif +} + +TEST_CASE("lexer string fast path") +{ + // Build a byte string from explicit values: a hex escape in a string + // literal swallows every following hex digit, which makes sequences like + // "\xC3\xA9b" mean something other than they look like. + const auto bytes = [](std::initializer_list values) + { + std::string result; + for (const int value : values) + { + result.push_back(static_cast(value)); + } + return result; + }; + +#if !defined(JSON_NOEXCEPTION) + // the full outcome of parsing @a doc: the parsed value, or the exact error + // message, so a mismatch in either is caught. Only usable with exceptions + // on: parsing invalid input aborts when they are off. + const auto outcome = [](const std::string & doc, bool streaming) -> std::string + { + try + { + if (streaming) + { + std::stringstream ss(doc); + const json j = json::parse(ss); + return j.dump(); + } + const json j = json::parse(doc); + return j.dump(); + } + // not just parse_error: if a bulk scanner ever let ill-formed UTF-8 + // through, dump() would throw type_error.316, and that has to surface + // as a reported mismatch rather than as an uncaught exception + catch (const json::exception& e) + { + return {e.what()}; + } + }; +#endif + + // once at the start of the string, once past the first 8-byte SWAR word, so + // the bulk scanner sees each case with and without a run behind it + const std::vector offsets{0, 9}; + +#if !defined(JSON_NOEXCEPTION) + SECTION("exhaustive contiguous vs streaming parity") + { + // ordinary ASCII, both specials, a control byte, characters that make + // the preceding backslash a valid escape, a UTF-8 lead byte of each + // length, a continuation byte, and a byte that is never valid + const std::vector alphabet = + { + "a", "\"", "\\", "n", "u", "0", bytes({0x01}), + bytes({0xC3}), bytes({0xA9}), bytes({0xE4}), bytes({0xF0}), + bytes({0x80}), bytes({0xFF}) + }; + + std::vector mismatches; + std::vector tokens{""}; + for (std::size_t length = 1; length <= 3; ++length) + { + std::vector next; + next.reserve(tokens.size() * alphabet.size()); + for (const auto& prefix : tokens) + { + for (const auto& symbol : alphabet) + { + next.push_back(prefix + symbol); + } + } + tokens = next; + + for (const auto& token : tokens) + { + for (const std::size_t offset : offsets) + { + const std::string doc = "[\"" + std::string(offset, 'a') + token + "\"]"; + if (outcome(doc, false) != outcome(doc, true)) + { + mismatches.push_back(doc); + } + } + } + } + + // 13 + 169 + 2197 tokens, each at two offsets + CHECK(tokens.size() == 2197); + CAPTURE(mismatches); + CHECK(mismatches.empty()); + } + + SECTION("special bytes at every offset of the SWAR stride") + { + // The bulk scanner consumes 8 bytes at a time and then a tail; place + // every kind of byte that ends a run at each offset across two words, + // so multibyte sequences also straddle the word boundary. + const std::vector specials = + { + "\"", "\\", bytes({0x01}), bytes({0x1F}), bytes({0x7F}), + bytes({0xC3, 0xA9}), bytes({0xE4, 0xB8, 0xAD}), bytes({0xF0, 0x9F, 0x98, 0x80}), + bytes({0xFF}), bytes({0xC3}), bytes({0xE4, 0xB8}) + }; + + std::vector mismatches; + for (std::size_t offset = 0; offset <= 17; ++offset) + { + for (const auto& special : specials) + { + const std::string doc = "[\"" + std::string(offset, 'a') + special + "\"]"; + if (outcome(doc, false) != outcome(doc, true)) + { + mismatches.push_back(doc); + } + } + } + CAPTURE(mismatches); + CHECK(mismatches.empty()); + } +#endif + + // json::accept() never throws, so the ranges stay covered without exceptions + SECTION("UTF-8 ranges are accepted and rejected as documented") + { + // The bulk validator must accept exactly what the byte-at-a-time + // scanner accepts, so pin the boundaries of every range it recognizes. + // aggregate, only ever brace-initialized below; default member + // initializers would stop it being an aggregate in C++11 + struct utf8_case // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init) + { + std::string sequence; + bool valid; + const char* description; + }; + const std::vector cases = + { + {bytes({0xC2, 0x80}), true, "U+0080, shortest two-byte"}, + {bytes({0xDF, 0xBF}), true, "U+07FF, longest two-byte"}, + {bytes({0xC1, 0xBF}), false, "overlong two-byte"}, + {bytes({0xC2, 0x7F}), false, "two-byte with bad continuation"}, + {bytes({0xE0, 0xA0, 0x80}), true, "U+0800, shortest three-byte"}, + {bytes({0xE0, 0x9F, 0xBF}), false, "overlong three-byte"}, + {bytes({0xED, 0x9F, 0xBF}), true, "U+D7FF, just below the surrogates"}, + {bytes({0xED, 0xA0, 0x80}), false, "surrogate U+D800"}, + {bytes({0xED, 0xBF, 0xBF}), false, "surrogate U+DFFF"}, + {bytes({0xEE, 0x80, 0x80}), true, "U+E000, just above the surrogates"}, + {bytes({0xEF, 0xBF, 0xBF}), true, "U+FFFF"}, + {bytes({0xF0, 0x90, 0x80, 0x80}), true, "U+10000, shortest four-byte"}, + {bytes({0xF0, 0x8F, 0xBF, 0xBF}), false, "overlong four-byte"}, + {bytes({0xF4, 0x8F, 0xBF, 0xBF}), true, "U+10FFFF, highest code point"}, + {bytes({0xF4, 0x90, 0x80, 0x80}), false, "above U+10FFFF"}, + {bytes({0xF5, 0x80, 0x80, 0x80}), false, "lead byte out of range"}, + {bytes({0x80}), false, "bare continuation byte"}, + {bytes({0xFF}), false, "byte that never appears in UTF-8"}, + {bytes({0xC3}), false, "truncated two-byte"}, + {bytes({0xE4, 0xB8}), false, "truncated three-byte"}, + {bytes({0xF0, 0x9F, 0x98}), false, "truncated four-byte"} + }; + + for (const auto& test_case : cases) + { + CAPTURE(test_case.description); + for (const std::size_t offset : offsets) + { + CAPTURE(offset); + const std::string doc = "[\"" + std::string(offset, 'a') + test_case.sequence + "\"]"; + CHECK(json::accept(doc) == test_case.valid); +#if !defined(JSON_NOEXCEPTION) + CHECK(outcome(doc, false) == outcome(doc, true)); +#endif + } + } + } +} diff --git a/tests/src/unit-class_parser.cpp b/tests/src/unit-class_parser.cpp index 8b3ea660e..34ca5db2b 100644 --- a/tests/src/unit-class_parser.cpp +++ b/tests/src/unit-class_parser.cpp @@ -23,6 +23,8 @@ using nlohmann::json; #include #include +#include "test_utils.hpp" + namespace { class SaxEventLogger @@ -344,6 +346,50 @@ void trailing_comma_helper(const std::string& s) } } +#if JSON_DIAGNOSTIC_POSITIONS +/** + * Validates that the generated JSON object is the same as expected + * Validates that the start position and end position match the start and end of the string + * + * This check assumes that there is no whitespace around the json object in the original string. + */ +void validate_generated_json_and_start_end_pos_helper(const std::string& original_string, const json& j, const json& check) +{ + CHECK(j == check); + CHECK(j.start_pos() == 0); + CHECK(j.end_pos() == original_string.size()); +} + +/** + * Parses the root object from the given root string and validates that the start and end positions for the nested object are correct. + * + * This checks that whitespace around the nested object is included in the start and end positions of the root object. + */ +void validate_start_end_pos_for_nested_obj_helper(const std::string& nested_type_json_str, const std::string& root_type_json_str, const json& expected_json, const json::parser_callback_t& cb = nullptr) +{ + json j; + + // 1. If callback is provided, use callback version of parse() + if (cb) + { + j = json::parse(root_type_json_str, cb); + } + else + { + j = json::parse(root_type_json_str); + } + + // 2. Check if the generated JSON is as expected + // Assumptions: The root_type_json_str does not have any whitespace around the json object + validate_generated_json_and_start_end_pos_helper(root_type_json_str, j, expected_json); + + // 3. Get the nested object + const auto& nested = j["nested"]; + // 4. Check if the start and end positions are generated correctly for nested objects and arrays + CHECK(nested_type_json_str == root_type_json_str.substr(nested.start_pos(), nested.end_pos() - nested.start_pos())); +} +#endif + } // namespace TEST_CASE("parser class") @@ -624,7 +670,8 @@ TEST_CASE("parser class") SECTION("overflow") { // overflows during parsing yield an exception - CHECK_THROWS_WITH_AS(parser_helper("1.18973e+4932").empty(), "[json.exception.out_of_range.406] number overflow parsing '1.18973e+4932'", json::out_of_range&); + // empty() is nodiscard; the exception is thrown by parser_helper() itself, before empty() would run + CHECK_THROWS_WITH_AS(utils::ignore_return_value(parser_helper("1.18973e+4932").empty()), "[json.exception.out_of_range.406] number overflow parsing '1.18973e+4932'", json::out_of_range&); } SECTION("invalid numbers") @@ -930,6 +977,98 @@ TEST_CASE("parser class") CHECK(accept_helper("+1") == false); CHECK(accept_helper("+0") == false); } + + SECTION("issue #5411 - skip conversion when accept() does not need the numeric value") + { + // lexer::scan_number() may skip strtoull()/strtoll() for + // value_unsigned/value_integer tokens when the caller (e.g. + // json::accept()) does not need the converted value, as long + // as the digit count alone guarantees no 64-bit overflow (see + // the "safe_digit_count" fast path in scan_number()). This + // differential test checks that json::accept() (which enables + // the fast path) and json::parse() (which never does) always + // agree, over a corpus that exercises both the fast path + // (<=18 digits) and the untouched, exact fallback path (>=19 + // digits) -- including reclassification of huge digit-only + // integers to a (possibly non-finite) floating-point value. + const std::vector> cases = + { + // normal small/large integers, both signs + {"0", true}, {"1", true}, {"-1", true}, {"42", true}, {"-42", true}, + {"123456789", true}, {"-123456789", true}, + + // digit-count boundary around the 18-digit safe cutoff (both signs) + {std::string(17, '9'), true}, + {std::string(18, '9'), true}, + {std::string(19, '9'), true}, + {std::string(20, '9'), true}, + {"-" + std::string(17, '9'), true}, + {"-" + std::string(18, '9'), true}, + {"-" + std::string(19, '9'), true}, + {"-" + std::string(20, '9'), true}, + + // 64-bit boundaries + {"9223372036854775807", true}, // INT64_MAX + {"-9223372036854775808", true}, // INT64_MIN + {"18446744073709551615", true}, // UINT64_MAX + {"18446744073709551616", true}, // UINT64_MAX + 1 (overflows uint64_t, finite double) + + // the 28-digit example from the issue: overflows uint64_t + // but is finite as a double, so the scanner reclassifies + // it to value_float and it is accepted + {"9999999999999999999999999999", true}, + + // huge digit-only integers that overflow even a double -> rejected + {std::string(309, '9'), false}, + {std::string(400, '9'), false}, + {"1" + std::string(400, '0'), false}, + + // 1e999 / 1e400 style overflow -> rejected + {"1e999", false}, + {"1e400", false}, + {"-1e999", false}, + {"1E999", false}, + + // values straddling DBL_MAX + {"1.7976931348623157e308", true}, // <= DBL_MAX, finite + {"1.7976931348623159e308", false}, // > DBL_MAX, overflows to inf + + // a mix of other valid/invalid numeric syntax + {"3.14159", true}, + {"-0.0", true}, + {"1.0e10", true}, + {"01", false}, + {"-", false}, + {"1.", false}, + {"1e", false}, + {"+1", false}, + }; + + for (const auto& c : cases) + { + const std::string& number = c.first; + const bool expected = c.second; + CAPTURE(number) + CAPTURE(expected) + + // accept() takes the fast path (skips conversion when possible) + CHECK(json::accept(number) == expected); + + // parse() always performs the full conversion; it must agree + json j; + CHECK_NOTHROW(json::parser(nlohmann::detail::input_adapter(number), nullptr, false).parse(true, j)); + CHECK(!j.is_discarded() == expected); + + // wrap in an array so get_token() is exercised beyond the + // very first (constructor-time) scan as well + std::string wrapped = "["; + wrapped += number; + wrapped += ","; + wrapped += number; + wrapped += "]"; + CHECK(json::accept(wrapped) == expected); + } + } } } @@ -1394,6 +1533,71 @@ TEST_CASE("parser class") CHECK(accept_helper("\"\\uD80C\\uFFFF\"") == false); } +#if !defined(JSON_NOEXCEPTION) + SECTION("issue #5412 - whitespace skipping bookkeeping (compact vs. pretty-printed)") + { + // lexer::skip_whitespace() reads its first character with get() (to + // honor a possibly pending unget() from the previous token) and every + // further whitespace character with get_ignoring_pending_unget() (a + // get() variant that skips the then-always-false next_unget check). + // This must not change the reported byte offset, line, or column of + // a syntax error, even when a long run of whitespace containing + // multiple newlines is skipped beforehand (as with pretty-printed + // input). The expected values below were captured from the + // unmodified do-while(get()) loop, so any regression that miscounts + // characters or newlines while skipping whitespace changes them. + const auto check_error = [](const std::string & input, std::size_t expected_byte, + const std::string & expected_what) + { + CAPTURE(input) + try + { + json _ = json::parse(input); + FAIL_CHECK("expected a parse_error, but parsing succeeded"); + } + catch (const json::parse_error& e) + { + CHECK(e.byte == expected_byte); + CHECK(std::string(e.what()) == expected_what); + } + }; + + // a nested document, serialized both compactly and pretty-printed + // (dump(4)), each truncated right before the final closing '}' so + // that the parser hits EOF after skipping all of the (in the + // pretty-printed case, substantial) indentation whitespace + const json doc = + { + {"a", 1}, + {"b", json::array({true, false, nullptr, "x"})}, + {"c", json::object({{"d", 3.14}, {"e", json::array({1, 2, 3})}})} + }; + + const std::string compact = doc.dump(); + const std::string pretty = doc.dump(4); + + check_error(compact.substr(0, compact.size() - 1), 60, + "[json.exception.parse_error.101] parse error at line 1, column 60: syntax error while parsing object - unexpected end of input; expected '}'"); + check_error(pretty.substr(0, pretty.size() - 1), 193, + "[json.exception.parse_error.101] parse error at line 17, column 1: syntax error while parsing object - unexpected end of input; expected '}'"); + + // an invalid token appearing after several indented, multi-line + // whitespace runs vs. the same document without any of that + // whitespace + check_error(R"({ + "a": 1, + "b": [ + true, + false + ], + "c": @ +})", 70, + "[json.exception.parse_error.101] parse error at line 7, column 10: syntax error while parsing value - invalid literal; last read: '\"c\": @'"); + check_error(R"({"a":1,"b":[true,false],"c":@})", 29, + "[json.exception.parse_error.101] parse error at line 1, column 29: syntax error while parsing value - invalid literal; last read: '\"c\":@'"); + } +#endif + SECTION("tests found by mutate++") { // test case to make sure no comma precedes the first key @@ -1564,6 +1768,58 @@ TEST_CASE("parser class") CHECK (j_filtered2 == json({{"foo", {1, 2}}})); } + SECTION("filter many members of one container") + { + // Rejecting a value makes the parser remove the placeholder its key + // event stored. Locating that placeholder used to be a scan of the + // whole parent, which made filtering a large container quadratic: + // 128k members took ~25 s. These cases keep many members alive + // while discarding many others, so the removal cost is the whole + // point; they run in milliseconds when the placeholder is erased + // directly. + constexpr int count = 20000; + + std::string s = "{"; + for (int i = 0; i < count; ++i) + { + // "a" is kept, "z" is discarded + s += "\"a" + std::to_string(i) + "\":" + std::to_string(i) + ","; + s += "\"z" + std::to_string(i) + "\":-1,"; + } + s.back() = '}'; + + const json j_values = json::parse(s, [](int /*unused*/, json::parse_event_t e, const json & parsed) noexcept + { + return !(e == json::parse_event_t::value && parsed == json(-1)); + }); + + CHECK(j_values.size() == count); + CHECK(j_values.at("a0") == json(0)); + CHECK(j_values.at("a" + std::to_string(count - 1)) == json(count - 1)); + CHECK_FALSE(j_values.contains("z0")); + CHECK_FALSE(j_values.contains("z" + std::to_string(count - 1))); + + // the same, but discarding whole containers rather than values, + // which takes the end_object()/end_array() removal path + std::string s_nested = "{"; + for (int i = 0; i < count; ++i) + { + s_nested += "\"a" + std::to_string(i) + "\":" + std::to_string(i) + ","; + s_nested += "\"z" + std::to_string(i) + "\":[1,2],"; + } + s_nested.back() = '}'; + + const json j_arrays = json::parse(s_nested, [](int /*unused*/, json::parse_event_t e, const json& /*unused*/) noexcept + { + return e != json::parse_event_t::array_end; + }); + + CHECK(j_arrays.size() == count); + CHECK(j_arrays.at("a0") == json(0)); + CHECK_FALSE(j_arrays.contains("z0")); + CHECK_FALSE(j_arrays.contains("z" + std::to_string(count - 1))); + } + SECTION("filter specific events") { SECTION("first closing event") @@ -1779,6 +2035,228 @@ TEST_CASE("parser class") CHECK_THROWS_WITH_AS(_ = json::parse("/a", nullptr, true, true), "[json.exception.parse_error.101] parse error at line 1, column 2: syntax error while parsing value - invalid comment; expecting '/' or '*' after '/'; last read: '/a'", json::parse_error); CHECK_THROWS_WITH_AS(_ = json::parse("/*", nullptr, true, true), "[json.exception.parse_error.101] parse error at line 1, column 3: syntax error while parsing value - invalid comment; missing closing '*/'; last read: '/*'", json::parse_error); } + +#if JSON_DIAGNOSTIC_POSITIONS + // Macro for all test cases for start_pos and end_pos +#define SETUP_TESTCASES() \ + SECTION("with callback") \ + { \ + SECTION("filter nothing") \ + { \ + json::parser_callback_t const cb = [](int /*unused*/, json::parse_event_t /*unused*/, json& /*unused*/) noexcept \ + { \ + return true; \ + }; \ + validate_start_end_pos_for_nested_obj_helper(nested_type_json_str, root_type_json_str, expected, cb); \ + } \ + SECTION("filter element") \ + { \ + json::parser_callback_t const cb = [](int /*unused*/, json::parse_event_t event, json& j) noexcept \ + { \ + return (event != json::parse_event_t::key && event != json::parse_event_t::value) || j != json("a"); \ + }; \ + validate_start_end_pos_for_nested_obj_helper(nested_type_json_str, root_type_json_str, filteredExpected, cb); \ + } \ + } \ + SECTION("without callback") \ + { \ + validate_start_end_pos_for_nested_obj_helper(nested_type_json_str, root_type_json_str, expected); \ + } + + SECTION("retrieve start position and end position") + { + SECTION("for object") + { + // Create an object with spaces to test the start and end positions. Spaces will not be included in the + // JSON object, however, the start and end positions should include the spaces from the input JSON string. + const std::string nested_type_json_str = R"({ "a": 1,"b" : "test1"})"; + const std::string root_type_json_str = R"({ "nested": )" + nested_type_json_str + R"(, "anotherValue": "test2"})"; + auto expected = json({{"nested", {{"a", 1}, {"b", "test1"}}}, {"anotherValue", "test2"}}); + auto filteredExpected = expected; + filteredExpected["nested"].erase("a"); + + SETUP_TESTCASES() + } + + SECTION("for array") + { + const std::string nested_type_json_str = R"(["a", "test", 45])"; + const std::string root_type_json_str = R"({ "nested": )" + nested_type_json_str + R"(, "anotherValue": "test" })"; + auto expected = json({{"nested", {"a", "test", 45}}, {"anotherValue", "test"}}); + auto filteredExpected = expected; + filteredExpected["nested"] = json({"test", 45}); + SETUP_TESTCASES() + } + + SECTION("for array with objects") + { + const std::string nested_type_json_str = R"([{"a": 1, "b": "test"}, {"c": 2, "d": "test2"}])"; + const std::string root_type_json_str = R"({ "nested": )" + nested_type_json_str + R"(, "anotherValue": "test" })"; + auto expected = json({{"nested", {{{"a", 1}, {"b", "test"}}, {{"c", 2}, {"d", "test2"}}}}, {"anotherValue", "test"}}); + auto filteredExpected = expected; + filteredExpected["nested"][0].erase("a"); + SETUP_TESTCASES() + + auto j = json::parse(root_type_json_str); + auto nested_array = j["nested"]; + const auto& nested_obj = nested_array[0]; + CHECK(nested_type_json_str.substr(1, 21) == root_type_json_str.substr(nested_obj.start_pos(), nested_obj.end_pos() - nested_obj.start_pos())); + CHECK(nested_type_json_str.substr(24, 22) == root_type_json_str.substr(nested_array[1].start_pos(), nested_array[1].end_pos() - nested_array[1].start_pos())); + } + + SECTION("for two levels of nesting objects") + { + const std::string nested_type_json_str = R"({"nested2": {"b": "test"}})"; + const std::string root_type_json_str = R"({ "a": 2, "nested": )" + nested_type_json_str + R"(, "anotherValue": "test" })"; + auto expected = json({{"a", 2}, {"nested", {{"nested2", {{"b", "test"}}}}}, {"anotherValue", "test"}}); + auto filteredExpected = expected; + filteredExpected.erase("a"); + SETUP_TESTCASES() + + auto j = json::parse(root_type_json_str); + auto nested_obj = j["nested"]["nested2"]; + CHECK(nested_type_json_str.substr(12, 13) == root_type_json_str.substr(nested_obj.start_pos(), nested_obj.end_pos() - nested_obj.start_pos())); + } + + SECTION("for simple types") + { + SECTION("no nested") + { + SECTION("with callback") + { + json::parser_callback_t const cb = [](int /*unused*/, json::parse_event_t /*unused*/, json& /*unused*/) noexcept + { + return true; + }; + + // 1. string type + std::string json_str = R"("test")"; + auto j = json::parse(json_str, cb); + validate_generated_json_and_start_end_pos_helper(json_str, j, "test"); + + // 2. number type + json_str = R"(1)"; + j = json::parse(json_str, cb); + validate_generated_json_and_start_end_pos_helper(json_str, j, 1); + + // 3. boolean type + json_str = R"(true)"; + j = json::parse(json_str, cb); + validate_generated_json_and_start_end_pos_helper(json_str, j, true); + + // 4. null type + json_str = R"(null)"; + j = json::parse(json_str, cb); + validate_generated_json_and_start_end_pos_helper(json_str, j, nullptr); + } + + SECTION("without callback") + { + // 1. string type + std::string json_str = R"("test")"; + auto j = json::parse(json_str); + validate_generated_json_and_start_end_pos_helper(json_str, j, "test"); + + // 2. number type + json_str = R"(1)"; + j = json::parse(json_str); + validate_generated_json_and_start_end_pos_helper(json_str, j, 1); + + json_str = R"(1.001239923)"; + j = json::parse(json_str); + validate_generated_json_and_start_end_pos_helper(json_str, j, 1.001239923); + + json_str = R"(1.123812389000000)"; + j = json::parse(json_str); + validate_generated_json_and_start_end_pos_helper(json_str, j, 1.123812389); + + // 3. boolean type + json_str = R"(true)"; + j = json::parse(json_str); + validate_generated_json_and_start_end_pos_helper(json_str, j, true); + + json_str = R"(false)"; + j = json::parse(json_str); + validate_generated_json_and_start_end_pos_helper(json_str, j, false); + + // 4. null type + json_str = R"(null)"; + j = json::parse(json_str); + validate_generated_json_and_start_end_pos_helper(json_str, j, nullptr); + } + } + + SECTION("string type") + { + const std::string nested_type_json_str = R"("test")"; + const std::string root_type_json_str = R"({ "a": 1, "nested": )" + nested_type_json_str + R"(, "anotherValue": "test" })"; + auto expected = json({{"nested", "test"}, {"anotherValue", "test"}, {"a", 1}}); + auto filteredExpected = expected; + filteredExpected.erase("a"); + SETUP_TESTCASES() + } + + SECTION("number type") + { + const std::string nested_type_json_str = R"(2)"; + const std::string root_type_json_str = R"({ "a": 1, "nested": )" + nested_type_json_str + R"(, "anotherValue": "test" })"; + auto expected = json({{"nested", 2}, {"anotherValue", "test"}, {"a", 1}}); + auto filteredExpected = expected; + filteredExpected.erase("a"); + SETUP_TESTCASES() + } + + SECTION("boolean type") + { + const std::string nested_type_json_str = R"(true)"; + const std::string root_type_json_str = R"({ "a": 1, "nested": )" + nested_type_json_str + R"(, "anotherValue": "test" })"; + auto expected = json({{"nested", true}, {"anotherValue", "test"}, {"a", 1}}); + auto filteredExpected = expected; + filteredExpected.erase("a"); + SETUP_TESTCASES() + } + + SECTION("null type") + { + const std::string nested_type_json_str = R"(null)"; + const std::string root_type_json_str = R"({ "a": 1, "nested": )" + nested_type_json_str + R"(, "anotherValue": "test" })"; + auto expected = json({{"nested", nullptr}, {"anotherValue", "test"}, {"a", 1}}); + auto filteredExpected = expected; + filteredExpected.erase("a"); + SETUP_TESTCASES() + } + } + SECTION("with leading whitespace and newlines around root JSON") + { + const std::string initial_whitespace = R"( + + )"; + const std::string nested_type_json_str = R"({ + "a": 1, + "nested": { + "b": "test" + }, + "anotherValue": "test" + })"; + const std::string end_whitespace = R"( + + )"; + const std::string root_type_json_str = initial_whitespace + nested_type_json_str + end_whitespace; + + auto expected = json({{"a", 1}, {"nested", {{"b", "test"}}}, {"anotherValue", "test"}}); + + auto j = json::parse(root_type_json_str); + + // 2. Check if the generated JSON is as expected + CHECK(j == expected); + + // 3. Check if the start and end positions do not include the surrounding whitespace + CHECK(j.start_pos() == initial_whitespace.size()); + CHECK(j.end_pos() == root_type_json_str.size() - end_whitespace.size()); + } + } +#undef SETUP_TESTCASES +#endif } // this test relies on parse errors being thrown, so it is skipped when diff --git a/tests/src/unit-class_parser_diagnostic_positions.cpp b/tests/src/unit-class_parser_diagnostic_positions.cpp deleted file mode 100644 index 2697ecf8a..000000000 --- a/tests/src/unit-class_parser_diagnostic_positions.cpp +++ /dev/null @@ -1,1957 +0,0 @@ -// __ _____ _____ _____ -// __| | __| | | | JSON for Modern C++ (supporting code) -// | | |__ | | | | | | version 3.12.0 -// |_____|_____|_____|_|___| https://github.com/nlohmann/json -// -// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann -// SPDX-License-Identifier: MIT - -#include "doctest_compatibility.h" -#define JSON_TESTS_PRIVATE -#ifdef JSON_DIAGNOSTIC_POSITIONS - #undef JSON_DIAGNOSTIC_POSITIONS -#endif - -#define JSON_DIAGNOSTIC_POSITIONS 1 -#include -using nlohmann::json; - -#ifdef JSON_TEST_NO_GLOBAL_UDLS - using namespace nlohmann::literals; // NOLINT(google-build-using-namespace) -#endif - -#include - -namespace -{ -class SaxEventLogger -{ - public: - bool null() - { - events.emplace_back("null()"); - return true; - } - - bool boolean(bool val) - { - events.emplace_back(val ? "boolean(true)" : "boolean(false)"); - return true; - } - - bool number_integer(json::number_integer_t val) - { - events.push_back("number_integer(" + std::to_string(val) + ")"); - return true; - } - - bool number_unsigned(json::number_unsigned_t val) - { - events.push_back("number_unsigned(" + std::to_string(val) + ")"); - return true; - } - - bool number_float(json::number_float_t /*unused*/, const std::string& s) - { - events.push_back("number_float(" + s + ")"); - return true; - } - - bool string(std::string& val) - { - events.push_back("string(" + val + ")"); - return true; - } - - bool binary(json::binary_t& val) - { - std::string binary_contents = "binary("; - std::string comma_space; - for (auto b : val) - { - binary_contents.append(comma_space); - binary_contents.append(std::to_string(static_cast(b))); - comma_space = ", "; - } - binary_contents.append(")"); - events.push_back(binary_contents); - return true; - } - - bool start_object(std::size_t elements) - { - if (elements == (std::numeric_limits::max)()) - { - events.emplace_back("start_object()"); - } - else - { - events.push_back("start_object(" + std::to_string(elements) + ")"); - } - return true; - } - - bool key(std::string& val) - { - events.push_back("key(" + val + ")"); - return true; - } - - bool end_object() - { - events.emplace_back("end_object()"); - return true; - } - - bool start_array(std::size_t elements) - { - if (elements == (std::numeric_limits::max)()) - { - events.emplace_back("start_array()"); - } - else - { - events.push_back("start_array(" + std::to_string(elements) + ")"); - } - return true; - } - - bool end_array() - { - events.emplace_back("end_array()"); - return true; - } - - bool parse_error(std::size_t position, const std::string& /*unused*/, const json::exception& /*unused*/) - { - errored = true; - events.push_back("parse_error(" + std::to_string(position) + ")"); - return false; - } - - std::vector events {}; // NOLINT(readability-redundant-member-init) - bool errored = false; -}; - -class SaxCountdown : public nlohmann::json::json_sax_t -{ - public: - explicit SaxCountdown(const int count) : events_left(count) - {} - - bool null() override - { - return events_left-- > 0; - } - - bool boolean(bool /*val*/) override - { - return events_left-- > 0; - } - - bool number_integer(json::number_integer_t /*val*/) override - { - return events_left-- > 0; - } - - bool number_unsigned(json::number_unsigned_t /*val*/) override - { - return events_left-- > 0; - } - - bool number_float(json::number_float_t /*val*/, const std::string& /*s*/) override - { - return events_left-- > 0; - } - - bool string(std::string& /*val*/) override - { - return events_left-- > 0; - } - - bool binary(json::binary_t& /*val*/) override - { - return events_left-- > 0; - } - - bool start_object(std::size_t /*elements*/) override - { - return events_left-- > 0; - } - - bool key(std::string& /*val*/) override - { - return events_left-- > 0; - } - - bool end_object() override - { - return events_left-- > 0; - } - - bool start_array(std::size_t /*elements*/) override - { - return events_left-- > 0; - } - - bool end_array() override - { - return events_left-- > 0; - } - - bool parse_error(std::size_t /*position*/, const std::string& /*last_token*/, const json::exception& /*ex*/) override - { - return false; - } - - private: - int events_left = 0; -}; - -json parser_helper(const std::string& s); -bool accept_helper(const std::string& s); -void comments_helper(const std::string& s); - -json parser_helper(const std::string& s) -{ - json j; - json::parser(nlohmann::detail::input_adapter(s)).parse(true, j); - - // if this line was reached, no exception occurred - // -> check if result is the same without exceptions - json j_nothrow; - CHECK_NOTHROW(json::parser(nlohmann::detail::input_adapter(s), nullptr, false).parse(true, j_nothrow)); - CHECK(j_nothrow == j); - - json j_sax; - nlohmann::detail::json_sax_dom_parser sdp(j_sax); - json::sax_parse(s, &sdp); - CHECK(j_sax == j); - - comments_helper(s); - - return j; -} - -bool accept_helper(const std::string& s) -{ - CAPTURE(s) - - // 1. parse s without exceptions - json j; - CHECK_NOTHROW(json::parser(nlohmann::detail::input_adapter(s), nullptr, false).parse(true, j)); - const bool ok_noexcept = !j.is_discarded(); - - // 2. accept s - const bool ok_accept = json::parser(nlohmann::detail::input_adapter(s)).accept(true); - - // 3. check if both approaches come to the same result - CHECK(ok_noexcept == ok_accept); - - // 4. parse with SAX (compare with relaxed accept result) - SaxEventLogger el; - CHECK_NOTHROW(json::sax_parse(s, &el, json::input_format_t::json, false)); - CHECK(json::parser(nlohmann::detail::input_adapter(s)).accept(false) == !el.errored); - - // 5. parse with simple callback - json::parser_callback_t const cb = [](int /*unused*/, json::parse_event_t /*unused*/, json& /*unused*/) noexcept - { - return true; - }; - json const j_cb = json::parse(s, cb, false); - const bool ok_noexcept_cb = !j_cb.is_discarded(); - - // 6. check if this approach came to the same result - CHECK(ok_noexcept == ok_noexcept_cb); - - // 7. check if comments are properly ignored - if (ok_accept) - { - comments_helper(s); - } - - // 8. return result - return ok_accept; -} - -void comments_helper(const std::string& s) -{ - json _; - - // parse/accept with default parser - CHECK_NOTHROW(_ = json::parse(s)); - CHECK(json::accept(s)); - - // parse/accept while skipping comments - CHECK_NOTHROW(_ = json::parse(s, nullptr, false, true)); - CHECK(json::accept(s, true)); - - std::vector json_with_comments; - - // start with a comment - json_with_comments.push_back(std::string("// this is a comment\n") + s); - json_with_comments.push_back(std::string("/* this is a comment */") + s); - // end with a comment - json_with_comments.push_back(s + "// this is a comment"); - json_with_comments.push_back(s + "/* this is a comment */"); - - // check all strings - for (const auto& json_with_comment : json_with_comments) - { - CAPTURE(json_with_comment) - CHECK_THROWS_AS(_ = json::parse(json_with_comment), json::parse_error); - CHECK(!json::accept(json_with_comment)); - - CHECK_NOTHROW(_ = json::parse(json_with_comment, nullptr, true, true)); - CHECK(json::accept(json_with_comment, true)); - } -} - -/** - * Validates that the generated JSON object is the same as expected - * Validates that the start position and end position match the start and end of the string - * - * This check assumes that there is no whitespace around the json object in the original string. - */ -void validate_generated_json_and_start_end_pos_helper(const std::string& original_string, const json& j, const json& check) -{ - CHECK(j == check); - CHECK(j.start_pos() == 0); - CHECK(j.end_pos() == original_string.size()); -} - -/** - * Parses the root object from the given root string and validates that the start and end positions for the nested object are correct. - * - * This checks that whitespace around the nested object is included in the start and end positions of the root object. - */ -void validate_start_end_pos_for_nested_obj_helper(const std::string& nested_type_json_str, const std::string& root_type_json_str, const json& expected_json, const json::parser_callback_t& cb = nullptr) -{ - json j; - - // 1. If callback is provided, use callback version of parse() - if (cb) - { - j = json::parse(root_type_json_str, cb); - } - else - { - j = json::parse(root_type_json_str); - } - - // 2. Check if the generated JSON is as expected - // Assumptions: The root_type_json_str does not have any whitespace around the json object - validate_generated_json_and_start_end_pos_helper(root_type_json_str, j, expected_json); - - // 3. Get the nested object - const auto& nested = j["nested"]; - // 4. Check if the start and end positions are generated correctly for nested objects and arrays - CHECK(nested_type_json_str == root_type_json_str.substr(nested.start_pos(), nested.end_pos() - nested.start_pos())); -} - -} // namespace - -TEST_CASE("parser class") -{ - SECTION("parse") - { - SECTION("null") - { - CHECK(parser_helper("null") == json(nullptr)); - } - - SECTION("true") - { - CHECK(parser_helper("true") == json(true)); - } - - SECTION("false") - { - CHECK(parser_helper("false") == json(false)); - } - - SECTION("array") - { - SECTION("empty array") - { - CHECK(parser_helper("[]") == json(json::value_t::array)); - CHECK(parser_helper("[ ]") == json(json::value_t::array)); - } - - SECTION("nonempty array") - { - CHECK(parser_helper("[true, false, null]") == json({true, false, nullptr})); - } - } - - SECTION("object") - { - SECTION("empty object") - { - CHECK(parser_helper("{}") == json(json::value_t::object)); - CHECK(parser_helper("{ }") == json(json::value_t::object)); - } - - SECTION("nonempty object") - { - CHECK(parser_helper("{\"\": true, \"one\": 1, \"two\": null}") == json({{"", true}, {"one", 1}, {"two", nullptr}})); - } - } - - SECTION("string") - { - // empty string - CHECK(parser_helper("\"\"") == json(json::value_t::string)); - - SECTION("errors") - { - // error: tab in string - CHECK_THROWS_WITH_AS(parser_helper("\"\t\""), "[json.exception.parse_error.101] parse error at line 1, column 2: syntax error while parsing value - invalid string: control character U+0009 (HT) must be escaped to \\u0009 or \\t; last read: '\"'", json::parse_error&); - // error: newline in string - CHECK_THROWS_WITH_AS(parser_helper("\"\n\""), "[json.exception.parse_error.101] parse error at line 2, column 0: syntax error while parsing value - invalid string: control character U+000A (LF) must be escaped to \\u000A or \\n; last read: '\"'", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("\"\r\""), "[json.exception.parse_error.101] parse error at line 1, column 2: syntax error while parsing value - invalid string: control character U+000D (CR) must be escaped to \\u000D or \\r; last read: '\"'", json::parse_error&); - // error: backspace in string - CHECK_THROWS_WITH_AS(parser_helper("\"\b\""), "[json.exception.parse_error.101] parse error at line 1, column 2: syntax error while parsing value - invalid string: control character U+0008 (BS) must be escaped to \\u0008 or \\b; last read: '\"'", json::parse_error&); - // improve code coverage - CHECK_THROWS_AS(parser_helper("\uFF01"), json::parse_error&); - CHECK_THROWS_AS(parser_helper("[-4:1,]"), json::parse_error&); - // unescaped control characters - CHECK_THROWS_WITH_AS(parser_helper("\"\x00\""), "[json.exception.parse_error.101] parse error at line 1, column 2: syntax error while parsing value - invalid string: missing closing quote; last read: '\"'", json::parse_error&); // NOLINT(bugprone-string-literal-with-embedded-nul) - CHECK_THROWS_WITH_AS(parser_helper("\"\x01\""), "[json.exception.parse_error.101] parse error at line 1, column 2: syntax error while parsing value - invalid string: control character U+0001 (SOH) must be escaped to \\u0001; last read: '\"'", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("\"\x02\""), "[json.exception.parse_error.101] parse error at line 1, column 2: syntax error while parsing value - invalid string: control character U+0002 (STX) must be escaped to \\u0002; last read: '\"'", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("\"\x03\""), "[json.exception.parse_error.101] parse error at line 1, column 2: syntax error while parsing value - invalid string: control character U+0003 (ETX) must be escaped to \\u0003; last read: '\"'", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("\"\x04\""), "[json.exception.parse_error.101] parse error at line 1, column 2: syntax error while parsing value - invalid string: control character U+0004 (EOT) must be escaped to \\u0004; last read: '\"'", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("\"\x05\""), "[json.exception.parse_error.101] parse error at line 1, column 2: syntax error while parsing value - invalid string: control character U+0005 (ENQ) must be escaped to \\u0005; last read: '\"'", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("\"\x06\""), "[json.exception.parse_error.101] parse error at line 1, column 2: syntax error while parsing value - invalid string: control character U+0006 (ACK) must be escaped to \\u0006; last read: '\"'", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("\"\x07\""), "[json.exception.parse_error.101] parse error at line 1, column 2: syntax error while parsing value - invalid string: control character U+0007 (BEL) must be escaped to \\u0007; last read: '\"'", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("\"\x08\""), "[json.exception.parse_error.101] parse error at line 1, column 2: syntax error while parsing value - invalid string: control character U+0008 (BS) must be escaped to \\u0008 or \\b; last read: '\"'", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("\"\x09\""), "[json.exception.parse_error.101] parse error at line 1, column 2: syntax error while parsing value - invalid string: control character U+0009 (HT) must be escaped to \\u0009 or \\t; last read: '\"'", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("\"\x0a\""), "[json.exception.parse_error.101] parse error at line 2, column 0: syntax error while parsing value - invalid string: control character U+000A (LF) must be escaped to \\u000A or \\n; last read: '\"'", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("\"\x0b\""), "[json.exception.parse_error.101] parse error at line 1, column 2: syntax error while parsing value - invalid string: control character U+000B (VT) must be escaped to \\u000B; last read: '\"'", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("\"\x0c\""), "[json.exception.parse_error.101] parse error at line 1, column 2: syntax error while parsing value - invalid string: control character U+000C (FF) must be escaped to \\u000C or \\f; last read: '\"'", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("\"\x0d\""), "[json.exception.parse_error.101] parse error at line 1, column 2: syntax error while parsing value - invalid string: control character U+000D (CR) must be escaped to \\u000D or \\r; last read: '\"'", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("\"\x0e\""), "[json.exception.parse_error.101] parse error at line 1, column 2: syntax error while parsing value - invalid string: control character U+000E (SO) must be escaped to \\u000E; last read: '\"'", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("\"\x0f\""), "[json.exception.parse_error.101] parse error at line 1, column 2: syntax error while parsing value - invalid string: control character U+000F (SI) must be escaped to \\u000F; last read: '\"'", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("\"\x10\""), "[json.exception.parse_error.101] parse error at line 1, column 2: syntax error while parsing value - invalid string: control character U+0010 (DLE) must be escaped to \\u0010; last read: '\"'", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("\"\x11\""), "[json.exception.parse_error.101] parse error at line 1, column 2: syntax error while parsing value - invalid string: control character U+0011 (DC1) must be escaped to \\u0011; last read: '\"'", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("\"\x12\""), "[json.exception.parse_error.101] parse error at line 1, column 2: syntax error while parsing value - invalid string: control character U+0012 (DC2) must be escaped to \\u0012; last read: '\"'", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("\"\x13\""), "[json.exception.parse_error.101] parse error at line 1, column 2: syntax error while parsing value - invalid string: control character U+0013 (DC3) must be escaped to \\u0013; last read: '\"'", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("\"\x14\""), "[json.exception.parse_error.101] parse error at line 1, column 2: syntax error while parsing value - invalid string: control character U+0014 (DC4) must be escaped to \\u0014; last read: '\"'", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("\"\x15\""), "[json.exception.parse_error.101] parse error at line 1, column 2: syntax error while parsing value - invalid string: control character U+0015 (NAK) must be escaped to \\u0015; last read: '\"'", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("\"\x16\""), "[json.exception.parse_error.101] parse error at line 1, column 2: syntax error while parsing value - invalid string: control character U+0016 (SYN) must be escaped to \\u0016; last read: '\"'", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("\"\x17\""), "[json.exception.parse_error.101] parse error at line 1, column 2: syntax error while parsing value - invalid string: control character U+0017 (ETB) must be escaped to \\u0017; last read: '\"'", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("\"\x18\""), "[json.exception.parse_error.101] parse error at line 1, column 2: syntax error while parsing value - invalid string: control character U+0018 (CAN) must be escaped to \\u0018; last read: '\"'", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("\"\x19\""), "[json.exception.parse_error.101] parse error at line 1, column 2: syntax error while parsing value - invalid string: control character U+0019 (EM) must be escaped to \\u0019; last read: '\"'", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("\"\x1a\""), "[json.exception.parse_error.101] parse error at line 1, column 2: syntax error while parsing value - invalid string: control character U+001A (SUB) must be escaped to \\u001A; last read: '\"'", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("\"\x1b\""), "[json.exception.parse_error.101] parse error at line 1, column 2: syntax error while parsing value - invalid string: control character U+001B (ESC) must be escaped to \\u001B; last read: '\"'", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("\"\x1c\""), "[json.exception.parse_error.101] parse error at line 1, column 2: syntax error while parsing value - invalid string: control character U+001C (FS) must be escaped to \\u001C; last read: '\"'", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("\"\x1d\""), "[json.exception.parse_error.101] parse error at line 1, column 2: syntax error while parsing value - invalid string: control character U+001D (GS) must be escaped to \\u001D; last read: '\"'", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("\"\x1e\""), "[json.exception.parse_error.101] parse error at line 1, column 2: syntax error while parsing value - invalid string: control character U+001E (RS) must be escaped to \\u001E; last read: '\"'", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("\"\x1f\""), "[json.exception.parse_error.101] parse error at line 1, column 2: syntax error while parsing value - invalid string: control character U+001F (US) must be escaped to \\u001F; last read: '\"'", json::parse_error&); - - SECTION("additional test for null byte") - { - // The test above for the null byte is wrong, because passing - // a string to the parser only reads int until it encounters - // a null byte. This test inserts the null byte later on and - // uses an iterator range. - std::string s = "\"1\""; - s[1] = '\0'; - json _; - CHECK_THROWS_WITH_AS(_ = json::parse(s.begin(), s.end()), "[json.exception.parse_error.101] parse error at line 1, column 2: syntax error while parsing value - invalid string: control character U+0000 (NUL) must be escaped to \\u0000; last read: '\"'", json::parse_error&); - } - } - - SECTION("escaped") - { - // quotation mark "\"" - auto r1 = R"("\"")"_json; - CHECK(parser_helper("\"\\\"\"") == r1); - // reverse solidus "\\" - auto r2 = R"("\\")"_json; - CHECK(parser_helper("\"\\\\\"") == r2); - // solidus - CHECK(parser_helper("\"\\/\"") == R"("/")"_json); - // backspace - CHECK(parser_helper("\"\\b\"") == json("\b")); - // formfeed - CHECK(parser_helper("\"\\f\"") == json("\f")); - // newline - CHECK(parser_helper("\"\\n\"") == json("\n")); - // carriage return - CHECK(parser_helper("\"\\r\"") == json("\r")); - // horizontal tab - CHECK(parser_helper("\"\\t\"") == json("\t")); - - CHECK(parser_helper("\"\\u0001\"").get() == "\x01"); - CHECK(parser_helper("\"\\u000a\"").get() == "\n"); - CHECK(parser_helper("\"\\u00b0\"").get() == "°"); - CHECK(parser_helper("\"\\u0c00\"").get() == "ఀ"); - CHECK(parser_helper("\"\\ud000\"").get() == "퀀"); - CHECK(parser_helper("\"\\u000E\"").get() == "\x0E"); - CHECK(parser_helper("\"\\u00F0\"").get() == "ð"); - CHECK(parser_helper("\"\\u0100\"").get() == "Ā"); - CHECK(parser_helper("\"\\u2000\"").get() == " "); - CHECK(parser_helper("\"\\uFFFF\"").get() == "￿"); - CHECK(parser_helper("\"\\u20AC\"").get() == "€"); - CHECK(parser_helper("\"€\"").get() == "€"); - CHECK(parser_helper("\"🎈\"").get() == "🎈"); - - CHECK(parser_helper("\"\\ud80c\\udc60\"").get() == "\xf0\x93\x81\xa0"); - CHECK(parser_helper("\"\\ud83c\\udf1e\"").get() == "🌞"); - } - } - - SECTION("number") - { - SECTION("integers") - { - SECTION("without exponent") - { - CHECK(parser_helper("-128") == json(-128)); - CHECK(parser_helper("-0") == json(-0)); - CHECK(parser_helper("0") == json(0)); - CHECK(parser_helper("128") == json(128)); - } - - SECTION("with exponent") - { - CHECK(parser_helper("0e1") == json(0e1)); - CHECK(parser_helper("0E1") == json(0e1)); - - CHECK(parser_helper("10000E-4") == json(10000e-4)); - CHECK(parser_helper("10000E-3") == json(10000e-3)); - CHECK(parser_helper("10000E-2") == json(10000e-2)); - CHECK(parser_helper("10000E-1") == json(10000e-1)); - CHECK(parser_helper("10000E0") == json(10000e0)); - CHECK(parser_helper("10000E1") == json(10000e1)); - CHECK(parser_helper("10000E2") == json(10000e2)); - CHECK(parser_helper("10000E3") == json(10000e3)); - CHECK(parser_helper("10000E4") == json(10000e4)); - - CHECK(parser_helper("10000e-4") == json(10000e-4)); - CHECK(parser_helper("10000e-3") == json(10000e-3)); - CHECK(parser_helper("10000e-2") == json(10000e-2)); - CHECK(parser_helper("10000e-1") == json(10000e-1)); - CHECK(parser_helper("10000e0") == json(10000e0)); - CHECK(parser_helper("10000e1") == json(10000e1)); - CHECK(parser_helper("10000e2") == json(10000e2)); - CHECK(parser_helper("10000e3") == json(10000e3)); - CHECK(parser_helper("10000e4") == json(10000e4)); - - CHECK(parser_helper("-0e1") == json(-0e1)); - CHECK(parser_helper("-0E1") == json(-0e1)); - CHECK(parser_helper("-0E123") == json(-0e123)); - - // numbers after exponent - CHECK(parser_helper("10E0") == json(10e0)); - CHECK(parser_helper("10E1") == json(10e1)); - CHECK(parser_helper("10E2") == json(10e2)); - CHECK(parser_helper("10E3") == json(10e3)); - CHECK(parser_helper("10E4") == json(10e4)); - CHECK(parser_helper("10E5") == json(10e5)); - CHECK(parser_helper("10E6") == json(10e6)); - CHECK(parser_helper("10E7") == json(10e7)); - CHECK(parser_helper("10E8") == json(10e8)); - CHECK(parser_helper("10E9") == json(10e9)); - CHECK(parser_helper("10E+0") == json(10e0)); - CHECK(parser_helper("10E+1") == json(10e1)); - CHECK(parser_helper("10E+2") == json(10e2)); - CHECK(parser_helper("10E+3") == json(10e3)); - CHECK(parser_helper("10E+4") == json(10e4)); - CHECK(parser_helper("10E+5") == json(10e5)); - CHECK(parser_helper("10E+6") == json(10e6)); - CHECK(parser_helper("10E+7") == json(10e7)); - CHECK(parser_helper("10E+8") == json(10e8)); - CHECK(parser_helper("10E+9") == json(10e9)); - CHECK(parser_helper("10E-1") == json(10e-1)); - CHECK(parser_helper("10E-2") == json(10e-2)); - CHECK(parser_helper("10E-3") == json(10e-3)); - CHECK(parser_helper("10E-4") == json(10e-4)); - CHECK(parser_helper("10E-5") == json(10e-5)); - CHECK(parser_helper("10E-6") == json(10e-6)); - CHECK(parser_helper("10E-7") == json(10e-7)); - CHECK(parser_helper("10E-8") == json(10e-8)); - CHECK(parser_helper("10E-9") == json(10e-9)); - } - - SECTION("edge cases") - { - // From RFC8259, Section 6: - // Note that when such software is used, numbers that are - // integers and are in the range [-(2**53)+1, (2**53)-1] - // are interoperable in the sense that implementations will - // agree exactly on their numeric values. - - // -(2**53)+1 - CHECK(parser_helper("-9007199254740991").get() == -9007199254740991); - // (2**53)-1 - CHECK(parser_helper("9007199254740991").get() == 9007199254740991); - } - - SECTION("over the edge cases") // issue #178 - Integer conversion to unsigned (incorrect handling of 64-bit integers) - { - // While RFC8259, Section 6 specifies a preference for support - // for ranges in range of IEEE 754-2008 binary64 (double precision) - // this does not accommodate 64-bit integers without loss of accuracy. - // As 64-bit integers are now widely used in software, it is desirable - // to expand support to the full 64 bit (signed and unsigned) range - // i.e. -(2**63) -> (2**64)-1. - - // -(2**63) ** Note: compilers see negative literals as negated positive numbers (hence the -1)) - CHECK(parser_helper("-9223372036854775808").get() == -9223372036854775807 - 1); - // (2**63)-1 - CHECK(parser_helper("9223372036854775807").get() == 9223372036854775807); - // (2**64)-1 - CHECK(parser_helper("18446744073709551615").get() == 18446744073709551615u); - } - } - - SECTION("floating-point") - { - SECTION("without exponent") - { - CHECK(parser_helper("-128.5") == json(-128.5)); - CHECK(parser_helper("0.999") == json(0.999)); - CHECK(parser_helper("128.5") == json(128.5)); - CHECK(parser_helper("-0.0") == json(-0.0)); - } - - SECTION("with exponent") - { - CHECK(parser_helper("-128.5E3") == json(-128.5E3)); - CHECK(parser_helper("-128.5E-3") == json(-128.5E-3)); - CHECK(parser_helper("-0.0e1") == json(-0.0e1)); - CHECK(parser_helper("-0.0E1") == json(-0.0e1)); - } - } - - SECTION("overflow") - { - // overflows during parsing yield an exception - CHECK_THROWS_WITH_AS(parser_helper("1.18973e+4932").empty(), "[json.exception.out_of_range.406] number overflow parsing '1.18973e+4932'", json::out_of_range&); - } - - SECTION("invalid numbers") - { - // numbers must not begin with "+" - CHECK_THROWS_AS(parser_helper("+1"), json::parse_error&); - CHECK_THROWS_AS(parser_helper("+0"), json::parse_error&); - - CHECK_THROWS_WITH_AS(parser_helper("01"), - "[json.exception.parse_error.101] parse error at line 1, column 2: syntax error while parsing value - unexpected number literal; expected end of input", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("-01"), - "[json.exception.parse_error.101] parse error at line 1, column 3: syntax error while parsing value - unexpected number literal; expected end of input", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("--1"), - "[json.exception.parse_error.101] parse error at line 1, column 2: syntax error while parsing value - invalid number; expected digit after '-'; last read: '--'", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("1."), - "[json.exception.parse_error.101] parse error at line 1, column 3: syntax error while parsing value - invalid number; expected digit after '.'; last read: '1.'", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("1E"), - "[json.exception.parse_error.101] parse error at line 1, column 3: syntax error while parsing value - invalid number; expected '+', '-', or digit after exponent; last read: '1E'", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("1E-"), - "[json.exception.parse_error.101] parse error at line 1, column 4: syntax error while parsing value - invalid number; expected digit after exponent sign; last read: '1E-'", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("1.E1"), - "[json.exception.parse_error.101] parse error at line 1, column 3: syntax error while parsing value - invalid number; expected digit after '.'; last read: '1.E'", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("-1E"), - "[json.exception.parse_error.101] parse error at line 1, column 4: syntax error while parsing value - invalid number; expected '+', '-', or digit after exponent; last read: '-1E'", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("-0E#"), - "[json.exception.parse_error.101] parse error at line 1, column 4: syntax error while parsing value - invalid number; expected '+', '-', or digit after exponent; last read: '-0E#'", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("-0E-#"), - "[json.exception.parse_error.101] parse error at line 1, column 5: syntax error while parsing value - invalid number; expected digit after exponent sign; last read: '-0E-#'", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("-0#"), - "[json.exception.parse_error.101] parse error at line 1, column 3: syntax error while parsing value - invalid literal; last read: '-0#'; expected end of input", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("-0.0:"), - "[json.exception.parse_error.101] parse error at line 1, column 5: syntax error while parsing value - unexpected ':'; expected end of input", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("-0.0Z"), - "[json.exception.parse_error.101] parse error at line 1, column 5: syntax error while parsing value - invalid literal; last read: '-0.0Z'; expected end of input", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("-0E123:"), - "[json.exception.parse_error.101] parse error at line 1, column 7: syntax error while parsing value - unexpected ':'; expected end of input", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("-0e0-:"), - "[json.exception.parse_error.101] parse error at line 1, column 6: syntax error while parsing value - invalid number; expected digit after '-'; last read: '-:'; expected end of input", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("-0e-:"), - "[json.exception.parse_error.101] parse error at line 1, column 5: syntax error while parsing value - invalid number; expected digit after exponent sign; last read: '-0e-:'", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("-0f"), - "[json.exception.parse_error.101] parse error at line 1, column 4: syntax error while parsing value - invalid literal; last read: '-0f'; expected end of input", json::parse_error&); - } - } - } - - SECTION("accept") - { - SECTION("null") - { - CHECK(accept_helper("null")); - } - - SECTION("true") - { - CHECK(accept_helper("true")); - } - - SECTION("false") - { - CHECK(accept_helper("false")); - } - - SECTION("array") - { - SECTION("empty array") - { - CHECK(accept_helper("[]")); - CHECK(accept_helper("[ ]")); - } - - SECTION("nonempty array") - { - CHECK(accept_helper("[true, false, null]")); - } - } - - SECTION("object") - { - SECTION("empty object") - { - CHECK(accept_helper("{}")); - CHECK(accept_helper("{ }")); - } - - SECTION("nonempty object") - { - CHECK(accept_helper("{\"\": true, \"one\": 1, \"two\": null}")); - } - } - - SECTION("string") - { - // empty string - CHECK(accept_helper("\"\"")); - - SECTION("errors") - { - // error: tab in string - CHECK(accept_helper("\"\t\"") == false); - // error: newline in string - CHECK(accept_helper("\"\n\"") == false); - CHECK(accept_helper("\"\r\"") == false); - // error: backspace in string - CHECK(accept_helper("\"\b\"") == false); - // improve code coverage - CHECK(accept_helper("\uFF01") == false); - CHECK(accept_helper("[-4:1,]") == false); - // unescaped control characters - CHECK(accept_helper("\"\x00\"") == false); // NOLINT(bugprone-string-literal-with-embedded-nul) - CHECK(accept_helper("\"\x01\"") == false); - CHECK(accept_helper("\"\x02\"") == false); - CHECK(accept_helper("\"\x03\"") == false); - CHECK(accept_helper("\"\x04\"") == false); - CHECK(accept_helper("\"\x05\"") == false); - CHECK(accept_helper("\"\x06\"") == false); - CHECK(accept_helper("\"\x07\"") == false); - CHECK(accept_helper("\"\x08\"") == false); - CHECK(accept_helper("\"\x09\"") == false); - CHECK(accept_helper("\"\x0a\"") == false); - CHECK(accept_helper("\"\x0b\"") == false); - CHECK(accept_helper("\"\x0c\"") == false); - CHECK(accept_helper("\"\x0d\"") == false); - CHECK(accept_helper("\"\x0e\"") == false); - CHECK(accept_helper("\"\x0f\"") == false); - CHECK(accept_helper("\"\x10\"") == false); - CHECK(accept_helper("\"\x11\"") == false); - CHECK(accept_helper("\"\x12\"") == false); - CHECK(accept_helper("\"\x13\"") == false); - CHECK(accept_helper("\"\x14\"") == false); - CHECK(accept_helper("\"\x15\"") == false); - CHECK(accept_helper("\"\x16\"") == false); - CHECK(accept_helper("\"\x17\"") == false); - CHECK(accept_helper("\"\x18\"") == false); - CHECK(accept_helper("\"\x19\"") == false); - CHECK(accept_helper("\"\x1a\"") == false); - CHECK(accept_helper("\"\x1b\"") == false); - CHECK(accept_helper("\"\x1c\"") == false); - CHECK(accept_helper("\"\x1d\"") == false); - CHECK(accept_helper("\"\x1e\"") == false); - CHECK(accept_helper("\"\x1f\"") == false); - } - - SECTION("escaped") - { - // quotation mark "\"" - auto r1 = R"("\"")"_json; - CHECK(accept_helper("\"\\\"\"")); - // reverse solidus "\\" - auto r2 = R"("\\")"_json; - CHECK(accept_helper("\"\\\\\"")); - // solidus - CHECK(accept_helper("\"\\/\"")); - // backspace - CHECK(accept_helper("\"\\b\"")); - // formfeed - CHECK(accept_helper("\"\\f\"")); - // newline - CHECK(accept_helper("\"\\n\"")); - // carriage return - CHECK(accept_helper("\"\\r\"")); - // horizontal tab - CHECK(accept_helper("\"\\t\"")); - - CHECK(accept_helper("\"\\u0001\"")); - CHECK(accept_helper("\"\\u000a\"")); - CHECK(accept_helper("\"\\u00b0\"")); - CHECK(accept_helper("\"\\u0c00\"")); - CHECK(accept_helper("\"\\ud000\"")); - CHECK(accept_helper("\"\\u000E\"")); - CHECK(accept_helper("\"\\u00F0\"")); - CHECK(accept_helper("\"\\u0100\"")); - CHECK(accept_helper("\"\\u2000\"")); - CHECK(accept_helper("\"\\uFFFF\"")); - CHECK(accept_helper("\"\\u20AC\"")); - CHECK(accept_helper("\"€\"")); - CHECK(accept_helper("\"🎈\"")); - - CHECK(accept_helper("\"\\ud80c\\udc60\"")); - CHECK(accept_helper("\"\\ud83c\\udf1e\"")); - } - } - - SECTION("number") - { - SECTION("integers") - { - SECTION("without exponent") - { - CHECK(accept_helper("-128")); - CHECK(accept_helper("-0")); - CHECK(accept_helper("0")); - CHECK(accept_helper("128")); - } - - SECTION("with exponent") - { - CHECK(accept_helper("0e1")); - CHECK(accept_helper("0E1")); - - CHECK(accept_helper("10000E-4")); - CHECK(accept_helper("10000E-3")); - CHECK(accept_helper("10000E-2")); - CHECK(accept_helper("10000E-1")); - CHECK(accept_helper("10000E0")); - CHECK(accept_helper("10000E1")); - CHECK(accept_helper("10000E2")); - CHECK(accept_helper("10000E3")); - CHECK(accept_helper("10000E4")); - - CHECK(accept_helper("10000e-4")); - CHECK(accept_helper("10000e-3")); - CHECK(accept_helper("10000e-2")); - CHECK(accept_helper("10000e-1")); - CHECK(accept_helper("10000e0")); - CHECK(accept_helper("10000e1")); - CHECK(accept_helper("10000e2")); - CHECK(accept_helper("10000e3")); - CHECK(accept_helper("10000e4")); - - CHECK(accept_helper("-0e1")); - CHECK(accept_helper("-0E1")); - CHECK(accept_helper("-0E123")); - } - - SECTION("edge cases") - { - // From RFC8259, Section 6: - // Note that when such software is used, numbers that are - // integers and are in the range [-(2**53)+1, (2**53)-1] - // are interoperable in the sense that implementations will - // agree exactly on their numeric values. - - // -(2**53)+1 - CHECK(accept_helper("-9007199254740991")); - // (2**53)-1 - CHECK(accept_helper("9007199254740991")); - } - - SECTION("over the edge cases") // issue #178 - Integer conversion to unsigned (incorrect handling of 64-bit integers) - { - // While RFC8259, Section 6 specifies a preference for support - // for ranges in range of IEEE 754-2008 binary64 (double precision) - // this does not accommodate 64 bit integers without loss of accuracy. - // As 64 bit integers are now widely used in software, it is desirable - // to expand support to the full 64 bit (signed and unsigned) range - // i.e. -(2**63) -> (2**64)-1. - - // -(2**63) ** Note: compilers see negative literals as negated positive numbers (hence the -1)) - CHECK(accept_helper("-9223372036854775808")); - // (2**63)-1 - CHECK(accept_helper("9223372036854775807")); - // (2**64)-1 - CHECK(accept_helper("18446744073709551615")); - } - } - - SECTION("floating-point") - { - SECTION("without exponent") - { - CHECK(accept_helper("-128.5")); - CHECK(accept_helper("0.999")); - CHECK(accept_helper("128.5")); - CHECK(accept_helper("-0.0")); - } - - SECTION("with exponent") - { - CHECK(accept_helper("-128.5E3")); - CHECK(accept_helper("-128.5E-3")); - CHECK(accept_helper("-0.0e1")); - CHECK(accept_helper("-0.0E1")); - } - } - - SECTION("overflow") - { - // overflows during parsing - CHECK(!accept_helper("1.18973e+4932")); - } - - SECTION("invalid numbers") - { - CHECK(accept_helper("01") == false); - CHECK(accept_helper("--1") == false); - CHECK(accept_helper("1.") == false); - CHECK(accept_helper("1E") == false); - CHECK(accept_helper("1E-") == false); - CHECK(accept_helper("1.E1") == false); - CHECK(accept_helper("-1E") == false); - CHECK(accept_helper("-0E#") == false); - CHECK(accept_helper("-0E-#") == false); - CHECK(accept_helper("-0#") == false); - CHECK(accept_helper("-0.0:") == false); - CHECK(accept_helper("-0.0Z") == false); - CHECK(accept_helper("-0E123:") == false); - CHECK(accept_helper("-0e0-:") == false); - CHECK(accept_helper("-0e-:") == false); - CHECK(accept_helper("-0f") == false); - - // numbers must not begin with "+" - CHECK(accept_helper("+1") == false); - CHECK(accept_helper("+0") == false); - } - } - } - - SECTION("parse errors") - { - // unexpected end of number - CHECK_THROWS_WITH_AS(parser_helper("0."), - "[json.exception.parse_error.101] parse error at line 1, column 3: syntax error while parsing value - invalid number; expected digit after '.'; last read: '0.'", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("-"), - "[json.exception.parse_error.101] parse error at line 1, column 2: syntax error while parsing value - invalid number; expected digit after '-'; last read: '-'", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("--"), - "[json.exception.parse_error.101] parse error at line 1, column 2: syntax error while parsing value - invalid number; expected digit after '-'; last read: '--'", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("-0."), - "[json.exception.parse_error.101] parse error at line 1, column 4: syntax error while parsing value - invalid number; expected digit after '.'; last read: '-0.'", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("-."), - "[json.exception.parse_error.101] parse error at line 1, column 2: syntax error while parsing value - invalid number; expected digit after '-'; last read: '-.'", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("-:"), - "[json.exception.parse_error.101] parse error at line 1, column 2: syntax error while parsing value - invalid number; expected digit after '-'; last read: '-:'", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("0.:"), - "[json.exception.parse_error.101] parse error at line 1, column 3: syntax error while parsing value - invalid number; expected digit after '.'; last read: '0.:'", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("e."), - "[json.exception.parse_error.101] parse error at line 1, column 1: syntax error while parsing value - invalid literal; last read: 'e'", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("1e."), - "[json.exception.parse_error.101] parse error at line 1, column 3: syntax error while parsing value - invalid number; expected '+', '-', or digit after exponent; last read: '1e.'", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("1e/"), - "[json.exception.parse_error.101] parse error at line 1, column 3: syntax error while parsing value - invalid number; expected '+', '-', or digit after exponent; last read: '1e/'", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("1e:"), - "[json.exception.parse_error.101] parse error at line 1, column 3: syntax error while parsing value - invalid number; expected '+', '-', or digit after exponent; last read: '1e:'", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("1E."), - "[json.exception.parse_error.101] parse error at line 1, column 3: syntax error while parsing value - invalid number; expected '+', '-', or digit after exponent; last read: '1E.'", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("1E/"), - "[json.exception.parse_error.101] parse error at line 1, column 3: syntax error while parsing value - invalid number; expected '+', '-', or digit after exponent; last read: '1E/'", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("1E:"), - "[json.exception.parse_error.101] parse error at line 1, column 3: syntax error while parsing value - invalid number; expected '+', '-', or digit after exponent; last read: '1E:'", json::parse_error&); - - // unexpected end of null - CHECK_THROWS_WITH_AS(parser_helper("n"), - "[json.exception.parse_error.101] parse error at line 1, column 2: syntax error while parsing value - invalid literal; last read: 'n'", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("nu"), - "[json.exception.parse_error.101] parse error at line 1, column 3: syntax error while parsing value - invalid literal; last read: 'nu'", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("nul"), - "[json.exception.parse_error.101] parse error at line 1, column 4: syntax error while parsing value - invalid literal; last read: 'nul'", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("nulk"), - "[json.exception.parse_error.101] parse error at line 1, column 4: syntax error while parsing value - invalid literal; last read: 'nulk'", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("nulm"), - "[json.exception.parse_error.101] parse error at line 1, column 4: syntax error while parsing value - invalid literal; last read: 'nulm'", json::parse_error&); - - // unexpected end of true - CHECK_THROWS_WITH_AS(parser_helper("t"), - "[json.exception.parse_error.101] parse error at line 1, column 2: syntax error while parsing value - invalid literal; last read: 't'", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("tr"), - "[json.exception.parse_error.101] parse error at line 1, column 3: syntax error while parsing value - invalid literal; last read: 'tr'", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("tru"), - "[json.exception.parse_error.101] parse error at line 1, column 4: syntax error while parsing value - invalid literal; last read: 'tru'", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("trud"), - "[json.exception.parse_error.101] parse error at line 1, column 4: syntax error while parsing value - invalid literal; last read: 'trud'", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("truf"), - "[json.exception.parse_error.101] parse error at line 1, column 4: syntax error while parsing value - invalid literal; last read: 'truf'", json::parse_error&); - - // unexpected end of false - CHECK_THROWS_WITH_AS(parser_helper("f"), - "[json.exception.parse_error.101] parse error at line 1, column 2: syntax error while parsing value - invalid literal; last read: 'f'", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("fa"), - "[json.exception.parse_error.101] parse error at line 1, column 3: syntax error while parsing value - invalid literal; last read: 'fa'", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("fal"), - "[json.exception.parse_error.101] parse error at line 1, column 4: syntax error while parsing value - invalid literal; last read: 'fal'", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("fals"), - "[json.exception.parse_error.101] parse error at line 1, column 5: syntax error while parsing value - invalid literal; last read: 'fals'", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("falsd"), - "[json.exception.parse_error.101] parse error at line 1, column 5: syntax error while parsing value - invalid literal; last read: 'falsd'", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("falsf"), - "[json.exception.parse_error.101] parse error at line 1, column 5: syntax error while parsing value - invalid literal; last read: 'falsf'", json::parse_error&); - - // missing/unexpected end of array - CHECK_THROWS_WITH_AS(parser_helper("["), - "[json.exception.parse_error.101] parse error at line 1, column 2: syntax error while parsing value - unexpected end of input; expected '[', '{', or a literal", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("[1"), - "[json.exception.parse_error.101] parse error at line 1, column 3: syntax error while parsing array - unexpected end of input; expected ']'", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("[1,"), - "[json.exception.parse_error.101] parse error at line 1, column 4: syntax error while parsing value - unexpected end of input; expected '[', '{', or a literal", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("[1,]"), - "[json.exception.parse_error.101] parse error at line 1, column 4: syntax error while parsing value - unexpected ']'; expected '[', '{', or a literal", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("]"), - "[json.exception.parse_error.101] parse error at line 1, column 1: syntax error while parsing value - unexpected ']'; expected '[', '{', or a literal", json::parse_error&); - - // missing/unexpected end of object - CHECK_THROWS_WITH_AS(parser_helper("{"), - "[json.exception.parse_error.101] parse error at line 1, column 2: syntax error while parsing object key - unexpected end of input; expected string literal", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("{\"foo\""), - "[json.exception.parse_error.101] parse error at line 1, column 7: syntax error while parsing object separator - unexpected end of input; expected ':'", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("{\"foo\":"), - "[json.exception.parse_error.101] parse error at line 1, column 8: syntax error while parsing value - unexpected end of input; expected '[', '{', or a literal", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("{\"foo\":}"), - "[json.exception.parse_error.101] parse error at line 1, column 8: syntax error while parsing value - unexpected '}'; expected '[', '{', or a literal", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("{\"foo\":1,}"), - "[json.exception.parse_error.101] parse error at line 1, column 10: syntax error while parsing object key - unexpected '}'; expected string literal", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("}"), - "[json.exception.parse_error.101] parse error at line 1, column 1: syntax error while parsing value - unexpected '}'; expected '[', '{', or a literal", json::parse_error&); - - // missing/unexpected end of string - CHECK_THROWS_WITH_AS(parser_helper("\""), - "[json.exception.parse_error.101] parse error at line 1, column 2: syntax error while parsing value - invalid string: missing closing quote; last read: '\"'", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("\"\\\""), - "[json.exception.parse_error.101] parse error at line 1, column 4: syntax error while parsing value - invalid string: missing closing quote; last read: '\"\\\"'", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("\"\\u\""), - "[json.exception.parse_error.101] parse error at line 1, column 4: syntax error while parsing value - invalid string: '\\u' must be followed by 4 hex digits; last read: '\"\\u\"'", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("\"\\u0\""), - "[json.exception.parse_error.101] parse error at line 1, column 5: syntax error while parsing value - invalid string: '\\u' must be followed by 4 hex digits; last read: '\"\\u0\"'", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("\"\\u01\""), - "[json.exception.parse_error.101] parse error at line 1, column 6: syntax error while parsing value - invalid string: '\\u' must be followed by 4 hex digits; last read: '\"\\u01\"'", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("\"\\u012\""), - "[json.exception.parse_error.101] parse error at line 1, column 7: syntax error while parsing value - invalid string: '\\u' must be followed by 4 hex digits; last read: '\"\\u012\"'", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("\"\\u"), - "[json.exception.parse_error.101] parse error at line 1, column 4: syntax error while parsing value - invalid string: '\\u' must be followed by 4 hex digits; last read: '\"\\u'", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("\"\\u0"), - "[json.exception.parse_error.101] parse error at line 1, column 5: syntax error while parsing value - invalid string: '\\u' must be followed by 4 hex digits; last read: '\"\\u0'", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("\"\\u01"), - "[json.exception.parse_error.101] parse error at line 1, column 6: syntax error while parsing value - invalid string: '\\u' must be followed by 4 hex digits; last read: '\"\\u01'", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("\"\\u012"), - "[json.exception.parse_error.101] parse error at line 1, column 7: syntax error while parsing value - invalid string: '\\u' must be followed by 4 hex digits; last read: '\"\\u012'", json::parse_error&); - - // invalid escapes - for (int c = 1; c < 128; ++c) - { - auto s = std::string("\"\\") + std::string(1, static_cast(c)) + "\""; - - switch (c) - { - // valid escapes - case ('"'): - case ('\\'): - case ('/'): - case ('b'): - case ('f'): - case ('n'): - case ('r'): - case ('t'): - { - CHECK_NOTHROW(parser_helper(s)); - break; - } - - // \u must be followed with four numbers, so we skip it here - case ('u'): - { - break; - } - - // any other combination of backslash and character is invalid - default: - { - CHECK_THROWS_AS(parser_helper(s), json::parse_error&); - // only check error message if c is not a control character - if (c > 0x1f) - { - CHECK_THROWS_WITH_STD_STR(parser_helper(s), - "[json.exception.parse_error.101] parse error at line 1, column 3: syntax error while parsing value - invalid string: forbidden character after backslash; last read: '\"\\" + std::string(1, static_cast(c)) + "'"); - } - break; - } - } - } - - // invalid \uxxxx escapes - { - // check whether character is a valid hex character - const auto valid = [](int c) - { - switch (c) - { - case ('0'): - case ('1'): - case ('2'): - case ('3'): - case ('4'): - case ('5'): - case ('6'): - case ('7'): - case ('8'): - case ('9'): - case ('a'): - case ('b'): - case ('c'): - case ('d'): - case ('e'): - case ('f'): - case ('A'): - case ('B'): - case ('C'): - case ('D'): - case ('E'): - case ('F'): - { - return true; - } - - default: - { - return false; - } - } - }; - - for (int c = 1; c < 128; ++c) - { - std::string const s = "\"\\u"; - - // create a string with the iterated character at each position - auto s1 = s + "000" + std::string(1, static_cast(c)) + "\""; - auto s2 = s + "00" + std::string(1, static_cast(c)) + "0\""; - auto s3 = s + "0" + std::string(1, static_cast(c)) + "00\""; - auto s4 = s + std::string(1, static_cast(c)) + "000\""; - - if (valid(c)) - { - CAPTURE(s1) - CHECK_NOTHROW(parser_helper(s1)); - CAPTURE(s2) - CHECK_NOTHROW(parser_helper(s2)); - CAPTURE(s3) - CHECK_NOTHROW(parser_helper(s3)); - CAPTURE(s4) - CHECK_NOTHROW(parser_helper(s4)); - } - else - { - CAPTURE(s1) - CHECK_THROWS_AS(parser_helper(s1), json::parse_error&); - // only check error message if c is not a control character - if (c > 0x1f) - { - CHECK_THROWS_WITH_STD_STR(parser_helper(s1), - "[json.exception.parse_error.101] parse error at line 1, column 7: syntax error while parsing value - invalid string: '\\u' must be followed by 4 hex digits; last read: '" + s1.substr(0, 7) + "'"); - } - - CAPTURE(s2) - CHECK_THROWS_AS(parser_helper(s2), json::parse_error&); - // only check error message if c is not a control character - if (c > 0x1f) - { - CHECK_THROWS_WITH_STD_STR(parser_helper(s2), - "[json.exception.parse_error.101] parse error at line 1, column 6: syntax error while parsing value - invalid string: '\\u' must be followed by 4 hex digits; last read: '" + s2.substr(0, 6) + "'"); - } - - CAPTURE(s3) - CHECK_THROWS_AS(parser_helper(s3), json::parse_error&); - // only check error message if c is not a control character - if (c > 0x1f) - { - CHECK_THROWS_WITH_STD_STR(parser_helper(s3), - "[json.exception.parse_error.101] parse error at line 1, column 5: syntax error while parsing value - invalid string: '\\u' must be followed by 4 hex digits; last read: '" + s3.substr(0, 5) + "'"); - } - - CAPTURE(s4) - CHECK_THROWS_AS(parser_helper(s4), json::parse_error&); - // only check error message if c is not a control character - if (c > 0x1f) - { - CHECK_THROWS_WITH_STD_STR(parser_helper(s4), - "[json.exception.parse_error.101] parse error at line 1, column 4: syntax error while parsing value - invalid string: '\\u' must be followed by 4 hex digits; last read: '" + s4.substr(0, 4) + "'"); - } - } - } - } - - json _; - - // missing part of a surrogate pair - CHECK_THROWS_WITH_AS(_ = json::parse("\"\\uD80C\""), "[json.exception.parse_error.101] parse error at line 1, column 8: syntax error while parsing value - invalid string: surrogate U+D800..U+DBFF must be followed by U+DC00..U+DFFF; last read: '\"\\uD80C\"'", json::parse_error&); - // invalid surrogate pair - CHECK_THROWS_WITH_AS(_ = json::parse("\"\\uD80C\\uD80C\""), - "[json.exception.parse_error.101] parse error at line 1, column 13: syntax error while parsing value - invalid string: surrogate U+D800..U+DBFF must be followed by U+DC00..U+DFFF; last read: '\"\\uD80C\\uD80C'", json::parse_error&); - CHECK_THROWS_WITH_AS(_ = json::parse("\"\\uD80C\\u0000\""), - "[json.exception.parse_error.101] parse error at line 1, column 13: syntax error while parsing value - invalid string: surrogate U+D800..U+DBFF must be followed by U+DC00..U+DFFF; last read: '\"\\uD80C\\u0000'", json::parse_error&); - CHECK_THROWS_WITH_AS(_ = json::parse("\"\\uD80C\\uFFFF\""), - "[json.exception.parse_error.101] parse error at line 1, column 13: syntax error while parsing value - invalid string: surrogate U+D800..U+DBFF must be followed by U+DC00..U+DFFF; last read: '\"\\uD80C\\uFFFF'", json::parse_error&); - } - - SECTION("parse errors (accept)") - { - // unexpected end of number - CHECK(accept_helper("0.") == false); - CHECK(accept_helper("-") == false); - CHECK(accept_helper("--") == false); - CHECK(accept_helper("-0.") == false); - CHECK(accept_helper("-.") == false); - CHECK(accept_helper("-:") == false); - CHECK(accept_helper("0.:") == false); - CHECK(accept_helper("e.") == false); - CHECK(accept_helper("1e.") == false); - CHECK(accept_helper("1e/") == false); - CHECK(accept_helper("1e:") == false); - CHECK(accept_helper("1E.") == false); - CHECK(accept_helper("1E/") == false); - CHECK(accept_helper("1E:") == false); - - // unexpected end of null - CHECK(accept_helper("n") == false); - CHECK(accept_helper("nu") == false); - CHECK(accept_helper("nul") == false); - - // unexpected end of true - CHECK(accept_helper("t") == false); - CHECK(accept_helper("tr") == false); - CHECK(accept_helper("tru") == false); - - // unexpected end of false - CHECK(accept_helper("f") == false); - CHECK(accept_helper("fa") == false); - CHECK(accept_helper("fal") == false); - CHECK(accept_helper("fals") == false); - - // missing/unexpected end of array - CHECK(accept_helper("[") == false); - CHECK(accept_helper("[1") == false); - CHECK(accept_helper("[1,") == false); - CHECK(accept_helper("[1,]") == false); - CHECK(accept_helper("]") == false); - - // missing/unexpected end of object - CHECK(accept_helper("{") == false); - CHECK(accept_helper("{\"foo\"") == false); - CHECK(accept_helper("{\"foo\":") == false); - CHECK(accept_helper("{\"foo\":}") == false); - CHECK(accept_helper("{\"foo\":1,}") == false); - CHECK(accept_helper("}") == false); - - // missing/unexpected end of string - CHECK(accept_helper("\"") == false); - CHECK(accept_helper("\"\\\"") == false); - CHECK(accept_helper("\"\\u\"") == false); - CHECK(accept_helper("\"\\u0\"") == false); - CHECK(accept_helper("\"\\u01\"") == false); - CHECK(accept_helper("\"\\u012\"") == false); - CHECK(accept_helper("\"\\u") == false); - CHECK(accept_helper("\"\\u0") == false); - CHECK(accept_helper("\"\\u01") == false); - CHECK(accept_helper("\"\\u012") == false); - - // unget of newline - CHECK(parser_helper("\n123\n") == 123); - - // invalid escapes - for (int c = 1; c < 128; ++c) - { - auto s = std::string("\"\\") + std::string(1, static_cast(c)) + "\""; - - switch (c) - { - // valid escapes - case ('"'): - case ('\\'): - case ('/'): - case ('b'): - case ('f'): - case ('n'): - case ('r'): - case ('t'): - { - CHECK(json::parser(nlohmann::detail::input_adapter(s)).accept()); - break; - } - - // \u must be followed with four numbers, so we skip it here - case ('u'): - { - break; - } - - // any other combination of backslash and character is invalid - default: - { - CHECK(json::parser(nlohmann::detail::input_adapter(s)).accept() == false); - break; - } - } - } - - // invalid \uxxxx escapes - { - // check whether character is a valid hex character - const auto valid = [](int c) - { - switch (c) - { - case ('0'): - case ('1'): - case ('2'): - case ('3'): - case ('4'): - case ('5'): - case ('6'): - case ('7'): - case ('8'): - case ('9'): - case ('a'): - case ('b'): - case ('c'): - case ('d'): - case ('e'): - case ('f'): - case ('A'): - case ('B'): - case ('C'): - case ('D'): - case ('E'): - case ('F'): - { - return true; - } - - default: - { - return false; - } - } - }; - - for (int c = 1; c < 128; ++c) - { - std::string const s = "\"\\u"; - - // create a string with the iterated character at each position - const auto s1 = s + "000" + std::string(1, static_cast(c)) + "\""; - const auto s2 = s + "00" + std::string(1, static_cast(c)) + "0\""; - const auto s3 = s + "0" + std::string(1, static_cast(c)) + "00\""; - const auto s4 = s + std::string(1, static_cast(c)) + "000\""; - - if (valid(c)) - { - CAPTURE(s1) - CHECK(json::parser(nlohmann::detail::input_adapter(s1)).accept()); - CAPTURE(s2) - CHECK(json::parser(nlohmann::detail::input_adapter(s2)).accept()); - CAPTURE(s3) - CHECK(json::parser(nlohmann::detail::input_adapter(s3)).accept()); - CAPTURE(s4) - CHECK(json::parser(nlohmann::detail::input_adapter(s4)).accept()); - } - else - { - CAPTURE(s1) - CHECK(json::parser(nlohmann::detail::input_adapter(s1)).accept() == false); - - CAPTURE(s2) - CHECK(json::parser(nlohmann::detail::input_adapter(s2)).accept() == false); - - CAPTURE(s3) - CHECK(json::parser(nlohmann::detail::input_adapter(s3)).accept() == false); - - CAPTURE(s4) - CHECK(json::parser(nlohmann::detail::input_adapter(s4)).accept() == false); - } - } - } - - // missing part of a surrogate pair - CHECK(accept_helper("\"\\uD80C\"") == false); - // invalid surrogate pair - CHECK(accept_helper("\"\\uD80C\\uD80C\"") == false); - CHECK(accept_helper("\"\\uD80C\\u0000\"") == false); - CHECK(accept_helper("\"\\uD80C\\uFFFF\"") == false); - } - - SECTION("tests found by mutate++") - { - // test case to make sure no comma precedes the first key - CHECK_THROWS_WITH_AS(parser_helper("{,\"key\": false}"), "[json.exception.parse_error.101] parse error at line 1, column 2: syntax error while parsing object key - unexpected ','; expected string literal", json::parse_error&); - // test case to make sure an object is properly closed - CHECK_THROWS_WITH_AS(parser_helper("[{\"key\": false true]"), "[json.exception.parse_error.101] parse error at line 1, column 19: syntax error while parsing object - unexpected true literal; expected '}'", json::parse_error&); - - // test case to make sure the callback is properly evaluated after reading a key - { - json::parser_callback_t const cb = [](int /*unused*/, json::parse_event_t event, json& /*unused*/) noexcept - { - return event != json::parse_event_t::key; - }; - - const json x = json::parse("{\"key\": false}", cb); - CHECK(x == json::object()); - } - } - - SECTION("callback function") - { - const auto* s_object = R"( - { - "foo": 2, - "bar": { - "baz": 1 - } - } - )"; - - const auto* s_array = R"( - [1,2,[3,4,5],4,5] - )"; - - const auto* structured_array = R"( - [ - 1, - { - "foo": "bar" - }, - { - "qux": "baz" - } - ] - )"; - - SECTION("filter nothing") - { - const json j_object = json::parse(s_object, [](int /*unused*/, json::parse_event_t /*unused*/, const json& /*unused*/) noexcept - { - return true; - }); - - CHECK (j_object == json({{"foo", 2}, {"bar", {{"baz", 1}}}})); - - const json j_array = json::parse(s_array, [](int /*unused*/, json::parse_event_t /*unused*/, const json& /*unused*/) noexcept - { - return true; - }); - - CHECK (j_array == json({1, 2, {3, 4, 5}, 4, 5})); - } - - SECTION("filter everything") - { - json const j_object = json::parse(s_object, [](int /*unused*/, json::parse_event_t /*unused*/, const json& /*unused*/) noexcept - { - return false; - }); - - // the top-level object will be discarded, leaving a null - CHECK (j_object.is_null()); - - json const j_array = json::parse(s_array, [](int /*unused*/, json::parse_event_t /*unused*/, const json& /*unused*/) noexcept - { - return false; - }); - - // the top-level array will be discarded, leaving a null - CHECK (j_array.is_null()); - } - - SECTION("filter specific element") - { - const json j_object = json::parse(s_object, [](int /*unused*/, json::parse_event_t event, const json & j) noexcept - { - // filter all number(2) elements - return event != json::parse_event_t::value || j != json(2); - }); - - CHECK (j_object == json({{"bar", {{"baz", 1}}}})); - - const json j_array = json::parse(s_array, [](int /*unused*/, json::parse_event_t event, const json & j) noexcept - { - return event != json::parse_event_t::value || j != json(2); - }); - - CHECK (j_array == json({1, {3, 4, 5}, 4, 5})); - } - - SECTION("filter object in array") - { - const json j_filtered1 = json::parse(structured_array, [](int /*unused*/, json::parse_event_t e, const json & parsed) - { - return !(e == json::parse_event_t::object_end && parsed.contains("foo")); - }); - - // the specified object will be discarded, and removed. - CHECK (j_filtered1.size() == 2); - CHECK (j_filtered1 == json({1, {{"qux", "baz"}}})); - - const json j_filtered2 = json::parse(structured_array, [](int /*unused*/, json::parse_event_t e, const json& /*parsed*/) noexcept - { - return e != json::parse_event_t::object_end; - }); - - // removed all objects in array. - CHECK (j_filtered2.size() == 1); - CHECK (j_filtered2 == json({1})); - } - - SECTION("filter specific events") - { - SECTION("first closing event") - { - { - const json j_object = json::parse(s_object, [](int /*unused*/, json::parse_event_t e, const json& /*unused*/) noexcept - { - static bool first = true; - if (e == json::parse_event_t::object_end && first) - { - first = false; - return false; - } - - return true; - }); - - // the first completed object will be discarded - CHECK (j_object == json({{"foo", 2}})); - } - - { - const json j_array = json::parse(s_array, [](int /*unused*/, json::parse_event_t e, const json& /*unused*/) noexcept - { - static bool first = true; - if (e == json::parse_event_t::array_end && first) - { - first = false; - return false; - } - - return true; - }); - - // the first completed array will be discarded - CHECK (j_array == json({1, 2, 4, 5})); - } - } - } - - SECTION("special cases") - { - // the following test cases cover the situation in which an empty - // object and array is discarded only after the closing character - // has been read - - const json j_empty_object = json::parse("{}", [](int /*unused*/, json::parse_event_t e, const json& /*unused*/) noexcept - { - return e != json::parse_event_t::object_end; - }); - CHECK(j_empty_object == json()); - - const json j_empty_array = json::parse("[]", [](int /*unused*/, json::parse_event_t e, const json& /*unused*/) noexcept - { - return e != json::parse_event_t::array_end; - }); - CHECK(j_empty_array == json()); - } - } - - SECTION("constructing from contiguous containers") - { - SECTION("from std::vector") - { - std::vector v = {'t', 'r', 'u', 'e'}; - json j; - json::parser(nlohmann::detail::input_adapter(std::begin(v), std::end(v))).parse(true, j); - CHECK(j == json(true)); - } - - SECTION("from std::array") - { - std::array v { {'t', 'r', 'u', 'e'} }; - json j; - json::parser(nlohmann::detail::input_adapter(std::begin(v), std::end(v))).parse(true, j); - CHECK(j == json(true)); - } - - SECTION("from array") - { - uint8_t v[] = {'t', 'r', 'u', 'e'}; // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays) - json j; - json::parser(nlohmann::detail::input_adapter(std::begin(v), std::end(v))).parse(true, j); - CHECK(j == json(true)); - } - - SECTION("from char literal") - { - CHECK(parser_helper("true") == json(true)); - } - - SECTION("from std::string") - { - std::string v = {'t', 'r', 'u', 'e'}; - json j; - json::parser(nlohmann::detail::input_adapter(std::begin(v), std::end(v))).parse(true, j); - CHECK(j == json(true)); - } - - SECTION("from std::initializer_list") - { - std::initializer_list const v = {'t', 'r', 'u', 'e'}; - json j; - json::parser(nlohmann::detail::input_adapter(std::begin(v), std::end(v))).parse(true, j); - CHECK(j == json(true)); - } - - SECTION("from std::valarray") - { - std::valarray v = {'t', 'r', 'u', 'e'}; - json j; - json::parser(nlohmann::detail::input_adapter(std::begin(v), std::end(v))).parse(true, j); - CHECK(j == json(true)); - } - } - - SECTION("improve test coverage") - { - SECTION("parser with callback") - { - json::parser_callback_t const cb = [](int /*unused*/, json::parse_event_t /*unused*/, json& /*unused*/) noexcept - { - return true; - }; - - CHECK(json::parse("{\"foo\": true:", cb, false).is_discarded()); - - json _; - CHECK_THROWS_WITH_AS(_ = json::parse("{\"foo\": true:", cb), "[json.exception.parse_error.101] parse error at line 1, column 13: syntax error while parsing object - unexpected ':'; expected '}'", json::parse_error&); - - CHECK_THROWS_WITH_AS(_ = json::parse("1.18973e+4932", cb), "[json.exception.out_of_range.406] number overflow parsing '1.18973e+4932'", json::out_of_range&); - } - - SECTION("SAX parser") - { - SECTION("} without value") - { - SaxCountdown s(1); - CHECK(json::sax_parse("{}", &s) == false); - } - - SECTION("} with value") - { - SaxCountdown s(3); - CHECK(json::sax_parse("{\"k1\": true}", &s) == false); - } - - SECTION("second key") - { - SaxCountdown s(3); - CHECK(json::sax_parse("{\"k1\": true, \"k2\": false}", &s) == false); - } - - SECTION("] without value") - { - SaxCountdown s(1); - CHECK(json::sax_parse("[]", &s) == false); - } - - SECTION("] with value") - { - SaxCountdown s(2); - CHECK(json::sax_parse("[1]", &s) == false); - } - - SECTION("float") - { - SaxCountdown s(0); - CHECK(json::sax_parse("3.14", &s) == false); - } - - SECTION("false") - { - SaxCountdown s(0); - CHECK(json::sax_parse("false", &s) == false); - } - - SECTION("null") - { - SaxCountdown s(0); - CHECK(json::sax_parse("null", &s) == false); - } - - SECTION("true") - { - SaxCountdown s(0); - CHECK(json::sax_parse("true", &s) == false); - } - - SECTION("unsigned") - { - SaxCountdown s(0); - CHECK(json::sax_parse("12", &s) == false); - } - - SECTION("integer") - { - SaxCountdown s(0); - CHECK(json::sax_parse("-12", &s) == false); - } - - SECTION("string") - { - SaxCountdown s(0); - CHECK(json::sax_parse("\"foo\"", &s) == false); - } - } - } - - SECTION("error messages for comments") - { - json _; - CHECK_THROWS_WITH_AS(_ = json::parse("/a", nullptr, true, true), "[json.exception.parse_error.101] parse error at line 1, column 2: syntax error while parsing value - invalid comment; expecting '/' or '*' after '/'; last read: '/a'", json::parse_error); - CHECK_THROWS_WITH_AS(_ = json::parse("/*", nullptr, true, true), "[json.exception.parse_error.101] parse error at line 1, column 3: syntax error while parsing value - invalid comment; missing closing '*/'; last read: '/*'", json::parse_error); - } - - // Macro for all test cases for start_pos and end_pos -#define SETUP_TESTCASES() \ - SECTION("with callback") \ - { \ - SECTION("filter nothing") \ - { \ - json::parser_callback_t const cb = [](int /*unused*/, json::parse_event_t /*unused*/, json& /*unused*/) noexcept \ - { \ - return true; \ - }; \ - validate_start_end_pos_for_nested_obj_helper(nested_type_json_str, root_type_json_str, expected, cb); \ - } \ - SECTION("filter element") \ - { \ - json::parser_callback_t const cb = [](int /*unused*/, json::parse_event_t event, json& j) noexcept \ - { \ - return (event != json::parse_event_t::key && event != json::parse_event_t::value) || j != json("a"); \ - }; \ - validate_start_end_pos_for_nested_obj_helper(nested_type_json_str, root_type_json_str, filteredExpected, cb); \ - } \ - } \ - SECTION("without callback") \ - { \ - validate_start_end_pos_for_nested_obj_helper(nested_type_json_str, root_type_json_str, expected); \ - } - - SECTION("retrieve start position and end position") - { - SECTION("for object") - { - // Create an object with spaces to test the start and end positions. Spaces will not be included in the - // JSON object, however, the start and end positions should include the spaces from the input JSON string. - const std::string nested_type_json_str = R"({ "a": 1,"b" : "test1"})"; - const std::string root_type_json_str = R"({ "nested": )" + nested_type_json_str + R"(, "anotherValue": "test2"})"; - auto expected = json({{"nested", {{"a", 1}, {"b", "test1"}}}, {"anotherValue", "test2"}}); - auto filteredExpected = expected; - filteredExpected["nested"].erase("a"); - - SETUP_TESTCASES() - } - - SECTION("for array") - { - const std::string nested_type_json_str = R"(["a", "test", 45])"; - const std::string root_type_json_str = R"({ "nested": )" + nested_type_json_str + R"(, "anotherValue": "test" })"; - auto expected = json({{"nested", {"a", "test", 45}}, {"anotherValue", "test"}}); - auto filteredExpected = expected; - filteredExpected["nested"] = json({"test", 45}); - SETUP_TESTCASES() - } - - SECTION("for array with objects") - { - const std::string nested_type_json_str = R"([{"a": 1, "b": "test"}, {"c": 2, "d": "test2"}])"; - const std::string root_type_json_str = R"({ "nested": )" + nested_type_json_str + R"(, "anotherValue": "test" })"; - auto expected = json({{"nested", {{{"a", 1}, {"b", "test"}}, {{"c", 2}, {"d", "test2"}}}}, {"anotherValue", "test"}}); - auto filteredExpected = expected; - filteredExpected["nested"][0].erase("a"); - SETUP_TESTCASES() - - auto j = json::parse(root_type_json_str); - auto nested_array = j["nested"]; - const auto& nested_obj = nested_array[0]; - CHECK(nested_type_json_str.substr(1, 21) == root_type_json_str.substr(nested_obj.start_pos(), nested_obj.end_pos() - nested_obj.start_pos())); - CHECK(nested_type_json_str.substr(24, 22) == root_type_json_str.substr(nested_array[1].start_pos(), nested_array[1].end_pos() - nested_array[1].start_pos())); - } - - SECTION("for two levels of nesting objects") - { - const std::string nested_type_json_str = R"({"nested2": {"b": "test"}})"; - const std::string root_type_json_str = R"({ "a": 2, "nested": )" + nested_type_json_str + R"(, "anotherValue": "test" })"; - auto expected = json({{"a", 2}, {"nested", {{"nested2", {{"b", "test"}}}}}, {"anotherValue", "test"}}); - auto filteredExpected = expected; - filteredExpected.erase("a"); - SETUP_TESTCASES() - - auto j = json::parse(root_type_json_str); - auto nested_obj = j["nested"]["nested2"]; - CHECK(nested_type_json_str.substr(12, 13) == root_type_json_str.substr(nested_obj.start_pos(), nested_obj.end_pos() - nested_obj.start_pos())); - } - - SECTION("for simple types") - { - SECTION("no nested") - { - SECTION("with callback") - { - json::parser_callback_t const cb = [](int /*unused*/, json::parse_event_t /*unused*/, json& /*unused*/) noexcept - { - return true; - }; - - // 1. string type - std::string json_str = R"("test")"; - auto j = json::parse(json_str, cb); - validate_generated_json_and_start_end_pos_helper(json_str, j, "test"); - - // 2. number type - json_str = R"(1)"; - j = json::parse(json_str, cb); - validate_generated_json_and_start_end_pos_helper(json_str, j, 1); - - // 3. boolean type - json_str = R"(true)"; - j = json::parse(json_str, cb); - validate_generated_json_and_start_end_pos_helper(json_str, j, true); - - // 4. null type - json_str = R"(null)"; - j = json::parse(json_str, cb); - validate_generated_json_and_start_end_pos_helper(json_str, j, nullptr); - } - - SECTION("without callback") - { - // 1. string type - std::string json_str = R"("test")"; - auto j = json::parse(json_str); - validate_generated_json_and_start_end_pos_helper(json_str, j, "test"); - - // 2. number type - json_str = R"(1)"; - j = json::parse(json_str); - validate_generated_json_and_start_end_pos_helper(json_str, j, 1); - - json_str = R"(1.001239923)"; - j = json::parse(json_str); - validate_generated_json_and_start_end_pos_helper(json_str, j, 1.001239923); - - json_str = R"(1.123812389000000)"; - j = json::parse(json_str); - validate_generated_json_and_start_end_pos_helper(json_str, j, 1.123812389); - - // 3. boolean type - json_str = R"(true)"; - j = json::parse(json_str); - validate_generated_json_and_start_end_pos_helper(json_str, j, true); - - json_str = R"(false)"; - j = json::parse(json_str); - validate_generated_json_and_start_end_pos_helper(json_str, j, false); - - // 4. null type - json_str = R"(null)"; - j = json::parse(json_str); - validate_generated_json_and_start_end_pos_helper(json_str, j, nullptr); - } - } - - SECTION("string type") - { - const std::string nested_type_json_str = R"("test")"; - const std::string root_type_json_str = R"({ "a": 1, "nested": )" + nested_type_json_str + R"(, "anotherValue": "test" })"; - auto expected = json({{"nested", "test"}, {"anotherValue", "test"}, {"a", 1}}); - auto filteredExpected = expected; - filteredExpected.erase("a"); - SETUP_TESTCASES() - } - - SECTION("number type") - { - const std::string nested_type_json_str = R"(2)"; - const std::string root_type_json_str = R"({ "a": 1, "nested": )" + nested_type_json_str + R"(, "anotherValue": "test" })"; - auto expected = json({{"nested", 2}, {"anotherValue", "test"}, {"a", 1}}); - auto filteredExpected = expected; - filteredExpected.erase("a"); - SETUP_TESTCASES() - } - - SECTION("boolean type") - { - const std::string nested_type_json_str = R"(true)"; - const std::string root_type_json_str = R"({ "a": 1, "nested": )" + nested_type_json_str + R"(, "anotherValue": "test" })"; - auto expected = json({{"nested", true}, {"anotherValue", "test"}, {"a", 1}}); - auto filteredExpected = expected; - filteredExpected.erase("a"); - SETUP_TESTCASES() - } - - SECTION("null type") - { - const std::string nested_type_json_str = R"(null)"; - const std::string root_type_json_str = R"({ "a": 1, "nested": )" + nested_type_json_str + R"(, "anotherValue": "test" })"; - auto expected = json({{"nested", nullptr}, {"anotherValue", "test"}, {"a", 1}}); - auto filteredExpected = expected; - filteredExpected.erase("a"); - SETUP_TESTCASES() - } - } - SECTION("with leading whitespace and newlines around root JSON") - { - const std::string initial_whitespace = R"( - - )"; - const std::string nested_type_json_str = R"({ - "a": 1, - "nested": { - "b": "test" - }, - "anotherValue": "test" - })"; - const std::string end_whitespace = R"( - - )"; - const std::string root_type_json_str = initial_whitespace + nested_type_json_str + end_whitespace; - - auto expected = json({{"a", 1}, {"nested", {{"b", "test"}}}, {"anotherValue", "test"}}); - - auto j = json::parse(root_type_json_str); - - // 2. Check if the generated JSON is as expected - CHECK(j == expected); - - // 3. Check if the start and end positions do not include the surrounding whitespace - CHECK(j.start_pos() == initial_whitespace.size()); - CHECK(j.end_pos() == root_type_json_str.size() - end_whitespace.size()); - } - } -} diff --git a/tests/src/unit-convenience.cpp b/tests/src/unit-convenience.cpp index 037a4e589..0ba57d846 100644 --- a/tests/src/unit-convenience.cpp +++ b/tests/src/unit-convenience.cpp @@ -98,8 +98,10 @@ void check_escaped(const char* original, const char* escaped = "", bool ensure_a void check_escaped(const char* original, const char* escaped, const bool ensure_ascii) { std::stringstream ss; - json::serializer s(nlohmann::detail::output_adapter(ss), ' '); - s.dump_escaped(original, ensure_ascii); + nlohmann::detail::output_stream_adapter adapter(ss); + json::serializer s(adapter, ' ', false, ensure_ascii); + s.dump_escaped(original); + s.flush(); // dump_escaped writes into the serializer's internal buffer CHECK(ss.str() == escaped); } } // namespace diff --git a/tests/src/unit-conversions.cpp b/tests/src/unit-conversions.cpp index 1937affbb..4975854c0 100644 --- a/tests/src/unit-conversions.cpp +++ b/tests/src/unit-conversions.cpp @@ -1389,6 +1389,37 @@ TEST_CASE("value conversion") // CHECK(m5["one"] == "eins"); } + SECTION("reserve is called on containers that support it (#5406)") + { + // build a larger object so that a missing/incorrect reserve() + // call would be more likely to corrupt or drop elements + json j_large; + for (int i = 0; i < 100; ++i) + { + j_large[std::to_string(i)] = i; + } + + SECTION("std::unordered_map (supports reserve)") + { + const auto m = j_large.get>(); + CHECK(m.size() == 100); + for (int i = 0; i < 100; ++i) + { + CHECK(m.at(std::to_string(i)) == i); + } + } + + SECTION("std::map (no reserve, fallback path)") + { + const auto m = j_large.get>(); + CHECK(m.size() == 100); + for (int i = 0; i < 100; ++i) + { + CHECK(m.at(std::to_string(i)) == i); + } + } + } + SECTION("std::multimap") { j1.get>(); diff --git a/tests/src/unit-diagnostic-positions-only.cpp b/tests/src/unit-diagnostic-positions-only.cpp deleted file mode 100644 index 735376514..000000000 --- a/tests/src/unit-diagnostic-positions-only.cpp +++ /dev/null @@ -1,44 +0,0 @@ -// __ _____ _____ _____ -// __| | __| | | | JSON for Modern C++ (supporting code) -// | | |__ | | | | | | version 3.12.0 -// |_____|_____|_____|_|___| https://github.com/nlohmann/json -// -// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann -// SPDX-License-Identifier: MIT - -#include "doctest_compatibility.h" - -#ifdef JSON_DIAGNOSTICS - #undef JSON_DIAGNOSTICS -#endif - -#define JSON_DIAGNOSTICS 0 -#define JSON_DIAGNOSTIC_POSITIONS 1 -#include - -using json = nlohmann::json; - -TEST_CASE("Better diagnostics with positions only") -{ - SECTION("invalid type") - { - const std::string json_invalid_string = R"( - { - "address": { - "street": "Fake Street", - "housenumber": "1" - } - } - )"; - json j = json::parse(json_invalid_string); - CHECK_THROWS_WITH_AS(j.at("address").at("housenumber").get(), - "[json.exception.type_error.302] (bytes 108-111) type must be number, but is string", json::type_error); - } - - SECTION("invalid type without positions") - { - const json j = "foo"; - CHECK_THROWS_WITH_AS(j.get(), - "[json.exception.type_error.302] type must be number, but is string", json::type_error); - } -} diff --git a/tests/src/unit-diagnostic-positions.cpp b/tests/src/unit-diagnostic-positions.cpp index ad9527540..4d2f50a98 100644 --- a/tests/src/unit-diagnostic-positions.cpp +++ b/tests/src/unit-diagnostic-positions.cpp @@ -8,7 +8,9 @@ #include "doctest_compatibility.h" -#define JSON_DIAGNOSTICS 1 +#ifndef JSON_DIAGNOSTICS + #define JSON_DIAGNOSTICS 1 +#endif #define JSON_DIAGNOSTIC_POSITIONS 1 #include @@ -27,8 +29,13 @@ TEST_CASE("Better diagnostics with positions") } )"; json j = json::parse(json_invalid_string); +#if JSON_DIAGNOSTICS CHECK_THROWS_WITH_AS(j.at("address").at("housenumber").get(), "[json.exception.type_error.302] (/address/housenumber) (bytes 108-111) type must be number, but is string", json::type_error); +#else + CHECK_THROWS_WITH_AS(j.at("address").at("housenumber").get(), + "[json.exception.type_error.302] (bytes 108-111) type must be number, but is string", json::type_error); +#endif } SECTION("invalid type without positions") @@ -74,7 +81,12 @@ TEST_CASE("Better diagnostics with positions") // (/foo/bar); the position of that parent is reported in the message const json doc = json::parse(R"({"foo":{"bar":"a string"}})"); const json patch = json::parse(R"([{"op":"add","path":"/foo/bar/baz","value":1}])"); +#if JSON_DIAGNOSTICS CHECK_THROWS_WITH_AS(doc.patch(patch), "[json.exception.out_of_range.411] (/foo/bar) (bytes 14-24) cannot add value: the JSON Patch 'add' target's parent is of type string, but must be an object or array", json::out_of_range); +#else + CHECK_THROWS_WITH_AS(doc.patch(patch), + "[json.exception.out_of_range.411] (bytes 14-24) cannot add value: the JSON Patch 'add' target's parent is of type string, but must be an object or array", json::out_of_range); +#endif } } diff --git a/tests/src/unit-items-cpp17.cpp b/tests/src/unit-items-cpp17.cpp new file mode 100644 index 000000000..577dcea56 --- /dev/null +++ b/tests/src/unit-items-cpp17.cpp @@ -0,0 +1,42 @@ +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ (supporting code) +// | | |__ | | | | | | version 3.12.0 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann +// SPDX-License-Identifier: MIT + +// This file contains the C++17-only part of unit-items.cpp (structured +// bindings support for json::items()). It is kept in a separate +// translation unit so the (much larger) unit-items.cpp does not need to +// be compiled a second time just for this one SECTION. + +#include "doctest_compatibility.h" + +#include +using nlohmann::json; + +#ifdef JSON_HAS_CPP_17 +#include +#include + +TEST_CASE("items()") +{ + SECTION("object") + { + SECTION("structured bindings") + { + json j = { {"A", 1}, {"B", 2} }; + + std::map m; + + for (auto const&[key, value] : j.items()) + { + m.emplace(key, value); + } + + CHECK(j.get() == m); + } + } +} +#endif diff --git a/tests/src/unit-items.cpp b/tests/src/unit-items.cpp index fa8948447..81959db8d 100644 --- a/tests/src/unit-items.cpp +++ b/tests/src/unit-items.cpp @@ -862,22 +862,6 @@ TEST_CASE("items()") CHECK(counter == 3); } - -#ifdef JSON_HAS_CPP_17 - SECTION("structured bindings") - { - json j = { {"A", 1}, {"B", 2} }; - - std::map m; - - for (auto const&[key, value] : j.items()) - { - m.emplace(key, value); - } - - CHECK(j.get() == m); - } -#endif } SECTION("const object") diff --git a/tests/src/unit-json_patch.cpp b/tests/src/unit-json_patch.cpp index 17d30f189..257e455aa 100644 --- a/tests/src/unit-json_patch.cpp +++ b/tests/src/unit-json_patch.cpp @@ -1389,6 +1389,192 @@ TEST_CASE("JSON patch - add to a primitive parent (regression #4292)") } } +TEST_CASE("JSON patch - remove with primitive or null parent (regression #5396)") +{ + // Regression test for https://github.com/nlohmann/json/issues/5396 + // + // RFC 6902 (§4.2) requires the target location of a "remove" operation + // to exist. When the target's parent resolves to a primitive value or + // null, the operation must fail. Previously operation_remove silently + // did nothing in this case (neither the "is_object" nor the "is_array" + // branch matched, and there was no final "else"), so the patch appeared + // to succeed without changing the document. It now throws + // out_of_range.413. + + SECTION("parent is a primitive (number)") + { + json const doc = {{"a", 1}}; + json const patch = {{{"op", "remove"}, {"path", "/a/b"}}}; +#if JSON_DIAGNOSTICS + CHECK_THROWS_WITH_AS(doc.patch(patch), "[json.exception.out_of_range.413] (/a) cannot remove value: the JSON Patch 'remove' target's parent is of type number, but must be an object or array", json::out_of_range&); +#else + CHECK_THROWS_WITH_AS(doc.patch(patch), "[json.exception.out_of_range.413] cannot remove value: the JSON Patch 'remove' target's parent is of type number, but must be an object or array", json::out_of_range&); +#endif + } + + SECTION("parent is a primitive (string)") + { + json const doc = {{"foo", {{"bar", "a string"}}}}; + json const patch = {{{"op", "remove"}, {"path", "/foo/bar/baz"}}}; +#if JSON_DIAGNOSTICS + CHECK_THROWS_WITH_AS(doc.patch(patch), "[json.exception.out_of_range.413] (/foo/bar) cannot remove value: the JSON Patch 'remove' target's parent is of type string, but must be an object or array", json::out_of_range&); +#else + CHECK_THROWS_WITH_AS(doc.patch(patch), "[json.exception.out_of_range.413] cannot remove value: the JSON Patch 'remove' target's parent is of type string, but must be an object or array", json::out_of_range&); +#endif + } + + SECTION("top-level document is null") + { + json const doc = nullptr; + json const patch = {{{"op", "remove"}, {"path", "/a"}}}; + CHECK_THROWS_WITH_AS(doc.patch(patch), "[json.exception.out_of_range.413] cannot remove value: the JSON Patch 'remove' target's parent is of type null, but must be an object or array", json::out_of_range&); + } + + SECTION("legitimate removes still work") + { + // object member + json const doc1 = {{"a", 1}, {"b", 2}}; + json const patch1 = {{{"op", "remove"}, {"path", "/a"}}}; + CHECK(doc1.patch(patch1) == json({{"b", 2}})); + + // array element + json const doc2 = R"([1, 2, 3])"_json; + json const patch2 = {{{"op", "remove"}, {"path", "/1"}}}; + CHECK(doc2.patch(patch2) == R"([1, 3])"_json); + } +} + +TEST_CASE("JSON patch - move where 'from' is a proper prefix of 'path' (regression #5397)") +{ + // Regression test for https://github.com/nlohmann/json/issues/5397 + // + // RFC 6902 (§4.4) forbids "from" from being a proper prefix of "path" + // for a "move" operation: "a location cannot be moved into one of its + // children." "move" is implemented as remove-then-add; for an object + // target this happened to throw anyway as a side effect of the "add" + // step re-resolving through the now-removed parent, but for an array + // target the removal shifted subsequent indices, so "path" silently + // re-resolved to a different element and the operation "succeeded" + // with a corrupted result. It now throws out_of_range.414 for both + // object and array targets. + + SECTION("array target (from the issue)") + { + json const doc = R"([[1,2],[3]])"_json; + json const patch = {{{"op", "move"}, {"from", "/0"}, {"path", "/0/0"}}}; +#if JSON_DIAGNOSTIC_POSITIONS + CHECK_THROWS_WITH_AS(doc.patch(patch), "[json.exception.out_of_range.414] (bytes 0-11) cannot move value: 'from' path '/0' is a proper prefix of 'path' '/0/0'", json::out_of_range&); +#else + CHECK_THROWS_WITH_AS(doc.patch(patch), "[json.exception.out_of_range.414] cannot move value: 'from' path '/0' is a proper prefix of 'path' '/0/0'", json::out_of_range&); +#endif + } + + SECTION("object target") + { + json const doc = R"({"a": {"b": 1}})"_json; + json const patch = {{{"op", "move"}, {"from", "/a"}, {"path", "/a/b"}}}; +#if JSON_DIAGNOSTIC_POSITIONS + CHECK_THROWS_WITH_AS(doc.patch(patch), "[json.exception.out_of_range.414] (bytes 0-15) cannot move value: 'from' path '/a' is a proper prefix of 'path' '/a/b'", json::out_of_range&); +#else + CHECK_THROWS_WITH_AS(doc.patch(patch), "[json.exception.out_of_range.414] cannot move value: 'from' path '/a' is a proper prefix of 'path' '/a/b'", json::out_of_range&); +#endif + } + + SECTION("from == path is not a proper prefix and must not be rejected") + { + // "from" equal to "path" is a no-op move; it is not a *proper* + // prefix relationship, so this new check must not reject it. + json const doc = R"({"a": 1, "b": 2})"_json; + json const patch = {{{"op", "move"}, {"from", "/a"}, {"path", "/a"}}}; + CHECK(doc.patch(patch) == doc); + } + + SECTION("raw string prefix that is not a pointer-token prefix must be allowed") + { + // "/ab" is a string-prefix of "/abc/x" as raw text, but "ab" and + // "abc" are different reference tokens, so this is NOT a + // pointer-token prefix relationship and the move must succeed. + // This is the key case proving the check compares tokens, not + // raw pointer text (a naive std::string prefix/rfind check on + // the undecoded pointer would wrongly reject this). + json const doc = R"({"ab": 1, "abc": {"x": 2}})"_json; + json const patch = {{{"op", "move"}, {"from", "/ab"}, {"path", "/abc/x"}}}; + json const result = R"({"abc": {"x": 1}})"_json; + CHECK(doc.patch(patch) == result); + } + + SECTION("escaped reference tokens are compared unescaped") + { + // "from" is the single token "a/b" (escaped as "a~1b"); "path" + // addresses member "x" of that same value, so "from" is a + // proper (token-level) prefix of "path" and must be rejected. + json const doc = R"({"a/b": {"x": 1}})"_json; + json const patch = {{{"op", "move"}, {"from", "/a~1b"}, {"path", "/a~1b/x"}}}; +#if JSON_DIAGNOSTIC_POSITIONS + CHECK_THROWS_WITH_AS(doc.patch(patch), "[json.exception.out_of_range.414] (bytes 0-17) cannot move value: 'from' path '/a~1b' is a proper prefix of 'path' '/a~1b/x'", json::out_of_range&); +#else + CHECK_THROWS_WITH_AS(doc.patch(patch), "[json.exception.out_of_range.414] cannot move value: 'from' path '/a~1b' is a proper prefix of 'path' '/a~1b/x'", json::out_of_range&); +#endif + } + + SECTION("ordinary valid moves still work") + { + // unrelated top-level members + json const doc1 = R"({"a": 1, "b": 2})"_json; + json const patch1 = {{{"op", "move"}, {"from", "/a"}, {"path", "/c"}}}; + CHECK(doc1.patch(patch1) == R"({"b": 2, "c": 1})"_json); + + // sibling paths that share a textual prefix but are unrelated + json const doc2 = R"({"a": {"x": 1}, "b": {"y": 2}})"_json; + json const patch2 = {{{"op", "move"}, {"from", "/a/x"}, {"path", "/b/z"}}}; + CHECK(doc2.patch(patch2) == R"({"a": {}, "b": {"y": 2, "z": 1}})"_json); + + // "path" is a proper prefix of "from" (the reverse relationship, + // which RFC 6902 does not forbid) + json const doc3 = R"({"a": {"b": 1}})"_json; + json const patch3 = {{{"op", "move"}, {"from", "/a/b"}, {"path", "/a"}}}; + CHECK(doc3.patch(patch3) == R"({"a": 1})"_json); + } + + SECTION("root 'from' is a proper prefix of every non-root 'path'") + { + // the whole document is a proper prefix of any location inside it + json const doc = R"({"a": 1})"_json; + json const patch = {{{"op", "move"}, {"from", ""}, {"path", "/a"}}}; +#if JSON_DIAGNOSTIC_POSITIONS + CHECK_THROWS_WITH_AS(doc.patch(patch), "[json.exception.out_of_range.414] (bytes 0-8) cannot move value: 'from' path '' is a proper prefix of 'path' '/a'", json::out_of_range&); +#else + CHECK_THROWS_WITH_AS(doc.patch(patch), "[json.exception.out_of_range.414] cannot move value: 'from' path '' is a proper prefix of 'path' '/a'", json::out_of_range&); +#endif + } + + SECTION("root 'path' is never a proper prefix violation for a non-root 'from'") + { + // the reverse of the above: moving a non-root location to the root + // is the "path is a prefix of from" relationship, which RFC 6902 + // permits (already covered generally above; this pins the root + // case specifically, since root is the one path with no reference + // tokens at all) + json const doc = R"({"a": {"b": 1}})"_json; + json const patch = {{{"op", "move"}, {"from", "/a"}, {"path", ""}}}; + CHECK(doc.patch(patch) == R"({"b": 1})"_json); + } + + SECTION("the array-append token '-' is an ordinary child token") + { + // "-" (append-to-array) addresses a location *inside* the array, + // so "from" pointing at the array is still a proper prefix of + // "path" ending in "-" and must be rejected like any other child. + json const doc = R"({"a": [1, 2]})"_json; + json const patch = {{{"op", "move"}, {"from", "/a"}, {"path", "/a/-"}}}; +#if JSON_DIAGNOSTIC_POSITIONS + CHECK_THROWS_WITH_AS(doc.patch(patch), "[json.exception.out_of_range.414] (bytes 0-13) cannot move value: 'from' path '/a' is a proper prefix of 'path' '/a/-'", json::out_of_range&); +#else + CHECK_THROWS_WITH_AS(doc.patch(patch), "[json.exception.out_of_range.414] cannot move value: 'from' path '/a' is a proper prefix of 'path' '/a/-'", json::out_of_range&); +#endif + } +} + TEST_CASE("JSON patch - diff emits array removals in descending index order") { SECTION("array shrunk to empty") diff --git a/tests/src/unit-json_pointer.cpp b/tests/src/unit-json_pointer.cpp index a8ed4a89e..4082de45c 100644 --- a/tests/src/unit-json_pointer.cpp +++ b/tests/src/unit-json_pointer.cpp @@ -319,6 +319,44 @@ TEST_CASE("JSON pointers") CHECK_THROWS_WITH_AS(j[jp] = 1, throw_msg.c_str(), json::out_of_range&); CHECK_THROWS_WITH_AS(j_const[jp] == 1, throw_msg.c_str(), json::out_of_range&); + + // #5395: contains() must not throw for a reference token that is a + // syntactically valid array index but numerically exceeds ULLONG_MAX + // (causing strtoull() to set errno to ERANGE) -- it should just report + // that the pointer does not resolve to an element + CHECK(!j.contains(jp)); + CHECK(!j_const.contains(jp)); + } + + { + // #5395: same as above, but using the exact reproduction from the issue + json::json_pointer const jp("/99999999999999999999"); + std::string const throw_msg = "[json.exception.out_of_range.404] unresolved reference token '99999999999999999999'"; + + CHECK_THROWS_WITH_AS(j[jp] = 1, throw_msg.c_str(), json::out_of_range&); + CHECK_THROWS_WITH_AS(j_const[jp] == 1, throw_msg.c_str(), json::out_of_range&); + CHECK_THROWS_WITH_AS(j.at(jp) = 1, throw_msg.c_str(), json::out_of_range&); + CHECK_THROWS_WITH_AS(j_const.at(jp) == 1, throw_msg.c_str(), json::out_of_range&); + + CHECK(!j.contains(jp)); + CHECK(!j_const.contains(jp)); + } + + { + // #5395: a reference token that is numerically representable in + // unsigned long long but exceeds size_type's max (e.g. ULLONG_MAX + // itself on typical 64-bit platforms, where size_type's max equals + // ULLONG_MAX) must not make contains() throw either + json::json_pointer const jp("/18446744073709551615"); + std::string const throw_msg = "[json.exception.out_of_range.410] array index 18446744073709551615 exceeds size_type"; + + CHECK_THROWS_WITH_AS(j[jp] = 1, throw_msg.c_str(), json::out_of_range&); + CHECK_THROWS_WITH_AS(j_const[jp] == 1, throw_msg.c_str(), json::out_of_range&); + CHECK_THROWS_WITH_AS(j.at(jp) = 1, throw_msg.c_str(), json::out_of_range&); + CHECK_THROWS_WITH_AS(j_const.at(jp) == 1, throw_msg.c_str(), json::out_of_range&); + + CHECK(!j.contains(jp)); + CHECK(!j_const.contains(jp)); } // on some machines, the check below is not constant @@ -334,6 +372,10 @@ TEST_CASE("JSON pointers") CHECK_THROWS_WITH_AS(j[jp] = 1, throw_msg.c_str(), json::out_of_range&); CHECK_THROWS_WITH_AS(j_const[jp] == 1, throw_msg.c_str(), json::out_of_range&); + + // #5395: contains() must not throw for a reference token exceeding size_type's max + CHECK(!j.contains(jp)); + CHECK(!j_const.contains(jp)); } DOCTEST_MSVC_SUPPRESS_WARNING_POP diff --git a/tests/src/unit-msgpack.cpp b/tests/src/unit-msgpack.cpp index c57f29299..74f7f4969 100644 --- a/tests/src/unit-msgpack.cpp +++ b/tests/src/unit-msgpack.cpp @@ -1683,6 +1683,67 @@ TEST_CASE("issue #5405 - array reserve for definite-length MessagePack arrays") } // use this testcase outside [hide] to run it with Valgrind +TEST_CASE("MessagePack nesting does not consume the call stack") +{ + // Reading a container used to call back into the value reader once per + // element, so the native call stack grew with the nesting depth of the + // input: one frame per byte for repeated 0x91 (a one-element array), which + // crashes the process long before the input is exhausted (#5104). The + // containers are kept on a heap stack now. + // + // Note that deeply nested values must not be compared, copied or dumped + // here: those operations are still recursive, and would reintroduce the + // very crash this checks for. Depth is measured by descending instead. + + SECTION("an unterminated chain is reported, not crashed on") + { + json _; + const std::vector input(300000, 0x91); + CHECK_THROWS_WITH_AS(_ = json::from_msgpack(input), "[json.exception.parse_error.110] parse error at byte 300001: syntax error while parsing MessagePack value: unexpected end of input", json::parse_error&); + CHECK(json::from_msgpack(input, true, false).is_discarded()); + } + + SECTION("a well-formed deep value is read through the SAX interface") + { + std::vector input(300000, 0x91); + input.push_back(0x01); // innermost value + + SaxCountdown accept_all(600001); + CHECK(json::sax_parse(input, &accept_all, json::input_format_t::msgpack)); + } + + SECTION("a well-formed deep value is read into a value") + { + const std::size_t depth = 10000; + std::vector input(depth, 0x91); + input.push_back(0x01); + + json j = json::from_msgpack(input); + + std::size_t measured = 0; + const json* p = &j; + while (p->is_array() && !p->empty()) + { + p = &p->front(); + ++measured; + } + CHECK(measured == depth); + CHECK(p->is_number()); + } + + SECTION("containers are still read the same way") + { + CHECK(json::from_msgpack(std::vector({0x90})) == json::array()); + CHECK(json::from_msgpack(std::vector({0x80})) == json::object()); + CHECK(json::from_msgpack(std::vector({0x92, 0x90, 0x80})) == json({json::array(), json::object()})); + CHECK(json::from_msgpack(std::vector({0x91, 0x91, 0x91, 0x90})) == json({{{json::array()}}})); + CHECK(json::from_msgpack(std::vector({0x81, 0xA1, 'a', 0x81, 0xA1, 'b', 0x92, 0x01, 0x02})) == json({{"a", {{"b", {1, 2}}}}})); + // array 16 and map 32, i.e. the counted forms + CHECK(json::from_msgpack(std::vector({0xDC, 0x00, 0x02, 0x01, 0x02})) == json({1, 2})); + CHECK(json::from_msgpack(std::vector({0xDF, 0x00, 0x00, 0x00, 0x01, 0xA1, 'k', 0xC3})) == json({{"k", true}})); + } +} + TEST_CASE("single MessagePack roundtrip") { SECTION("sample.json") diff --git a/tests/src/unit-no-macro-leak.cpp b/tests/src/unit-no-macro-leak.cpp new file mode 100644 index 000000000..c5184c52f --- /dev/null +++ b/tests/src/unit-no-macro-leak.cpp @@ -0,0 +1,34 @@ +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ +// | | |__ | | | | | | version 3.12.0 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann +// SPDX-License-Identifier: MIT + +// This file makes sure that none of the internal JSON_HEDLEY_* macros (vendored +// from https://nemequ.github.io/hedley/, see +// include/nlohmann/thirdparty/hedley/hedley.hpp) leak into the including +// translation unit. include/nlohmann/detail/macro_unscope.hpp is supposed to +// #undef every JSON_HEDLEY_* macro (via hedley_undef.hpp) once json.hpp has +// been fully processed. See https://github.com/nlohmann/json/issues/5408, +// where JSON_HEDLEY_PRAGMA, JSON_HEDLEY_PREDICT_TRUE, JSON_HEDLEY_PREDICT_FALSE, +// and JSON_HEDLEY_CLANG_HAS_DECLSPEC_ATTRIBUTE escaped this cleanup because +// hedley_undef.hpp had no matching #undef for them. +// +// hedley_undef_checks.inc (included below) is generated at CMake configure/ +// build time by cmake/scripts/gen_hedley_undef_check.cmake, which derives the +// full list of JSON_HEDLEY_* macro names directly from hedley.hpp. That way +// this test covers every macro Hedley actually defines -- not a hardcoded +// snapshot that would silently go stale the next time `make update_hedley` +// runs -- and can never drift from the vendored header. + +#include "doctest_compatibility.h" + +#include + +TEST_CASE("JSON_HEDLEY macros do not leak after including json.hpp") +{ +#include "hedley_undef_checks.inc" + CHECK(true); // keep an assertion when nothing leaked +} diff --git a/tests/src/unit-regression1.cpp b/tests/src/unit-regression1.cpp index 475ef511f..0529f83dd 100644 --- a/tests/src/unit-regression1.cpp +++ b/tests/src/unit-regression1.cpp @@ -29,10 +29,7 @@ using nlohmann::json; #include #include #include "make_test_data_available.hpp" - -#ifdef JSON_HAS_CPP_17 - #include -#endif +#include "test_utils.hpp" #include "fifo_map.hpp" @@ -1373,7 +1370,8 @@ TEST_CASE("regression tests 1") std::array key1 = {{ 103, 92, 117, 48, 48, 48, 55, 92, 114, 215, 126, 214, 95, 92, 34, 174, 40, 71, 38, 174, 40, 71, 38, 223, 134, 247, 127, 0 }}; std::string const key1_str(reinterpret_cast(key1.data())); json const j = key1_str; - CHECK_THROWS_WITH_AS(j.dump(), "[json.exception.type_error.316] invalid UTF-8 byte at index 10: 0x7E", json::type_error&); + // dump() is nodiscard; the exception is thrown by dump() itself before it would return + CHECK_THROWS_WITH_AS(utils::ignore_return_value(j.dump()), "[json.exception.type_error.316] invalid UTF-8 byte at index 10: 0x7E", json::type_error&); } #if JSON_USE_IMPLICIT_CONVERSIONS diff --git a/tests/src/unit-regression2.cpp b/tests/src/unit-regression2.cpp index 2e7450e2e..4280ec361 100644 --- a/tests/src/unit-regression2.cpp +++ b/tests/src/unit-regression2.cpp @@ -31,6 +31,8 @@ using ordered_json = nlohmann::ordered_json; #include #include +#include "test_utils.hpp" + #ifdef JSON_HAS_CPP_17 #include #include @@ -239,209 +241,6 @@ class my_allocator : public std::allocator }; }; -///////////////////////////////////////////////////////////////////// -// for #3077 -///////////////////////////////////////////////////////////////////// - -class FooAlloc -{}; - -class Foo -{ - public: - explicit Foo(const FooAlloc& /* unused */ = FooAlloc()) {} - - bool value = false; -}; - -class FooBar -{ - public: - Foo foo{}; // NOLINT(readability-redundant-member-init) -}; - -inline void from_json(const nlohmann::json& j, FooBar& fb) // NOLINT(misc-use-internal-linkage) -{ - j.at("value").get_to(fb.foo.value); -} - -///////////////////////////////////////////////////////////////////// -// for #3171 -///////////////////////////////////////////////////////////////////// - -struct for_3171_base // NOLINT(cppcoreguidelines-special-member-functions) -{ - for_3171_base(const std::string& /*unused*/ = {}) {} - virtual ~for_3171_base(); - - for_3171_base(const for_3171_base& other) // NOLINT(hicpp-use-equals-default,modernize-use-equals-default) - : str(other.str) - {} - - for_3171_base& operator=(const for_3171_base& other) - { - if (this != &other) - { - str = other.str; - } - return *this; - } - - for_3171_base(for_3171_base&& other) noexcept - : str(std::move(other.str)) - {} - - for_3171_base& operator=(for_3171_base&& other) noexcept - { - if (this != &other) - { - str = std::move(other.str); - } - return *this; - } - - virtual void _from_json(const json& j) - { - j.at("str").get_to(str); - } - - std::string str{}; // NOLINT(readability-redundant-member-init) -}; - -for_3171_base::~for_3171_base() = default; - -struct for_3171_derived : public for_3171_base -{ - for_3171_derived() = default; - ~for_3171_derived() override; - explicit for_3171_derived(const std::string& /*unused*/) { } - - for_3171_derived(const for_3171_derived& other) // NOLINT(hicpp-use-equals-default,modernize-use-equals-default) - : for_3171_base(other) - {} - - for_3171_derived& operator=(const for_3171_derived& other) - { - if (this != &other) - { - for_3171_base::operator=(other); // Call base class assignment operator - } - return *this; - } - - for_3171_derived(for_3171_derived&& other) noexcept - : for_3171_base(std::move(other)) - {} - - for_3171_derived& operator=(for_3171_derived&& other) noexcept - { - if (this != &other) - { - for_3171_base::operator=(std::move(other)); // Call base class move assignment operator - } - return *this; - } -}; - -for_3171_derived::~for_3171_derived() = default; - -inline void from_json(const json& j, for_3171_base& tb) // NOLINT(misc-use-internal-linkage) -{ - tb._from_json(j); -} - -///////////////////////////////////////////////////////////////////// -// for #3312 -///////////////////////////////////////////////////////////////////// - -#ifdef JSON_HAS_CPP_20 -struct for_3312 -{ - std::string name; -}; - -inline void from_json(const json& j, for_3312& obj) // NOLINT(misc-use-internal-linkage) -{ - j.at("name").get_to(obj.name); -} -#endif - -///////////////////////////////////////////////////////////////////// -// for #3204 -///////////////////////////////////////////////////////////////////// - -struct for_3204_foo -{ - for_3204_foo() = default; - explicit for_3204_foo(std::string /*unused*/) {} // NOLINT(performance-unnecessary-value-param) -}; - -struct for_3204_bar -{ - enum constructed_from_t // NOLINT(cppcoreguidelines-use-enum-class) - { - constructed_from_none = 0, - constructed_from_foo = 1, - constructed_from_json = 2 - }; - - explicit for_3204_bar(std::function /*unused*/) noexcept // NOLINT(performance-unnecessary-value-param) - : constructed_from(constructed_from_foo) {} - explicit for_3204_bar(std::function /*unused*/) noexcept // NOLINT(performance-unnecessary-value-param) - : constructed_from(constructed_from_json) {} - - constructed_from_t constructed_from = constructed_from_none; -}; - -///////////////////////////////////////////////////////////////////// -// for #3333 -///////////////////////////////////////////////////////////////////// - -struct for_3333 final -{ - for_3333(int x_ = 0, int y_ = 0) : x(x_), y(y_) {} - - template - for_3333(const T& /*unused*/) - { - CHECK(false); - } - - int x = 0; - int y = 0; -}; - -template <> -inline for_3333::for_3333(const json& j) - : for_3333(j.value("x", 0), j.value("y", 0)) -{} - -///////////////////////////////////////////////////////////////////// -// for #3810 -///////////////////////////////////////////////////////////////////// - -struct Example_3810 -{ - int bla{}; - - Example_3810() = default; -}; - -NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(Example_3810, bla) // NOLINT(misc-use-internal-linkage) - -///////////////////////////////////////////////////////////////////// -// for #4740 -///////////////////////////////////////////////////////////////////// - -#ifdef JSON_HAS_CPP_17 -struct Example_4740 -{ - std::optional host = std::nullopt; - std::optional port = std::nullopt; - NLOHMANN_DEFINE_TYPE_INTRUSIVE_WITH_DEFAULT(Example_4740, host, port) -}; -#endif - TEST_CASE("regression tests 2") { SECTION("issue #1001 - Fix memory leak during parser callback") @@ -639,7 +438,8 @@ TEST_CASE("regression tests 2") s += static_cast(i); } dump_test["1"] = s; - dump_test.dump(-1, ' ', true, nlohmann::json::error_handler_t::replace); + // dump() is nodiscard; this only checks that dumping does not throw/crash + utils::ignore_return_value(dump_test.dump(-1, ' ', true, nlohmann::json::error_handler_t::replace)); } } @@ -731,12 +531,14 @@ TEST_CASE("regression tests 2") { const std::array data = {{0x81, 0xA4, 0x64, 0x61, 0x74, 0x61, 0xC4, 0x0F, 0x33, 0x30, 0x30, 0x32, 0x33, 0x34, 0x30, 0x31, 0x30, 0x37, 0x30, 0x35, 0x30, 0x31, 0x30}}; const json j = json::from_msgpack(data.data(), data.size()); + // dump() is nodiscard; this only checks that dumping does not throw CHECK_NOTHROW( - j.dump(4, // Indent - ' ', // Indent char - false, // Ensure ascii - json::error_handler_t::strict // Error - )); + utils::ignore_return_value( + j.dump(4, // Indent + ' ', // Indent char + false, // Ensure ascii + json::error_handler_t::strict // Error + ))); } SECTION("PR #2181 - regression bug with lvalue") @@ -959,611 +761,6 @@ TEST_CASE("regression tests 2") CHECK(j == k); } -#if JSON_HAS_FILESYSTEM || JSON_HAS_EXPERIMENTAL_FILESYSTEM - // JSON_HAS_CPP_17 (do not remove; see note at top of file) - SECTION("issue #3070 - Version 3.10.3 breaks backward-compatibility with 3.10.2 ") - { - nlohmann::detail::std_fs::path text_path("/tmp/text.txt"); - const json j(text_path); - - const auto j_path = j.get(); - CHECK(j_path == text_path); - -#if DOCTEST_CLANG || DOCTEST_GCC >= DOCTEST_COMPILER(8, 4, 0) - // only known to work on Clang and GCC >=8.4 - CHECK_THROWS_WITH_AS(nlohmann::detail::std_fs::path(json(1)), "[json.exception.type_error.302] type must be string, but is number", json::type_error); -#endif - } -#endif - - SECTION("issue #3077 - explicit constructor with default does not compile") - { - json j; - j[0]["value"] = true; - std::vector foo; - j.get_to(foo); - } - - SECTION("issue #3108 - ordered_json doesn't support range based erase") - { - ordered_json j = {1, 2, 2, 4}; - - auto last = std::unique(j.begin(), j.end()); - j.erase(last, j.end()); - - CHECK(j.dump() == "[1,2,4]"); - - j.erase(std::remove_if(j.begin(), j.end(), [](const ordered_json & val) - { - return val == 2; - }), j.end()); - - CHECK(j.dump() == "[1,4]"); - } - - SECTION("issue #3343 - json and ordered_json are not interchangeable") - { - json::object_t jobj({ { "product", "one" } }); - ordered_json::object_t ojobj({{"product", "one"}}); - - auto jit = jobj.begin(); - auto ojit = ojobj.begin(); - - CHECK(jit->first == ojit->first); - CHECK(jit->second.get() == ojit->second.get()); - } - - SECTION("issue #3171 - if class is_constructible from std::string wrong from_json overload is being selected, compilation failed") - { - const json j{{ "str", "value"}}; - - // failed with: error: no match for ‘operator=’ (operand types are ‘for_3171_derived’ and ‘const nlohmann::basic_json<>::string_t’ - // {aka ‘const std::__cxx11::basic_string’}) - // s = *j.template get_ptr(); - auto td = j.get(); - - CHECK(td.str == "value"); - } - -#ifdef JSON_HAS_CPP_20 - SECTION("issue #3312 - Parse to custom class from unordered_json breaks on G++11.2.0 with C++20") - { - // see test for #3171 - const ordered_json j = {{"name", "class"}}; - for_3312 obj{}; - - j.get_to(obj); - - CHECK(obj.name == "class"); - } -#endif - -#if defined(JSON_HAS_CPP_17) && JSON_USE_IMPLICIT_CONVERSIONS - SECTION("issue #3428 - Error occurred when converting nlohmann::json to std::any") - { - const json j; - const std::any a1 = j; - std::any&& a2 = j; - - CHECK(a1.type() == typeid(j)); - CHECK(a2.type() == typeid(j)); - } -#endif - - SECTION("issue #3204 - ambiguous regression") - { - const for_3204_bar bar_from_foo([](for_3204_foo) noexcept {}); // NOLINT(performance-unnecessary-value-param) - const for_3204_bar bar_from_json([](json) noexcept {}); // NOLINT(performance-unnecessary-value-param) - - CHECK(bar_from_foo.constructed_from == for_3204_bar::constructed_from_foo); - CHECK(bar_from_json.constructed_from == for_3204_bar::constructed_from_json); - } - - SECTION("issue #3333 - Ambiguous conversion from nlohmann::basic_json<> to custom class") - { - const json j - { - {"x", 1}, - {"y", 2} - }; - const for_3333 p = j; - - CHECK(p.x == 1); - CHECK(p.y == 2); - } - - SECTION("issue #3810 - ordered_json doesn't support construction from C array of custom type") - { - Example_3810 states[45]; // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays) - - // fix "not used" warning - states[0].bla = 1; - - const auto* const expected = R"([{"bla":1},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0}])"; - - // This works: - nlohmann::json j; - j["test"] = states; - CHECK(j["test"].dump() == expected); - - // This doesn't compile: - nlohmann::ordered_json oj; - oj["test"] = states; - CHECK(oj["test"].dump() == expected); - } - -#ifdef JSON_HAS_CPP_17 - SECTION("issue #4740 - build issue with std::optional") - { - const auto t1 = Example_4740(); - const auto j1 = nlohmann::json(t1); - CHECK(j1.dump() == "{\"host\":null,\"port\":null}"); - const auto t2 = j1.get(); - CHECK(!t2.host.has_value()); - CHECK(!t2.port.has_value()); - - // improve coverage - auto t3 = Example_4740(); - t3.port = 80; - t3.host = "example.com"; - const auto j2 = nlohmann::json(t3); - CHECK(j2.dump() == "{\"host\":\"example.com\",\"port\":80}"); - const auto t4 = j2.get(); - CHECK(t4.host.has_value()); - CHECK(t4.port.has_value()); - } -#endif - -#if !defined(_MSVC_LANG) - // MSVC returns garbage on invalid enum values, so this test is excluded - // there. - SECTION("issue #4762 - json exception 302 with unhelpful explanation : type must be number, but is number") - { - // In #4762, the main issue was that a json object with an invalid type - // returned "number" as type_name(), because this was the default case. - // This test makes sure we now return "invalid" instead. - json j; - j.m_data.m_type = static_cast(100); // NOLINT(clang-analyzer-optin.core.EnumCastOutOfRange) - CHECK(j.type_name() == "invalid"); - } -#endif - -#ifdef JSON_HAS_CPP_17 - SECTION("issue #4804: from_cbor incompatible with std::vector as binary_t") - { - const std::vector data = {0x80}; - const auto decoded = json_4804::from_cbor(data); - CHECK((decoded == json_4804::array())); - } - - SECTION("discussion #4209 - custom BinaryType direct assignment and round-tripping") - { - // Test that assigning a custom BinaryType directly creates a binary value, not an array - const std::vector original{std::byte{1}, std::byte{2}, std::byte{3}}; - const json_4804 j = original; - CHECK(j.is_binary()); - CHECK(!j.is_array()); - - // Test round-tripping: extracting the binary value back as the custom container type - const auto extracted = j.get>(); - CHECK(extracted == original); - - // Test that the default json alias behavior is unchanged: std::vector -> array - const json default_json = std::vector {1, 2, 3}; - CHECK(default_json.is_array()); - CHECK(!default_json.is_binary()); - } - - SECTION("discussion #4209 - custom BinaryType extraction from parsed array") - { - // Test that extracting a custom BinaryType from a parsed JSON array still works - // (not just from a binary-typed node) - const auto j = json_4804::parse("[1,2,3]"); - CHECK(j.is_array()); - CHECK(!j.is_binary()); - - // Extracting as custom BinaryType should work from arrays - const auto extracted = j.get>(); - CHECK(extracted.size() == 3); - CHECK(extracted[0] == std::byte{1}); - CHECK(extracted[1] == std::byte{2}); - CHECK(extracted[2] == std::byte{3}); - } - - SECTION("issue #5046 - implicit conversion of return json to std::optional no longer implicit") - { - const json jval{}; - auto GetValue = [](const json & valRoot) -> std::optional - { - if (valRoot.contains("default")) - { - return valRoot.at("default"); - } - return std::nullopt; - }; - auto result = GetValue(jval); - CHECK(!result.has_value()); - } -#endif - -#if JSON_HAS_RANGES == 1 - SECTION("issue #4440 - assert when using std::views::filter and GCC 10") - { - auto noOpFilter = std::views::filter([](auto&&) noexcept - { - return true; - }); - json j = {1, 2, 3}; - auto filtered = j | noOpFilter; - CHECK(*filtered.begin() == 1); - } -#endif - -#if JSON_HAS_RANGES && !defined(__MINGW32__) - SECTION("issue #4916 - constructing array from C++20 ranges view does not work") - { - std::vector nums{1, 2, 37, 42, 21}; - auto filteredNums = nums | std::views::filter([](int i) - { - return i > 10; - }); - json const j(filteredNums); - CHECK(j.type() == json::value_t::array); - CHECK(j == json({37, 42, 21})); - } -#endif - - // owning_view is not available in libstdc++ < 12 -#if JSON_HAS_RANGES && !defined(__MINGW32__) && !(defined(__GLIBCXX__) && _GLIBCXX_RELEASE < 12) - SECTION("issue #4916 - constructing array from prvalue C++20 ranges view (owning_view)") - { - json const j(std::vector {1, 2, 37, 42, 21} | std::views::filter([](int i) - { - return i > 10; - })); - CHECK(j.type() == json::value_t::array); - CHECK(j == json({37, 42, 21})); - } -#endif - -#if JSON_HAS_RANGES && !defined(__MINGW32__) - SECTION("issue #4916 - constructing array from C++20 transform view (prvalue elements)") - { - std::vector nums{1, 2, 3}; - auto t = nums | std::views::transform([](int i) noexcept - { - return i * 2; - }); - json const j(t); - CHECK(j.type() == json::value_t::array); - CHECK(j == json({2, 4, 6})); - } -#endif -} - -TEST_CASE_TEMPLATE("issue #4798 - nlohmann::json::to_msgpack() encode float NaN as double", T, double, float) // NOLINT(readability-math-missing-parentheses, bugprone-throwing-static-initialization) -{ - // With issue #4798, we encode NaN, infinity, and -infinity as float instead - // of double to allow for smaller encodings. - const json jx = std::numeric_limits::quiet_NaN(); - const json jy = std::numeric_limits::infinity(); - const json jz = -std::numeric_limits::infinity(); - - ///////////////////////////////////////////////////////////////////////// - // MessagePack - ///////////////////////////////////////////////////////////////////////// - - // expected MessagePack values - const std::vector msgpack_x = {{0xCA, 0x7F, 0xC0, 0x00, 0x00}}; - const std::vector msgpack_y = {{0xCA, 0x7F, 0x80, 0x00, 0x00}}; - const std::vector msgpack_z = {{0xCA, 0xFF, 0x80, 0x00, 0x00}}; - - CHECK(json::to_msgpack(jx) == msgpack_x); - CHECK(json::to_msgpack(jy) == msgpack_y); - CHECK(json::to_msgpack(jz) == msgpack_z); - - CHECK(std::isnan(json::from_msgpack(msgpack_x).get())); - CHECK(json::from_msgpack(msgpack_y).get() == std::numeric_limits::infinity()); - CHECK(json::from_msgpack(msgpack_z).get() == -std::numeric_limits::infinity()); - - // Make sure the other MessagePakc encodings for NaN, infinity, and - // -infinity are still supported. - const std::vector msgpack_x_2 = {{0xCB, 0x7F, 0xF8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}}; - const std::vector msgpack_y_2 = {{0xCB, 0x7F, 0xF0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}}; - const std::vector msgpack_z_2 = {{0xCB, 0xFF, 0xF0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}}; - CHECK(std::isnan(json::from_msgpack(msgpack_x_2).get())); - CHECK(json::from_msgpack(msgpack_y_2).get() == std::numeric_limits::infinity()); - CHECK(json::from_msgpack(msgpack_z_2).get() == -std::numeric_limits::infinity()); - - ///////////////////////////////////////////////////////////////////////// - // CBOR - ///////////////////////////////////////////////////////////////////////// - - // expected CBOR values - const std::vector cbor_x = {{0xF9, 0x7E, 0x00}}; - const std::vector cbor_y = {{0xF9, 0x7C, 0x00}}; - const std::vector cbor_z = {{0xF9, 0xfC, 0x00}}; - - CHECK(json::to_cbor(jx) == cbor_x); - CHECK(json::to_cbor(jy) == cbor_y); - CHECK(json::to_cbor(jz) == cbor_z); - - CHECK(std::isnan(json::from_cbor(cbor_x).get())); - CHECK(json::from_cbor(cbor_y).get() == std::numeric_limits::infinity()); - CHECK(json::from_cbor(cbor_z).get() == -std::numeric_limits::infinity()); - - // Make sure the other CBOR encodings for NaN, infinity, and -infinity are - // still supported. - const std::vector cbor_x_2 = {{0xFA, 0x7F, 0xC0, 0x00, 0x00}}; - const std::vector cbor_y_2 = {{0xFA, 0x7F, 0x80, 0x00, 0x00}}; - const std::vector cbor_z_2 = {{0xFA, 0xFF, 0x80, 0x00, 0x00}}; - const std::vector cbor_x_3 = {{0xFB, 0x7F, 0xF8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}}; - const std::vector cbor_y_3 = {{0xFB, 0x7F, 0xF0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}}; - const std::vector cbor_z_3 = {{0xFB, 0xFF, 0xF0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}}; - CHECK(std::isnan(json::from_cbor(cbor_x_2).get())); - CHECK(json::from_cbor(cbor_y_2).get() == std::numeric_limits::infinity()); - CHECK(json::from_cbor(cbor_z_2).get() == -std::numeric_limits::infinity()); - CHECK(std::isnan(json::from_cbor(cbor_x_3).get())); - CHECK(json::from_cbor(cbor_y_3).get() == std::numeric_limits::infinity()); - CHECK(json::from_cbor(cbor_z_3).get() == -std::numeric_limits::infinity()); -} - -TEST_CASE("regression test #5074 - portable workaround for single-element brace init") -{ - json const j_obj = {{"key", "value"}}; - - json const j = json::array({j_obj}); - CHECK(j.is_array()); - CHECK(j.size() == 1); - CHECK(j[0] == j_obj); -} - -#if defined(JSON_BRACE_INIT_COPY_SEMANTICS) && (JSON_BRACE_INIT_COPY_SEMANTICS == 1) -TEST_CASE("regression test #5074 - single-element brace init with JSON_BRACE_INIT_COPY_SEMANTICS") -{ - // with JSON_BRACE_INIT_COPY_SEMANTICS: single-element brace init copies/moves - json const j_obj = {{"key", "value"}, {"num", 42}}; - json const j_arr = {1, 2, 3}; - - // object: brace init copies instead of wrapping - json const j1{j_obj}; - CHECK(j1.is_object()); - CHECK(j1 == j_obj); - - // array: brace init copies instead of wrapping - json const j2{j_arr}; - CHECK(j2.is_array()); - CHECK(j2.size() == 3); - CHECK(j2 == j_arr); - - // primitives still work as initializer lists - json const j3{true}; - CHECK(j3.is_boolean()); - - json const j4{42}; - CHECK(j4.is_number_integer()); -} -#endif - -struct Example_5122 -{ - float b = 2; - nlohmann::ordered_map c{}; // NOLINT(readability-redundant-member-init): needed for GCC -Weffc++ - int a = 1; - NLOHMANN_DEFINE_TYPE_INTRUSIVE_WITH_DEFAULT(Example_5122, b, c, a) -}; - -TEST_CASE("regression test #5122 - from_json into types holding nlohmann::ordered_map") -{ - Example_5122 src; - src.c.emplace("first", "1"); - src.c.emplace("second", "2"); - - ordered_json const j = src; - Example_5122 const dst = j.get(); - - CHECK(dst.b == src.b); - CHECK(dst.a == src.a); - REQUIRE(dst.c.size() == src.c.size()); - auto src_it = src.c.begin(); - auto dst_it = dst.c.begin(); - for (; src_it != src.c.end(); ++src_it, ++dst_it) - { - CHECK(dst_it->first == src_it->first); - CHECK(dst_it->second == src_it->second); - } -} - -// -Wself-assign-overloaded was introduced in Clang 7. Gate the pragma on -// __has_warning so older Clang versions do not error with "unknown warning -// group". The __has_warning check has to stay inside the __clang__ branch -// because GCC does not provide it and would tokenize-error on the argument. -#if defined(__clang__) && defined(__has_warning) - #if __has_warning("-Wself-assign-overloaded") - DOCTEST_CLANG_SUPPRESS_WARNING_PUSH - DOCTEST_CLANG_SUPPRESS_WARNING("-Wself-assign-overloaded") - #endif -#endif - -TEST_CASE("regression test #5122 - nlohmann::ordered_map copy-assignment is self-assignment safe") -{ - nlohmann::ordered_map m; - m.emplace("first", "1"); - m.emplace("second", "2"); - - // Insertion order is preserved by ordered_map, so we can check it directly. - m = m; - - REQUIRE(m.size() == 2); - auto it = m.begin(); - CHECK(it->first == "first"); - CHECK(it->second == "1"); - ++it; - CHECK(it->first == "second"); - CHECK(it->second == "2"); -} - -#if defined(__clang__) && defined(__has_warning) - #if __has_warning("-Wself-assign-overloaded") - DOCTEST_CLANG_SUPPRESS_WARNING_POP - #endif -#endif - -TEST_CASE("regression test #5122 - nlohmann::ordered_map move-assignment transfers contents") -{ - nlohmann::ordered_map src; - src.emplace("first", "1"); - src.emplace("second", "2"); - - nlohmann::ordered_map dst; - dst.emplace("stale", "x"); - dst = std::move(src); - - REQUIRE(dst.size() == 2); - auto it = dst.begin(); - CHECK(it->first == "first"); - CHECK(it->second == "1"); - ++it; - CHECK(it->first == "second"); - CHECK(it->second == "2"); - - // Re-assigning into the moved-from object must leave it in a usable state. - src = nlohmann::ordered_map {}; - src.emplace("after-move", "3"); - REQUIRE(src.size() == 1); - CHECK(src.begin()->first == "after-move"); -} - -// Stand-in for a third-party library (e.g., Eigen as of 3.4, which added -// STL-compatible begin()/end() to its vector types), living in its own -// namespace with its own to_json overload for its vector type. -namespace issue_4320_eigen -{ -// "array-compatible" from the library's point of view (it has begin()/end()), -// but for which this (fake) third-party namespace provides its own to_json. -struct vector3 -{ - double v[3]; // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays,cppcoreguidelines-use-default-member-init,modernize-use-default-member-init) - vector3(double x, double y, double z) : v{x, y, z} {} // NOLINT(hicpp-member-init,cppcoreguidelines-pro-type-member-init) - double x() const - { - return v[0]; - } - double y() const - { - return v[1]; - } - double z() const - { - return v[2]; - } - double* begin() - { - return v; - } - double* end() - { - return v + 3; - } - const double* begin() const - { - return v; - } - const double* end() const - { - return v + 3; - } -}; - -inline void to_json(json& j, const vector3& v) // NOLINT(misc-use-internal-linkage) -{ - j = {{"x", v.x()}, {"y", v.y()}, {"z", v.z()}}; -} -} // namespace issue_4320_eigen - -// The user's own namespace, using the (fake) Eigen type as an implementation -// detail behind a payload type that has nothing to do with vectors/arrays. -namespace issue_4320 -{ -// Publicly derives from issue_4320_eigen::vector3 but does *not* define its -// own to_json - it is only ever used as a temporary to reach the base -// class's to_json via ADL. -struct vector3_wrapper : issue_4320_eigen::vector3 -{ - using issue_4320_eigen::vector3::vector3; -}; - -struct payload -{ - double x, y, z; -}; - -inline vector3_wrapper to_eigen(const payload& p) // NOLINT(misc-use-internal-linkage) -{ - return {p.x, p.y, p.z}; -} - -inline void to_json(json& j, const payload& p) // NOLINT(misc-use-internal-linkage) -{ - // Unqualified call, passing a *derived* vector3_wrapper: relies on ADL - // finding issue_4320_eigen::to_json(json&, const vector3&) through the - // vector3 base class, via a derived-to-base conversion. Must NOT resolve - // to the library's own generic array-compatible to_json (an exact-match - // template for vector3_wrapper, since it also has begin()/end()), which - // would serialize this as [x, y, z] instead of {"x":x, "y":y, "z":z}. - to_json(j, to_eigen(p)); -} -} // namespace issue_4320 - -TEST_CASE("issue #4320 - custom base class must not leak nlohmann::detail into ADL") -{ - // Before the fix, basic_json unconditionally derived from a type living in - // nlohmann::detail (json_default_base), which made nlohmann::detail an - // associated namespace of every basic_json for ADL purposes. That leaked - // the library's internal generic-array to_json overload into unqualified - // to_json() calls made from user code, silently bypassing user-defined - // to_json overloads reached via a derived-to-base conversion. - const issue_4320::payload p{1.0, 2.0, 3.0}; - - json j; - to_json(j, p); - CHECK(j == json({{"x", 1.0}, {"y", 2.0}, {"z", 3.0}})); -} - -TEST_CASE("issue #5338 - truncated CBOR tagged binary subtype is rejected") -{ - const std::vector> truncated_tags = - { - {0xD8}, - {0xD9, 0x00}, - {0xDA, 0x00, 0x00, 0x00}, - {0xDB, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00} - }; - - for (const auto& data : truncated_tags) - { - CAPTURE(data); - for (const auto tag_handler : - { - json::cbor_tag_handler_t::ignore, json::cbor_tag_handler_t::store - }) - { - CAPTURE(tag_handler); - const auto result = json::from_cbor(data, true, false, tag_handler); - CHECK(result.is_discarded()); - } - } -} - -TEST_CASE("issue #5402 - update(merge_objects=true) overwrites a primitive with an object") -{ - json t = {{"k", 1}}; - t.update(json{{"k", {{"x", 2}}}}, true); - CHECK(t == json({{"k", {{"x", 2}}}})); - - json mixed = {{"keep", {{"a", 1}}}, {"replace", 1}}; - mixed.update(json{{"keep", {{"b", 2}}}, {"replace", {{"x", 2}}}}, true); - CHECK(mixed == json({{"keep", {{"a", 1}, {"b", 2}}}, {"replace", {{"x", 2}}}})); } DOCTEST_CLANG_SUPPRESS_WARNING_POP diff --git a/tests/src/unit-regression3.cpp b/tests/src/unit-regression3.cpp new file mode 100644 index 000000000..cb2ed59a6 --- /dev/null +++ b/tests/src/unit-regression3.cpp @@ -0,0 +1,899 @@ +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ (supporting code) +// | | |__ | | | | | | version 3.12.0 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann +// SPDX-License-Identifier: MIT + +// cmake/test.cmake selects the C++ standard versions with which to build a +// unit test based on the presence of JSON_HAS_CPP_ macros. +// When using macros that are only defined for particular versions of the standard +// (e.g., JSON_HAS_FILESYSTEM for C++17 and up), please mention the corresponding +// version macro in a comment close by, like this: +// JSON_HAS_CPP_ (do not remove; see note at top of file) + +#include "doctest_compatibility.h" + +// for some reason including this after the json header leads to linker errors with VS 2017... +#include + +#define JSON_TESTS_PRIVATE +#include +using json = nlohmann::json; +using ordered_json = nlohmann::ordered_json; +#ifdef JSON_TEST_NO_GLOBAL_UDLS + using namespace nlohmann::literals; // NOLINT(google-build-using-namespace) +#endif + +#include +#include +#include +#include + +#ifdef JSON_HAS_CPP_17 + #include + #include +#endif + +#ifdef JSON_HAS_CPP_17 + #if __has_include() + #include + #elif __has_include() + #endif + + ///////////////////////////////////////////////////////////////////// + // for #4804 + ///////////////////////////////////////////////////////////////////// + using json_4804 = nlohmann::basic_json, // BinaryType + void // CustomBaseClass + >; +#endif + +#ifdef JSON_HAS_CPP_20 + #if __has_include() + #include + #endif +#endif + +///////////////////////////////////////////////////////////////////// +// for #4825 - explicitly instantiating basic_json must compile; this +// forces instantiation of binary_writer::write_bjdata_ndarray, whose +// static_cast was ambiguous under explicit instantiation on +// C++17. Merely compiling this translation unit is the regression test. +///////////////////////////////////////////////////////////////////// +template class nlohmann::basic_json<>; + +///////////////////////////////////////////////////////////////////// +// for #4440 +///////////////////////////////////////////////////////////////////// +#if JSON_HAS_RANGES == 1 + #include +#endif + +// NLOHMANN_JSON_SERIALIZE_ENUM uses a static std::pair +DOCTEST_CLANG_SUPPRESS_WARNING_PUSH +DOCTEST_CLANG_SUPPRESS_WARNING("-Wexit-time-destructors") +///////////////////////////////////////////////////////////////////// +// for #3077 +///////////////////////////////////////////////////////////////////// + +class FooAlloc +{}; + +class Foo +{ + public: + explicit Foo(const FooAlloc& /* unused */ = FooAlloc()) {} + + bool value = false; +}; + +class FooBar +{ + public: + Foo foo{}; // NOLINT(readability-redundant-member-init) +}; + +inline void from_json(const nlohmann::json& j, FooBar& fb) // NOLINT(misc-use-internal-linkage) +{ + j.at("value").get_to(fb.foo.value); +} + +///////////////////////////////////////////////////////////////////// +// for #3171 +///////////////////////////////////////////////////////////////////// + +struct for_3171_base // NOLINT(cppcoreguidelines-special-member-functions) +{ + for_3171_base(const std::string& /*unused*/ = {}) {} + virtual ~for_3171_base(); + + for_3171_base(const for_3171_base& other) // NOLINT(hicpp-use-equals-default,modernize-use-equals-default) + : str(other.str) + {} + + for_3171_base& operator=(const for_3171_base& other) + { + if (this != &other) + { + str = other.str; + } + return *this; + } + + for_3171_base(for_3171_base&& other) noexcept + : str(std::move(other.str)) + {} + + for_3171_base& operator=(for_3171_base&& other) noexcept + { + if (this != &other) + { + str = std::move(other.str); + } + return *this; + } + + virtual void _from_json(const json& j) + { + j.at("str").get_to(str); + } + + std::string str{}; // NOLINT(readability-redundant-member-init) +}; + +for_3171_base::~for_3171_base() = default; + +struct for_3171_derived : public for_3171_base +{ + for_3171_derived() = default; + ~for_3171_derived() override; + explicit for_3171_derived(const std::string& /*unused*/) { } + + for_3171_derived(const for_3171_derived& other) // NOLINT(hicpp-use-equals-default,modernize-use-equals-default) + : for_3171_base(other) + {} + + for_3171_derived& operator=(const for_3171_derived& other) + { + if (this != &other) + { + for_3171_base::operator=(other); // Call base class assignment operator + } + return *this; + } + + for_3171_derived(for_3171_derived&& other) noexcept + : for_3171_base(std::move(other)) + {} + + for_3171_derived& operator=(for_3171_derived&& other) noexcept + { + if (this != &other) + { + for_3171_base::operator=(std::move(other)); // Call base class move assignment operator + } + return *this; + } +}; + +for_3171_derived::~for_3171_derived() = default; + +inline void from_json(const json& j, for_3171_base& tb) // NOLINT(misc-use-internal-linkage) +{ + tb._from_json(j); +} + +///////////////////////////////////////////////////////////////////// +// for #3312 +///////////////////////////////////////////////////////////////////// + +#ifdef JSON_HAS_CPP_20 +struct for_3312 +{ + std::string name; +}; + +inline void from_json(const json& j, for_3312& obj) // NOLINT(misc-use-internal-linkage) +{ + j.at("name").get_to(obj.name); +} +#endif + +///////////////////////////////////////////////////////////////////// +// for #3204 +///////////////////////////////////////////////////////////////////// + +struct for_3204_foo +{ + for_3204_foo() = default; + explicit for_3204_foo(std::string /*unused*/) {} // NOLINT(performance-unnecessary-value-param) +}; + +struct for_3204_bar +{ + enum constructed_from_t // NOLINT(cppcoreguidelines-use-enum-class) + { + constructed_from_none = 0, + constructed_from_foo = 1, + constructed_from_json = 2 + }; + + explicit for_3204_bar(std::function /*unused*/) noexcept // NOLINT(performance-unnecessary-value-param) + : constructed_from(constructed_from_foo) {} + explicit for_3204_bar(std::function /*unused*/) noexcept // NOLINT(performance-unnecessary-value-param) + : constructed_from(constructed_from_json) {} + + constructed_from_t constructed_from = constructed_from_none; +}; + +///////////////////////////////////////////////////////////////////// +// for #3333 +///////////////////////////////////////////////////////////////////// + +struct for_3333 final +{ + for_3333(int x_ = 0, int y_ = 0) : x(x_), y(y_) {} + + template + for_3333(const T& /*unused*/) + { + CHECK(false); + } + + int x = 0; + int y = 0; +}; + +template <> +inline for_3333::for_3333(const json& j) + : for_3333(j.value("x", 0), j.value("y", 0)) +{} + +///////////////////////////////////////////////////////////////////// +// for #3810 +///////////////////////////////////////////////////////////////////// + +struct Example_3810 +{ + int bla{}; + + Example_3810() = default; +}; + +NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(Example_3810, bla) // NOLINT(misc-use-internal-linkage) + +///////////////////////////////////////////////////////////////////// +// for #4740 +///////////////////////////////////////////////////////////////////// + +#ifdef JSON_HAS_CPP_17 +struct Example_4740 +{ + std::optional host = std::nullopt; + std::optional port = std::nullopt; + NLOHMANN_DEFINE_TYPE_INTRUSIVE_WITH_DEFAULT(Example_4740, host, port) +}; +#endif + +TEST_CASE("regression tests 3") +{ +#if JSON_HAS_FILESYSTEM || JSON_HAS_EXPERIMENTAL_FILESYSTEM + // JSON_HAS_CPP_17 (do not remove; see note at top of file) + SECTION("issue #3070 - Version 3.10.3 breaks backward-compatibility with 3.10.2 ") + { + nlohmann::detail::std_fs::path text_path("/tmp/text.txt"); + const json j(text_path); + + const auto j_path = j.get(); + CHECK(j_path == text_path); + +#if DOCTEST_CLANG || DOCTEST_GCC >= DOCTEST_COMPILER(8, 4, 0) + // only known to work on Clang and GCC >=8.4 + CHECK_THROWS_WITH_AS(nlohmann::detail::std_fs::path(json(1)), "[json.exception.type_error.302] type must be string, but is number", json::type_error); +#endif + } +#endif + + SECTION("issue #3077 - explicit constructor with default does not compile") + { + json j; + j[0]["value"] = true; + std::vector foo; + j.get_to(foo); + } + + SECTION("issue #3108 - ordered_json doesn't support range based erase") + { + ordered_json j = {1, 2, 2, 4}; + + auto last = std::unique(j.begin(), j.end()); + j.erase(last, j.end()); + + CHECK(j.dump() == "[1,2,4]"); + + j.erase(std::remove_if(j.begin(), j.end(), [](const ordered_json & val) + { + return val == 2; + }), j.end()); + + CHECK(j.dump() == "[1,4]"); + } + + SECTION("issue #3343 - json and ordered_json are not interchangeable") + { + json::object_t jobj({ { "product", "one" } }); + ordered_json::object_t ojobj({{"product", "one"}}); + + auto jit = jobj.begin(); + auto ojit = ojobj.begin(); + + CHECK(jit->first == ojit->first); + CHECK(jit->second.get() == ojit->second.get()); + } + + SECTION("issue #3171 - if class is_constructible from std::string wrong from_json overload is being selected, compilation failed") + { + const json j{{ "str", "value"}}; + + // failed with: error: no match for ‘operator=’ (operand types are ‘for_3171_derived’ and ‘const nlohmann::basic_json<>::string_t’ + // {aka ‘const std::__cxx11::basic_string’}) + // s = *j.template get_ptr(); + auto td = j.get(); + + CHECK(td.str == "value"); + } + +#ifdef JSON_HAS_CPP_20 + SECTION("issue #3312 - Parse to custom class from unordered_json breaks on G++11.2.0 with C++20") + { + // see test for #3171 + const ordered_json j = {{"name", "class"}}; + for_3312 obj{}; + + j.get_to(obj); + + CHECK(obj.name == "class"); + } +#endif + +#if defined(JSON_HAS_CPP_17) && JSON_USE_IMPLICIT_CONVERSIONS + SECTION("issue #3428 - Error occurred when converting nlohmann::json to std::any") + { + const json j; + const std::any a1 = j; + std::any&& a2 = j; + + CHECK(a1.type() == typeid(j)); + CHECK(a2.type() == typeid(j)); + } +#endif + + SECTION("issue #3204 - ambiguous regression") + { + const for_3204_bar bar_from_foo([](for_3204_foo) noexcept {}); // NOLINT(performance-unnecessary-value-param) + const for_3204_bar bar_from_json([](json) noexcept {}); // NOLINT(performance-unnecessary-value-param) + + CHECK(bar_from_foo.constructed_from == for_3204_bar::constructed_from_foo); + CHECK(bar_from_json.constructed_from == for_3204_bar::constructed_from_json); + } + + SECTION("issue #3333 - Ambiguous conversion from nlohmann::basic_json<> to custom class") + { + const json j + { + {"x", 1}, + {"y", 2} + }; + const for_3333 p = j; + + CHECK(p.x == 1); + CHECK(p.y == 2); + } + + SECTION("issue #3810 - ordered_json doesn't support construction from C array of custom type") + { + Example_3810 states[45]; // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays) + + // fix "not used" warning + states[0].bla = 1; + + const auto* const expected = R"([{"bla":1},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0}])"; + + // This works: + nlohmann::json j; + j["test"] = states; + CHECK(j["test"].dump() == expected); + + // This doesn't compile: + nlohmann::ordered_json oj; + oj["test"] = states; + CHECK(oj["test"].dump() == expected); + } + +#ifdef JSON_HAS_CPP_17 + SECTION("issue #4740 - build issue with std::optional") + { + const auto t1 = Example_4740(); + const auto j1 = nlohmann::json(t1); + CHECK(j1.dump() == "{\"host\":null,\"port\":null}"); + const auto t2 = j1.get(); + CHECK(!t2.host.has_value()); + CHECK(!t2.port.has_value()); + + // improve coverage + auto t3 = Example_4740(); + t3.port = 80; + t3.host = "example.com"; + const auto j2 = nlohmann::json(t3); + CHECK(j2.dump() == "{\"host\":\"example.com\",\"port\":80}"); + const auto t4 = j2.get(); + CHECK(t4.host.has_value()); + CHECK(t4.port.has_value()); + } +#endif + +#if !defined(_MSVC_LANG) + // MSVC returns garbage on invalid enum values, so this test is excluded + // there. + SECTION("issue #4762 - json exception 302 with unhelpful explanation : type must be number, but is number") + { + // In #4762, the main issue was that a json object with an invalid type + // returned "number" as type_name(), because this was the default case. + // This test makes sure we now return "invalid" instead. + json j; + j.m_data.m_type = static_cast(100); // NOLINT(clang-analyzer-optin.core.EnumCastOutOfRange) + CHECK(j.type_name() == "invalid"); + } +#endif + +#ifdef JSON_HAS_CPP_17 + SECTION("issue #4804: from_cbor incompatible with std::vector as binary_t") + { + const std::vector data = {0x80}; + const auto decoded = json_4804::from_cbor(data); + CHECK((decoded == json_4804::array())); + } + + SECTION("discussion #4209 - custom BinaryType direct assignment and round-tripping") + { + // Test that assigning a custom BinaryType directly creates a binary value, not an array + const std::vector original{std::byte{1}, std::byte{2}, std::byte{3}}; + const json_4804 j = original; + CHECK(j.is_binary()); + CHECK(!j.is_array()); + + // Test round-tripping: extracting the binary value back as the custom container type + const auto extracted = j.get>(); + CHECK(extracted == original); + + // Test that the default json alias behavior is unchanged: std::vector -> array + const json default_json = std::vector {1, 2, 3}; + CHECK(default_json.is_array()); + CHECK(!default_json.is_binary()); + } + + SECTION("discussion #4209 - custom BinaryType extraction from parsed array") + { + // Test that extracting a custom BinaryType from a parsed JSON array still works + // (not just from a binary-typed node) + const auto j = json_4804::parse("[1,2,3]"); + CHECK(j.is_array()); + CHECK(!j.is_binary()); + + // Extracting as custom BinaryType should work from arrays + const auto extracted = j.get>(); + CHECK(extracted.size() == 3); + CHECK(extracted[0] == std::byte{1}); + CHECK(extracted[1] == std::byte{2}); + CHECK(extracted[2] == std::byte{3}); + } + + SECTION("issue #5046 - implicit conversion of return json to std::optional no longer implicit") + { + const json jval{}; + auto GetValue = [](const json & valRoot) -> std::optional + { + if (valRoot.contains("default")) + { + return valRoot.at("default"); + } + return std::nullopt; + }; + auto result = GetValue(jval); + CHECK(!result.has_value()); + } +#endif + +#if JSON_HAS_RANGES == 1 + SECTION("issue #4440 - assert when using std::views::filter and GCC 10") + { + auto noOpFilter = std::views::filter([](auto&&) noexcept + { + return true; + }); + json j = {1, 2, 3}; + auto filtered = j | noOpFilter; + CHECK(*filtered.begin() == 1); + } +#endif + +#if JSON_HAS_RANGES && !defined(__MINGW32__) + SECTION("issue #4916 - constructing array from C++20 ranges view does not work") + { + std::vector nums{1, 2, 37, 42, 21}; + auto filteredNums = nums | std::views::filter([](int i) + { + return i > 10; + }); + json const j(filteredNums); + CHECK(j.type() == json::value_t::array); + CHECK(j == json({37, 42, 21})); + } +#endif + + // owning_view is not available in libstdc++ < 12 +#if JSON_HAS_RANGES && !defined(__MINGW32__) && !(defined(__GLIBCXX__) && _GLIBCXX_RELEASE < 12) + SECTION("issue #4916 - constructing array from prvalue C++20 ranges view (owning_view)") + { + json const j(std::vector {1, 2, 37, 42, 21} | std::views::filter([](int i) + { + return i > 10; + })); + CHECK(j.type() == json::value_t::array); + CHECK(j == json({37, 42, 21})); + } +#endif + +#if JSON_HAS_RANGES && !defined(__MINGW32__) + SECTION("issue #4916 - constructing array from C++20 transform view (prvalue elements)") + { + std::vector nums{1, 2, 3}; + auto t = nums | std::views::transform([](int i) noexcept + { + return i * 2; + }); + json const j(t); + CHECK(j.type() == json::value_t::array); + CHECK(j == json({2, 4, 6})); + } +#endif +} + +TEST_CASE_TEMPLATE("issue #4798 - nlohmann::json::to_msgpack() encode float NaN as double", T, double, float) // NOLINT(readability-math-missing-parentheses, bugprone-throwing-static-initialization) +{ + // With issue #4798, we encode NaN, infinity, and -infinity as float instead + // of double to allow for smaller encodings. + const json jx = std::numeric_limits::quiet_NaN(); + const json jy = std::numeric_limits::infinity(); + const json jz = -std::numeric_limits::infinity(); + + ///////////////////////////////////////////////////////////////////////// + // MessagePack + ///////////////////////////////////////////////////////////////////////// + + // expected MessagePack values + const std::vector msgpack_x = {{0xCA, 0x7F, 0xC0, 0x00, 0x00}}; + const std::vector msgpack_y = {{0xCA, 0x7F, 0x80, 0x00, 0x00}}; + const std::vector msgpack_z = {{0xCA, 0xFF, 0x80, 0x00, 0x00}}; + + CHECK(json::to_msgpack(jx) == msgpack_x); + CHECK(json::to_msgpack(jy) == msgpack_y); + CHECK(json::to_msgpack(jz) == msgpack_z); + + CHECK(std::isnan(json::from_msgpack(msgpack_x).get())); + CHECK(json::from_msgpack(msgpack_y).get() == std::numeric_limits::infinity()); + CHECK(json::from_msgpack(msgpack_z).get() == -std::numeric_limits::infinity()); + + // Make sure the other MessagePakc encodings for NaN, infinity, and + // -infinity are still supported. + const std::vector msgpack_x_2 = {{0xCB, 0x7F, 0xF8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}}; + const std::vector msgpack_y_2 = {{0xCB, 0x7F, 0xF0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}}; + const std::vector msgpack_z_2 = {{0xCB, 0xFF, 0xF0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}}; + CHECK(std::isnan(json::from_msgpack(msgpack_x_2).get())); + CHECK(json::from_msgpack(msgpack_y_2).get() == std::numeric_limits::infinity()); + CHECK(json::from_msgpack(msgpack_z_2).get() == -std::numeric_limits::infinity()); + + ///////////////////////////////////////////////////////////////////////// + // CBOR + ///////////////////////////////////////////////////////////////////////// + + // expected CBOR values + const std::vector cbor_x = {{0xF9, 0x7E, 0x00}}; + const std::vector cbor_y = {{0xF9, 0x7C, 0x00}}; + const std::vector cbor_z = {{0xF9, 0xfC, 0x00}}; + + CHECK(json::to_cbor(jx) == cbor_x); + CHECK(json::to_cbor(jy) == cbor_y); + CHECK(json::to_cbor(jz) == cbor_z); + + CHECK(std::isnan(json::from_cbor(cbor_x).get())); + CHECK(json::from_cbor(cbor_y).get() == std::numeric_limits::infinity()); + CHECK(json::from_cbor(cbor_z).get() == -std::numeric_limits::infinity()); + + // Make sure the other CBOR encodings for NaN, infinity, and -infinity are + // still supported. + const std::vector cbor_x_2 = {{0xFA, 0x7F, 0xC0, 0x00, 0x00}}; + const std::vector cbor_y_2 = {{0xFA, 0x7F, 0x80, 0x00, 0x00}}; + const std::vector cbor_z_2 = {{0xFA, 0xFF, 0x80, 0x00, 0x00}}; + const std::vector cbor_x_3 = {{0xFB, 0x7F, 0xF8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}}; + const std::vector cbor_y_3 = {{0xFB, 0x7F, 0xF0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}}; + const std::vector cbor_z_3 = {{0xFB, 0xFF, 0xF0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}}; + CHECK(std::isnan(json::from_cbor(cbor_x_2).get())); + CHECK(json::from_cbor(cbor_y_2).get() == std::numeric_limits::infinity()); + CHECK(json::from_cbor(cbor_z_2).get() == -std::numeric_limits::infinity()); + CHECK(std::isnan(json::from_cbor(cbor_x_3).get())); + CHECK(json::from_cbor(cbor_y_3).get() == std::numeric_limits::infinity()); + CHECK(json::from_cbor(cbor_z_3).get() == -std::numeric_limits::infinity()); +} + +TEST_CASE("regression test #5074 - portable workaround for single-element brace init") +{ + json const j_obj = {{"key", "value"}}; + + json const j = json::array({j_obj}); + CHECK(j.is_array()); + CHECK(j.size() == 1); + CHECK(j[0] == j_obj); +} + +#if defined(JSON_BRACE_INIT_COPY_SEMANTICS) && (JSON_BRACE_INIT_COPY_SEMANTICS == 1) +TEST_CASE("regression test #5074 - single-element brace init with JSON_BRACE_INIT_COPY_SEMANTICS") +{ + // with JSON_BRACE_INIT_COPY_SEMANTICS: single-element brace init copies/moves + json const j_obj = {{"key", "value"}, {"num", 42}}; + json const j_arr = {1, 2, 3}; + + // object: brace init copies instead of wrapping + json const j1{j_obj}; + CHECK(j1.is_object()); + CHECK(j1 == j_obj); + + // array: brace init copies instead of wrapping + json const j2{j_arr}; + CHECK(j2.is_array()); + CHECK(j2.size() == 3); + CHECK(j2 == j_arr); + + // primitives still work as initializer lists + json const j3{true}; + CHECK(j3.is_boolean()); + + json const j4{42}; + CHECK(j4.is_number_integer()); +} +#endif + +struct Example_5122 +{ + float b = 2; + nlohmann::ordered_map c{}; // NOLINT(readability-redundant-member-init): needed for GCC -Weffc++ + int a = 1; + NLOHMANN_DEFINE_TYPE_INTRUSIVE_WITH_DEFAULT(Example_5122, b, c, a) +}; + +TEST_CASE("regression test #5122 - from_json into types holding nlohmann::ordered_map") +{ + Example_5122 src; + src.c.emplace("first", "1"); + src.c.emplace("second", "2"); + + ordered_json const j = src; + Example_5122 const dst = j.get(); + + CHECK(dst.b == src.b); + CHECK(dst.a == src.a); + REQUIRE(dst.c.size() == src.c.size()); + auto src_it = src.c.begin(); + auto dst_it = dst.c.begin(); + for (; src_it != src.c.end(); ++src_it, ++dst_it) + { + CHECK(dst_it->first == src_it->first); + CHECK(dst_it->second == src_it->second); + } +} + +// -Wself-assign-overloaded was introduced in Clang 7. Gate the pragma on +// __has_warning so older Clang versions do not error with "unknown warning +// group". The __has_warning check has to stay inside the __clang__ branch +// because GCC does not provide it and would tokenize-error on the argument. +#if defined(__clang__) && defined(__has_warning) + #if __has_warning("-Wself-assign-overloaded") + DOCTEST_CLANG_SUPPRESS_WARNING_PUSH + DOCTEST_CLANG_SUPPRESS_WARNING("-Wself-assign-overloaded") + #endif +#endif + +TEST_CASE("regression test #5122 - nlohmann::ordered_map copy-assignment is self-assignment safe") +{ + nlohmann::ordered_map m; + m.emplace("first", "1"); + m.emplace("second", "2"); + + // Insertion order is preserved by ordered_map, so we can check it directly. + m = m; + + REQUIRE(m.size() == 2); + auto it = m.begin(); + CHECK(it->first == "first"); + CHECK(it->second == "1"); + ++it; + CHECK(it->first == "second"); + CHECK(it->second == "2"); +} + +#if defined(__clang__) && defined(__has_warning) + #if __has_warning("-Wself-assign-overloaded") + DOCTEST_CLANG_SUPPRESS_WARNING_POP + #endif +#endif + +TEST_CASE("regression test #5122 - nlohmann::ordered_map move-assignment transfers contents") +{ + nlohmann::ordered_map src; + src.emplace("first", "1"); + src.emplace("second", "2"); + + nlohmann::ordered_map dst; + dst.emplace("stale", "x"); + dst = std::move(src); + + REQUIRE(dst.size() == 2); + auto it = dst.begin(); + CHECK(it->first == "first"); + CHECK(it->second == "1"); + ++it; + CHECK(it->first == "second"); + CHECK(it->second == "2"); + + // Re-assigning into the moved-from object must leave it in a usable state. + src = nlohmann::ordered_map {}; + src.emplace("after-move", "3"); + REQUIRE(src.size() == 1); + CHECK(src.begin()->first == "after-move"); +} + +// Stand-in for a third-party library (e.g., Eigen as of 3.4, which added +// STL-compatible begin()/end() to its vector types), living in its own +// namespace with its own to_json overload for its vector type. +namespace issue_4320_eigen +{ +// "array-compatible" from the library's point of view (it has begin()/end()), +// but for which this (fake) third-party namespace provides its own to_json. +struct vector3 +{ + double v[3]; // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays,cppcoreguidelines-use-default-member-init,modernize-use-default-member-init) + vector3(double x, double y, double z) : v{x, y, z} {} // NOLINT(hicpp-member-init,cppcoreguidelines-pro-type-member-init) + double x() const + { + return v[0]; + } + double y() const + { + return v[1]; + } + double z() const + { + return v[2]; + } + double* begin() + { + return v; + } + double* end() + { + return v + 3; + } + const double* begin() const + { + return v; + } + const double* end() const + { + return v + 3; + } +}; + +inline void to_json(json& j, const vector3& v) // NOLINT(misc-use-internal-linkage) +{ + j = {{"x", v.x()}, {"y", v.y()}, {"z", v.z()}}; +} +} // namespace issue_4320_eigen + +// The user's own namespace, using the (fake) Eigen type as an implementation +// detail behind a payload type that has nothing to do with vectors/arrays. +namespace issue_4320 +{ +// Publicly derives from issue_4320_eigen::vector3 but does *not* define its +// own to_json - it is only ever used as a temporary to reach the base +// class's to_json via ADL. +struct vector3_wrapper : issue_4320_eigen::vector3 +{ + using issue_4320_eigen::vector3::vector3; +}; + +struct payload +{ + double x, y, z; +}; + +inline vector3_wrapper to_eigen(const payload& p) // NOLINT(misc-use-internal-linkage) +{ + return {p.x, p.y, p.z}; +} + +inline void to_json(json& j, const payload& p) // NOLINT(misc-use-internal-linkage) +{ + // Unqualified call, passing a *derived* vector3_wrapper: relies on ADL + // finding issue_4320_eigen::to_json(json&, const vector3&) through the + // vector3 base class, via a derived-to-base conversion. Must NOT resolve + // to the library's own generic array-compatible to_json (an exact-match + // template for vector3_wrapper, since it also has begin()/end()), which + // would serialize this as [x, y, z] instead of {"x":x, "y":y, "z":z}. + to_json(j, to_eigen(p)); +} +} // namespace issue_4320 + +TEST_CASE("issue #4320 - custom base class must not leak nlohmann::detail into ADL") +{ + // Before the fix, basic_json unconditionally derived from a type living in + // nlohmann::detail (json_default_base), which made nlohmann::detail an + // associated namespace of every basic_json for ADL purposes. That leaked + // the library's internal generic-array to_json overload into unqualified + // to_json() calls made from user code, silently bypassing user-defined + // to_json overloads reached via a derived-to-base conversion. + const issue_4320::payload p{1.0, 2.0, 3.0}; + + json j; + to_json(j, p); + CHECK(j == json({{"x", 1.0}, {"y", 2.0}, {"z", 3.0}})); +} + +TEST_CASE("issue #5338 - truncated CBOR tagged binary subtype is rejected") +{ + const std::vector> truncated_tags = + { + {0xD8}, + {0xD9, 0x00}, + {0xDA, 0x00, 0x00, 0x00}, + {0xDB, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00} + }; + + for (const auto& data : truncated_tags) + { + CAPTURE(data); + for (const auto tag_handler : + { + json::cbor_tag_handler_t::ignore, json::cbor_tag_handler_t::store + }) + { + CAPTURE(tag_handler); + const auto result = json::from_cbor(data, true, false, tag_handler); + CHECK(result.is_discarded()); + } + } +} + +TEST_CASE("issue #5402 - update(merge_objects=true) overwrites a primitive with an object") +{ + json t = {{"k", 1}}; + t.update(json{{"k", {{"x", 2}}}}, true); + CHECK(t == json({{"k", {{"x", 2}}}})); + + json mixed = {{"keep", {{"a", 1}}}, {"replace", 1}}; + mixed.update(json{{"keep", {{"b", 2}}}, {"replace", {{"x", 2}}}}, true); + CHECK(mixed == json({{"keep", {{"a", 1}, {"b", 2}}}, {"replace", {{"x", 2}}}})); +} + + +DOCTEST_CLANG_SUPPRESS_WARNING_POP diff --git a/tests/src/unit-serialization.cpp b/tests/src/unit-serialization.cpp index f55ed8470..eddf59f2c 100644 --- a/tests/src/unit-serialization.cpp +++ b/tests/src/unit-serialization.cpp @@ -15,6 +15,8 @@ using nlohmann::json; #include #include +#include "test_utils.hpp" + TEST_CASE("serialization") { SECTION("operator<<") @@ -84,8 +86,9 @@ TEST_CASE("serialization") { const json j = "ä\xA9ü"; - CHECK_THROWS_WITH_AS(j.dump(), "[json.exception.type_error.316] invalid UTF-8 byte at index 2: 0xA9", json::type_error&); - CHECK_THROWS_WITH_AS(j.dump(1, ' ', false, json::error_handler_t::strict), "[json.exception.type_error.316] invalid UTF-8 byte at index 2: 0xA9", json::type_error&); + // dump() is nodiscard; the exception is thrown by dump() itself before it would return + CHECK_THROWS_WITH_AS(utils::ignore_return_value(j.dump()), "[json.exception.type_error.316] invalid UTF-8 byte at index 2: 0xA9", json::type_error&); + CHECK_THROWS_WITH_AS(utils::ignore_return_value(j.dump(1, ' ', false, json::error_handler_t::strict)), "[json.exception.type_error.316] invalid UTF-8 byte at index 2: 0xA9", json::type_error&); CHECK(j.dump(-1, ' ', false, json::error_handler_t::ignore) == "\"äü\""); CHECK(j.dump(-1, ' ', false, json::error_handler_t::replace) == "\"ä\xEF\xBF\xBDü\""); CHECK(j.dump(-1, ' ', true, json::error_handler_t::replace) == "\"\\u00e4\\ufffd\\u00fc\""); @@ -95,8 +98,9 @@ TEST_CASE("serialization") { const json j = "123\xC2"; - CHECK_THROWS_WITH_AS(j.dump(), "[json.exception.type_error.316] incomplete UTF-8 string; last byte: 0xC2", json::type_error&); - CHECK_THROWS_AS(j.dump(1, ' ', false, json::error_handler_t::strict), json::type_error&); + // dump() is nodiscard; the exception is thrown by dump() itself before it would return + CHECK_THROWS_WITH_AS(utils::ignore_return_value(j.dump()), "[json.exception.type_error.316] incomplete UTF-8 string; last byte: 0xC2", json::type_error&); + CHECK_THROWS_AS(utils::ignore_return_value(j.dump(1, ' ', false, json::error_handler_t::strict)), json::type_error&); CHECK(j.dump(-1, ' ', false, json::error_handler_t::ignore) == "\"123\""); CHECK(j.dump(-1, ' ', false, json::error_handler_t::replace) == "\"123\xEF\xBF\xBD\""); CHECK(j.dump(-1, ' ', true, json::error_handler_t::replace) == "\"123\\ufffd\""); @@ -106,8 +110,9 @@ TEST_CASE("serialization") { const json j = "123\xF1\xB0\x34\x35\x36"; - CHECK_THROWS_WITH_AS(j.dump(), "[json.exception.type_error.316] invalid UTF-8 byte at index 5: 0x34", json::type_error&); - CHECK_THROWS_AS(j.dump(1, ' ', false, json::error_handler_t::strict), json::type_error&); + // dump() is nodiscard; the exception is thrown by dump() itself before it would return + CHECK_THROWS_WITH_AS(utils::ignore_return_value(j.dump()), "[json.exception.type_error.316] invalid UTF-8 byte at index 5: 0x34", json::type_error&); + CHECK_THROWS_AS(utils::ignore_return_value(j.dump(1, ' ', false, json::error_handler_t::strict)), json::type_error&); CHECK(j.dump(-1, ' ', false, json::error_handler_t::ignore) == "\"123456\""); CHECK(j.dump(-1, ' ', false, json::error_handler_t::replace) == "\"123\xEF\xBF\xBD\x34\x35\x36\""); CHECK(j.dump(-1, ' ', true, json::error_handler_t::replace) == "\"123\\ufffd456\""); @@ -382,3 +387,232 @@ TEST_CASE("dump for basic_json with long double number_float_t") check_same(100.0L, 100.0); } } + +TEST_CASE("serialization of strings (bulk fast path)") +{ + // These cases exercise the SWAR bulk-copy fast path in dump_escaped and the + // internal write buffer: long runs, escapes interrupting runs, 0x7F/DEL, + // multibyte UTF-8 under both ensure_ascii settings, and payloads larger than + // the write buffer. + + SECTION("long unescaped ASCII exceeds the write buffer") + { + const std::string big(3000, 'a'); + const json j = big; + CHECK(j.dump() == '"' + big + '"'); + CHECK(j.dump(-1, ' ', true) == '"' + big + '"'); + // round-trips + CHECK(json::parse(j.dump()) == j); + } + + SECTION("runs interrupted by escapes") + { + const json j = std::string(500, 'x') + "\n\"\\" + std::string(500, 'y'); + const std::string out = j.dump(); + CHECK(out == '"' + std::string(500, 'x') + "\\n\\\"\\\\" + std::string(500, 'y') + '"'); + CHECK(json::parse(out) == j); + } + + SECTION("DEL (0x7F) depends on ensure_ascii") + { + const json j = std::string("a\x7f" "b"); + CHECK(j.dump(-1, ' ', false) == "\"a\x7f" "b\""); // copied verbatim + CHECK(j.dump(-1, ' ', true) == "\"a\\u007fb\""); // escaped + } + + SECTION("multibyte UTF-8 under both ensure_ascii settings") + { + const json j = std::string("A\xc3\xa9\xe4\xbd\xa0\xf0\x9f\x98\x80Z"); // A é 你 😀 Z + // not escaping non-ASCII: bytes are copied through the bulk validator + CHECK(j.dump(-1, ' ', false) == "\"A\xc3\xa9\xe4\xbd\xa0\xf0\x9f\x98\x80Z\""); + // ensure_ascii: escaped (with a surrogate pair for the emoji) + CHECK(j.dump(-1, ' ', true) == "\"A\\u00e9\\u4f60\\ud83d\\ude00Z\""); + CHECK(json::parse(j.dump(-1, ' ', true)) == j); + } + + SECTION("many small structural writes exceed the write buffer") + { + json arr = json::array(); + for (int i = 0; i < 2000; ++i) + { + arr.push_back(i); + } + const std::string out = arr.dump(); + CHECK(out.front() == '['); + CHECK(out.back() == ']'); + CHECK(json::parse(out) == arr); + + json obj = json::object(); + for (int i = 0; i < 500; ++i) + { + obj["key" + std::to_string(i)] = i; + } + CHECK(json::parse(obj.dump()) == obj); + CHECK(json::parse(obj.dump(2)) == obj); + + // an array of many empty strings emits a long run of single-character + // writes ('"', '"', ',') at shallow nesting depth, so the write buffer + // fills and flushes mid-run without the deep recursion that would + // overflow the stack on some debug builds + json many_empty = json::array(); + for (int i = 0; i < 500; ++i) + { + many_empty.push_back(""); + } + const std::string out2 = many_empty.dump(); + CHECK(out2.size() > 1024); // spans multiple write-buffer flushes + CHECK(out2.front() == '['); + CHECK(out2.back() == ']'); + CHECK(json::parse(out2) == many_empty); + } + + SECTION("invalid UTF-8 handling is unaffected by the fast path") + { + const json j = std::string("valid\xff" "more"); + CHECK_THROWS_WITH_AS(j.dump(), "[json.exception.type_error.316] invalid UTF-8 byte at index 5: 0xFF", json::type_error&); + CHECK(j.dump(-1, ' ', false, json::error_handler_t::replace) == "\"valid\xef\xbf\xbd" "more\""); + CHECK(j.dump(-1, ' ', true, json::error_handler_t::replace) == "\"valid\\ufffdmore\""); + CHECK(j.dump(-1, ' ', false, json::error_handler_t::ignore) == "\"validmore\""); + } +} + +TEST_CASE("indentation is written straight into the write buffer") +{ + // put_indent() memsets the indentation into the write buffer instead of + // copying it out of a pre-grown indentation string. These cases cover an + // indentation wider than the buffer, a non-space indentation character, and + // nesting deep enough that the accumulated indentation spans several + // buffer-fulls - the situations the old grow-a-string approach got wrong. + + SECTION("indent_step wider than the write buffer") + { + const json j = {{"a", 1}}; + // 2000 > the 1024-byte write buffer, and > the 512 the indentation + // string used to start at + CHECK(j.dump(2000) == "{\n" + std::string(2000, ' ') + "\"a\": 1\n}"); + // several whole buffer-fulls, so the buffer is refilled once and then + // flushed repeatedly + CHECK(j.dump(5000) == "{\n" + std::string(5000, ' ') + "\"a\": 1\n}"); + CHECK(j.dump(5000, '\t') == "{\n" + std::string(5000, '\t') + "\"a\": 1\n}"); + // an exact multiple of the buffer size + CHECK(j.dump(4096) == "{\n" + std::string(4096, ' ') + "\"a\": 1\n}"); + } + + SECTION("a non-space indentation character is used throughout") + { + const json j = {{"a", 1}}; + // 600 is past the point where the indentation used to be grown, which + // is where a hard-coded space would have shown up + CHECK(j.dump(600, '\t') == "{\n" + std::string(600, '\t') + "\"a\": 1\n}"); + CHECK(j.dump(3, '.') == "{\n...\"a\": 1\n}"); + } + + SECTION("accumulated indentation spans several buffer-fulls") + { + // five levels deep at 400 per level: the innermost value is indented by + // 2000 characters, reached in steps that each straddle the buffer end + json j = json::array({1}); + for (int i = 0; i < 4; ++i) + { + j = json::array({j}); + } + + const std::string out = j.dump(400); + CHECK(out.find(std::string("\n") + std::string(2000, ' ') + "1\n") != std::string::npos); + CHECK(json::parse(out) == j); + } + + SECTION("indentation is unchanged for ordinary widths") + { + const json j = {{"a", {1, 2}}, {"b", nullptr}}; + CHECK(j.dump(2) == "{\n \"a\": [\n 1,\n 2\n ],\n \"b\": null\n}"); + CHECK(j.dump(0) == "{\n\"a\": [\n1,\n2\n],\n\"b\": null\n}"); + } +} + +TEST_CASE("serialization of deeply nested values") +{ + // dump() descends into a bounded number of levels and writes out whatever + // is nested deeper than that without the call stack; see + // https://github.com/nlohmann/json/issues/5387 + + SECTION("nested deeper than the call stack could follow") + { + // parsing is iterative, so building these costs little + const std::size_t depth = 100000; + + const std::string array_text = std::string(depth, '[') + '0' + std::string(depth, ']'); + CHECK(json::parse(array_text).dump() == array_text); + + std::string object_text; + object_text.reserve((6 * depth) + 1); + for (std::size_t i = 0; i < depth; ++i) + { + object_text += "{\"a\":"; + } + object_text += '1'; + object_text.append(depth, '}'); + CHECK(json::parse(object_text).dump() == object_text); + } + + SECTION("depths around the bound of the descent") + { + // Cover every depth around the bound, so that the two ways of writing a + // value are known to meet cleanly - wherever the bound is set. + for (std::size_t d = 1; d <= 300; ++d) + { + CAPTURE(d); + + const std::string array_text = std::string(d, '[') + '7' + std::string(d, ']'); + CHECK(json::parse(array_text).dump() == array_text); + + std::string object_text; + for (std::size_t i = 0; i < d; ++i) + { + object_text += "{\"k\":"; + } + object_text += '7'; + object_text.append(d, '}'); + CHECK(json::parse(object_text).dump() == object_text); + } + } + + SECTION("pretty-printing across the bound") + { + for (std::size_t d = 120; d <= 140; ++d) + { + CAPTURE(d); + + const json j = json::parse(std::string(d, '[') + '7' + std::string(d, ']')); + + std::string expected; + for (std::size_t i = 0; i < d; ++i) + { + expected += std::string(2 * i, ' ') + "[\n"; + } + expected += std::string(2 * d, ' ') + '7'; + for (std::size_t i = d; i > 0; --i) + { + expected += '\n' + std::string(2 * (i - 1), ' ') + ']'; + } + + CHECK(j.dump(2) == expected); + } + } + + SECTION("an empty container below the bound") + { + // an empty container is written out in full and never descended into, + // so it must not gain a newline when it is reached iteratively + for (std::size_t d = 125; d <= 135; ++d) + { + CAPTURE(d); + + const std::string compact = std::string(d, '[') + "[]" + std::string(d, ']'); + CHECK(json::parse(compact).dump() == compact); + + const std::string with_object = std::string(d, '[') + "{}" + std::string(d, ']'); + CHECK(json::parse(with_object).dump() == with_object); + } + } +} diff --git a/tests/src/unit-ubjson.cpp b/tests/src/unit-ubjson.cpp index a07df4cef..ae7effcc3 100644 --- a/tests/src/unit-ubjson.cpp +++ b/tests/src/unit-ubjson.cpp @@ -2254,6 +2254,172 @@ TEST_CASE("issue #5405 - array reserve for definite-length UBJSON arrays") } } +TEST_CASE("UBJSON nesting does not consume the call stack") +{ + // Containers used to be read by calling back into the value reader once + // per element, so the native call stack grew with the nesting depth of the + // input. '[' alone opens a container, so a payload of repeated '[' crashed + // the process (#5104), as did the optimized forms, which reach the same + // path through a type or size annotation. The containers are kept on a + // heap stack now. + // + // Deeply nested values must not be compared, copied or dumped here: those + // operations are still recursive and would reintroduce the crash. + json _; + + SECTION("containers that end at a marker") + { + const std::vector input(500000, '['); + CHECK_THROWS_WITH_AS(_ = json::from_ubjson(input), "[json.exception.parse_error.110] parse error at byte 500001: syntax error while parsing UBJSON value: unexpected end of input", json::parse_error&); + CHECK(json::from_ubjson(input, true, false).is_discarded()); + } + + SECTION("containers with a size") + { + std::vector input; + for (std::size_t i = 0; i < 100000; ++i) + { + input.push_back('['); + input.push_back('#'); + input.push_back('i'); + input.push_back(1); + } + CHECK_THROWS_AS(_ = json::from_ubjson(input), json::parse_error&); + CHECK(json::from_ubjson(input, true, false).is_discarded()); + } + + SECTION("containers with a type and a size") + { + // '[' is a permitted optimized type in UBJSON, so each element of such + // a container is itself a container, read without a marker of its own + std::vector input; + for (std::size_t i = 0; i < 100000; ++i) + { + const std::vector level = {'[', '$', '[', '#', 'i', 1}; + input.insert(input.end(), level.begin(), level.end()); + } + CHECK_THROWS_AS(_ = json::from_ubjson(input), json::parse_error&); + CHECK(json::from_ubjson(input, true, false).is_discarded()); + } + + SECTION("a well-formed deep value is read through the SAX interface") + { + std::vector input(100000, '['); + input.insert(input.end(), 100000, ']'); + + SaxCountdown accept_all(1000000); + CHECK(json::sax_parse(input, &accept_all, json::input_format_t::ubjson)); + } + + SECTION("a well-formed deep value is read into a value") + { + const std::size_t depth = 10000; + std::vector input(depth, '['); + input.insert(input.end(), depth, ']'); + + json j = json::from_ubjson(input); + + std::size_t measured = 0; + const json* p = &j; + while (p->is_array() && !p->empty()) + { + p = &p->front(); + ++measured; + } + // the innermost array is empty, so the descent stops one level short + CHECK(measured == depth - 1); + } + + SECTION("containers are still read the same way") + { + CHECK(json::from_ubjson(std::vector({'[', ']'})) == json::array()); + CHECK(json::from_ubjson(std::vector({'{', '}'})) == json::object()); + CHECK(json::from_ubjson(std::vector({'[', '#', 'i', 0})) == json::array()); + CHECK(json::from_ubjson(std::vector({'{', '#', 'i', 0})) == json::object()); + CHECK(json::from_ubjson(std::vector({'[', '$', 'i', '#', 'i', 2, 1, 2})) == json({1, 2})); + CHECK(json::from_ubjson(std::vector({'[', '#', 'i', 2, 'i', 1, 'i', 2})) == json({1, 2})); + CHECK(json::from_ubjson(std::vector({'{', '$', 'i', '#', 'i', 1, 'i', 1, 'a', 1})) == json({{"a", 1}})); + // a no-op is not a value, so a container of them holds none + CHECK(json::from_ubjson(std::vector({'[', '$', 'N', '#', 'i', 2})) == json::array()); + // sized and unsized forms nested inside one another + CHECK(json::from_ubjson(std::vector({'[', '[', '#', 'i', 2, 'i', 1, 'i', 2, ']'})) == json({{1, 2}})); + CHECK(json::from_ubjson(std::vector({'[', '#', 'i', 1, '[', 'i', 1, ']'})) == json({{1}})); + // an optimized container of containers + CHECK(json::from_ubjson(std::vector({'[', '$', '[', '#', 'i', 2, 'i', 1, ']', 'i', 2, ']'})) == json({{1}, {2}})); + } + + SECTION("BJData containers are still read the same way") + { + // the ND-array wrapper and the binary shortcut are complete values, + // not containers the reader descends into + CHECK(json::from_bjdata(std::vector({'[', '$', 'U', '#', '[', '$', 'i', '#', 'i', 2, 2, 3, 1, 2, 3, 4, 5, 6})) == + json({{"_ArrayType_", "uint8"}, {"_ArraySize_", {2, 3}}, {"_ArrayData_", {1, 2, 3, 4, 5, 6}}})); + CHECK(json::from_bjdata(std::vector({'[', '$', 'i', '#', 'i', 2, 1, 2})) == json({1, 2})); + CHECK(json::from_bjdata(std::vector({'[', '[', 'i', 1, ']', ']'})) == json({{1}})); + } +} + +TEST_CASE("UBJSON optimized arrays of a valueless type are bounded") +{ + // An element of type 'Z', 'T' or 'F' is encoded by its marker alone, so an + // optimized array of one of those has no payload and the declared count is + // the only thing deciding how much is allocated. Ten bytes used to produce + // billions of values (#2793); every other type costs at least one byte per + // element and is bounded by the end of the input. + json _; + + SECTION("an excessive count is rejected") + { + // 'l' is a big-endian int32: 0x7FFFFFFF elements, about 34 GB of value + for (const auto marker : + {'Z', 'T', 'F' + }) + { + const std::vector input = {'[', '$', static_cast(marker), '#', 'l', 0x7F, 0xFF, 0xFF, 0xFF}; + CHECK_THROWS_WITH_AS(_ = json::from_ubjson(input), "[json.exception.out_of_range.408] syntax error while parsing UBJSON size: excessive array size", json::out_of_range&); + CHECK(json::from_ubjson(input, true, false).is_discarded()); + } + } + + SECTION("ordinary counts are unaffected") + { + CHECK(json::from_ubjson(std::vector({'[', '$', 'Z', '#', 'i', 3})) == json({nullptr, nullptr, nullptr})); + CHECK(json::from_ubjson(std::vector({'[', '$', 'T', '#', 'i', 2})) == json({true, true})); + CHECK(json::from_ubjson(std::vector({'[', '$', 'F', '#', 'i', 2})) == json({false, false})); + // 'N' is a no-op rather than a value, and still yields an empty array + CHECK(json::from_ubjson(std::vector({'[', '$', 'N', '#', 'i', 2})) == json::array()); + } + + SECTION("a type with a payload is unaffected") + { + // A count past the limit is not rejected for 'U', which costs a byte + // per element and is bounded by the end of the input instead. The + // count is kept just past the limit rather than made huge, because a + // count that also exceeds the array's max_size() is reported as + // out_of_range before the input runs out, and max_size() depends on + // the width of std::size_t. + const std::vector input = {'[', '$', 'U', '#', 'l', 0x00, 0x10, 0x00, 0x01}; + CHECK_THROWS_WITH_AS(_ = json::from_ubjson(input), "[json.exception.parse_error.110] parse error at byte 10: syntax error while parsing UBJSON number: unexpected end of input", json::parse_error&); + CHECK(json::from_ubjson(input, true, false).is_discarded()); + } + + SECTION("the writer stays within what the reader accepts") + { + // below the limit the optimized form is used and is tiny; above it the + // writer falls back so that the result can still be read back + json const at_limit(1048576, nullptr); + const auto v_at_limit = json::to_ubjson(at_limit, true, true); + CHECK(v_at_limit.size() == 9); + CHECK(v_at_limit.at(1) == '$'); + CHECK(json::from_ubjson(v_at_limit) == at_limit); + + json const above_limit(1048577, nullptr); + const auto v_above_limit = json::to_ubjson(above_limit, true, true); + CHECK(v_above_limit.at(1) != '$'); + CHECK(json::from_ubjson(v_above_limit) == above_limit); + } +} + TEST_CASE("Universal Binary JSON Specification Examples 1") { SECTION("Null Value") diff --git a/tests/src/unit-unicode1.cpp b/tests/src/unit-unicode1.cpp index 174ce1395..2d744003a 100644 --- a/tests/src/unit-unicode1.cpp +++ b/tests/src/unit-unicode1.cpp @@ -17,6 +17,7 @@ using nlohmann::json; #include #include #include "make_test_data_available.hpp" +#include "test_utils.hpp" TEST_CASE("Unicode (1/5)" * doctest::skip()) { @@ -240,7 +241,8 @@ void roundtrip(bool success_expected, const std::string& s) if (success_expected) { // serialization succeeds - CHECK_NOTHROW(j.dump()); + // dump() is nodiscard; this only checks that dumping does not throw + CHECK_NOTHROW(utils::ignore_return_value(j.dump())); // exclude parse test for U+0000 if (s[0] != '\0') @@ -259,7 +261,8 @@ void roundtrip(bool success_expected, const std::string& s) else { // serialization fails - CHECK_THROWS_AS(j.dump(), json::type_error&); + // dump() is nodiscard; the exception is thrown by dump() itself before it would return + CHECK_THROWS_AS(utils::ignore_return_value(j.dump()), json::type_error&); // parsing JSON text fails CHECK_THROWS_AS(_ = json::parse(ps), json::parse_error&); diff --git a/tests/src/unit-unicode2.cpp b/tests/src/unit-unicode2.cpp index fb68815ba..a9649b4de 100644 --- a/tests/src/unit-unicode2.cpp +++ b/tests/src/unit-unicode2.cpp @@ -19,6 +19,7 @@ using nlohmann::json; #include #include #include "make_test_data_available.hpp" +#include "test_utils.hpp" // this test suite uses static variables with non-trivial destructors DOCTEST_CLANG_SUPPRESS_WARNING_PUSH @@ -97,7 +98,8 @@ void check_utf8dump(bool success_expected, int byte1, int byte2 = -1, int byte3 else { // strict mode must throw if success is not expected - CHECK_THROWS_AS(j.dump(), json::type_error&); + // dump() is nodiscard; the exception is thrown by dump() itself before it would return + CHECK_THROWS_AS(utils::ignore_return_value(j.dump()), json::type_error&); // ignore and replace must create different dumps CHECK(s_ignored != s_replaced); diff --git a/tests/src/unit-unicode3.cpp b/tests/src/unit-unicode3.cpp index 739a3dad3..d5627d8cc 100644 --- a/tests/src/unit-unicode3.cpp +++ b/tests/src/unit-unicode3.cpp @@ -19,6 +19,7 @@ using nlohmann::json; #include #include #include "make_test_data_available.hpp" +#include "test_utils.hpp" // this test suite uses static variables with non-trivial destructors DOCTEST_CLANG_SUPPRESS_WARNING_PUSH @@ -97,7 +98,8 @@ void check_utf8dump(bool success_expected, int byte1, int byte2 = -1, int byte3 else { // strict mode must throw if success is not expected - CHECK_THROWS_AS(j.dump(), json::type_error&); + // dump() is nodiscard; the exception is thrown by dump() itself before it would return + CHECK_THROWS_AS(utils::ignore_return_value(j.dump()), json::type_error&); // ignore and replace must create different dumps CHECK(s_ignored != s_replaced); diff --git a/tests/src/unit-unicode4.cpp b/tests/src/unit-unicode4.cpp index f7047201c..f15a1499f 100644 --- a/tests/src/unit-unicode4.cpp +++ b/tests/src/unit-unicode4.cpp @@ -19,6 +19,7 @@ using nlohmann::json; #include #include #include "make_test_data_available.hpp" +#include "test_utils.hpp" // this test suite uses static variables with non-trivial destructors DOCTEST_CLANG_SUPPRESS_WARNING_PUSH @@ -97,7 +98,8 @@ void check_utf8dump(bool success_expected, int byte1, int byte2 = -1, int byte3 else { // strict mode must throw if success is not expected - CHECK_THROWS_AS(j.dump(), json::type_error&); + // dump() is nodiscard; the exception is thrown by dump() itself before it would return + CHECK_THROWS_AS(utils::ignore_return_value(j.dump()), json::type_error&); // ignore and replace must create different dumps CHECK(s_ignored != s_replaced); diff --git a/tests/src/unit-unicode5.cpp b/tests/src/unit-unicode5.cpp index e4dcc2131..e35801823 100644 --- a/tests/src/unit-unicode5.cpp +++ b/tests/src/unit-unicode5.cpp @@ -19,6 +19,7 @@ using nlohmann::json; #include #include #include "make_test_data_available.hpp" +#include "test_utils.hpp" // this test suite uses static variables with non-trivial destructors DOCTEST_CLANG_SUPPRESS_WARNING_PUSH @@ -97,7 +98,8 @@ void check_utf8dump(bool success_expected, int byte1, int byte2 = -1, int byte3 else { // strict mode must throw if success is not expected - CHECK_THROWS_AS(j.dump(), json::type_error&); + // dump() is nodiscard; the exception is thrown by dump() itself before it would return + CHECK_THROWS_AS(utils::ignore_return_value(j.dump()), json::type_error&); // ignore and replace must create different dumps CHECK(s_ignored != s_replaced); diff --git a/tests/src/unit-user_defined_input.cpp b/tests/src/unit-user_defined_input.cpp index 823e82862..f07a8a608 100644 --- a/tests/src/unit-user_defined_input.cpp +++ b/tests/src/unit-user_defined_input.cpp @@ -18,7 +18,12 @@ #include using nlohmann::json; +#include // array +#include // size_t +#include // uint8_t #include +#include // string +#include // vector #if defined(__cpp_lib_concepts) && defined(JSON_HAS_CPP_20) #include @@ -212,6 +217,66 @@ TEST_CASE("Parse with heterogeneous iterator and sentinel types") CHECK(j2.at(0) == 1); } +// A type whose data() hands out raw bytes but whose size() counts something +// else - here fixed-size records. Reading [data(), data() + size()) as bytes +// would silently truncate the input, so data() and size() alone must not be +// taken as evidence of contiguous byte storage. +struct record_buffer +{ + using value_type = std::array; + + std::string bytes; + + const char* data() const noexcept + { + return bytes.data(); + } + std::size_t size() const noexcept + { + return bytes.size() / sizeof(value_type); + } + const char* begin() const noexcept + { + return bytes.data(); + } + const char* end() const noexcept + { + return bytes.data() + bytes.size(); + } +}; + +TEST_CASE("Contiguous byte containers take the pointer adapter") +{ + // Containers with contiguous single-byte storage are routed through the + // pointer-based adapter so the bulk fast paths apply in every standard, not + // only in C++20 where the library iterators model std::contiguous_iterator. + CHECK(nlohmann::detail::is_contiguous_byte_container::value); + CHECK(nlohmann::detail::is_contiguous_byte_container>::value); + CHECK(nlohmann::detail::is_contiguous_byte_container>::value); + CHECK(nlohmann::detail::is_contiguous_byte_container>::value); + + // input_adapter() takes its container by forwarding reference, so the trait + // is also asked about reference types + CHECK(nlohmann::detail::is_contiguous_byte_container::value); + CHECK(nlohmann::detail::is_contiguous_byte_container::value); + + // everything else keeps the iterator-based adapter + CHECK_FALSE(nlohmann::detail::is_contiguous_byte_container>::value); + CHECK_FALSE(nlohmann::detail::is_contiguous_byte_container>::value); + CHECK_FALSE(nlohmann::detail::is_contiguous_byte_container::value); + + // including a type that has data() and size() but whose size() does not + // count the units data() points at: its value_type says so + CHECK_FALSE(nlohmann::detail::is_contiguous_byte_container::value); + + // and such a container still parses through its iterators, in full - taking + // it for a byte container would stop after data() + size() bytes + const record_buffer buffer{"[1,2,3,4,5]"}; + CHECK(buffer.data() == buffer.bytes.data()); + CHECK(buffer.size() * sizeof(record_buffer::value_type) < buffer.bytes.size()); + CHECK(json::parse(buffer) == json({1, 2, 3, 4, 5})); +} + #if defined(__cpp_lib_concepts) && defined(JSON_HAS_CPP_20) // JSON_HAS_CPP_20 (do not remove; see note at top of file) TEST_CASE("Parse with std::counted_iterator and std::default_sentinel_t") @@ -228,6 +293,180 @@ TEST_CASE("Parse with std::counted_iterator and std::default_sentinel_t") const std::counted_iterator first2(json_str.begin(), len); CHECK(json::accept(first2, std::default_sentinel)); } + +TEST_CASE("std::counted_iterator reaches the contiguous fast paths") +{ + // A sized sentinel makes the remaining element count computable in O(1), so + // std::counted_iterator over a contiguous iterator must reach the same bulk + // string/number scanners as a plain pointer - not just the byte-at-a-time + // fallback (see #5268 for the equivalent memcpy fast path). +#if JSON_HAS_RANGES + // JSON_HAS_RANGES is 0 on standard libraries with an incomplete + // (libstdc++ < 11, libc++ < 16), where the adapter deliberately falls back + // to the byte-at-a-time scanner; everything below still has to work there. + using adapter_type = nlohmann::detail::iterator_input_adapter, std::default_sentinel_t>; + CHECK(adapter_type::supports_bulk_scan); + CHECK(adapter_type::supports_seek); +#endif + + // exercise every fast path: long ASCII run, multibyte UTF-8, escapes, and + // integer/floating-point numbers + const std::string json_str = + R"({"ascii":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",)" + "\"utf8\":\"\xe4\xb8\xad\xe6\x96\x87\xf0\x9f\x98\x80\xc3\xa9\"," + R"("escaped":"aéb\n\\","ints":[0,-1,18446744073709551615,-9223372036854775808],)" + R"("floats":[1.5,-2.25e3,0.30000000000000004]})"; + const auto len = static_cast>(json_str.size()); + + const std::counted_iterator first(json_str.data(), len); + const json j = json::parse(first, std::default_sentinel); + + // parsing through the pointer adapter must give exactly the same result + CHECK(j == json::parse(json_str)); + +#if !defined(JSON_NOEXCEPTION) + // Diagnostics that quote the offending token are reconstructed from the + // already-consumed input (supports_seek), a path a sized sentinel only + // reaches now; check a few that include the "last read" text. Parsing + // invalid input aborts when exceptions are off, hence the guard. + // Raw strings and explicit bytes: an escaped literal and two literals + // written next to each other both read as mistakes to static analysis. + const auto byte = [](int value) + { + return std::string(1, static_cast(value)); + }; + const std::vector diagnostic_docs = + { + "1\nx", + "truX", + "[tru]", + R"("abc)", + R"(["\ud834"])", + R"(["a)" + byte(0x01) + R"(b"])", + R"([")" + byte(0xC3) + byte(0x28) + R"("])", + "[1e]", + R"(["aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaX)" + }; + + for (const auto& text : diagnostic_docs) + { + CAPTURE(text); + const std::counted_iterator it(text.data(), static_cast>(text.size())); + std::string counted_message; + std::string string_message; + try + { + const json counted_result = json::parse(it, std::default_sentinel); + static_cast(counted_result); + } + catch (const json::parse_error& e) + { + counted_message = e.what(); + } + try + { + const json string_result = json::parse(text); + static_cast(string_result); + } + catch (const json::parse_error& e) + { + string_message = e.what(); + } + CHECK_FALSE(counted_message.empty()); + CHECK(counted_message == string_message); + } + + // and errors must still be reported identically + const std::string bad = "[01\n]"; + const std::counted_iterator bad_first(bad.data(), static_cast>(bad.size())); + std::string counted_what; + std::string string_what; + try + { + const json counted_result = json::parse(bad_first, std::default_sentinel); + static_cast(counted_result); + } + catch (const json::parse_error& e) + { + counted_what = e.what(); + } + try + { + const json string_result = json::parse(bad); + static_cast(string_result); + } + catch (const json::parse_error& e) + { + string_what = e.what(); + } + CHECK_FALSE(counted_what.empty()); + CHECK(counted_what == string_what); +#endif +} + +#if !defined(JSON_NOEXCEPTION) +// several cases below are truncated on purpose, and parsing invalid input +// aborts when exceptions are off +TEST_CASE("std::counted_iterator bulk scanning stops at the counted end") +{ + // The count, not the size of the underlying buffer, is the end of the + // input: the bulk scanners must never look at the bytes behind it, even + // though they are readable. Each case is compared against parsing the + // equivalent prefix as a std::string. + const auto via_counted = [](const std::string & buf, std::size_t n) -> std::string + { + const std::counted_iterator first(buf.data(), static_cast>(n)); + try + { + const json j = json::parse(first, std::default_sentinel); + return "OK|" + j.dump(); + } + catch (const json::parse_error& e) + { + return {e.what()}; + } + }; + const auto via_prefix = [](const std::string & buf, std::size_t n) -> std::string + { + try + { + const json j = json::parse(buf.substr(0, n)); + return "OK|" + j.dump(); + } + catch (const json::parse_error& e) + { + return {e.what()}; + } + }; + + struct testcase // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init) + { + const char* buffer; + std::size_t count; + }; + const std::vector cases = + { + {"[\"abc\"]____TRAILING____", 7}, // exact fit, tail hidden + {"[\"abcdefghijklmnop\"]____", 8}, // cut inside a string + {"[\"abc\"]____", 6}, // cut just before the closing quote + {"[12345]xxxxx", 4}, // cut inside a number + {"[123]999999", 5}, // number ends exactly at the count + {"[\"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\"]", 12}, // closing quote only behind the count + {"[\"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\"]", 19}, // cut inside an 8-byte SWAR stride + {"[\"\xe4\xb8\xad\xe6\x96\x87\"]", 5}, // cut inside a UTF-8 sequence + {"[\"\xe4\xb8\xad\xe6\x96\x87\"]____", 10}, // complete UTF-8, tail hidden + {"[1.25e3]TRAILINGDIGITS999", 7}, // number token reaches the count + }; + + for (const auto& tc : cases) + { + CAPTURE(tc.buffer); + CAPTURE(tc.count); + const std::string buffer = tc.buffer; + CHECK(via_counted(buffer, tc.count) == via_prefix(buffer, tc.count)); + } +} +#endif #endif } // namespace