diff --git a/.github/workflows/cancel_closed_pr_runs.yml b/.github/workflows/cancel_closed_pr_runs.yml index 4d8ee33e5..3d3a3dd57 100644 --- a/.github/workflows/cancel_closed_pr_runs.yml +++ b/.github/workflows/cancel_closed_pr_runs.yml @@ -28,7 +28,7 @@ jobs: steps: - name: Harden Runner - uses: step-security/harden-runner@e14015d583714f6e62063499dc959a02595150a1 # v2.21.1 + uses: step-security/harden-runner@351661ca32ac09a36dc5ee2d536e3128f2a3c8ed # v2.22.0 with: egress-policy: audit diff --git a/.github/workflows/check_amalgamation.yml b/.github/workflows/check_amalgamation.yml index eb964c979..f4a546a2b 100644 --- a/.github/workflows/check_amalgamation.yml +++ b/.github/workflows/check_amalgamation.yml @@ -22,7 +22,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Harden Runner - uses: step-security/harden-runner@e14015d583714f6e62063499dc959a02595150a1 # v2.21.1 + uses: step-security/harden-runner@351661ca32ac09a36dc5ee2d536e3128f2a3c8ed # v2.22.0 with: egress-policy: audit @@ -46,7 +46,7 @@ jobs: steps: - name: Harden Runner - uses: step-security/harden-runner@e14015d583714f6e62063499dc959a02595150a1 # v2.21.1 + uses: step-security/harden-runner@351661ca32ac09a36dc5ee2d536e3128f2a3c8ed # v2.22.0 with: egress-policy: audit diff --git a/.github/workflows/check_docs_links.yml b/.github/workflows/check_docs_links.yml index 342c96bff..425a9b03e 100644 --- a/.github/workflows/check_docs_links.yml +++ b/.github/workflows/check_docs_links.yml @@ -20,7 +20,7 @@ jobs: timeout-minutes: 30 steps: - name: Harden Runner - uses: step-security/harden-runner@e14015d583714f6e62063499dc959a02595150a1 # v2.21.1 + uses: step-security/harden-runner@351661ca32ac09a36dc5ee2d536e3128f2a3c8ed # v2.22.0 with: egress-policy: audit diff --git a/.github/workflows/cifuzz.yml b/.github/workflows/cifuzz.yml index 9398fdbaf..e523eddbc 100644 --- a/.github/workflows/cifuzz.yml +++ b/.github/workflows/cifuzz.yml @@ -13,7 +13,7 @@ jobs: runs-on: ubuntu-22.04 steps: - name: Harden Runner - uses: step-security/harden-runner@e14015d583714f6e62063499dc959a02595150a1 # v2.21.1 + uses: step-security/harden-runner@351661ca32ac09a36dc5ee2d536e3128f2a3c8ed # v2.22.0 with: egress-policy: audit diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 9e86353c6..f2c9f8f5d 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -27,7 +27,7 @@ jobs: steps: - name: Harden Runner - uses: step-security/harden-runner@e14015d583714f6e62063499dc959a02595150a1 # v2.21.1 + uses: step-security/harden-runner@351661ca32ac09a36dc5ee2d536e3128f2a3c8ed # v2.22.0 with: egress-policy: audit diff --git a/.github/workflows/comment_check_amalgamation.yml b/.github/workflows/comment_check_amalgamation.yml index 32bca8486..41bfa222e 100644 --- a/.github/workflows/comment_check_amalgamation.yml +++ b/.github/workflows/comment_check_amalgamation.yml @@ -20,7 +20,7 @@ jobs: pull-requests: write steps: - name: Harden Runner - uses: step-security/harden-runner@e14015d583714f6e62063499dc959a02595150a1 # v2.21.1 + uses: step-security/harden-runner@351661ca32ac09a36dc5ee2d536e3128f2a3c8ed # v2.22.0 with: egress-policy: audit diff --git a/.github/workflows/dependency-review.yml b/.github/workflows/dependency-review.yml index 2052c1d35..8ccc91e04 100644 --- a/.github/workflows/dependency-review.yml +++ b/.github/workflows/dependency-review.yml @@ -21,7 +21,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Harden Runner - uses: step-security/harden-runner@e14015d583714f6e62063499dc959a02595150a1 # v2.21.1 + uses: step-security/harden-runner@351661ca32ac09a36dc5ee2d536e3128f2a3c8ed # v2.22.0 with: egress-policy: audit diff --git a/.github/workflows/flawfinder.yml b/.github/workflows/flawfinder.yml index f37539a73..bab3a8264 100644 --- a/.github/workflows/flawfinder.yml +++ b/.github/workflows/flawfinder.yml @@ -31,7 +31,7 @@ jobs: security-events: write steps: - name: Harden Runner - uses: step-security/harden-runner@e14015d583714f6e62063499dc959a02595150a1 # v2.21.1 + uses: step-security/harden-runner@351661ca32ac09a36dc5ee2d536e3128f2a3c8ed # v2.22.0 with: egress-policy: audit diff --git a/.github/workflows/labeler.yml b/.github/workflows/labeler.yml index a93c86ae2..3954235fc 100644 --- a/.github/workflows/labeler.yml +++ b/.github/workflows/labeler.yml @@ -23,7 +23,7 @@ jobs: steps: - name: Harden Runner - uses: step-security/harden-runner@e14015d583714f6e62063499dc959a02595150a1 # v2.21.1 + uses: step-security/harden-runner@351661ca32ac09a36dc5ee2d536e3128f2a3c8ed # v2.22.0 with: egress-policy: audit diff --git a/.github/workflows/publish_documentation.yml b/.github/workflows/publish_documentation.yml index 3256c819b..dc13f42dc 100644 --- a/.github/workflows/publish_documentation.yml +++ b/.github/workflows/publish_documentation.yml @@ -37,7 +37,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Harden Runner - uses: step-security/harden-runner@e14015d583714f6e62063499dc959a02595150a1 # v2.21.1 + uses: step-security/harden-runner@351661ca32ac09a36dc5ee2d536e3128f2a3c8ed # v2.22.0 with: egress-policy: audit diff --git a/.github/workflows/scorecards.yml b/.github/workflows/scorecards.yml index bbf11dd28..1549f768e 100644 --- a/.github/workflows/scorecards.yml +++ b/.github/workflows/scorecards.yml @@ -40,7 +40,7 @@ jobs: steps: - name: Harden Runner - uses: step-security/harden-runner@e14015d583714f6e62063499dc959a02595150a1 # v2.21.1 + uses: step-security/harden-runner@351661ca32ac09a36dc5ee2d536e3128f2a3c8ed # v2.22.0 with: egress-policy: audit diff --git a/.github/workflows/semgrep.yml b/.github/workflows/semgrep.yml index d8c717e83..0eafc21d9 100644 --- a/.github/workflows/semgrep.yml +++ b/.github/workflows/semgrep.yml @@ -36,7 +36,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Harden Runner - uses: step-security/harden-runner@e14015d583714f6e62063499dc959a02595150a1 # v2.21.1 + uses: step-security/harden-runner@351661ca32ac09a36dc5ee2d536e3128f2a3c8ed # v2.22.0 with: egress-policy: audit diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index 1e690ccdd..74e7c011f 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -16,7 +16,7 @@ jobs: steps: - name: Harden Runner - uses: step-security/harden-runner@e14015d583714f6e62063499dc959a02595150a1 # v2.21.1 + uses: step-security/harden-runner@351661ca32ac09a36dc5ee2d536e3128f2a3c8ed # v2.22.0 with: egress-policy: audit diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml index edce983a5..5216e60b2 100644 --- a/.github/workflows/ubuntu.yml +++ b/.github/workflows/ubuntu.yml @@ -80,7 +80,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Harden Runner - uses: step-security/harden-runner@e14015d583714f6e62063499dc959a02595150a1 # v2.21.1 + uses: step-security/harden-runner@351661ca32ac09a36dc5ee2d536e3128f2a3c8ed # v2.22.0 with: egress-policy: audit @@ -112,7 +112,7 @@ jobs: target: [ci_test_single_header, ci_cppcheck, ci_cpplint, ci_reproducible_tests, ci_non_git_tests, ci_offline_testdata, ci_reuse_compliance, ci_test_valgrind] steps: - name: Harden Runner - uses: step-security/harden-runner@e14015d583714f6e62063499dc959a02595150a1 # v2.21.1 + uses: step-security/harden-runner@351661ca32ac09a36dc5ee2d536e3128f2a3c8ed # v2.22.0 with: egress-policy: audit @@ -178,7 +178,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Harden Runner - uses: step-security/harden-runner@e14015d583714f6e62063499dc959a02595150a1 # v2.21.1 + uses: step-security/harden-runner@351661ca32ac09a36dc5ee2d536e3128f2a3c8ed # v2.22.0 with: egress-policy: audit @@ -433,7 +433,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Harden Runner - uses: step-security/harden-runner@e14015d583714f6e62063499dc959a02595150a1 # v2.21.1 + uses: step-security/harden-runner@351661ca32ac09a36dc5ee2d536e3128f2a3c8ed # v2.22.0 with: egress-policy: audit @@ -456,7 +456,7 @@ jobs: target: [ci_test_examples, ci_test_build_documentation, ci_test_documentation_mermaid] steps: - name: Harden Runner - uses: step-security/harden-runner@e14015d583714f6e62063499dc959a02595150a1 # v2.21.1 + uses: step-security/harden-runner@351661ca32ac09a36dc5ee2d536e3128f2a3c8ed # v2.22.0 with: egress-policy: audit diff --git a/BUILD.bazel b/BUILD.bazel index 08d242eef..8fa0a9a9c 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -25,6 +25,7 @@ cc_library( "include/nlohmann/detail/conversions/from_json.hpp", "include/nlohmann/detail/conversions/to_chars.hpp", "include/nlohmann/detail/conversions/to_json.hpp", + "include/nlohmann/detail/conversions/zmij.hpp", "include/nlohmann/detail/exceptions.hpp", "include/nlohmann/detail/hash.hpp", "include/nlohmann/detail/input/binary_reader.hpp", diff --git a/README.md b/README.md index e4b8932fc..24fc5b75f 100644 --- a/README.md +++ b/README.md @@ -116,7 +116,7 @@ There is also a [**docset**](https://github.com/Kapeli/Dash-User-Contributions/t - **Constructors**: [basic_json](https://json.nlohmann.me/api/basic_json/basic_json), [array](https://json.nlohmann.me/api/basic_json/array), [binary](https://json.nlohmann.me/api/basic_json/binary), [object](https://json.nlohmann.me/api/basic_json/object) - **Object inspection**: [type](https://json.nlohmann.me/api/basic_json/type), [operator value_t](https://json.nlohmann.me/api/basic_json/operator_value_t), [type_name](https://json.nlohmann.me/api/basic_json/type_name), [is_primitive](https://json.nlohmann.me/api/basic_json/is_primitive), [is_structured](https://json.nlohmann.me/api/basic_json/is_structured), [is_null](https://json.nlohmann.me/api/basic_json/is_null), [is_boolean](https://json.nlohmann.me/api/basic_json/is_boolean), [is_number](https://json.nlohmann.me/api/basic_json/is_number), [is_number_integer](https://json.nlohmann.me/api/basic_json/is_number_integer), [is_number_unsigned](https://json.nlohmann.me/api/basic_json/is_number_unsigned), [is_number_float](https://json.nlohmann.me/api/basic_json/is_number_float), [is_object](https://json.nlohmann.me/api/basic_json/is_object), [is_array](https://json.nlohmann.me/api/basic_json/is_array), [is_string](https://json.nlohmann.me/api/basic_json/is_string), [is_binary](https://json.nlohmann.me/api/basic_json/is_binary), [is_discarded](https://json.nlohmann.me/api/basic_json/is_discarded) - **Value access**: [get](https://json.nlohmann.me/api/basic_json/get), [get_to](https://json.nlohmann.me/api/basic_json/get_to), [get_ptr](https://json.nlohmann.me/api/basic_json/get_ptr), [get_ref](https://json.nlohmann.me/api/basic_json/get_ref), [operator ValueType](https://json.nlohmann.me/api/basic_json/operator_ValueType), [get_binary](https://json.nlohmann.me/api/basic_json/get_binary) -- **Element access**: [at](https://json.nlohmann.me/api/basic_json/at), [operator[]](https://json.nlohmann.me/api/basic_json/operator[]), [value](https://json.nlohmann.me/api/basic_json/value), [front](https://json.nlohmann.me/api/basic_json/front), [back](https://json.nlohmann.me/api/basic_json/back) +- **Element access**: [at](https://json.nlohmann.me/api/basic_json/at), [operator[]](https://json.nlohmann.me/api/basic_json/operator%5B%5D/), [value](https://json.nlohmann.me/api/basic_json/value), [front](https://json.nlohmann.me/api/basic_json/front), [back](https://json.nlohmann.me/api/basic_json/back) - **Lookup**: [find](https://json.nlohmann.me/api/basic_json/find), [count](https://json.nlohmann.me/api/basic_json/count), [contains](https://json.nlohmann.me/api/basic_json/contains) - **Iterators**: [begin](https://json.nlohmann.me/api/basic_json/begin), [cbegin](https://json.nlohmann.me/api/basic_json/cbegin), [end](https://json.nlohmann.me/api/basic_json/end), [cend](https://json.nlohmann.me/api/basic_json/cend), [rbegin](https://json.nlohmann.me/api/basic_json/rbegin), [rend](https://json.nlohmann.me/api/basic_json/rend), [crbegin](https://json.nlohmann.me/api/basic_json/crbegin), [crend](https://json.nlohmann.me/api/basic_json/crend), [items](https://json.nlohmann.me/api/basic_json/items) - **Capacity**: [empty](https://json.nlohmann.me/api/basic_json/empty), [size](https://json.nlohmann.me/api/basic_json/size), [max_size](https://json.nlohmann.me/api/basic_json/max_size) @@ -125,7 +125,7 @@ There is also a [**docset**](https://github.com/Kapeli/Dash-User-Contributions/t - **Serialization / Dumping**: [dump](https://json.nlohmann.me/api/basic_json/dump) - **Deserialization / Parsing**: [parse](https://json.nlohmann.me/api/basic_json/parse), [accept](https://json.nlohmann.me/api/basic_json/accept), [sax_parse](https://json.nlohmann.me/api/basic_json/sax_parse) - **JSON Pointer functions**: [flatten](https://json.nlohmann.me/api/basic_json/flatten), [unflatten](https://json.nlohmann.me/api/basic_json/unflatten) -- **JSON Patch functions**: [patch](https://json.nlohmann.me/api/basic_json/patch), [patch_inplace](https://json.nlohmann.me/api/basic_json/patch_inplace), [diff](https://json.nlohmann.me/api/basic_json/diff), [merge_patch](https://json.nlohmann.me/api/basic_json/merge_patch) +- **JSON Patch functions**: [patch](https://json.nlohmann.me/api/basic_json/patch), [patch_inplace](https://json.nlohmann.me/api/basic_json/patch_inplace), [diff](https://json.nlohmann.me/api/basic_json/diff), [merge_patch](https://json.nlohmann.me/api/basic_json/merge_patch), [merge_diff](https://json.nlohmann.me/api/basic_json/merge_diff) - **Static functions**: [meta](https://json.nlohmann.me/api/basic_json/meta), [get_allocator](https://json.nlohmann.me/api/basic_json/get_allocator) - **Binary formats**: [from_bjdata](https://json.nlohmann.me/api/basic_json/from_bjdata), [from_bon8](https://json.nlohmann.me/api/basic_json/from_bon8), [from_bson](https://json.nlohmann.me/api/basic_json/from_bson), [from_cbor](https://json.nlohmann.me/api/basic_json/from_cbor), [from_msgpack](https://json.nlohmann.me/api/basic_json/from_msgpack), [from_ubjson](https://json.nlohmann.me/api/basic_json/from_ubjson), [to_bjdata](https://json.nlohmann.me/api/basic_json/to_bjdata), [to_bon8](https://json.nlohmann.me/api/basic_json/to_bon8), [to_bson](https://json.nlohmann.me/api/basic_json/to_bson), [to_cbor](https://json.nlohmann.me/api/basic_json/to_cbor), [to_msgpack](https://json.nlohmann.me/api/basic_json/to_msgpack), [to_ubjson](https://json.nlohmann.me/api/basic_json/to_ubjson) - **Non-member functions**: [operator<<](https://json.nlohmann.me/api/operator_ltlt/), [operator>>](https://json.nlohmann.me/api/operator_gtgt/), [to_string](https://json.nlohmann.me/api/basic_json/to_string) @@ -360,7 +360,7 @@ std::cout << cpp_string << " == " << cpp_string2 << " == " << j_string.get()`](https://json.nlohmann.me/api/basic_json/get/) to retrieve the originally stored string value. Note the library only supports UTF-8. When you store strings with different encodings in the library, calling [`dump()`](https://json.nlohmann.me/api/basic_json/dump/) may throw an exception unless `json::error_handler_t::replace`, `json::error_handler_t::ignore`, or `json::error_handler_t::keep` are used as error handlers. @@ -687,15 +687,15 @@ json j_result = j_original.patch(j_patch); // calculate a JSON patch from two JSON values json::diff(j_result, j_original); // [ -// { "op":" replace", "path": "/baz", "value": ["one", "two", "three"] }, -// { "op": "remove","path": "/hello" }, +// { "op": "replace", "path": "/baz", "value": ["one", "two", "three"] }, +// { "op": "remove", "path": "/hello" }, // { "op": "add", "path": "/foo", "value": "bar" } // ] ``` ### JSON Merge Patch -The library supports **JSON Merge Patch** ([RFC 7386](https://tools.ietf.org/html/rfc7386)) as a patch format. Instead of using JSON Pointer (see above) to specify values to be manipulated, it describes the changes using a syntax that closely mimics the document being modified. +The library supports **JSON Merge Patch** ([RFC 7396](https://tools.ietf.org/html/rfc7396)) as a patch format. Instead of using JSON Pointer (see above) to specify values to be manipulated, it describes the changes using a syntax that closely mimics the document being modified. ```cpp // a JSON value @@ -725,6 +725,20 @@ j_document.merge_patch(j_patch); // } ``` +Conversely, [`merge_diff`](https://json.nlohmann.me/api/basic_json/merge_diff/) creates a merge patch from two JSON values: + +```cpp +// create a merge patch that turns j_original into j_document +json j_original = R"({"a": "b", "c": {"d": "e", "f": "g"}})"_json; +json::merge_diff(j_original, j_document); +// { +// "a": "z", +// "c": { +// "f": null +// } +// } +``` + ### Implicit conversions Supported types can be implicitly converted to JSON values. @@ -1266,13 +1280,9 @@ The package configuration file, `nlohmann_jsonConfig.cmake`, can be used either To embed the library directly into an existing CMake project, place the entire source tree in a subdirectory and call `add_subdirectory()` in your `CMakeLists.txt` file: ```cmake -# Typically you don't care so much for a third party library's tests to be -# run from your own project's code. -set(JSON_BuildTests OFF CACHE INTERNAL "") - # If you only include this third party in PRIVATE source files, you do not # need to install it when your main project gets installed. -# set(JSON_Install OFF CACHE INTERNAL "") +set(JSON_Install OFF CACHE INTERNAL "") # Don't use include(nlohmann_json/CMakeLists.txt) since that carries with it # unintended consequences that will break the build. It's generally @@ -1287,8 +1297,8 @@ target_link_libraries(foo PRIVATE nlohmann_json::nlohmann_json) ##### Embedded (FetchContent) -Since CMake v3.11, -[FetchContent](https://cmake.org/cmake/help/v3.11/module/FetchContent.html) can +Since CMake v3.14, +[FetchContent](https://cmake.org/cmake/help/v3.14/module/FetchContent.html) can be used to automatically download a release as a dependency at configure time. Example: @@ -1353,7 +1363,9 @@ Use your favorite [**package manager**](https://json.nlohmann.me/integration/pac -  [**Swift Package Manager**](https://json.nlohmann.me/integration/package_managers/#swift-package-manager) `nlohmann/json` -  [**Nuget**](https://json.nlohmann.me/integration/package_managers/#nuget) `nlohmann.json` -  [**Conda**](https://json.nlohmann.me/integration/package_managers/#conda) `nlohmann_json` +- [**MSYS2**](https://json.nlohmann.me/integration/package_managers/#msys2) `mingw-w64-nlohmann-json` -  [**MacPorts**](https://json.nlohmann.me/integration/package_managers/#macports) `nlohmann-json` +- [**build2**](https://json.nlohmann.me/integration/package_managers/#build2) `nlohmann-json` -  [**cpm.cmake**](https://json.nlohmann.me/integration/package_managers/#cpmcmake) `gh:nlohmann/json` -  [**xmake**](https://json.nlohmann.me/integration/package_managers/#xmake) `nlohmann_json` @@ -1385,9 +1397,10 @@ THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR I - The class contains the UTF-8 Decoder from Bjoern Hoehrmann which is licensed under the [MIT License](https://opensource.org/licenses/MIT) (see above). Copyright © 2008-2009 [Björn Hoehrmann](https://bjoern.hoehrmann.de/) - The class contains a slightly modified version of the Grisu2 algorithm from Florian Loitsch which is licensed under the [MIT License](https://opensource.org/licenses/MIT) (see above). Copyright © 2009 [Florian Loitsch](https://florian.loitsch.com/) +- The class contains a port of the shortest double-to-decimal conversion of [Żmij](https://github.com/vitaut/zmij) by Victor Zverovich, including the conversion of the digits to text by Xiang JunBo and the SIMD instruction sequence of Dougall Johnson, which is licensed under the [MIT License](https://opensource.org/licenses/MIT) (see above). Copyright © 2025 [Victor Zverovich](https://github.com/vitaut) - The class contains a copy of [Hedley](https://nemequ.github.io/hedley/) from Evan Nemerson which is licensed as [CC0-1.0](https://creativecommons.org/publicdomain/zero/1.0/). - The class contains parts of [Google Abseil](https://github.com/abseil/abseil-cpp) which is licensed under the [Apache 2.0 License](https://opensource.org/licenses/Apache-2.0). -- The class contains an adapted version of the Eisel-Lemire algorithm and its table of powers of five from [fast_float](https://github.com/fastfloat/fast_float) by Daniel Lemire and contributors, which is available under the [MIT License](https://opensource.org/licenses/MIT) (used here), the Apache 2.0 License, and the Boost Software License. Copyright © 2021 The fast_float authors +- The class contains an adapted version of the Eisel-Lemire algorithm, its table of powers of five, and its digit comparison for long numbers from [fast_float](https://github.com/fastfloat/fast_float) by Daniel Lemire and contributors, which is available under the [MIT License](https://opensource.org/licenses/MIT) (used here), the Apache 2.0 License, and the Boost Software License. Copyright © 2021 The fast_float authors REUSE Software diff --git a/cmake/ci.cmake b/cmake/ci.cmake index 5affec6c5..cc8aaf620 100644 --- a/cmake/ci.cmake +++ b/cmake/ci.cmake @@ -610,9 +610,8 @@ foreach(SRC_FILE ${SRC_FILES}) add_executable(single_${RELATIVE_SRC_FILE} EXCLUDE_FROM_ALL ${PROJECT_BINARY_DIR}/src_single/${RELATIVE_SRC_FILE}.cpp) target_include_directories(single_${RELATIVE_SRC_FILE} PRIVATE ${PROJECT_SOURCE_DIR}/include) target_compile_features(single_${RELATIVE_SRC_FILE} PRIVATE cxx_std_11) - if(RELATIVE_SRC_FILE STREQUAL "json" OR RELATIVE_SRC_FILE STREQUAL "json_literals") - # see below: report the diagnostics of json.hpp and json_literals.hpp without --error, so they - # do not fail the build + if(RELATIVE_SRC_FILE STREQUAL "json") + # see below: report the diagnostics of json.hpp without --error, so they do not fail the build set_property(TARGET single_${RELATIVE_SRC_FILE} PROPERTY CXX_INCLUDE_WHAT_YOU_USE ${IWYU_TOOL} -Xiwyu --max_line_length=300) else() set_property(TARGET single_${RELATIVE_SRC_FILE} PROPERTY CXX_INCLUDE_WHAT_YOU_USE "${iwyu_path_and_options}") @@ -626,18 +625,31 @@ foreach(SRC_FILE ${SRC_FILES}) # reporting its diagnostics (informational, via CXX_INCLUDE_WHAT_YOU_USE above) but exclude it # from the hard gate below so a fresh IWYU/compiler combination does not fail this target on a # nondeterministic suggestion for a header that already re-exports everything on purpose. - # json_literals.hpp and json.hpp include each other on purpose (json.hpp includes it at its end - # unless JSON_NO_AUTOMATIC_UDLS is defined), and IWYU, not following the cycle, suggests replacing - # json.hpp with json_fwd.hpp although the literals need the complete basic_json; exclude it, too. - if(NOT RELATIVE_SRC_FILE STREQUAL "json" AND NOT RELATIVE_SRC_FILE STREQUAL "json_literals") + if(NOT RELATIVE_SRC_FILE STREQUAL "json") list(APPEND single_binaries_tus src_single/${RELATIVE_SRC_FILE}.cpp) endif() endforeach() +# The checks above look at each header on its own. Client code that includes only +# must not be told by include analyzers to include another nlohmann header for what it uses +# (issue #5812): json.hpp marks those includes "IWYU pragma: export", and both IWYU and clang-tidy's +# misc-include-cleaner (disabled in .clang-tidy for the library itself) check these clients. +file(GLOB CONSUMER_FILES ${PROJECT_SOURCE_DIR}/tests/iwyu_consumer/*.cpp) +foreach(CONSUMER_FILE ${CONSUMER_FILES}) + get_filename_component(CONSUMER_NAME "${CONSUMER_FILE}" NAME_WE) + add_executable(iwyu_consumer_${CONSUMER_NAME} EXCLUDE_FROM_ALL ${CONSUMER_FILE}) + target_include_directories(iwyu_consumer_${CONSUMER_NAME} PRIVATE ${PROJECT_SOURCE_DIR}/include) + target_compile_features(iwyu_consumer_${CONSUMER_NAME} PRIVATE cxx_std_11) + list(APPEND single_binaries iwyu_consumer_${CONSUMER_NAME}) + list(APPEND single_binaries_tus ${CONSUMER_FILE}) +endforeach() + if(IWYU_TOOL_PY) add_custom_target(ci_single_binaries DEPENDS ${single_binaries} COMMAND ${IWYU_TOOL_PY} -p ${PROJECT_BINARY_DIR} ${single_binaries_tus} -- ${iwyu_options} + COMMAND ${CLANG_TIDY_TOOL} -p ${PROJECT_BINARY_DIR} --checks=-*,misc-include-cleaner --warnings-as-errors=* ${CONSUMER_FILES} + VERBATIM COMMENT "Check if headers are self-contained" ) else() diff --git a/docs/mkdocs/docs/api/basic_json/array_t.md b/docs/mkdocs/docs/api/basic_json/array_t.md index caa0e9162..f621f7b58 100644 --- a/docs/mkdocs/docs/api/basic_json/array_t.md +++ b/docs/mkdocs/docs/api/basic_json/array_t.md @@ -27,7 +27,7 @@ To store objects in C++, a type is defined by the template parameters explained ### Default type -With the default values for `ArrayType` (`std::vector`) and `AllocatorType` (`std::allocator`), the default value for +With the default values for `ArrayType` (`std::vector`) and `AllocatorType` (`std::allocator`), the default type for `array_t` is: ```cpp diff --git a/docs/mkdocs/docs/api/basic_json/boolean_t.md b/docs/mkdocs/docs/api/basic_json/boolean_t.md index f79a7622e..2fd1beb4e 100644 --- a/docs/mkdocs/docs/api/basic_json/boolean_t.md +++ b/docs/mkdocs/docs/api/basic_json/boolean_t.md @@ -23,7 +23,7 @@ To store boolean values in C++, a type is defined by the template parameter `Bo ### Default type -With the default values for `BooleanType` (`#!cpp bool`), the default value for `boolean_t` is `#!cpp bool`. +With the default values for `BooleanType` (`#!cpp bool`), the default type for `boolean_t` is `#!cpp bool`. ### Storage diff --git a/docs/mkdocs/docs/api/basic_json/diff.md b/docs/mkdocs/docs/api/basic_json/diff.md index ae22454dd..c4e710a7d 100644 --- a/docs/mkdocs/docs/api/basic_json/diff.md +++ b/docs/mkdocs/docs/api/basic_json/diff.md @@ -59,6 +59,7 @@ Currently, only `remove`, `add`, and `replace` operations are generated. - [patch](patch.md) applies a JSON Patch - [patch_inplace](patch_inplace.md) applies a JSON Patch in place - [merge_patch](merge_patch.md) applies a JSON Merge Patch +- [merge_diff](merge_diff.md) creates a diff as a JSON Merge Patch ## Version history diff --git a/docs/mkdocs/docs/api/basic_json/dump.md b/docs/mkdocs/docs/api/basic_json/dump.md index d805f3db1..eb13717e4 100644 --- a/docs/mkdocs/docs/api/basic_json/dump.md +++ b/docs/mkdocs/docs/api/basic_json/dump.md @@ -62,6 +62,9 @@ Linear. ## Notes +Floating-point numbers are written with the fewest digits that read back as the same value (for `#!cpp double`; see +[number handling](../../features/types/number_handling.md#number-serialization)). + Binary values are serialized as an object containing two keys: - "bytes": an array of bytes as integers @@ -97,3 +100,5 @@ Binary values are serialized as an object containing two keys: - Error handlers added in version 3.4.0. - Serialization of binary values added in version 3.8.0. - Error handler `keep` added in version 3.13.0. +- Doubles are written with the shortest digits (Żmij instead of Grisu2) since version 3.13.0; about 0.1% of doubles are + written differently, most of them with fewer digits. diff --git a/docs/mkdocs/docs/api/basic_json/index.md b/docs/mkdocs/docs/api/basic_json/index.md index fdb57105c..8d945de9b 100644 --- a/docs/mkdocs/docs/api/basic_json/index.md +++ b/docs/mkdocs/docs/api/basic_json/index.md @@ -85,7 +85,7 @@ The class satisfies the following concept requirements: - [EqualityComparable](https://en.cppreference.com/w/cpp/named_req/EqualityComparable): JSON values can be compared with `==`, see [`operator==`](operator_eq.md). - [LessThanComparable](https://en.cppreference.com/w/cpp/named_req/LessThanComparable): JSON values can be compared with - `<`, see [`operator<`](operator_le.md). + `<`, see [`operator<`](operator_lt.md). - [Swappable](https://en.cppreference.com/w/cpp/named_req/Swappable): Any JSON lvalue or rvalue of can be swapped with any lvalue or rvalue of other compatible types, using unqualified function `swap`. - [NullablePointer](https://en.cppreference.com/w/cpp/named_req/NullablePointer): JSON values can be compared against @@ -285,6 +285,7 @@ Access to the JSON value ### JSON Merge Patch functions - [**merge_patch**](merge_patch.md) - applies a JSON Merge Patch +- [**merge_diff**](merge_diff.md) (_static_) - creates a diff as a JSON Merge Patch ## Static functions diff --git a/docs/mkdocs/docs/api/basic_json/merge_diff.md b/docs/mkdocs/docs/api/basic_json/merge_diff.md new file mode 100644 index 000000000..b02ac3e54 --- /dev/null +++ b/docs/mkdocs/docs/api/basic_json/merge_diff.md @@ -0,0 +1,79 @@ +# nlohmann::basic_json::merge_diff + +```cpp +static basic_json merge_diff(const basic_json& source, + const basic_json& target); +``` + +Creates a [JSON Merge Patch](https://tools.ietf.org/html/rfc7396) so that value `source` can be changed into the value +`target` by calling the [`merge_patch`](merge_patch.md) function. The patch contains only what differs: changed and +added members with their new values, and removed members with the value `#!json null`. Nested objects are compared +member by member; any other value that differs, including an array, is replaced as a whole. + +For two JSON values `source` and `target`, where `target` contains no object member whose value is `#!json null`, the +following code always yields `#!cpp true` (for [`ordered_json`](../ordered_json.md), see the notes below): +```cpp +basic_json patched = source; +patched.merge_patch(merge_diff(source, target)); +patched == target; +``` + +## Parameters + +`source` (in) +: JSON value to compare from + +`target` (in) +: JSON value to compare against + +## Return value + +a JSON Merge Patch to convert the `source` to `target` + +## Exception safety + +Strong guarantee: `source` and `target` are never modified. + +## Complexity + +Linear in the total number of values (including nested ones) in `source` and `target`, times the cost of a key-based +object operation (looking up a key, or adding one to the patch): logarithmic for [`json`](../json.md), and linear for +[`ordered_json`](../ordered_json.md) (see [`ordered_map` complexity](../ordered_map.md#complexity)). For +`ordered_json`, diffing two objects with `n` and `m` members therefore takes O((n+m)²). + +## Notes + +- A merge patch uses `#!json null` to remove a member, so it cannot set a member to `#!json null`. As in RFC 7396, a + `#!json null` member of `target` is treated as absent: the patch removes the member from `source` instead of setting + it to `#!json null`. A member that is `#!json null` in both `source` and `target` is unchanged and left out of the + patch, so `source` keeps it. +- A merge patch cannot reorder object members, and [`merge_patch`](merge_patch.md) appends added members at the end. + For [`ordered_json`](../ordered_json.md), the patched value therefore has the same members as `target`, but not + necessarily in the same order, and may compare unequal to it. The patch lists changed and removed members in the + order of `source`, followed by added members in the order of `target`. + +## Examples + +??? example + + The following code shows how a JSON Merge Patch is created as a diff for two JSON values. + + ```cpp + --8<-- "examples/merge_diff.cpp" + ``` + + Output: + + ```json + --8<-- "examples/merge_diff.output" + ``` + +## See also + +- [RFC 7396 (JSON Merge Patch)](https://tools.ietf.org/html/rfc7396) +- [merge_patch](merge_patch.md) applies a JSON Merge Patch +- [diff](diff.md) creates a diff as a JSON Patch + +## Version history + +- Added in version 3.13.0. diff --git a/docs/mkdocs/docs/api/basic_json/merge_patch.md b/docs/mkdocs/docs/api/basic_json/merge_patch.md index dbda23f42..00e1bc8e5 100644 --- a/docs/mkdocs/docs/api/basic_json/merge_patch.md +++ b/docs/mkdocs/docs/api/basic_json/merge_patch.md @@ -67,6 +67,7 @@ returned by `#!cpp (*this)[key]`); it is read as it was when `merge_patch()` was - [RFC 7396 (JSON Merge Patch)](https://tools.ietf.org/html/rfc7396) - [patch](patch.md) apply a JSON patch +- [merge_diff](merge_diff.md) creates a diff as a JSON Merge Patch ## Version history diff --git a/docs/mkdocs/docs/api/basic_json/number_float_t.md b/docs/mkdocs/docs/api/basic_json/number_float_t.md index 282bb6123..cf482d6f0 100644 --- a/docs/mkdocs/docs/api/basic_json/number_float_t.md +++ b/docs/mkdocs/docs/api/basic_json/number_float_t.md @@ -23,9 +23,10 @@ type to use. ## Template parameters `NumberFloatType` -: the type to store floating-point numbers. Parsing and serialization are implemented in terms of - `#!cpp std::strtof`/`#!cpp std::strtod`/`#!cpp std::strtold` and `#!cpp std::snprintf`, so the type must be - `#!cpp float`, `#!cpp double`, or `#!cpp long double`. The +: the type to store floating-point numbers. The type must be `#!cpp float`, `#!cpp double`, or + `#!cpp long double`. The parser converts `#!cpp float`, `#!cpp double`, and a `#!cpp long double` that is IEEE 754 + binary64 itself. It converts other `#!cpp long double` formats with `#!cpp std::from_chars` where available, or + with `#!cpp std::strtold` otherwise. Serialization falls back to `#!cpp std::snprintf`. The [binary formats](../../features/binary_formats/index.md) additionally require `#!cpp float` or `#!cpp double`, because they have no encoding for `#!cpp long double`. See [Template Parameter Requirements](../../features/types/template_parameters.md#numberfloattype). @@ -34,7 +35,7 @@ type to use. ### Default type -With the default values for `NumberFloatType` (`double`), the default value for `number_float_t` is `#!cpp double`. +With the default values for `NumberFloatType` (`double`), the default type for `number_float_t` is `#!cpp double`. ### Default behavior @@ -51,9 +52,11 @@ With the default values for `NumberFloatType` (`double`), the default value for > interoperability can be achieved by implementations that expect no more precision or range than these provide, in the > sense that implementations will approximate JSON numbers within the expected precision. -This implementation does exactly follow this approach, as it uses double precision floating-point numbers. Note values -smaller than `-1.79769313486232e+308` and values greater than `1.79769313486232e+308` will be stored as NaN internally -and be serialized to `null`. +This implementation does exactly follow this approach, as it uses double precision floating-point numbers by default. +Note that with the default `double`, numbers smaller than `-1.79769313486232e+308` or greater than +`1.79769313486232e+308` are rejected during parsing with +[`out_of_range.406`](../../home/exceptions.md#jsonexceptionout_of_range406). Non-finite values (NaN and infinity) can +only be stored by assigning them directly and are serialized to `null`. During deserialization (from JSON text or any of the binary formats), a finite number that does not fit into `number_float_t` is rejected with [`out_of_range.406`](../../home/exceptions.md#jsonexceptionout_of_range406), for diff --git a/docs/mkdocs/docs/api/basic_json/number_integer_t.md b/docs/mkdocs/docs/api/basic_json/number_integer_t.md index 48d2873ed..6bd18a74b 100644 --- a/docs/mkdocs/docs/api/basic_json/number_integer_t.md +++ b/docs/mkdocs/docs/api/basic_json/number_integer_t.md @@ -31,7 +31,7 @@ to use. ### Default type -With the default values for `NumberIntegerType` (`std::int64_t`), the default value for `number_integer_t` is +With the default values for `NumberIntegerType` (`std::int64_t`), the default type for `number_integer_t` is `#!cpp std::int64_t`. ### Default behavior diff --git a/docs/mkdocs/docs/api/basic_json/number_unsigned_t.md b/docs/mkdocs/docs/api/basic_json/number_unsigned_t.md index 81cc00f77..217a59f5a 100644 --- a/docs/mkdocs/docs/api/basic_json/number_unsigned_t.md +++ b/docs/mkdocs/docs/api/basic_json/number_unsigned_t.md @@ -32,7 +32,7 @@ the type to use. ### Default type -With the default values for `NumberUnsignedType` (`std::uint64_t`), the default value for `number_unsigned_t` is +With the default values for `NumberUnsignedType` (`std::uint64_t`), the default type for `number_unsigned_t` is `#!cpp std::uint64_t`. ### Default behavior diff --git a/docs/mkdocs/docs/api/basic_json/object_t.md b/docs/mkdocs/docs/api/basic_json/object_t.md index 204e5ab28..66fa9c110 100644 --- a/docs/mkdocs/docs/api/basic_json/object_t.md +++ b/docs/mkdocs/docs/api/basic_json/object_t.md @@ -36,7 +36,7 @@ To store objects in C++, a type is defined by the template parameters described ### Default type With the default values for `ObjectType` (`std::map`), `StringType` (`std::string`), and `AllocatorType` -(`std::allocator`), the default value for `object_t` is: +(`std::allocator`), the default type for `object_t` is: ```cpp // until C++14 diff --git a/docs/mkdocs/docs/api/basic_json/operator_eq.md b/docs/mkdocs/docs/api/basic_json/operator_eq.md index 58a7e74da..778945f38 100644 --- a/docs/mkdocs/docs/api/basic_json/operator_eq.md +++ b/docs/mkdocs/docs/api/basic_json/operator_eq.md @@ -11,18 +11,17 @@ template bool operator==(ScalarType lhs, const const_reference rhs) noexcept(/* see below */); // (2) // since C++20 -class basic_json { - bool operator==(const_reference rhs) const noexcept; // (1) +bool operator==(const_reference rhs) const noexcept; // (1) - template - bool operator==(ScalarType rhs) const noexcept(/* see below */); // (2) -}; +template +bool operator==(ScalarType rhs) const noexcept(/* see below */); // (2) ``` 1. Compares two JSON values for equality according to the following rules: - Two JSON values are equal if (1) neither value is discarded, and (2) they are of the same type and their stored values are the same according to their respective `operator==`. - - Integer and floating-point numbers are automatically converted before comparison. + - Integer and floating-point numbers are compared by their exact values, without converting the integer to + floating-point. For instance, `9007199254740993` and `9007199254740992.0` are not equal. 2. Compares a JSON value and a scalar or a scalar and a JSON value for equality by converting the scalar to a JSON value and comparing both JSON values according to 1. diff --git a/docs/mkdocs/docs/api/basic_json/operator_lt.md b/docs/mkdocs/docs/api/basic_json/operator_lt.md index 1b8e65225..baa04964c 100644 --- a/docs/mkdocs/docs/api/basic_json/operator_lt.md +++ b/docs/mkdocs/docs/api/basic_json/operator_lt.md @@ -15,7 +15,8 @@ bool operator<(ScalarType lhs, const const_reference rhs) noexcept(/* see below following rules: - If either operand is discarded, the comparison yields `#!cpp false`. - If both operands have the same type, the values are compared using their respective `operator<`. - - Integer and floating-point numbers are automatically converted before comparison. + - Integer and floating-point numbers are compared by their exact values, without converting the integer to + floating-point. For instance, `9007199254740992.0` is less than `9007199254740993`. - In case `lhs` and `rhs` have different types, the values are ignored and the order of the types is considered, which is: 1. null diff --git a/docs/mkdocs/docs/api/basic_json/operator_spaceship.md b/docs/mkdocs/docs/api/basic_json/operator_spaceship.md index 47ca22484..731372f97 100644 --- a/docs/mkdocs/docs/api/basic_json/operator_spaceship.md +++ b/docs/mkdocs/docs/api/basic_json/operator_spaceship.md @@ -2,22 +2,18 @@ ```cpp // since C++20 -class basic_json { - std::partial_ordering operator<=>(const_reference rhs) const noexcept; // (1) +std::partial_ordering operator<=>(const_reference rhs) const noexcept; // (1) - template - std::partial_ordering operator<=>(const ScalarType rhs) const noexcept(/* see below */); // (2) -}; +template +std::partial_ordering operator<=>(const ScalarType rhs) const noexcept(/* see below */); // (2) ``` 1. 3-way compares two JSON values producing a result of type `std::partial_ordering` according to the following rules: - Two JSON values compare with a result of `std::partial_ordering::unordered` if either value is discarded. - If both JSON values are of the same type, the result is produced by 3-way comparing their stored values using their respective `operator<=>`. - - Integer and floating-point numbers are converted to their common type and then 3-way compared using their - respective `operator<=>`. - For instance, comparing an integer and a floating-point value will 3-way compare the first value converted to - floating-point with the second value. + - Integer and floating-point numbers are 3-way compared by their exact values, without converting the integer to + floating-point. For instance, `9007199254740993` compares greater than `9007199254740992.0`. - Otherwise, yields a result by comparing the type (see [`value_t`](value_t.md)). 2. 3-way compares a JSON value and a scalar or a scalar and a JSON value by converting the scalar to a JSON value and diff --git a/docs/mkdocs/docs/api/basic_json/patch.md b/docs/mkdocs/docs/api/basic_json/patch.md index 951ee46e6..ed26135a9 100644 --- a/docs/mkdocs/docs/api/basic_json/patch.md +++ b/docs/mkdocs/docs/api/basic_json/patch.md @@ -45,7 +45,7 @@ Strong guarantee: if an exception is thrown, there are no changes in the JSON va pointer inside the patch cannot be resolved, e.g., `-` in a "remove" operation or `1a` for an array; example: `"unresolved reference token '-'"`. - Throws [`out_of_range.405`](../../home/exceptions.md#jsonexceptionout_of_range405) if JSON pointer has no parent - ("add", "remove", "move") + ("remove", "move") - Throws [`out_of_range.411`](../../home/exceptions.md#jsonexceptionout_of_range411) if an "add" operation's target location has a parent that is neither an object nor an array. - Throws [`out_of_range.413`](../../home/exceptions.md#jsonexceptionout_of_range413) if a "remove" operation's target diff --git a/docs/mkdocs/docs/api/basic_json/patch_inplace.md b/docs/mkdocs/docs/api/basic_json/patch_inplace.md index dff8b095b..5b0d501bb 100644 --- a/docs/mkdocs/docs/api/basic_json/patch_inplace.md +++ b/docs/mkdocs/docs/api/basic_json/patch_inplace.md @@ -41,7 +41,7 @@ No guarantees, value may be corrupted by an unsuccessful patch operation. pointer inside the patch cannot be resolved, e.g., `-` in a "remove" operation or `1a` for an array; example: `"unresolved reference token '-'"`. - Throws [`out_of_range.405`](../../home/exceptions.md#jsonexceptionout_of_range405) if JSON pointer has no parent - ("add", "remove", "move") + ("remove", "move") - Throws [`out_of_range.411`](../../home/exceptions.md#jsonexceptionout_of_range411) if an "add" operation's target location has a parent that is neither an object nor an array. - Throws [`out_of_range.413`](../../home/exceptions.md#jsonexceptionout_of_range413) if a "remove" operation's target diff --git a/docs/mkdocs/docs/api/basic_json/string_t.md b/docs/mkdocs/docs/api/basic_json/string_t.md index 13177cef8..b65e95508 100644 --- a/docs/mkdocs/docs/api/basic_json/string_t.md +++ b/docs/mkdocs/docs/api/basic_json/string_t.md @@ -32,7 +32,7 @@ JSON class into byte-sized characters during deserialization. ### Default type -With the default values for `StringType` (`std::string`), the default value for `string_t` is `#!cpp std::string`. +With the default values for `StringType` (`std::string`), the default type for `string_t` is `#!cpp std::string`. ### Encoding diff --git a/docs/mkdocs/docs/api/json_pointer/operator_eq.md b/docs/mkdocs/docs/api/json_pointer/operator_eq.md index a9f40891f..44cda4354 100644 --- a/docs/mkdocs/docs/api/json_pointer/operator_eq.md +++ b/docs/mkdocs/docs/api/json_pointer/operator_eq.md @@ -18,13 +18,11 @@ bool operator==( const json_pointer& rhs); // (2) // since C++20 -class json_pointer { - template - bool operator==( - const json_pointer& rhs) const noexcept; // (1) +template +bool operator==( + const json_pointer& rhs) const noexcept; // (1) - bool operator==(const string_t& rhs) const; // (2) -}; +bool operator==(const string_t& rhs) const; // (2) ``` 1. Compares two JSON pointers for equality by comparing their reference tokens. diff --git a/docs/mkdocs/docs/api/json_pointer/operator_spaceship.md b/docs/mkdocs/docs/api/json_pointer/operator_spaceship.md index 2ec917b72..ffc3f0e93 100644 --- a/docs/mkdocs/docs/api/json_pointer/operator_spaceship.md +++ b/docs/mkdocs/docs/api/json_pointer/operator_spaceship.md @@ -2,10 +2,8 @@ ```cpp // since C++20 -class json_pointer { - template - std::strong_ordering operator<=>(const json_pointer& rhs) const noexcept; // *NOPAD* -}; +template +std::strong_ordering operator<=>(const json_pointer& rhs) const noexcept; // *NOPAD* ``` 3-way compares two JSON pointers by lexicographically comparing their sequences of reference tokens: corresponding diff --git a/docs/mkdocs/docs/api/macros/json_use_legacy_discarded_value_comparison.md b/docs/mkdocs/docs/api/macros/json_use_legacy_discarded_value_comparison.md index e61ce1a3e..381e1c644 100644 --- a/docs/mkdocs/docs/api/macros/json_use_legacy_discarded_value_comparison.md +++ b/docs/mkdocs/docs/api/macros/json_use_legacy_discarded_value_comparison.md @@ -18,7 +18,8 @@ When enabled, comparisons involving at least one discarded JSON value yield resu | `>=` | `#!cpp true` | | `>` | `#!cpp false` | -Otherwise, comparisons involving at least one discarded JSON value always yield `#!cpp false`. +Otherwise, comparisons involving at least one discarded JSON value yield `#!cpp false`, except for `!=`, which is the +negation of `==` and yields `#!cpp true`. ## Default definition diff --git a/docs/mkdocs/docs/community/assurance_case.md b/docs/mkdocs/docs/community/assurance_case.md index 46ebd5ebe..54da5c04c 100644 --- a/docs/mkdocs/docs/community/assurance_case.md +++ b/docs/mkdocs/docs/community/assurance_case.md @@ -54,10 +54,9 @@ flowchart LR user code. The destructor does not recurse, so destroying a deeply nested value does not exhaust the stack. - **Bounded recursion.** The JSON parser and the binary readers keep their state in explicit stacks instead of recursing per nesting level. Operations that walk a value, such as [`dump`](../api/basic_json/dump.md), copying, - comparison, hashing, and [`merge_patch`](../api/basic_json/merge_patch.md), recurse only up to a fixed depth and - continue with an explicit stack below it. Some operations, such as [`diff`](../api/basic_json/diff.md), - [`flatten`](../api/basic_json/flatten.md), and the binary writers, still recurse once per nesting level; work on them - is in progress. Applications that process untrusted input can limit its nesting depth with a + comparison, hashing, [`merge_patch`](../api/basic_json/merge_patch.md), [`diff`](../api/basic_json/diff.md), + [`flatten`](../api/basic_json/flatten.md), and the binary writers, recurse only up to a fixed depth and continue with + an explicit stack below it. Applications that process untrusted input can limit its nesting depth with a [parser callback](../features/parsing/parser_callbacks.md). - **Invariants are checked.** The class invariant (for instance, that the pointer for the stored type is never null) is checked with runtime assertions throughout the test suite. diff --git a/docs/mkdocs/docs/examples/merge_diff.cpp b/docs/mkdocs/docs/examples/merge_diff.cpp new file mode 100644 index 000000000..22ae9b087 --- /dev/null +++ b/docs/mkdocs/docs/examples/merge_diff.cpp @@ -0,0 +1,51 @@ +#include +#include +#include + +using json = nlohmann::json; +using namespace nlohmann::literals; + +int main() +{ + // the source document + json source = R"( + { + "title": "Goodbye!", + "author": { + "givenName": "John", + "familyName": "Doe" + }, + "tags": [ + "example", + "sample" + ], + "content": "This will be unchanged" + } + )"_json; + + // the target document + json target = R"( + { + "title": "Hello!", + "author": { + "givenName": "John" + }, + "tags": [ + "example" + ], + "content": "This will be unchanged", + "phoneNumber": "+01-123-456-7890" + } + )"_json; + + // create the patch + json patch = json::merge_diff(source, target); + + // roundtrip + json patched_source = source; + patched_source.merge_patch(patch); + + // output patch and roundtrip result + std::cout << std::setw(4) << patch << "\n\n" + << std::setw(4) << patched_source << std::endl; +} diff --git a/docs/mkdocs/docs/examples/merge_diff.output b/docs/mkdocs/docs/examples/merge_diff.output new file mode 100644 index 000000000..ca7c59bf2 --- /dev/null +++ b/docs/mkdocs/docs/examples/merge_diff.output @@ -0,0 +1,22 @@ +{ + "author": { + "familyName": null + }, + "phoneNumber": "+01-123-456-7890", + "tags": [ + "example" + ], + "title": "Hello!" +} + +{ + "author": { + "givenName": "John" + }, + "content": "This will be unchanged", + "phoneNumber": "+01-123-456-7890", + "tags": [ + "example" + ], + "title": "Hello!" +} diff --git a/docs/mkdocs/docs/features/binary_values.md b/docs/mkdocs/docs/features/binary_values.md index 98e8e71c3..b4c3bab6d 100644 --- a/docs/mkdocs/docs/features/binary_values.md +++ b/docs/mkdocs/docs/features/binary_values.md @@ -267,9 +267,10 @@ unsigned 8-bit integer. If no subtype is given, the generic binary subtype 0x00 ### CBOR -[CBOR](binary_formats/cbor.md) supports binary values, but no subtypes. Subtypes will be serialized as tags. Any binary -value will be serialized as byte strings. The library will choose the smallest representation using the length of the -byte array. +[CBOR](binary_formats/cbor.md) supports binary values and subtypes. Any binary value will be serialized as a byte +string; a subtype is serialized as a tag preceding it. When parsing, tags are only read back as subtypes with +`cbor_tag_handler_t::store` (see [`from_cbor`](../api/basic_json/from_cbor.md)). The library will choose the smallest +representation using the length of the byte array. ??? example "Example: serialize a binary value to CBOR" diff --git a/docs/mkdocs/docs/features/macros.md b/docs/mkdocs/docs/features/macros.md index 5df3671e2..0f676b501 100644 --- a/docs/mkdocs/docs/features/macros.md +++ b/docs/mkdocs/docs/features/macros.md @@ -243,9 +243,9 @@ See [full documentation of `JSON_USE_GLOBAL_UDLS`](../api/macros/json_use_global ## `JSON_USE_LEGACY_DISCARDED_VALUE_COMPARISON` -When defined to `1`, the library restores the legacy behavior in which a discarded value compared equal to itself. This -behavior is [deprecated](../integration/migration_guide.md#miscellaneous-functions) and switched off (`0`) by -default. +When defined to `1`, the library restores the legacy behavior in which comparing a discarded value with `<=` or `>=` +yields `#!cpp true` instead of `#!cpp false`. This behavior is +[deprecated](../integration/migration_guide.md#miscellaneous-functions) and switched off (`0`) by default. See [full documentation of `JSON_USE_LEGACY_DISCARDED_VALUE_COMPARISON`](../api/macros/json_use_legacy_discarded_value_comparison.md). diff --git a/docs/mkdocs/docs/features/merge_patch.md b/docs/mkdocs/docs/features/merge_patch.md index c0a316e65..ac732c7bf 100644 --- a/docs/mkdocs/docs/features/merge_patch.md +++ b/docs/mkdocs/docs/features/merge_patch.md @@ -1,6 +1,6 @@ # JSON Merge Patch -The library supports JSON Merge Patch ([RFC 7386](https://tools.ietf.org/html/rfc7386)) as a patch format. +The library supports JSON Merge Patch ([RFC 7396](https://tools.ietf.org/html/rfc7396)) as a patch format. The merge patch format is primarily intended for use with the HTTP PATCH method as a means of describing a set of modifications to a target resource's content. This function applies a merge patch to the current JSON value. @@ -9,6 +9,9 @@ syntax that closely mimics the document being modified. Unlike [JSON Patch](json express every kind of change (e.g., it cannot reorder array elements or remove a specific array element), but it is easier to read and write for object-shaped documents. +The function [`merge_diff`](../api/basic_json/merge_diff.md) creates a merge patch that changes one JSON value into +another. + !!! tip "Not a general deep merge" A merge patch is not a general deep merge: a `#!json null` value in the patch deletes the key from the target. @@ -35,4 +38,5 @@ easier to read and write for object-shaped documents. - [JSON Patch and Diff](json_patch.md) - a more expressive alternative that describes a sequence of operations - [JSON Pointer](json_pointer.md) - the addressing scheme used by JSON Patch - Function [`merge_patch`](../api/basic_json/merge_patch.md) +- Function [`merge_diff`](../api/basic_json/merge_diff.md) - Function [`update`](../api/basic_json/update.md) - merge objects, optionally recursively diff --git a/docs/mkdocs/docs/features/modifying_values.md b/docs/mkdocs/docs/features/modifying_values.md index cf763a4b3..10814ac2e 100644 --- a/docs/mkdocs/docs/features/modifying_values.md +++ b/docs/mkdocs/docs/features/modifying_values.md @@ -66,7 +66,7 @@ config.update(user_settings, true); // {"log":{"file":"app.log","level":"debug"},"retries":3} ``` -[JSON Merge Patch](merge_patch.md) ([RFC 7386](https://tools.ietf.org/html/rfc7386)) also merges objects recursively, +[JSON Merge Patch](merge_patch.md) ([RFC 7396](https://tools.ietf.org/html/rfc7396)) also merges objects recursively, but it is a different tool: a `#!json null` in the patch means "remove this key". It is meant for applying merge patch documents (e.g., received via HTTP PATCH). To merge configuration-like objects, use `#!cpp update(..., true)`. To apply a sequence of well-defined edit operations, see [JSON Patch](json_patch.md). @@ -90,6 +90,6 @@ a.erase(1); // [1,3,4] (erase by index) - [`push_back`](../api/basic_json/push_back.md) / [`emplace_back`](../api/basic_json/emplace_back.md) - append to an array - [`emplace`](../api/basic_json/emplace.md) - insert into an object if the key is absent - [`update`](../api/basic_json/update.md) - merge objects (shallow, or recursive with `merge_objects`) -- [`merge_patch`](../api/basic_json/merge_patch.md) - apply an RFC 7386 merge patch +- [`merge_patch`](../api/basic_json/merge_patch.md) - apply an RFC 7396 merge patch - [`erase`](../api/basic_json/erase.md) / [`clear`](../api/basic_json/clear.md) - remove elements - [JSON Patch and Diff](json_patch.md) and [JSON Merge Patch](merge_patch.md) - structured modifications diff --git a/docs/mkdocs/docs/features/parsing/untrusted_input.md b/docs/mkdocs/docs/features/parsing/untrusted_input.md index d39a4179e..9c5fa4c9e 100644 --- a/docs/mkdocs/docs/features/parsing/untrusted_input.md +++ b/docs/mkdocs/docs/features/parsing/untrusted_input.md @@ -50,19 +50,13 @@ stack, however deeply it is nested. Once a value has been parsed, operations that walk it recursively -- serializing it with [`dump`](../../api/basic_json/dump.md), hashing it, copying it, comparing two values with `#!cpp ==`, `#!cpp <`, or (in -C++20) `#!cpp <=>`, merging with [`update`](../../api/basic_json/update.md), and applying a -[`merge_patch`](../../api/basic_json/merge_patch.md) -- descend at most 128 levels on the call stack and continue +C++20) `#!cpp <=>`, merging with [`update`](../../api/basic_json/update.md), applying a +[`merge_patch`](../../api/basic_json/merge_patch.md), creating a patch with [`diff`](../../api/basic_json/diff.md) or +[`merge_diff`](../../api/basic_json/merge_diff.md), [`flatten`](../../api/basic_json/flatten.md), and the binary +writers (`to_cbor`, `to_msgpack`, ...) -- descend at most 128 levels on the call stack and continue below that with an explicit stack instead, so none of them can exhaust the stack either, however deeply the value is nested. Destroying a value (its destructor) never recurses at all, regardless of nesting depth, for the same reason. -!!! note "Not every operation is bounded yet" - - [`diff`](../../api/basic_json/diff.md), [`flatten`](../../api/basic_json/flatten.md), and the binary writers - (`to_cbor`, `to_msgpack`, ...) still recurse once per nesting level; this is called out as work in progress in the - [assurance case](../../community/assurance_case.md#secure-design). A value deep enough to matter for these - operations would typically first have to survive parsing without hitting a self-imposed depth limit, as described - above. - ## Input size The library does not limit the overall size of a JSON text; a value nested or wide enough will use memory diff --git a/docs/mkdocs/docs/features/serialization.md b/docs/mkdocs/docs/features/serialization.md index 1190c9005..5e1aedac0 100644 --- a/docs/mkdocs/docs/features/serialization.md +++ b/docs/mkdocs/docs/features/serialization.md @@ -52,7 +52,7 @@ with `\uXXXX` sequences, so that the output contains only ASCII characters: ```cpp json j = "苹果"; j.dump(); // "苹果" -j.dump(-1, ' ', true); // "苹果" +j.dump(-1, ' ', true); // "\u82f9\u679c" ``` ## Handling invalid UTF-8 diff --git a/docs/mkdocs/docs/features/types/index.md b/docs/mkdocs/docs/features/types/index.md index 354acd5ac..67e97df6e 100644 --- a/docs/mkdocs/docs/features/types/index.md +++ b/docs/mkdocs/docs/features/types/index.md @@ -120,7 +120,7 @@ requirements are collected in [Template Parameter Requirements](template_paramet ### Default type -With the default values for *ObjectType* (`std::map`), *StringType* (`std::string`), and *AllocatorType* (`std::allocator`), the default value for `object_t` is: +With the default values for *ObjectType* (`std::map`), *StringType* (`std::string`), and *AllocatorType* (`std::allocator`), the default type for `object_t` is: ```cpp std::map< @@ -177,7 +177,7 @@ json j = m; ### Default type -With the default values for *ArrayType* (`std::vector`) and *AllocatorType* (`std::allocator`), the default value for `array_t` is: +With the default values for *ArrayType* (`std::vector`) and *AllocatorType* (`std::allocator`), the default type for `array_t` is: ```cpp std::vector< @@ -209,7 +209,7 @@ Unicode values are split by the JSON class into byte-sized characters during des ### Default type -With the default values for *StringType* (`std::string`), the default value for `string_t` is `#!cpp std::string`. +With the default values for *StringType* (`std::string`), the default type for `string_t` is `#!cpp std::string`. ### Encoding @@ -234,7 +234,7 @@ String values are stored as pointers in a `basic_json` type. That is, for any ac ### Default type -With the default values for *BooleanType* (`#!cpp bool`), the default value for `boolean_t` is `#!cpp bool`. +With the default values for *BooleanType* (`#!cpp bool`), the default type for `boolean_t` is `#!cpp bool`. ### Storage @@ -252,9 +252,9 @@ This description includes both integer and floating-point numbers. However, C++ ### Default types -With the default values for *NumberIntegerType* (`std::int64_t`), the default value for `number_integer_t` is `std::int64_t`. -With the default values for *NumberUnsignedType* (`std::uint64_t`), the default value for `number_unsigned_t` is `std::uint64_t`. -With the default values for *NumberFloatType* (`#!cpp double`), the default value for `number_float_t` is `#!cpp double`. +With the default values for *NumberIntegerType* (`std::int64_t`), the default type for `number_integer_t` is `std::int64_t`. +With the default values for *NumberUnsignedType* (`std::uint64_t`), the default type for `number_unsigned_t` is `std::uint64_t`. +With the default values for *NumberFloatType* (`#!cpp double`), the default type for `number_float_t` is `#!cpp double`. ### Default behavior @@ -281,7 +281,7 @@ As this range is a subrange of the exactly supported range [`INT64_MIN`, `INT64_ > This specification allows implementations to set limits on the range and precision of numbers accepted. Since software that implements IEEE 754-2008 binary64 (double precision) numbers is generally available and widely used, good interoperability can be achieved by implementations that expect no more precision or range than these provide, in the sense that implementations will approximate JSON numbers within the expected precision. -This implementation does exactly follow this approach, as it uses double precision floating-point numbers. Note values smaller than `#!c -1.79769313486232e+308` and values greater than `#!c 1.79769313486232e+308` will be stored as NaN internally and be serialized to `#!json null`. +This implementation does exactly follow this approach, as it uses double precision floating-point numbers by default (see *NumberFloatType* above). Note that with the default `#!cpp double`, numbers smaller than `#!c -1.79769313486232e+308` or greater than `#!c 1.79769313486232e+308` are rejected during parsing with [`out_of_range.406`](../../home/exceptions.md#jsonexceptionout_of_range406). Non-finite values (NaN and infinity) can only be stored by assigning them directly and are serialized to `#!json null`. ### Storage diff --git a/docs/mkdocs/docs/features/types/number_handling.md b/docs/mkdocs/docs/features/types/number_handling.md index 13b6dfd43..b65708040 100644 --- a/docs/mkdocs/docs/features/types/number_handling.md +++ b/docs/mkdocs/docs/features/types/number_handling.md @@ -82,12 +82,13 @@ flowchart TD - Numbers with a decimal digit or scientific notation are always stored as `#!c double`. - The number types can be changed, see [Template number types](#template-number-types). - - Integers are converted by the library's own digit parser. Floating-point numbers are converted with - [`std::from_chars`](https://en.cppreference.com/w/cpp/utility/from_chars) if the library is compiled with C++17 - and the standard library supports it, then with an exact fast path for `#!c double` values with few significant - digits, and otherwise with the locale-aware - [`std::strtod`](https://en.cppreference.com/w/cpp/string/byte/strtof) (`std::strtof`/`std::strtold` for the - other floating-point types). Before version 3.13.0, the conversion was realized by + - The library converts integers and floating-point numbers itself, independent of the locale. Floating-point + numbers are correctly rounded (to nearest, ties to even). Only a `#!c long double` that is not IEEE 754 binary64 + (e.g., the 80-bit x87 format) is converted with `#!cpp std::from_chars` where available, or else with + [`std::strtold`](https://en.cppreference.com/w/cpp/string/byte/strtof). For that call, the library temporarily + replaces the `.` with the decimal point of the current locale (which may be longer than one byte, e.g., in + `fa_IR.UTF-8`), so the result does not depend on the locale either. Changing the locale in another thread during + parsing is undefined behavior of the C library, though. Before version 3.13.0, the conversion was realized by [`std::strtoull`](https://en.cppreference.com/w/cpp/string/byte/strtoul), [`std::strtoll`](https://en.cppreference.com/w/cpp/string/byte/strtol), and `std::strtod`, respectively. @@ -100,10 +101,10 @@ flowchart TD ### Number limits - Any 64-bit signed or unsigned integer can be stored without loss of precision. -- Numbers exceeding the limits of `#!c double` (i.e., numbers that after conversion via -[`std::strtod`](https://en.cppreference.com/w/cpp/string/byte/strtof) are not satisfying +- Numbers exceeding the limits of `#!c double` (i.e., numbers whose rounded value is not satisfying [`std::isfinite`](https://en.cppreference.com/w/cpp/numeric/math/isfinite) such as `#!c 1E400`) will throw exception -[`json.exception.out_of_range.406`](../../home/exceptions.md#jsonexceptionout_of_range406) during parsing. +[`json.exception.out_of_range.406`](../../home/exceptions.md#jsonexceptionout_of_range406) during parsing. Numbers too +small for `#!c double` (such as `#!c 1E-400`) become zero, with the sign of the number. - Floating-point numbers are rounded to the next number representable as `double`. For instance `#!c 3.141592653589793238462643383279` is stored as [`0x400921fb54442d18`](https://float.exposed/0x400921fb54442d18). This is the same behavior as the code `#!c double x = 3.141592653589793238462643383279;`. @@ -133,9 +134,10 @@ That is, `-0` is stored as a signed integer, but the serialization does not repr ### Number serialization - Integer numbers are serialized as is; that is, no scientific notation is used. -- Floating-point numbers are serialized as specified by the `#!c %g` printf modifier with - [`std::numeric_limits::max_digits10`](https://en.cppreference.com/w/cpp/types/numeric_limits/max_digits10) - significant digits. The rationale is to use the shortest representation while still allowing round-tripping. +- Floating-point numbers are serialized with the fewest digits that read back as the same value (the closest such + digits if there are several), in the layout of the `#!c %g` printf modifier: `#!c 1.5`, `#!c 100.0`, `#!c 1e+100`. + Doubles are converted with the algorithm of [Żmij](https://github.com/vitaut/zmij), floats with Grisu2, which + can write more digits than necessary. !!! hint "Notes regarding precision of floating-point numbers" diff --git a/docs/mkdocs/docs/features/types/template_parameters.md b/docs/mkdocs/docs/features/types/template_parameters.md index f20102878..162cfbad5 100644 --- a/docs/mkdocs/docs/features/types/template_parameters.md +++ b/docs/mkdocs/docs/features/types/template_parameters.md @@ -30,9 +30,9 @@ Requirements are split into two groups: diagnosed with dedicated error messages, and violating most of them results in a compiler error somewhere inside the library. Four violations are not caught at compile time at all: - - A [`StringType`](#stringtype) whose `data()` is not null-terminated compiles and can silently misparse - floating-point numbers, because the lexer may hand the buffer to `#!cpp std::strtod`, which reads up to the - terminating null character. + - A [`StringType`](#stringtype) whose `data()` is not null-terminated compiles and silently misparses numbers + stored as a `#!cpp long double` that is not IEEE 754 binary64 (e.g., the 80-bit x87 format), because the lexer + hands the buffer to `#!cpp std::strtold`. - A stateful [`AllocatorType`](#allocatortype) compiles and silently ignores its state: allocation, deallocation, and [`get_allocator()`](../../api/basic_json/get_allocator.md) each use a different default-constructed instance. - The two [cross-specialization conversions](#cross-specialization-conversions) below. These abort on an assertion @@ -353,16 +353,21 @@ using array_t = ArrayType>; ### Always required - A member type `value_type` that is one byte wide and `char`-compatible. The library stores and processes UTF-8 - encoded `char` data and passes `data()` to functions that take a `#!cpp const char*`, such as `#!cpp std::strtod`. + encoded `char` data and passes `data()` to functions that take a `#!cpp const char*`, such as `#!cpp std::strtold` + (only used to parse a `#!cpp long double` that is not IEEE 754 binary64, see + [`NumberFloatType`](#numberfloattype)). `#!cpp std::wstring`, `#!cpp std::u16string`, and `#!cpp std::u32string` are **not** valid choices; see the FAQ on [wide string handling](../../home/faq.md#wide-string-handling). - Constructors: default, copy, move, from `#!cpp const char*` (which must not be `#!cpp explicit`), from `#!cpp (const char*, size_type)`, and from `#!cpp (size_type, char)`; and copy or move assignment. - Member functions `size()`, `clear()`, `resize(n, c)`, `data()`, `push_back(char)`, and `operator[]` (const and non-const, returning references). `c_str()` and `back()` are **not** required. -- `data()` must return a pointer to a contiguous, **null-terminated** buffer -- the parser may hand it to - `#!cpp std::strtod`, which reads up to the null character. A type whose `data()` is not null-terminated does not - fail to compile; it can silently misparse floating-point numbers. +- `data()` must return a pointer to a contiguous, **null-terminated** buffer. `#!cpp float`, `#!cpp double`, and a + `#!cpp long double` that is IEEE 754 binary64 are converted by the library itself and do not depend on this. For any + other `NumberFloatType` (a `#!cpp long double` of another format), the parser falls back to `#!cpp std::strtold` when + `#!cpp std::from_chars` is not available or declines the token, and `std::strtold` reads up to the null character. A type whose `data()` + is not null-terminated does not fail to compile; with such a `NumberFloatType` it can silently misparse + floating-point numbers. - `append(const char*, size_type)`, used by [`dump`](../../api/basic_json/dump.md), and `append(const StringType&)`, used by the CBOR reader for indefinite-length strings. The library's internal string concatenation additionally has to append a `#!cpp char` and a `#!cpp const char*`; for each it selects between `append(arg)`, `#!cpp operator+=`, @@ -541,16 +546,18 @@ therefore silently changes parse results rather than raising an error. See `NumberFloatType` must be one of `#!cpp float`, `#!cpp double`, or `#!cpp long double`: -- The [parser](../parsing/index.md) converts number literals with `#!cpp std::from_chars` or, as a fallback, with - `#!cpp std::strtof`, `#!cpp std::strtod`, or `#!cpp std::strtold`; the library provides overloads for exactly these - three types. +- The [parser](../parsing/index.md) converts number literals to `#!cpp float`, `#!cpp double`, and a + `#!cpp long double` that is IEEE 754 binary64 itself; other `#!cpp long double` formats are converted with + `#!cpp std::from_chars` where available, or with `#!cpp std::strtold`. The library provides overloads for exactly + these three types. - [`dump`](../../api/basic_json/dump.md) falls back to `#!cpp std::snprintf` with the `%g` and `%Lg` conversion specifiers, for which the library likewise provides only `#!cpp double` and `#!cpp long double` overloads (`#!cpp float` is promoted to `#!cpp double`). -If `#!cpp std::numeric_limits` describes an IEEE 754 binary32 or binary64 number, `dump` uses the -Grisu2 algorithm, which produces the shortest representation that round-trips. Otherwise the `snprintf` fallback with -`max_digits10` digits is used. +If `#!cpp std::numeric_limits` describes an IEEE 754 binary64 number, `dump` uses the algorithm of +Żmij, which produces the shortest representation that round-trips. For IEEE 754 binary32 numbers, it uses Grisu2, +which produces a short representation that round-trips. Otherwise the `snprintf` fallback with `max_digits10` digits is +used. ### Required for the binary formats @@ -562,7 +569,7 @@ binary32 or binary64 field and have no encoding for `#!cpp long double`. | Type | Support | |--------------------------|-----------------------------------------------------------------------------------------------------------------------| -| `#!cpp double` (default) | full; short round-trip output through Grisu2 | +| `#!cpp double` (default) | full; shortest round-trip output through Żmij | | `#!cpp float` | full; short round-trip output through Grisu2 | | `#!cpp long double` | `dump` and `parse` only; the binary format writers do not compile, as they only handle IEEE 754 binary32 and binary64 | | any other type | not usable | diff --git a/docs/mkdocs/docs/home/architecture.md b/docs/mkdocs/docs/home/architecture.md index ea47bab6e..4975c1d1b 100644 --- a/docs/mkdocs/docs/home/architecture.md +++ b/docs/mkdocs/docs/home/architecture.md @@ -185,9 +185,11 @@ The lexer detects two optional extensions at compile time. Only `iterator_input_ random-access input of single-byte characters: - `supports_seek`, `get_consumed_count()`, and `copy_consumed_range()` let the lexer reconstruct already consumed input - for error messages instead of copying every character it reads. + for error messages instead of copying every character it reads. With `get_read_count()` and + `count_consumed_lines()`, the lexer also derives the read position from the adapter and counts lines only when an + error needs them, instead of counting characters and lines on every read. - `supports_bulk_scan`, `bulk_data()`, `bulk_remaining()`, and `bulk_skip()` let the lexer scan strings directly in - contiguous memory, several bytes at a time. + contiguous memory, several bytes at a time, and skip the spaces that indent a line in 8-byte steps. The function `input_adapter` picks the right adapter for the argument passed to `parse`, `accept`, `sax_parse`, or the `from_*` functions: diff --git a/docs/mkdocs/docs/home/exceptions.md b/docs/mkdocs/docs/home/exceptions.md index c0f78574c..f4245b660 100644 --- a/docs/mkdocs/docs/home/exceptions.md +++ b/docs/mkdocs/docs/home/exceptions.md @@ -889,7 +889,9 @@ contains other characters (e.g., `1a`), or `-` where it cannot be used. ### json.exception.out_of_range.405 -The JSON Patch operations 'remove' and 'add' cannot be applied to the root element of the JSON value. +The JSON Patch operations 'remove' and 'move' cannot be applied to the root element of the JSON value. The exception is +also thrown when the parent or a reference token of an empty JSON Pointer is requested, for instance by +[`back`](../api/json_pointer/back.md) or [`pop_back`](../api/json_pointer/pop_back.md). !!! failure "Example message" @@ -916,7 +918,8 @@ double-precision number when `number_float_t` is `#!cpp float`. An integer number cannot be represented by the binary format it is serialized to: -- [BON8](../features/binary_formats/bon8.md) only stores integers that fit into int64. +- [BON8](../features/binary_formats/bon8.md) only stores integers that fit into int64, so + [`to_bon8`](../api/basic_json/to_bon8.md) throws for unsigned integers greater than 9223372036854775807. - [CBOR](../features/binary_formats/cbor.md), [MessagePack](../features/binary_formats/messagepack.md), and [BSON](../features/binary_formats/bson.md) store integers in at most 64 bits. With the default number types, every integer fits, but a [`number_integer_t`](../api/basic_json/number_integer_t.md) or @@ -927,22 +930,25 @@ An integer number cannot be represented by the binary format it is serialized to [UBJSON](../features/binary_formats/ubjson.md) and [BJData](../features/binary_formats/bjdata.md) never throw this exception, because they serialize integers beyond 64 bits as high-precision numbers. +!!! note "Former occurrences" + + The UBJSON and BSON writers used to throw this exception as well: + + - Since version 3.9.0, integer numbers beyond int64 are serialized as high-precision UBJSON numbers. + - Since version 3.12.0, integer numbers beyond int64 are serialized as uint64 BSON numbers. + + Before version 3.13.0, CBOR, MessagePack, and BSON silently truncated integers wider than 64 bits, and BJData + truncated unsigned integers wider than 64 bits. + !!! failure "Example messages" ``` - integer number 9223372036854775808 cannot be represented by BON8 as it does not fit int64 + integer number 18446744073709551615 cannot be represented by BON8 as it does not fit int64 ``` ``` integer number 1267650600228229401496703205376 cannot be represented by CBOR as it does not fit [-2^64, 2^64-1] ``` -!!! note - - Before version 3.13.0, CBOR, MessagePack, and BSON silently truncated integers wider than 64 bits, and BJData - truncated unsigned integers wider than 64 bits. This exception was previously thrown by UBJSON and BSON for - integers greater than 9223372036854775807; since version 3.9.0, such integers are serialized as high-precision - UBJSON numbers, and since version 3.12.0 as uint64 BSON numbers. - ### json.exception.out_of_range.408 The size of an array or object in a [binary format](../features/binary_formats/index.md) exceeds the maximal capacity: diff --git a/docs/mkdocs/docs/home/faq.md b/docs/mkdocs/docs/home/faq.md index eec13a481..2381f4f7e 100644 --- a/docs/mkdocs/docs/home/faq.md +++ b/docs/mkdocs/docs/home/faq.md @@ -66,7 +66,9 @@ Without the macro (default behavior), `json j{obj}` creates `[{"key":"value"}]`. Can you add an option to ignore trailing commas? -This library does not support any feature that would jeopardize interoperability. +Yes, trailing commas in arrays and objects can be allowed by setting parameter `ignore_trailing_commas` of +[`parse`](../api/basic_json/parse.md) to `#!cpp true`; see [trailing commas](../features/trailing_commas.md). They are +rejected by default, and the library never adds them when serializing. ### Parse errors reading non-ASCII characters @@ -230,11 +232,11 @@ Yes, you can. Please define the symbol [`JSON_DIAGNOSTICS`](../api/macros/json_d - It seems that precision is lost when serializing a double. - Can I change the precision for floating-point serialization? -The library uses `std::numeric_limits::digits10` (15 for IEEE `double`s) digits for serialization. This value is sufficient to guarantee roundtripping. If one uses more than this number of digits of precision, then string -> value -> string is not guaranteed to round-trip. +The library serializes floating-point numbers with the shortest representation that parses back to the same value, using at most `std::numeric_limits::max_digits10` (17 for IEEE `double`s) significant digits. This guarantees that value -> string -> value round-trips. The reverse direction, string -> value -> string, is only guaranteed for numbers with at most `std::numeric_limits::digits10` (15 for IEEE `double`s) significant digits; a number with more digits may not be representable exactly and is then serialized as the closest `double`. -!!! quote "[cppreference.com](https://en.cppreference.com/w/cpp/types/numeric_limits/digits10)" +!!! quote "[cppreference.com](https://en.cppreference.com/w/cpp/types/numeric_limits/max_digits10)" - The value of `std::numeric_limits::digits10` is the number of base-10 digits that can be represented by the type T without change, that is, any number with this many significant decimal digits can be converted to a value of type T and back to decimal form, without change due to rounding or overflow. + The value of `std::numeric_limits::max_digits10` is the number of base-10 digits that are necessary to uniquely represent all distinct values of the type T, such as necessary for serialization/deserialization to text. !!! tip diff --git a/docs/mkdocs/docs/home/license.md b/docs/mkdocs/docs/home/license.md index 11d0ec529..715c6ca51 100644 --- a/docs/mkdocs/docs/home/license.md +++ b/docs/mkdocs/docs/home/license.md @@ -18,6 +18,8 @@ The class contains the UTF-8 Decoder from Bjoern Hoehrmann which is licensed und The class contains a slightly modified version of the Grisu2 algorithm from Florian Loitsch which is licensed under the [MIT License](https://opensource.org/licenses/MIT) (see above). Copyright © 2009 [Florian Loitsch](https://florian.loitsch.com/) +The class contains a port of the shortest double-to-decimal conversion of [Żmij](https://github.com/vitaut/zmij) by Victor Zverovich, including the conversion of the digits to text by Xiang JunBo and the SIMD instruction sequence of Dougall Johnson, which is licensed under the [MIT License](https://opensource.org/licenses/MIT) (see above). Copyright © 2025 [Victor Zverovich](https://github.com/vitaut) + The class contains a copy of [Hedley](https://nemequ.github.io/hedley/) from Evan Nemerson which is licensed as [CC0-1.0](https://creativecommons.org/publicdomain/zero/1.0/). -The class contains an adapted version of the Eisel-Lemire algorithm and its table of powers of five from [fast_float](https://github.com/fastfloat/fast_float) by Daniel Lemire and contributors, which is available under the [MIT License](https://opensource.org/licenses/MIT) (used here), the Apache 2.0 License, and the Boost Software License. Copyright © 2021 The fast_float authors +The class contains an adapted version of the Eisel-Lemire algorithm, its table of powers of five, and its digit comparison for long numbers from [fast_float](https://github.com/fastfloat/fast_float) by Daniel Lemire and contributors, which is available under the [MIT License](https://opensource.org/licenses/MIT) (used here), the Apache 2.0 License, and the Boost Software License. Copyright © 2021 The fast_float authors diff --git a/docs/mkdocs/docs/integration/cmake.md b/docs/mkdocs/docs/integration/cmake.md index 9492efe9f..65eed72a8 100644 --- a/docs/mkdocs/docs/integration/cmake.md +++ b/docs/mkdocs/docs/integration/cmake.md @@ -90,13 +90,13 @@ to the following. ### FetchContent -Since CMake v3.11, [FetchContent](https://cmake.org/cmake/help/v3.11/module/FetchContent.html) can be used to +Since CMake v3.14, [FetchContent](https://cmake.org/cmake/help/v3.14/module/FetchContent.html) can be used to automatically download a release as a dependency at configure time. !!! example ```cmake title="CMakeLists.txt" - cmake_minimum_required(VERSION 3.11) + cmake_minimum_required(VERSION 3.14) project(ExampleProject LANGUAGES CXX) include(FetchContent) diff --git a/docs/mkdocs/docs/integration/compile_times.md b/docs/mkdocs/docs/integration/compile_times.md index 0e3be638c..279a00ec7 100644 --- a/docs/mkdocs/docs/integration/compile_times.md +++ b/docs/mkdocs/docs/integration/compile_times.md @@ -14,7 +14,8 @@ that cost, ordered by how much they typically save. Header files that only need to *name* the `json` type — for function declarations, members held by pointer or reference, or friend declarations — can include `` instead of ``. It only -forward-declares `basic_json`, `json`, `ordered_json`, `json_pointer`, and `adl_serializer`. The translation units that +forward-declares `basic_json`, `json_pointer`, `adl_serializer`, and `ordered_map`, and defines the aliases `json` and +`ordered_json`. The translation units that actually use the values then include ``. ```cpp title="person.hpp" diff --git a/docs/mkdocs/mkdocs.yml b/docs/mkdocs/mkdocs.yml index 111e0da81..ead3228e5 100644 --- a/docs/mkdocs/mkdocs.yml +++ b/docs/mkdocs/mkdocs.yml @@ -182,6 +182,7 @@ nav: - 'json_serializer': api/basic_json/json_serializer.md - 'max_size': api/basic_json/max_size.md - 'meta': api/basic_json/meta.md + - 'merge_diff': api/basic_json/merge_diff.md - 'merge_patch': api/basic_json/merge_patch.md - 'number_float_t': api/basic_json/number_float_t.md - 'number_integer_t': api/basic_json/number_integer_t.md @@ -452,8 +453,12 @@ plugins: # package badges (the fetch fails, then reading the missing cache entry # raises FileNotFoundError). Readers' browsers are served normally, so # leave these badges as external references instead of self-hosting them. + # The same goes for img.shields.io, whose version badges time out from + # time to time (strict mode turns the failed download into an error) and + # would go stale between builds if they were self-hosted anyway. assets_exclude: - repology.org/* + - img.shields.io/* - llmstxt: markdown_description: > JSON for Modern C++ is a C++11 header-only library implementing a JSON diff --git a/docs/mkdocs/requirements.txt b/docs/mkdocs/requirements.txt index d9e7098ed..245ebc387 100644 --- a/docs/mkdocs/requirements.txt +++ b/docs/mkdocs/requirements.txt @@ -7,6 +7,6 @@ mkdocs-material-extensions==1.3.1 # extensions mkdocs-minify-plugin==0.8.0 # plugin "minify" mkdocs-redirects==1.2.3 # plugin "redirects" mkdocs-htmlproofer-plugin==1.6.0 # plugin "htmlproofer" -mkdocs-llmstxt==0.5.0 # plugin "llmstxt" +mkdocs-llmstxt==0.5.1 # plugin "llmstxt" PyYAML==6.0.3 # linter diff --git a/docs/mkdocs/scripts/mermaid/package-lock.json b/docs/mkdocs/scripts/mermaid/package-lock.json index 4f9dc0395..d7d3cad06 100644 --- a/docs/mkdocs/scripts/mermaid/package-lock.json +++ b/docs/mkdocs/scripts/mermaid/package-lock.json @@ -6,7 +6,7 @@ "": { "name": "check-mermaid", "dependencies": { - "jsdom": "30.1.1", + "jsdom": "30.1.2", "mermaid": "11.17.2" } }, @@ -24,14 +24,14 @@ } }, "node_modules/@asamuzakjp/css-color": { - "version": "7.1.2", - "resolved": "https://registry.npmjs.org/@asamuzakjp/css-color/-/css-color-7.1.2.tgz", - "integrity": "sha512-99DHAnXDB5z6EEK+9GMpVI7Mw4oxj97dY5bpOzMnjADQWxI8rN6TvTduuFLUhUMlS7/CfVZ06tcZsus6cltnNw==", + "version": "7.1.3", + "resolved": "https://registry.npmjs.org/@asamuzakjp/css-color/-/css-color-7.1.3.tgz", + "integrity": "sha512-1t1U8Cm3RBl9vh3RAZhjkPaduzLSvQJRToRrxpGR/eDRW99mOREleuu+mWfiFa795bUzjwht7NgphCKTs2ianQ==", "license": "MIT", "dependencies": { - "@csstools/css-calc": "^3.4.1", - "@csstools/css-color-parser": "^4.2.4", - "@csstools/css-parser-algorithms": "^4.0.1", + "@csstools/css-calc": "^3.4.2", + "@csstools/css-color-parser": "^4.2.5", + "@csstools/css-parser-algorithms": "^4.0.2", "@csstools/css-tokenizer": "^4.0.2", "lru-cache": "^11.5.3" }, @@ -40,9 +40,9 @@ } }, "node_modules/@asamuzakjp/dom-selector": { - "version": "9.2.2", - "resolved": "https://registry.npmjs.org/@asamuzakjp/dom-selector/-/dom-selector-9.2.2.tgz", - "integrity": "sha512-lSWTBMjAcmu2xn5yEDU7jh6QDV+C8GEKtdJ4pIQhXh26RkKQ7S3FuQlt+zZkUbTdJ4d3XyV1kPI/G0zWXxqaqw==", + "version": "9.2.4", + "resolved": "https://registry.npmjs.org/@asamuzakjp/dom-selector/-/dom-selector-9.2.4.tgz", + "integrity": "sha512-YnVzxDxLaqL0t7Q4wfcgHZjg55Wm6y5EaO7Bhu3q9J7wq/8iw3PhsnWiOQhYekx4VexnbkgWpTT6n+QIV7kczw==", "license": "MIT", "dependencies": { "bidi-js": "^1.1.0", @@ -98,9 +98,9 @@ } }, "node_modules/@csstools/css-calc": { - "version": "3.4.1", - "resolved": "https://registry.npmjs.org/@csstools/css-calc/-/css-calc-3.4.1.tgz", - "integrity": "sha512-EtC7SoN1j6J4E4DCwg5QgbO5TGxgxIA1RXqe+W+qUM+BUcezx9wT+/tiQ/WO2yCX4i5X+Cuf9ciJ22aP4UEwWw==", + "version": "3.4.3", + "resolved": "https://registry.npmjs.org/@csstools/css-calc/-/css-calc-3.4.3.tgz", + "integrity": "sha512-iex20d8CHVkyvg6B7UKV7uHnI2Bqo9g+EFfT9E0y+GvTvhZ/DwONJ+9aKb1dlqm0ZiGsL5RXjp0fCoJYnkeDjA==", "funding": [ { "type": "github", @@ -116,14 +116,14 @@ "node": ">=20.19.0" }, "peerDependencies": { - "@csstools/css-parser-algorithms": "^4.0.1", + "@csstools/css-parser-algorithms": "^4.0.2", "@csstools/css-tokenizer": "^4.0.2" } }, "node_modules/@csstools/css-color-parser": { - "version": "4.2.4", - "resolved": "https://registry.npmjs.org/@csstools/css-color-parser/-/css-color-parser-4.2.4.tgz", - "integrity": "sha512-DyefytAZ735mX4Dq/WcDAXFtXhaEFvme0ZS9tVEBAc2whxUthXr0R0L2rmEPm59SrMBkGrFzQviBXMs/UtnABQ==", + "version": "4.2.6", + "resolved": "https://registry.npmjs.org/@csstools/css-color-parser/-/css-color-parser-4.2.6.tgz", + "integrity": "sha512-iiPQ3iRWwnJkeEn6RIu6SJPr7hYrLz6XZ9s/QZl+2/LI5KQVjpl2fdmDSZKuD4xP6GMmMPgHFFXg6k1Wkz0Trg==", "funding": [ { "type": "github", @@ -137,20 +137,20 @@ "license": "MIT", "dependencies": { "@csstools/color-helpers": "^6.1.2", - "@csstools/css-calc": "^3.4.1" + "@csstools/css-calc": "^3.4.3" }, "engines": { "node": ">=20.19.0" }, "peerDependencies": { - "@csstools/css-parser-algorithms": "^4.0.1", + "@csstools/css-parser-algorithms": "^4.0.2", "@csstools/css-tokenizer": "^4.0.2" } }, "node_modules/@csstools/css-parser-algorithms": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/@csstools/css-parser-algorithms/-/css-parser-algorithms-4.0.1.tgz", - "integrity": "sha512-ShL8BqPfbKJrJiKFH0xBbN0i7Nrh9HXYRuF+pzyj93R/BL2YAsUxJeqANErzqM+0JGl7vjHp+3OIgd/DL69YIA==", + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/@csstools/css-parser-algorithms/-/css-parser-algorithms-4.0.2.tgz", + "integrity": "sha512-40cSKyMvK+tq4qz6Awrlye2WGuOKt3FwPgtGg6KTfbHOWNw+Rk1rzbAtZnZ6IBhsY491HLRnDXwoyBAijmmILA==", "funding": [ { "type": "github", @@ -170,9 +170,9 @@ } }, "node_modules/@csstools/css-syntax-patches-for-csstree": { - "version": "1.1.14", - "resolved": "https://registry.npmjs.org/@csstools/css-syntax-patches-for-csstree/-/css-syntax-patches-for-csstree-1.1.14.tgz", - "integrity": "sha512-HpbVXyrofRXpHpgkNIjU/3EWR4WJvOkO3emNK/L6X/mTJU7bGUI3AkkpoTNXznQLp0KRjLHELTGeKI5dIkI9JQ==", + "version": "1.1.15", + "resolved": "https://registry.npmjs.org/@csstools/css-syntax-patches-for-csstree/-/css-syntax-patches-for-csstree-1.1.15.tgz", + "integrity": "sha512-J0u7HkVl2nzSlhsiTOp4AmwcUQ3D+mGEEKfBy/7To5/y7F2OHwyLrXfrhR0SMgr4p5Lo+eaMVSeai24zUcBIxA==", "funding": [ { "type": "github", @@ -1071,30 +1071,16 @@ } }, "node_modules/data-urls": { - "version": "7.0.0", - "resolved": "https://registry.npmjs.org/data-urls/-/data-urls-7.0.0.tgz", - "integrity": "sha512-23XHcCF+coGYevirZceTVD7NdJOqVn+49IHyxgszm+JIiHLoB2TkmPtsYkNWT1pvRSGkc35L6NHs0yHkN2SumA==", + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/data-urls/-/data-urls-8.0.0.tgz", + "integrity": "sha512-JQUCCK+/CAGbGGx618PlFZMXH81Jtj7t73CHv9dd/zVWvxgGT2zLFRjnKb+Ng46GIwcy+f7f88qd4duqllK+WQ==", "license": "MIT", "dependencies": { "whatwg-mimetype": "^5.0.0", - "whatwg-url": "^16.0.0" + "whatwg-url": "^17.1.2" }, "engines": { - "node": "^20.19.0 || ^22.12.0 || >=24.0.0" - } - }, - "node_modules/data-urls/node_modules/whatwg-url": { - "version": "16.0.1", - "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-16.0.1.tgz", - "integrity": "sha512-1to4zXBxmXHV3IiSSEInrreIlu02vUOvrhxJJH5vcxYTBDAx51cqZiKdyTxlecdKNSjj8EcxGBxNf6Vg+945gw==", - "license": "MIT", - "dependencies": { - "@exodus/bytes": "^1.11.0", - "tr46": "^6.0.0", - "webidl-conversions": "^8.0.1" - }, - "engines": { - "node": "^20.19.0 || ^22.12.0 || >=24.0.0" + "node": "^22.14.0 || >=24.0.0" } }, "node_modules/dayjs": { @@ -1216,30 +1202,30 @@ "license": "MIT" }, "node_modules/jsdom": { - "version": "30.1.1", - "resolved": "https://registry.npmjs.org/jsdom/-/jsdom-30.1.1.tgz", - "integrity": "sha512-FahmoPK5vbPc+jxV1iErMHmAZypCZ942NHF4+qqaWAuvaKKTBZxawnmAtrbGWLU7MtlxfqIP0qw6aSI+aWGtLg==", + "version": "30.1.2", + "resolved": "https://registry.npmjs.org/jsdom/-/jsdom-30.1.2.tgz", + "integrity": "sha512-0FFE/jE1rppmVfUrJUgxqXjcwolZYIGtAgj1pTussMhNZxIxi7W/PvfWaMNroGi2B36X1IKHbexpZ9DhkoOPiQ==", "license": "MIT", "dependencies": { - "@asamuzakjp/css-color": "^7.0.0", - "@asamuzakjp/dom-selector": "^9.2.1", + "@asamuzakjp/css-color": "^7.1.3", + "@asamuzakjp/dom-selector": "^9.2.4", "@bramus/specificity": "^2.4.2", - "@csstools/css-syntax-patches-for-csstree": "^1.1.13", - "@exodus/bytes": "^1.15.1", + "@csstools/css-syntax-patches-for-csstree": "^1.1.15", + "@exodus/bytes": "^1.16.0", "css-tree": "^3.2.1", - "data-urls": "^7.0.0", + "data-urls": "^8.0.0", "decimal.js": "^10.6.0", "html-encoding-sniffer": "^7.0.0", "is-potential-custom-element-name": "^1.0.1", - "lru-cache": "^11.5.2", + "lru-cache": "^11.5.3", "parse5": "^8.0.1", "saxes": "^6.0.0", "tough-cookie": "^6.0.2", - "undici": "^8.10.2", + "undici": "^8.11.2", "w3c-xmlserializer": "^6.0.0", "webidl-conversions": "^8.0.1", "whatwg-mimetype": "^5.0.0", - "whatwg-url": "^17.1.1", + "whatwg-url": "^17.2.0", "xml-name-validator": "^5.0.0" }, "engines": { @@ -1514,15 +1500,15 @@ } }, "node_modules/tr46": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/tr46/-/tr46-6.0.0.tgz", - "integrity": "sha512-bLVMLPtstlZ4iMQHpFHTR7GAGj2jxi8Dg0s2h2MafAE4uSWF98FC/3MomU51iQAMf8/qDUbKWf5GxuvvVcXEhw==", + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-7.0.0.tgz", + "integrity": "sha512-BcDSDW+SIGWVOns1Zm6Yg91q9AEj7eaID0QDY9oIqJD1A8dscNwMcl9Sms2DeuWCNov1X0/oXanHGYQBSLwZsg==", "license": "MIT", "dependencies": { "punycode": "^2.3.1" }, "engines": { - "node": ">=20" + "node": "^22.14.0 || >=24.0.0" } }, "node_modules/ts-dedent": { @@ -1587,13 +1573,13 @@ } }, "node_modules/whatwg-url": { - "version": "17.1.2", - "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-17.1.2.tgz", - "integrity": "sha512-TEZA+Zqxin7Jjsm2cjRohCmen5awh+hT6Zi3VZdqZlNRk7zvOI/9WpBFg/DWlA56bWnzwm6DuB8NS0EsxQH9uQ==", + "version": "17.2.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-17.2.0.tgz", + "integrity": "sha512-IhLIHNcBrzIixC5XXsbr4DxdjajyA5vQ5RwGZMtrUp+GbtE5Nrsge9QKau7jXrC0aTXLYGV0xiW7Tb4YS2aomw==", "license": "MIT", "dependencies": { - "@exodus/bytes": "^1.15.1", - "tr46": "^6.0.0", + "@exodus/bytes": "^1.16.0", + "tr46": "^7.0.0", "webidl-conversions": "^8.0.1" }, "engines": { diff --git a/docs/mkdocs/scripts/mermaid/package.json b/docs/mkdocs/scripts/mermaid/package.json index 6cf40b9f6..c79f4fa89 100644 --- a/docs/mkdocs/scripts/mermaid/package.json +++ b/docs/mkdocs/scripts/mermaid/package.json @@ -4,7 +4,7 @@ "description": "Validate the Mermaid diagrams of the documentation (see check_mermaid.mjs)", "type": "module", "dependencies": { - "jsdom": "30.1.1", + "jsdom": "30.1.2", "mermaid": "11.17.2" } } diff --git a/include/nlohmann/detail/bit_ops.hpp b/include/nlohmann/detail/bit_ops.hpp index 9ccb82c66..c9424a6d1 100644 --- a/include/nlohmann/detail/bit_ops.hpp +++ b/include/nlohmann/detail/bit_ops.hpp @@ -9,11 +9,12 @@ #pragma once #include // uint64_t -#if !defined(__SIZEOF_INT128__) && defined(_MSC_VER) && (defined(_M_X64) || defined(_M_ARM64)) - #include // __umulh, _umul128 +#include // memcpy +#if defined(_MSC_VER) && (defined(_M_X64) || defined(_M_ARM64)) && (!defined(__SIZEOF_INT128__) || (!defined(__GNUC__) && !defined(__clang__))) + #include // __umulh, _umul128, _BitScanForward64, _BitScanReverse64 #endif -#include +#include // JSON_HEDLEY_ALWAYS_INLINE, NLOHMANN_JSON_NAMESPACE_BEGIN // Portable bit-level helpers for the number and string scanners. They use // compiler builtins or platform-specific intrinsics where available and plain @@ -28,6 +29,10 @@ inline int count_leading_zeros(std::uint64_t x) noexcept { #if defined(__GNUC__) || defined(__clang__) return __builtin_clzll(x); +#elif defined(_MSC_VER) && (defined(_M_X64) || defined(_M_ARM64)) + unsigned long index = 0; // NOLINT(runtime/int): the type _BitScan*64 takes + _BitScanReverse64(&index, x); + return 63 - static_cast(index); #else int n = 0; for (int shift = 32; shift != 0; shift >>= 1) @@ -42,6 +47,29 @@ inline int count_leading_zeros(std::uint64_t x) noexcept #endif } +/// number of trailing zero bits of x (x != 0) +inline int count_trailing_zeros(std::uint64_t x) noexcept +{ +#if defined(__GNUC__) || defined(__clang__) + return __builtin_ctzll(x); +#elif defined(_MSC_VER) && (defined(_M_X64) || defined(_M_ARM64)) + unsigned long index = 0; // NOLINT(runtime/int): the type _BitScan*64 takes + _BitScanForward64(&index, x); + return static_cast(index); +#else + int n = 0; + for (int shift = 32; shift != 0; shift >>= 1) + { + if ((x << (64 - shift)) == 0) + { + n += shift; + x >>= shift; + } + } + return n; +#endif +} + /// the 128-bit product of two 64-bit numbers struct uint128_parts { @@ -75,15 +103,27 @@ inline uint128_parts full_multiplication(std::uint64_t a, std::uint64_t b) noexc #endif } -/// eight bytes as a little-endian word (compilers fold this into one load on -/// little-endian targets) -inline std::uint64_t read_eight_bytes(const char* p) noexcept +/// eight bytes as a little-endian word (a single load on little-endian +/// targets; always inlined, as GCC otherwise calls it in the number loops) +JSON_HEDLEY_ALWAYS_INLINE std::uint64_t read_eight_bytes(const unsigned char* b) noexcept { - const auto* b = reinterpret_cast(p); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) +#if defined(_MSC_VER) || defined(__x86_64__) || defined(__i386__) || (defined(__BYTE_ORDER__) && defined(__ORDER_LITTLE_ENDIAN__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__) + // the byte order already matches (all MSVC targets are little-endian) + std::uint64_t result = 0; + std::memcpy(&result, b, sizeof(result)); + return result; +#else return static_cast(b[0]) | (static_cast(b[1]) << 8u) | (static_cast(b[2]) << 16u) | (static_cast(b[3]) << 24u) | (static_cast(b[4]) << 32u) | (static_cast(b[5]) << 40u) | (static_cast(b[6]) << 48u) | (static_cast(b[7]) << 56u); +#endif +} + +/// eight bytes as a little-endian word +JSON_HEDLEY_ALWAYS_INLINE std::uint64_t read_eight_bytes(const char* p) noexcept +{ + return read_eight_bytes(reinterpret_cast(p)); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) } } // namespace detail diff --git a/include/nlohmann/detail/conversions/to_chars.hpp b/include/nlohmann/detail/conversions/to_chars.hpp index 52342db40..db50795b5 100644 --- a/include/nlohmann/detail/conversions/to_chars.hpp +++ b/include/nlohmann/detail/conversions/to_chars.hpp @@ -4,6 +4,7 @@ // |_____|_____|_____|_|___| https://github.com/nlohmann/json // // SPDX-FileCopyrightText: 2009 Florian Loitsch +// SPDX-FileCopyrightText: 2025 Victor Zverovich // SPDX-FileCopyrightText: 2013-2026 Niels Lohmann // SPDX-License-Identifier: MIT @@ -11,11 +12,32 @@ #include // array #include // signbit, isfinite +#include // size_t #include // intN_t, uintN_t #include // memcpy, memmove #include // numeric_limits #include // conditional +#ifdef _MSC_VER + #include // _byteswap_uint64 +#endif + +// SSE2 (every x86-64 CPU) and NEON (every 64-bit Arm CPU) convert the 16 +// digits of a double at once +#if defined(__x86_64__) || (defined(_M_X64) && !defined(_M_ARM64EC)) + #include + #define JSON_DTOA_SSE2 1 + #define JSON_DTOA_NEON 0 +#elif (defined(__aarch64__) || defined(_M_ARM64)) && !defined(_M_ARM64EC) && !defined(__ARM_BIG_ENDIAN) + #include + #define JSON_DTOA_SSE2 0 + #define JSON_DTOA_NEON 1 +#else + #define JSON_DTOA_SSE2 0 + #define JSON_DTOA_NEON 0 +#endif + +#include #include NLOHMANN_JSON_NAMESPACE_BEGIN @@ -1047,6 +1069,381 @@ inline char* format_buffer(char* buf, int len, int decimal_exponent, return append_exponent(buf, n - 1); } +/// eight decimal digits (a value below 10^8) as bytes 0..9, the first digit +/// in the most significant byte: three steps that divide all lanes at once +/// by a multiplication (the conversion of Xiang JunBo, as in Zmij) +inline std::uint64_t eight_digit_bytes(std::uint64_t abcdefgh) noexcept +{ + const std::uint64_t abcd_efgh = abcdefgh + (((std::uint64_t{1} << 32u) - 10000u) * ((abcdefgh * (((std::uint64_t{1} << 40u) / 10000u) + 1u)) >> 40u)); + const std::uint64_t ab_cd_ef_gh = abcd_efgh + (((std::uint64_t{1} << 16u) - 100u) * (((abcd_efgh * (((std::uint64_t{1} << 19u) / 100u) + 1u)) >> 19u) & 0x7F0000007Fu)); + return ab_cd_ef_gh + (((std::uint64_t{1} << 8u) - 10u) * (((ab_cd_ef_gh * (((std::uint64_t{1} << 10u) / 10u) + 1u)) >> 10u) & 0x000F000F000F000Fu)); +} + +/// store the bytes of v, the most significant one first (one byte swap and +/// one store where the byte order is known: compilers do not reliably merge +/// the byte stores once this is inlined) +inline void store_msb_first(char* p, std::uint64_t v) noexcept +{ +#if defined(__BYTE_ORDER__) && defined(__ORDER_LITTLE_ENDIAN__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ + v = __builtin_bswap64(v); + std::memcpy(p, &v, sizeof(v)); +#elif defined(__BYTE_ORDER__) && defined(__ORDER_BIG_ENDIAN__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + std::memcpy(p, &v, sizeof(v)); +#elif defined(_MSC_VER) // (little-endian on all its targets) + v = _byteswap_uint64(v); + std::memcpy(p, &v, sizeof(v)); +#else + for (unsigned i = 0; i < 8; ++i) + { + p[i] = static_cast(v >> (56u - (8u * i))); + } +#endif +} + +/*! +@brief digits * 10^exp for a double, in the layout of format_buffer() + +The layout is that of format_buffer() with min_exp -4 and max_exp 15 (the +digits10 of double). The digits are converted eight at a time and placed +with fixed-size moves instead of per-digit loops and moves of the buffer. + +@param[in] digits the digits (not 0, at most 17 digits; trailing zeros allowed) +@param[in] exp the decimal exponent of the last digit +@return a pointer past the text; up to 41 bytes at @a first are written + (some beyond the returned end) +*/ +JSON_HEDLEY_NON_NULL(1) +JSON_HEDLEY_RETURNS_NON_NULL +inline char* write_decimal(char* first, std::uint64_t digits, int exp) noexcept +{ + JSON_ASSERT(digits != 0 && digits < 100000000000000000u); + const std::uint64_t upper = digits / 100000000u; + const std::uint64_t b0 = upper / 100000000u; // (one digit: it is its own byte) + const std::uint64_t b1 = eight_digit_bytes(upper % 100000000u); + const std::uint64_t b2 = eight_digit_bytes(digits % 100000000u); + // leading and trailing zero digits: zero bytes, counted without division + int leading = 16; + int zeros = 16; + if (b0 != 0) + { + leading = count_leading_zeros(b0) / 8; + } + else if (b1 != 0) + { + leading = 8 + (count_leading_zeros(b1) / 8); + } + else + { + leading += count_leading_zeros(b2) / 8; + } + if (b2 != 0) + { + zeros = count_trailing_zeros(b2) / 8; + } + else if (b1 != 0) + { + zeros = 8 + (count_trailing_zeros(b1) / 8); + } + // (else: 16, b0 is the one digit that is not 0) + // the digits as text at text + leading, then '0's, so that fixed-size + // moves need not check how many digits there are + std::array text; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init): written before read + store_msb_first(text.data(), b0 + 0x3030303030303030u); + store_msb_first(text.data() + 8, b1 + 0x3030303030303030u); + store_msb_first(text.data() + 16, b2 + 0x3030303030303030u); + std::memset(text.data() + 24, '0', 40); + const int k = 24 - leading - zeros; // significant digits + const int n = k + exp + zeros; // position of the decimal point after the first digit + const char* const s0 = text.data() + leading; + + if (-4 < n && n <= 15) + { + // "0.[000]digits" (n <= 0) is the digits after 1 - n leading '0's + // with the point after the first; "digits[000].0" (n >= k) and + // "dig.its" put the point after n characters + const int pad = n <= 0 ? 1 - n : 0; + const char* const s = s0 - pad; + const int len = k + pad; + const int point = n + pad; + std::memcpy(first, s, 16); + std::memcpy(first + point + 1, s + point, 24); + first[point] = '.'; + return first + (point >= len ? point + 2 : len + 1); + } + + // d.igitse+XX, with at least two exponent digits (as append_exponent()) + std::memcpy(first, s0, 16); + std::memcpy(first + 2, s0 + 1, 16); + first[1] = '.'; + char* const end = first + (k == 1 ? 1 : k + 1); + const int e = n - 1; + const auto ea = e < 0 ? 0u - static_cast(e) : static_cast(e); // (unsigned: no signed overflow to assume) + const bool three = ea >= 100; + end[0] = 'e'; + end[1] = e < 0 ? '-' : '+'; + end[2] = static_cast('0' + (three ? ea / 100 : (ea / 10) % 10)); + end[3] = static_cast('0' + (three ? (ea / 10) % 10 : ea % 10)); + end[4] = static_cast('0' + (ea % 10)); + return end + (three ? 5 : 4); +} + +/*! +@brief the shortest decimal of a positive double (Zmij), as write_decimal() +writes it + +For a normal double, the shorter candidate has 15 or 16 digits: they are +converted at once (two halves of eight digits) and followed by the digit +after them, if there is one, without the multiplication and division by 10 +that counting the digits of one number would take. The fixed layouts move +the digits after the point by one byte. + +@return a pointer past the text; up to 41 bytes at @a first are written + (some beyond the returned end) +*/ +JSON_HEDLEY_NON_NULL(1) +JSON_HEDLEY_RETURNS_NON_NULL +inline char* write_shortest(char* first, const zmij::shortest_decimal d) noexcept +{ + const std::uint64_t sig = d.integral; + if (JSON_HEDLEY_UNLIKELY(sig < 100000000000000u || sig >= 10000000000000000u)) + { + // (subnormals) + return d.has_digit ? write_decimal(first, (sig * 10) + d.digit, d.exponent) : write_decimal(first, sig, d.exponent + 1); + } + const bool sixteen = sig >= 1000000000000000u; // (else 15 digits) + const int last = d.has_digit ? d.digit : 0; + const std::uint64_t upper = sig / 100000000u; +#if JSON_DTOA_SSE2 + // NOLINTBEGIN(portability-simd-intrinsics) + // the two halves in the 64-bit lanes, each as abcd * 2^32 + efgh, then as + // bytes (as eight_digit_bytes(), one lane each) + const __m128i x = _mm_set_epi64x(static_cast(sig - (upper * 100000000u)), static_cast(upper)); // NOLINT(runtime/int) + const __m128i abcd = _mm_srli_epi64(_mm_mul_epu32(x, _mm_set1_epi64x(109951163)), 40); // 2^40 / 10000 + 1 + const __m128i abcd_efgh = _mm_add_epi64(x, _mm_mul_epu32(abcd, _mm_set1_epi64x(4294957296))); // 2^32 - 10000 + // 32-bit lanes in the order of the text: abcd, efgh of both halves + const __m128i fours = _mm_shuffle_epi32(abcd_efgh, _MM_SHUFFLE(2, 3, 0, 1)); + const __m128i ab = _mm_srli_epi16(_mm_mulhi_epu16(fours, _mm_set1_epi32(5243)), 3); + const __m128i ab_cd = _mm_or_si128(_mm_slli_epi32(_mm_sub_epi16(fours, _mm_mullo_epi16(ab, _mm_set1_epi32(100))), 16), ab); + // 16-bit lanes ab (< 100) -> bytes a, b: 256 * ab - 2559 * (ab / 10) + const __m128i bytes = _mm_sub_epi16(_mm_slli_epi16(ab_cd, 8), _mm_mullo_epi16(_mm_set1_epi16(2559), _mm_mulhi_epu16(ab_cd, _mm_set1_epi16(6554)))); + // the last digit that is not 0 (sig is not 0) + const auto nonzero = static_cast(_mm_movemask_epi8(_mm_cmpgt_epi8(bytes, _mm_setzero_si128()))); + const int digits = 63 - count_leading_zeros(nonzero) + (sixteen ? 1 : 0); // without trailing zeros + const __m128i chars = _mm_add_epi8(bytes, _mm_set1_epi8('0')); + // the 16 characters from the first digit + const __m128i s = sixteen ? chars : _mm_or_si128(_mm_srli_si128(chars, 1), _mm_slli_si128(_mm_cvtsi32_si128('0' + last), 15)); + const char s16 = static_cast(sixteen ? '0' + last : '0'); // the 17th + const auto store_16 = [&s](char* p) noexcept + { + std::memcpy(p, &s, 16); + }; + const char first_digit = static_cast(_mm_cvtsi128_si32(s)); + // NOLINTEND(portability-simd-intrinsics) +#elif JSON_DTOA_NEON + // as with SSE2: the halves in 32-bit lanes, then abcd, efgh of both + const uint32x2_t halves = vcreate_u32(upper | ((sig - (upper * 100000000u)) << 32u)); + const uint32x2_t abcd = vmovn_u64(vshrq_n_u64(vmull_n_u32(halves, static_cast(((std::uint64_t{1} << 40u) / 10000u) + 1u)), 40)); + const uint32x2_t efgh = vmls_n_u32(halves, abcd, 10000u); + const uint32x4_t fours = vcombine_u32(vzip1_u32(abcd, efgh), vzip2_u32(abcd, efgh)); + const uint32x4_t ab = vshrq_n_u32(vmulq_n_u32(fours, 5243u), 19); + const uint16x8_t ab_cd = vreinterpretq_u16_u32(vorrq_u32(ab, vshlq_n_u32(vmlsq_n_u32(fours, ab, 100u), 16))); + const uint16x8_t tens = vshrq_n_u16(vmulq_n_u16(ab_cd, 103u), 10); + const uint8x16_t bytes = vreinterpretq_u8_u16(vorrq_u16(tens, vshlq_n_u16(vmlsq_n_u16(ab_cd, tens, 10u), 8))); + // the last digit that is not 0 (sig is not 0): a nibble per byte + const std::uint64_t nonzero = vget_lane_u64(vreinterpret_u64_u8(vshrn_n_u16(vreinterpretq_u16_u8(vtstq_u8(bytes, bytes)), 4)), 0); + const int digits = ((63 - count_leading_zeros(nonzero)) / 4) + (sixteen ? 1 : 0); // without trailing zeros + const uint8x16_t chars = vaddq_u8(bytes, vdupq_n_u8('0')); + // the 16 characters from the first digit + const uint8x16_t s = sixteen ? chars : vextq_u8(chars, vdupq_n_u8(static_cast('0' + last)), 1); + const char s16 = static_cast(sixteen ? '0' + last : '0'); // the 17th + const auto store_16 = [&s](char* p) noexcept + { + vst1q_u8(reinterpret_cast(p), s); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) + }; + const auto first_digit = static_cast(vgetq_lane_u8(s, 0)); +#else + const std::uint64_t hi = eight_digit_bytes(upper); + const std::uint64_t lo = eight_digit_bytes(sig - (upper * 100000000u)); + // trailing zero digits: zero bytes (sig is not 0) + const int zeros = lo != 0 ? count_trailing_zeros(lo) / 8 : 8 + (count_trailing_zeros(hi) / 8); + const int digits = 15 - zeros + (sixteen ? 1 : 0); // without trailing zeros + // the 16 characters from the first digit + const std::uint64_t s_hi = (sixteen ? hi : (hi << 8u) | (lo >> 56u)) + 0x3030303030303030u; + const std::uint64_t s_lo = (sixteen ? lo : (lo << 8u) | static_cast(last)) + 0x3030303030303030u; + const char s16 = static_cast(sixteen ? '0' + last : '0'); // the 17th + const auto store_16 = [s_hi, s_lo](char* p) noexcept + { + store_msb_first(p, s_hi); + store_msb_first(p + 8, s_lo); + }; + const auto first_digit = static_cast(s_hi >> 56u); +#endif + const int len = d.has_digit ? 16 + (sixteen ? 1 : 0) : digits; // significant digits + const int n = 16 + (sixteen ? 1 : 0) + d.exponent; // digits before the point + + if (JSON_HEDLEY_LIKELY(n >= 1 && n <= 15)) + { + // "dig.its" and "digits[000].0": the digits after the point move by + // one byte ('0's follow the digits) +#if JSON_DTOA_SSE2 + // NOLINTBEGIN(portability-simd-intrinsics) + // (in the register: reading the digits back from memory right after + // storing them waits until the stores are done) + const __m128i at = _mm_set1_epi8(static_cast(n)); + const __m128i index = _mm_setr_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + const __m128i before = _mm_cmpgt_epi8(at, index); + const __m128i after = _mm_cmpgt_epi8(index, at); + const __m128i text = _mm_or_si128(_mm_or_si128(_mm_and_si128(s, before), _mm_and_si128(_mm_slli_si128(s, 1), after)), + _mm_andnot_si128(_mm_or_si128(before, after), _mm_set1_epi8('.'))); + std::memcpy(first, &text, 16); + first[16] = static_cast(_mm_extract_epi16(s, 7) >> 8); + first[17] = s16; + // NOLINTEND(portability-simd-intrinsics) +#elif JSON_DTOA_NEON + const uint8x16_t index = vcombine_u8(vcreate_u8(0x0706050403020100u), vcreate_u8(0x0F0E0D0C0B0A0908u)); + const uint8x16_t at = vdupq_n_u8(static_cast(n)); + const uint8x16_t after_point = vbslq_u8(vcgtq_u8(index, at), vextq_u8(vdupq_n_u8(0), s, 15), vdupq_n_u8('.')); + vst1q_u8(reinterpret_cast(first), vbslq_u8(vcltq_u8(index, at), s, after_point)); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) + first[16] = static_cast(vgetq_lane_u8(s, 15)); + first[17] = s16; +#else + store_16(first); + first[16] = s16; + std::uint64_t after_point[2]; // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays,cppcoreguidelines-pro-type-member-init,hicpp-member-init): written before read + std::memcpy(after_point, first + n, 16); + std::memcpy(first + n + 1, after_point, 16); + first[n] = '.'; +#endif + return first + (n >= len ? n + 2 : len + 1); + } + if (n <= 0 && n > -4) + { + // "0.[000]digits" + std::memset(first, '0', 8); + first[1] = '.'; + store_16(first + 2 - n); + first[18 - n] = s16; + return first + 2 - n + len; + } + // d.igitse+XX, with at least two exponent digits (as append_exponent()) + store_16(first + 1); + first[17] = s16; + first[0] = first_digit; + first[1] = '.'; + char* const end = first + (len == 1 ? 1 : len + 1); + const int e = n - 1; + const auto ea = e < 0 ? 0u - static_cast(e) : static_cast(e); // (unsigned: no signed overflow to assume) + const bool three = ea >= 100; + end[0] = 'e'; + end[1] = e < 0 ? '-' : '+'; + end[2] = static_cast('0' + (three ? ea / 100 : (ea / 10) % 10)); + end[3] = static_cast('0' + (three ? (ea / 10) % 10 : ea % 10)); + end[4] = static_cast('0' + (ea % 10)); + return end + (three ? 5 : 4); +} + +/*! +@brief whether FloatType is an IEEE 754 binary64 type (a double, or a long double +that has the same format, as with MSVC and on Apple's Arm CPUs) + +These are the types the conversion of Zmij (see zmij.hpp) is used for; all +others (binary32, or a format the library does not know) use Grisu2. +*/ +template +constexpr bool has_binary64_format() noexcept +{ + return std::numeric_limits::is_iec559 + && std::numeric_limits::digits == 53 + && std::numeric_limits::max_exponent == 1024 + && sizeof(FloatType) == sizeof(std::uint64_t); +} + +template +struct is_binary64 : std::integral_constant()> {}; + +/// a positive finite float (other than binary64): Grisu2 and format_buffer() +template +JSON_HEDLEY_NON_NULL(1, 2) +JSON_HEDLEY_RETURNS_NON_NULL +char* write_positive_grisu2(char* first, const char* last, FloatType value) +{ + JSON_ASSERT(last - first >= std::numeric_limits::max_digits10); + static_cast(last); // (only used in the assertion) + + // Compute v = buffer * 10^decimal_exponent. + // The decimal digits are stored in the buffer, which needs to be interpreted + // as an unsigned decimal integer. + // len is the length of the buffer, i.e., the number of decimal digits. + int len = 0; + int decimal_exponent = 0; + grisu2(first, len, decimal_exponent, value); + + JSON_ASSERT(len <= std::numeric_limits::max_digits10); + + // Format the buffer like printf("%.*g", prec, value) + constexpr int kMinExp = -4; + // Use digits10 here to increase compatibility with version 2. + constexpr int kMaxExp = std::numeric_limits::digits10; + + JSON_ASSERT(last - first >= kMaxExp + 2); + JSON_ASSERT(last - first >= 2 + (-kMinExp - 1) + std::numeric_limits::max_digits10); + JSON_ASSERT(last - first >= std::numeric_limits::max_digits10 + 6); + + return format_buffer(first, len, decimal_exponent, kMinExp, kMaxExp); +} + +/// a positive finite binary64 number: the shortest digits (Zmij), laid out by +/// write_shortest() (through a local buffer if [first, last) is shorter than +/// the 41 bytes it may write) +template +JSON_HEDLEY_NON_NULL(1, 2) +JSON_HEDLEY_RETURNS_NON_NULL +char* write_positive_zmij(char* first, const char* last, FloatType value) +{ + static_assert(is_binary64::value, + "internal error: the conversion of Zmij needs IEEE 754 binary64 numbers"); + std::uint64_t bits = 0; + std::memcpy(&bits, &value, sizeof(bits)); + const zmij::shortest_decimal d = zmij::to_shortest(bits); + if (JSON_HEDLEY_LIKELY(last - first >= 41)) + { + return write_shortest(first, d); + } + std::array buf; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init): written before read + const auto len = static_cast(write_shortest(buf.data(), d) - buf.data()); + JSON_ASSERT(last - first >= static_cast(len)); + std::memcpy(first, buf.data(), len); + return first + len; +} + +/// a positive finite binary64 number: Zmij (as a long double has the format of +/// a double here, its bits are those of the double of the same value) +template +JSON_HEDLEY_NON_NULL(1, 2) +JSON_HEDLEY_RETURNS_NON_NULL +char* write_positive(char* first, const char* last, FloatType value, std::true_type /*is_binary64*/) +{ + return write_positive_zmij(first, last, value); +} + +/// a positive finite float of any other format: Grisu2 +template +JSON_HEDLEY_NON_NULL(1, 2) +JSON_HEDLEY_RETURNS_NON_NULL +char* write_positive(char* first, const char* last, FloatType value, std::false_type /*is_binary64*/) +{ + return write_positive_grisu2(first, last, value); +} + +/// a positive finite float: Zmij for binary64 numbers, Grisu2 otherwise +template +JSON_HEDLEY_NON_NULL(1, 2) +JSON_HEDLEY_RETURNS_NON_NULL +char* write_positive(char* first, const char* last, FloatType value) +{ + return write_positive(first, last, value, is_binary64 {}); +} + } // namespace dtoa_impl /*! @@ -1064,7 +1461,6 @@ JSON_HEDLEY_NON_NULL(1, 2) JSON_HEDLEY_RETURNS_NON_NULL char* to_chars(char* first, const char* last, FloatType value) { - static_cast(last); // maybe unused - fix warning JSON_ASSERT(std::isfinite(value)); // Use signbit(value) instead of (value < 0) since signbit works for -0. @@ -1090,28 +1486,7 @@ char* to_chars(char* first, const char* last, FloatType value) JSON_HEDLEY_DIAGNOSTIC_POP #endif - JSON_ASSERT(last - first >= std::numeric_limits::max_digits10); - - // Compute v = buffer * 10^decimal_exponent. - // The decimal digits are stored in the buffer, which needs to be interpreted - // as an unsigned decimal integer. - // len is the length of the buffer, i.e., the number of decimal digits. - int len = 0; - int decimal_exponent = 0; - dtoa_impl::grisu2(first, len, decimal_exponent, value); - - JSON_ASSERT(len <= std::numeric_limits::max_digits10); - - // Format the buffer like printf("%.*g", prec, value) - constexpr int kMinExp = -4; - // Use digits10 here to increase compatibility with version 2. - constexpr int kMaxExp = std::numeric_limits::digits10; - - JSON_ASSERT(last - first >= kMaxExp + 2); - JSON_ASSERT(last - first >= 2 + (-kMinExp - 1) + std::numeric_limits::max_digits10); - JSON_ASSERT(last - first >= std::numeric_limits::max_digits10 + 6); - - return dtoa_impl::format_buffer(first, len, decimal_exponent, kMinExp, kMaxExp); + return dtoa_impl::write_positive(first, last, value); } } // namespace detail diff --git a/include/nlohmann/detail/conversions/zmij.hpp b/include/nlohmann/detail/conversions/zmij.hpp new file mode 100644 index 000000000..40e477f7b --- /dev/null +++ b/include/nlohmann/detail/conversions/zmij.hpp @@ -0,0 +1,219 @@ +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ +// | | |__ | | | | | | version 3.12.0 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2025 Victor Zverovich +// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann +// SPDX-License-Identifier: MIT + +#pragma once + +#include // array +#include // size_t +#include // uint32_t, uint64_t + +#include +#include +#include +#include + +NLOHMANN_JSON_NAMESPACE_BEGIN +namespace detail +{ + +/*! +@brief the shortest decimal representation of a double + +A C++11 port of the conversion of Zmij by Victor Zverovich +(https://github.com/vitaut/zmij, MIT license): the shortest decimal in the +rounding interval of a double, the closest one if there are several. Zmij +credits Xiang JunBo (producing the shorter candidate without a division) and +Dougall Johnson (the compressed powers of ten). The powers of ten are taken +from the table for number parsing (pow5_table.hpp) where it holds them, and +computed from the compressed tables of Zmij beyond it. +*/ +namespace zmij +{ + +/// the compressed powers of ten of Zmij +inline const std::array& pow10_minor() noexcept +{ + static const std::array table = + { + { + 0x8000000000000000u, 0xa000000000000000u, 0xc800000000000000u, 0xfa00000000000000u, 0x9c40000000000000u, + 0xc350000000000000u, 0xf424000000000000u, 0x9896800000000000u, 0xbebc200000000000u, 0xee6b280000000000u, + 0x9502f90000000000u, 0xba43b74000000000u, 0xe8d4a51000000000u, 0x9184e72a00000000u, 0xb5e620f480000000u, + 0xe35fa931a0000000u, 0x8e1bc9bf04000000u, 0xb1a2bc2ec5000000u, 0xde0b6b3a76400000u, 0x8ac7230489e80000u, + 0xad78ebc5ac620000u, 0xd8d726b7177a8000u, 0x878678326eac9000u, 0xa968163f0a57b400u, 0xd3c21bcecceda100u, + 0x84595161401484a0u, 0xa56fa5b99019a5c8u, 0xcecb8f27f4200f3au + } + }; + return table; +} + +/// (high, low) pairs +inline const std::array& pow10_major() noexcept +{ + static const std::array table = + { + { + 0xaddcb9e83c6b1793u, 0xdf4abe242a1bbf3eu, 0xaf8e5410288e1b6fu, 0x07ecf0ae5ee44ddau, 0xb1442798f49ffb4au, 0x99cd11cfdf41779du, + 0xb2fe3f0b8599ef07u, 0x861fa7e6dcb4aa15u, 0xb4bca50b065abe63u, 0x0fed077a756b53aau, 0xb67f6455292cbf08u, 0x1a3bc84c17b1d543u, + 0xb84687c269ef3bfbu, 0x3d5d514f40eea742u, 0xba121a4650e4ddebu, 0x92f34d62616ce413u, 0xbbe226efb628afeau, 0x890489f70a55368cu, + 0xbdb6b8e905cb600fu, 0x5400e987bbc1c921u, 0xbf8fdb78849a5f96u, 0xde98520472bdd034u, 0xc16d9a0095928a27u, 0x75b7053c0f178294u, + 0xc350000000000000u, 0x0000000000000000u, 0xc5371912364ce305u, 0x6c28000000000000u, 0xc722f0ef9d80aad6u, 0x424d3ad2b7b97ef6u, + 0xc913936dd571c84cu, 0x03bc3a19cd1e38eau, 0xcb090c8001ab551cu, 0x5cadf5bfd3072cc6u, 0xcd036837130890a1u, 0x36dba887c37a8c10u, + 0xcf02b2c21207ef2eu, 0x94f967e45e03f4bcu, 0xd106f86e69d785c7u, 0xe13336d701beba52u, 0xd31045a8341ca07cu, 0x1ede48111209a051u, + 0xd51ea6fa85785631u, 0x552a74227f3ea566u, 0xd732290fbacaf133u, 0xa97c177947ad4096u, 0xd94ad8b1c7380874u, 0x18375281ae7822bdu, + 0xdb68c2ca82ed2a05u, 0xa67398db9f6820e1u + } + }; + return table; +} + +/// one bit per power: whether the computed value is one unit too large +inline const std::array& pow10_fixups() noexcept +{ + static const std::array table = + { + { + 0x8d8fc810u, 0x06100293u, 0x19000000u, 0x00100000u, 0x00000908u, 0x00000000u, 0x04e00300u, 0x3807e0b2u, 0x3d83d793u, 0x0006f5ccu, + 0x00000000u, 0xffff0000u, 0x8076337du, 0x4ff45ba0u, 0x09405033u, 0x034376d9u, 0x09000000u, 0x4e100501u, 0x076d14dcu, 0xf964f45eu, + 0x0000003du + } + }; + return table; +} + +/// the 128-bit significand of 10^k, rounded down, for k in [-307, 341] +/// (compute_pow10 of Zmij) +inline uint128_parts compute_pow10(int k) noexcept +{ + const auto i = static_cast(k + 307); + const std::uint64_t m = pow10_minor()[(i + 24) % 28]; + const std::size_t j = 2 * static_cast((i + 24) / 28); + const std::uint64_t h_hi = pow10_major()[j]; + const std::uint64_t h_lo = pow10_major()[j + 1]; + const std::uint64_t h1 = full_multiplication(h_lo, m).high; + const std::uint64_t c0 = h_lo * m; + const std::uint64_t c1 = h1 + (h_hi * m); + const std::uint64_t c2 = (c1 < h1 ? 1u : 0u) + full_multiplication(h_hi, m).high; + uint128_parts r{}; + if ((c2 >> 63u) != 0) + { + r.high = c2; + r.low = c1; + } + else + { + r.high = (c2 << 1u) | (c1 >> 63u); + r.low = (c1 << 1u) | (c0 >> 63u); + } + r.low -= (pow10_fixups()[i >> 5u] >> (i & 31u)) & 1u; + return r; +} + +/// The 128-bit significand of 10^k, rounded down, for k in [-342, 341]. +/// Up to 10^308, the table for number parsing holds the same significands +/// (those of 5^k), except for k in [-27, -1], where it holds them one unit +/// larger (as the Eisel-Lemire algorithm needs them). +inline uint128_parts pow10(int k) noexcept +{ + if (k > pow5_128_largest_power) + { + return compute_pow10(k); // (only for the smallest doubles) + } + const auto i = 2 * static_cast(k - pow5_128_smallest_power); + uint128_parts r{pow5_128()[i + 1], pow5_128()[i]}; + const std::uint64_t adjust = static_cast(k + 27) < 27u ? 1u : 0u; + r.high -= r.low < adjust ? 1u : 0u; + r.low -= adjust; + return r; +} + +/// (x_hi * 2^64 + x_lo) * y >> 64, as 128 bits +inline uint128_parts umul192_hi128(std::uint64_t x_hi, std::uint64_t x_lo, std::uint64_t y) noexcept +{ + const uint128_parts p = full_multiplication(x_hi, y); + uint128_parts r{}; + r.low = p.low + full_multiplication(x_lo, y).high; + r.high = p.high + (r.low < p.low ? 1u : 0u); + return r; +} + +/// (x * y + c) >> 64 +inline std::uint64_t umul128_add_hi64(std::uint64_t x, std::uint64_t y, std::uint64_t c) noexcept +{ + const uint128_parts p = full_multiplication(x, y); + return p.high + (p.low + c < p.low ? 1u : 0u); +} + +/// the result of Zmij: the shorter candidate and, if that is outside the +/// rounding interval, the digit after it (16 bytes: returned in registers) +struct shortest_decimal +{ + std::uint64_t integral; ///< the shorter candidate (15 or 16 digits for normal doubles) + int exponent; ///< the decimal exponent of the digit after it + unsigned char digit; ///< the digit after it (if has_digit) + bool has_digit; ///< whether the shortest decimal is integral * 10 + digit +}; + +/// The shortest decimal in the rounding interval of a positive finite double +/// given by its bits, the closest one if there are several (to_decimal of +/// Zmij, which keeps the last digit apart: the 15 or 16 digits before it can be +/// converted without a multiplication by 10 first). Always inlined: GCC +/// otherwise calls it, and its result goes through memory. +JSON_HEDLEY_ALWAYS_INLINE shortest_decimal to_shortest(std::uint64_t bits) noexcept +{ + constexpr int extra_shift = 9; + const auto raw_exp = static_cast((bits >> 52u) & 0x7FFu); + std::uint64_t bin_sig = bits & ((std::uint64_t{1} << 52u) - 1); + // a power of two has a narrower interval below (except the smallest normal) + const bool regular = bin_sig != 0 || raw_exp <= 1; + const int bin_exp = (raw_exp == 0 ? 1 : raw_exp) - 1075; + if (raw_exp != 0) + { + bin_sig |= std::uint64_t{1} << 52u; + } + // floor(log10(2^bin_exp)), or floor(log10(3/4 * 2^bin_exp)) for the irregular case + const int dec_exp = ((bin_exp * 315653) - (regular ? 0 : 131072)) >> 20; + // scaled by 10^(-dec_exp - 1): the integral part is the shorter candidate + const int shift = bin_exp + ((-(dec_exp + 1) * 217707) >> 16) + 1 + extra_shift; + const uint128_parts p10 = pow10(-dec_exp - 1); + const uint128_parts p = umul192_hi128(p10.high, p10.low, bin_sig << static_cast(shift)); + std::uint64_t integral = p.high >> static_cast(extra_shift); + const std::uint64_t fractional = (p.high << static_cast(64 - extra_shift)) | (p.low >> static_cast(extra_shift)); + std::uint64_t digit = 0; + bool round_up = false; + bool round_down = false; + if (JSON_HEDLEY_LIKELY(regular)) + { + const std::uint64_t half_ulp = (p10.high >> static_cast(extra_shift + 1 - shift)) + (1 - (bin_sig & 1u)); + round_up = fractional + half_ulp < fractional; + round_down = half_ulp > fractional; + // the last digit of the longer candidate, rounded to nearest + digit = umul128_add_hi64(fractional, 10, (std::uint64_t{1} << 63u) + 6); + if (fractional == (std::uint64_t{1} << 62u)) + { + digit = 2; // 2.5 rounds to 2 + } + } + else + { + const std::uint64_t half_ulp = p10.high >> static_cast(extra_shift + 1 - shift); + round_up = half_ulp > ~std::uint64_t{0} - fractional; + round_down = (half_ulp >> 1u) > fractional; + digit = umul128_add_hi64(fractional, 10, (std::uint64_t{1} << 63u) - 1); + const std::uint64_t lowest = umul128_add_hi64(fractional - (half_ulp >> 1u), 10, ~std::uint64_t{0}); + digit = digit < lowest ? lowest : digit; + } + integral += round_up ? 1u : 0u; + // if the shorter candidate is outside the rounding interval: one digit more + return shortest_decimal{integral, dec_exp, static_cast(digit), !round_up && !round_down}; +} + +} // namespace zmij +} // namespace detail +NLOHMANN_JSON_NAMESPACE_END diff --git a/include/nlohmann/detail/input/input_adapters.hpp b/include/nlohmann/detail/input/input_adapters.hpp index 847890f7a..2df78dabb 100644 --- a/include/nlohmann/detail/input/input_adapters.hpp +++ b/include/nlohmann/detail/input/input_adapters.hpp @@ -8,12 +8,12 @@ #pragma once -#include // min +#include // count, find, min #include // array #include // size_t #include // uint32_t #include // strlen -#include // begin, end, iterator_traits, random_access_iterator_tag, distance, next +#include // begin, end, iterator_traits, random_access_iterator_tag, distance, next, reverse_iterator #include // streambuf #include // string, char_traits #include // enable_if, is_base_of, is_pointer, is_integral, remove_pointer @@ -297,6 +297,7 @@ class iterator_input_adapter return result; } + ++eof_reads; return char_traits::eof(); } @@ -306,6 +307,26 @@ class iterator_input_adapter return static_cast(std::distance(begin, current)); } + // number of get_character() calls so far, including those that returned + // EOF; the lexer derives its position from it (see lexer::lazy_position) + std::size_t get_read_count() const + { + return get_consumed_count() + eof_reads; + } + + // number of newlines in the consumed range [0, last_index); @a line_start + // is set to the index after the last of them (0 if there is none); only + // valid when supports_seek + std::size_t count_consumed_lines(std::size_t last_index, std::size_t& line_start) const + { + const auto to = std::next(begin, static_cast::difference_type>(last_index)); + const auto newline = char_traits::to_char_type('\n'); + using reverse_it = std::reverse_iterator; + const auto last_newline = std::find(reverse_it(to), reverse_it(begin), newline); + line_start = static_cast(std::distance(begin, last_newline.base())); + return static_cast(std::count(begin, last_newline.base(), newline)); + } + // append the already-consumed characters in the half-open range // [first_index, last_index) to @a out; only valid when supports_seek template @@ -422,6 +443,8 @@ class iterator_input_adapter IteratorType begin; IteratorType current; SentinelType end; + /// number of get_character() calls that returned EOF + std::size_t eof_reads = 0; template friend struct wide_string_input_helper; diff --git a/include/nlohmann/detail/input/json_sax.hpp b/include/nlohmann/detail/input/json_sax.hpp index 816359a60..161568ded 100644 --- a/include/nlohmann/detail/input/json_sax.hpp +++ b/include/nlohmann/detail/input/json_sax.hpp @@ -217,7 +217,7 @@ struct diagnostic_positions { // Lexer has read the first character of the container, so // subtract 1 from the position to get the correct start position. - v.start_position = lexer->get_position() - 1; + v.start_position = lexer->get_chars_read_total() - 1; } } @@ -231,7 +231,7 @@ struct diagnostic_positions if (lexer) { // Lexer's position is past the closing brace or bracket, so set that as the end position. - v.end_position = lexer->get_position(); + v.end_position = lexer->get_chars_read_total(); } } @@ -247,7 +247,7 @@ struct diagnostic_positions // Lexer has read past the current field value, so set the end position to the current position. // The start position will be set below based on the length of the string representation // of the value. - v.end_position = lexer->get_position(); + v.end_position = lexer->get_chars_read_total(); switch (v.type()) { diff --git a/include/nlohmann/detail/input/lexer.hpp b/include/nlohmann/detail/input/lexer.hpp index 361baa642..d0e90d32b 100644 --- a/include/nlohmann/detail/input/lexer.hpp +++ b/include/nlohmann/detail/input/lexer.hpp @@ -17,6 +17,7 @@ #include // move #include // vector +#include #include #include #include @@ -186,6 +187,15 @@ class lexer : public lexer_base static constexpr bool lazy_token_string = input_adapter_supports_seek(is_detected {}); + /// whether the read position is derived from the input adapter on demand + /// instead of being counted on every scanned character. Seekable adapters + /// know how much input they consumed and can revisit it, so + /// chars_read_total is the adapter's read count (minus a pending unget), + /// and line and column are recomputed from the consumed input only when + /// get_position() is called (errors). Counting per character kept two + /// increments and a newline check in the hot path of every get(). + static constexpr bool lazy_position = lazy_token_string; + /// whether a simulated unget can be passed on to the input adapter, which /// then leaves the character in the input; see /// input_adapter_supports_lookahead @@ -195,7 +205,9 @@ class lexer : public lexer_base /// whether string scanning may bulk-consume runs of ordinary characters /// directly from a contiguous input buffer (SWAR fast path). This requires /// the token to be reconstructible lazily (lazy_token_string), so bypassing - /// the per-character capture in get() cannot lose error diagnostics. + /// the per-character capture in get() cannot lose error diagnostics. It + /// also implies lazy_position, so the bulk paths need not advance the + /// position counters. static constexpr bool bulk_scan = lazy_token_string && input_adapter_supports_bulk_scan(is_detected {}); @@ -221,6 +233,41 @@ class lexer : public lexer_base // scan functions ///////////////////// + /// contiguous input: try to decode the 4 hex digits following `\\u` + /// directly from the input buffer via hex_codepoint(), instead of 4 calls + /// to get(). On success, advances the adapter exactly as those 4 get() + /// calls would (bulk_scan implies lazy_position, so there are no position + /// counters to advance) and leaves @a current holding the last of + /// the 4 digits, just as the last such get() would; the codepoint is + /// written to @a out. Makes no state change and returns false for a + /// pending unget, fewer than 4 remaining bytes, or any of the 4 bytes not + /// being a hex digit, so the caller falls back unchanged to the + /// per-character loop, which then reports the same diagnostic (stopping + /// at the first invalid digit) as before this optimization. + bool get_codepoint_bulk(std::true_type /*bulk*/, int& out) + { + if (next_unget || ia.bulk_remaining() < 4) + { + return false; + } + const char_type* const raw = ia.bulk_data(); + const int codepoint = hex_codepoint(reinterpret_cast(raw)); + if (codepoint < 0) + { + return false; + } + ia.bulk_skip(4); + current = char_traits::to_int_type(raw[3]); + out = codepoint; + return true; + } + + /// streaming input: no bulk fast path + bool get_codepoint_bulk(std::false_type /*bulk*/, int& /*out*/) const noexcept + { + return false; + } + /*! @brief get codepoint from 4 hex characters following `\\u` @@ -240,6 +287,14 @@ class lexer : public lexer_base { // this function only makes sense after reading `\u` JSON_ASSERT(current == 'u'); + + // contiguous input: decode all 4 hex digits directly from the buffer + int fast_codepoint = 0; + if (get_codepoint_bulk(std::integral_constant {}, fast_codepoint)) + { + return fast_codepoint; + } + int codepoint = 0; const auto factors = { 12u, 8u, 4u, 0u }; @@ -331,10 +386,6 @@ class lexer : public lexer_base } token_buffer.append(reinterpret_cast(data), pos); ia.bulk_skip(pos); - // the run contains no newline (all bytes < 0x20 are treated as special), - // so only the flat character counters advance - position.chars_read_total += pos; - position.chars_read_current_line += pos; } /// streaming input: no bulk fast path @@ -1044,9 +1095,11 @@ class lexer : public lexer_base token_type::parse_error otherwise @note The scanner is independent of the current locale: token_buffer - always holds `.`. Only the std::strtod fallback of convert_number() - depends on the locale, and it looks up the decimal point right - before converting (see detail::convert_float_locale_aware()). + always holds `.`. The conversion of float and double does not use + the locale either. Only the std::strtold fallback of + convert_number() for long double formats other than binary64 + depends on it, and it looks up the decimal point right before + converting (see detail::convert_float_locale_aware()). */ token_type scan_number() // lgtm [cpp/use-of-goto] `goto` is used in this function to implement the number-parsing state machine described above. By design, any finite input will eventually reach the "done" state or return token_type::parse_error. In each intermediate state, 1 byte of the input is appended to the token_buffer vector, and only the already initialized variables token_buffer, number_type, and error_message are manipulated. { @@ -1059,7 +1112,7 @@ class lexer : public lexer_base // offset just past the last mantissa byte in token_buffer (i.e. the // index of 'e'/'E', or the whole token when there is no exponent). - // convert_number() uses it to count significant digits; npos means + // convert_number() uses it to split the token; npos means // "not seen an exponent yet" and is resolved at scan_number_done std::size_t mantissa_end = std::string::npos; @@ -1389,8 +1442,8 @@ scan_number_done: @param[in] mantissa_end offset just past the last mantissa byte in token_buffer (the index of 'e'/'E', or token_buffer.size() when there is no exponent); - used to skip Clinger's fast path when it cannot - possibly succeed - see detail::mantissa_fits_clinger() + with decimal_point_position, it locates the parts + of a float token without scanning it again */ token_type convert_number(token_type number_type, std::size_t mantissa_end) { @@ -1444,10 +1497,11 @@ scan_number_done: } // this code is reached if we parse a floating-point number or if an - // integer conversion above overflowed. Prefer std::from_chars - // (Eisel-Lemire, locale-independent, correctly rounded) when available; - // otherwise the exact Clinger fast path (double only); otherwise the - // locale-aware strtof/strtod/strtold. + // integer conversion above overflowed. float and double (and long + // double where it is binary64) are converted by the library itself, + // correctly rounded and independent of the locale; other long double + // formats use std::from_chars when available, otherwise the + // locale-aware strtold. if (convert_float_fast(num_begin, num_end, decimal_point_position, mantissa_end, value_float)) { return token_type::value_float; @@ -1567,8 +1621,6 @@ scan_number_done: if (JSON_HEDLEY_LIKELY(integer_result != token_type::uninitialized)) { ia.bulk_skip(len - 1); - position.chars_read_total += (len - 1); - position.chars_read_current_line += (len - 1); return integer_result; } // The value does not fit an integer, so this token converts as a @@ -1585,8 +1637,6 @@ scan_number_done: decimal_point_position = dot_index; ia.bulk_skip(len - 1); - position.chars_read_total += (len - 1); - position.chars_read_current_line += (len - 1); return convert_number(number_type, mantissa_end); } @@ -1637,7 +1687,7 @@ scan_number_done: #if JSON_DIAGNOSTIC_POSITIONS // the first character of the token has already been read, hence the -1 - token_start_position = position.chars_read_total - 1; + token_start_position = get_chars_read_total() - 1; #endif note_token_start(std::integral_constant {}); @@ -1690,6 +1740,15 @@ scan_number_done: /// per-character position counters (line-count-on-'\n' bookkeeping is /// handled afterwards, in track_after_read(), once `current` is known) void advance_position() noexcept + { + advance_position_impl(std::integral_constant {}); + } + + /// seekable adapter: the position is derived from the adapter on demand + void advance_position_impl(std::true_type /*lazy*/) const noexcept {} + + /// streaming adapter: count the character + void advance_position_impl(std::false_type /*lazy*/) noexcept { ++position.chars_read_total; ++position.chars_read_current_line; @@ -1703,7 +1762,16 @@ scan_number_done: // seekable adapters reconstruct the token lazily on error (see // get_token_string), so the eager per-character copy is skipped capture_char(std::integral_constant {}); + track_newline(std::integral_constant {}); + return current; + } + /// seekable adapter: lines are counted on demand in get_position() + void track_newline(std::true_type /*lazy*/) const noexcept {} + + /// streaming adapter: start a new line after reading a newline + void track_newline(std::false_type /*lazy*/) noexcept + { if (current == '\n') { ++position.lines_read; @@ -1712,8 +1780,6 @@ scan_number_done: chars_read_before_newline = position.chars_read_current_line; position.chars_read_current_line = 0; } - - return current; } /*! @@ -1757,12 +1823,22 @@ scan_number_done: We implement unget by setting variable next_unget to true. The input is not changed - we just simulate ungetting by modifying chars_read_total, chars_read_current_line, and token_string. The next call to get() will - behave as if the unget character is read again. + behave as if the unget character is read again. With lazy_position, the + pending next_unget is all get_chars_read_total() needs. */ void unget() { next_unget = true; + unget_position(std::integral_constant {}); + uncapture_char(std::integral_constant {}); + } + /// seekable adapter: get_chars_read_total() accounts for next_unget + void unget_position(std::true_type /*lazy*/) const noexcept {} + + /// streaming adapter: step the counters back over the ungotten character + void unget_position(std::false_type /*lazy*/) noexcept + { --position.chars_read_total; // in case we "unget" a newline, we have to also decrement the lines_read @@ -1785,8 +1861,6 @@ scan_number_done: { --position.chars_read_current_line; } - - uncapture_char(std::integral_constant {}); } /// adapter without lookahead: nothing to do (see release_lookahead) @@ -1857,10 +1931,21 @@ scan_number_done: // diagnostics ///////////////////// - /// return position of last read token - constexpr position_t get_position() const noexcept + /*! + @brief return the number of characters read so far + + Unlike get_position(), this is O(1) for all adapters, so it may be called + for every value (JSON_DIAGNOSTIC_POSITIONS). + */ + std::size_t get_chars_read_total() const noexcept { - return position; + return get_chars_read_total_impl(std::integral_constant {}); + } + + /// return position of last read token + position_t get_position() const + { + return get_position_impl(std::integral_constant {}); } /*! @@ -1897,6 +1982,43 @@ scan_number_done: } #endif + private: + /// seekable adapter: every get() either read from the adapter (including + /// reads at EOF, which get() counts as characters as well) or replayed an + /// ungotten character, so the count is the adapter's read count minus a + /// still pending unget + std::size_t get_chars_read_total_impl(std::true_type /*lazy*/) const noexcept + { + return ia.get_read_count() - (next_unget ? 1u : 0u); + } + + /// streaming adapter: the counter is maintained by get() and unget() + std::size_t get_chars_read_total_impl(std::false_type /*lazy*/) const noexcept + { + return position.chars_read_total; + } + + /// seekable adapter: count the lines in the consumed input; O(input read + /// so far), but only called on errors + position_t get_position_impl(std::true_type /*lazy*/) const + { + position_t result; + result.chars_read_total = get_chars_read_total(); + // reads at EOF count as characters but are not part of the input + const std::size_t consumed = (std::min)(result.chars_read_total, ia.get_consumed_count()); + std::size_t line_start = 0; + result.lines_read = ia.count_consumed_lines(consumed, line_start); + result.chars_read_current_line = result.chars_read_total - line_start; + return result; + } + + /// streaming adapter: the position is maintained by get() and unget() + position_t get_position_impl(std::false_type /*lazy*/) const noexcept + { + return position; + } + + public: /// seekable adapter: rebuild the last read token from the input on demand const std::vector& collect_token_chars(std::vector& out, std::true_type /*lazy*/) const { @@ -2004,15 +2126,57 @@ scan_number_done: // into one that reloads the pointer from memory every character do { + if (current == '\n') + { + skip_indentation(std::integral_constant {}); + } get_ignoring_pending_unget(); } while (current_is_whitespace()); } + /*! + @brief contiguous input: skip the space indentation that follows a newline + 8 bytes at a time + + Pretty-printed JSON indents every line with a run of spaces, which the loop + in skip_whitespace() would otherwise read one character at a time. The + spaces are skipped 8 bytes at a time; the last 7 bytes of the input and any + other whitespace are left to that loop. Skipping bypasses get(), which is + fine because bulk_scan implies lazy_position: the position is derived from + the adapter, not counted per character. + */ + void skip_indentation(std::true_type /*bulk*/) + { + JSON_ASSERT(!next_unget); + const std::size_t remaining = ia.bulk_remaining(); + if (remaining < 8) + { + return; + } + const char* const data = reinterpret_cast(ia.bulk_data()); + std::size_t pos = 0; + for (; pos + 8 <= remaining; pos += 8) + { + // little-endian, so the first byte is the lowest + const std::uint64_t non_space = read_eight_bytes(data + pos) ^ 0x2020202020202020ull; + if (non_space != 0) + { + // skip the spaces before the first other byte of this word + pos += static_cast(count_trailing_zeros(non_space)) / 8; + break; + } + } + ia.bulk_skip(pos); + } + + /// streaming input: no bulk fast path + void skip_indentation(std::false_type /*bulk*/) const noexcept {} + token_type scan() { // initially, skip the BOM - if (position.chars_read_total == 0 && !skip_bom()) + if (get_chars_read_total() == 0 && !skip_bom()) { error_message = "invalid BOM; must be 0xEF 0xBB 0xBF if given"; return token_type::parse_error; @@ -2039,7 +2203,7 @@ scan_number_done: token_type scan_expecting(token_type expected_type) { JSON_ASSERT(expected_type == token_type::name_separator || expected_type == token_type::value_separator); - JSON_ASSERT(position.chars_read_total > 0); + JSON_ASSERT(get_chars_read_total() > 0); const char_int_type expected_char = static_cast((expected_type == token_type::name_separator) ? ':' : ','); skip_whitespace(); if (JSON_HEDLEY_LIKELY(current == expected_char)) diff --git a/include/nlohmann/detail/input/number_parse.hpp b/include/nlohmann/detail/input/number_parse.hpp index 8971f7193..e6e5bd58b 100644 --- a/include/nlohmann/detail/input/number_parse.hpp +++ b/include/nlohmann/detail/input/number_parse.hpp @@ -9,6 +9,7 @@ #pragma once +#include // any_of #include // array #include // FLT_EVAL_METHOD #include // localeconv @@ -18,6 +19,8 @@ #include // memcpy #include // numeric_limits #include // string +#include // conditional, integral_constant, true_type, false_type +#include // move #include #include @@ -35,10 +38,13 @@ #endif // This file contains the value-conversion helpers used by the lexer to turn an -// already-validated number token into a value, without the locale/errno -// overhead of std::strtoull/std::strtod where possible. They are free functions -// so the lexer stays focused on scanning (see lexer::convert_number()) and so -// that other parsers of JSON text can convert tokens exactly like it does. +// already-validated number token into a value. Integers and binary32/binary64 +// floats (float, double, and long double where it is binary64) are converted +// by the library itself, without the locale/errno overhead of +// std::strtoull/std::strtod and correctly rounded; other long double formats +// use std::from_chars or std::strtold. They are free functions so the lexer +// stays focused on scanning (see lexer::convert_number()) and so that other +// parsers of JSON text can convert tokens exactly like it does. NLOHMANN_JSON_NAMESPACE_BEGIN namespace detail @@ -115,233 +121,291 @@ bool parse_integer_signed(const char* first, const char* last, NumberIntegerType } /*! -@brief exact fast path for parsing a `double` (Clinger's algorithm) - -For the common case - at most 19 significant digits, a decimal exponent in -[-22, 22], and a significand below 2^53 - the value equals significand * -10^exp computed in IEEE-754 double arithmetic, which is exact under -round-to-nearest because both operands are exactly representable. This is the -same fast path used by fast_float/simdjson; the general cases are left to -std::strtod. The parser only activates for number_float_t == double; float and -long double keep the std::strtof/std::strtold paths (see the templated overload -below). - -@param[in] first pointer to the first character of the number -@param[in] last pointer past the last character -@param[out] out the parsed value on success -@return true if the value was parsed exactly; false to fall back to strtod +@brief parameters of the IEEE-754 binary32 and binary64 formats for the float + conversion (after fast_float's binary_format) */ -inline bool parse_float_fast(const char* first, const char* last, double& out) noexcept +template +struct ieee_binary_format; + +template<> +struct ieee_binary_format<24> // binary32 { -#if defined(FLT_EVAL_METHOD) && FLT_EVAL_METHOD != 0 - // Clinger's fast path is only exact when double operations are evaluated in - // true double precision. On platforms that keep intermediates in extended - // precision (e.g. the x87 FPU on 32-bit x86, where FLT_EVAL_METHOD == 2) the - // single significand * 10^scale step is double-rounded and can be 1 ULP off, - // so decline and let the caller fall back to the correctly-rounded - // std::from_chars / std::strtod path. - static_cast(first); - static_cast(last); - static_cast(out); - return false; -#else - static const std::array powers_of_ten = + static constexpr int mantissa_bits() noexcept { - { - 1e0, 1e1, 1e2, 1e3, 1e4, 1e5, 1e6, 1e7, 1e8, 1e9, 1e10, 1e11, - 1e12, 1e13, 1e14, 1e15, 1e16, 1e17, 1e18, 1e19, 1e20, 1e21, 1e22 - } - }; + return 23; + } + static constexpr int sign_bit() noexcept + { + return 31; + } + static constexpr int minimum_exponent() noexcept + { + return -127; + } + static constexpr int infinite_power() noexcept + { + return 0xFF; + } + // w * 10^q with w < 2^64 is below half the smallest subnormal number for + // q < smallest_power_of_ten() and at least infinity for q > largest_power_of_ten() + static constexpr int smallest_power_of_ten() noexcept + { + return -64; + } + static constexpr int largest_power_of_ten() noexcept + { + return 38; + } + // w * 10^q can only be exactly between two numbers for q in this range + static constexpr int min_exponent_round_to_even() noexcept + { + return -17; + } + static constexpr int max_exponent_round_to_even() noexcept + { + return 10; + } + // Clinger's fast path: w and 10^|q| are exact + static constexpr int max_exponent_fast_path() noexcept + { + return 10; + } + static constexpr std::uint64_t max_mantissa_fast_path() noexcept + { + return std::uint64_t{2} << 23u; + } + // a midpoint between two numbers has at most this many significant digits + static constexpr std::int64_t max_digits() noexcept + { + return 114; + } +}; - const char* p = first; - bool negative = false; - if (p != last && (*p == '-' || *p == '+')) - { - negative = (*p == '-'); - ++p; - } - - std::uint64_t significand = 0; - int num_digits = 0; - int fractional_digits = 0; - bool seen_dot = false; - bool any_digit = false; - for (; p != last; ++p) - { - const char c = *p; - if (c >= '0' && c <= '9') - { - any_digit = true; - if (JSON_HEDLEY_UNLIKELY(num_digits >= 19)) - { - return false; // significand may not fit into uint64_t - } - significand = (significand * 10u) + static_cast(c - '0'); - ++num_digits; - fractional_digits += static_cast(seen_dot); - } - else if (c == '.') - { - if (JSON_HEDLEY_UNLIKELY(seen_dot)) - { - return false; - } - seen_dot = true; - } - else if (c == 'e' || c == 'E') - { - ++p; - break; - } - else - { - return false; - } - } - if (JSON_HEDLEY_UNLIKELY(!any_digit)) - { - return false; - } - - int exponent = 0; - if (p != last) // an exponent part remains - { - bool exp_negative = false; - if (p != last && (*p == '-' || *p == '+')) - { - exp_negative = (*p == '-'); - ++p; - } - bool any_exp_digit = false; - for (; p != last; ++p) - { - if (JSON_HEDLEY_UNLIKELY(*p < '0' || *p > '9')) - { - return false; - } - exponent = (exponent * 10) + (*p - '0'); - any_exp_digit = true; - if (JSON_HEDLEY_UNLIKELY(exponent > 9999)) - { - return false; - } - } - if (JSON_HEDLEY_UNLIKELY(!any_exp_digit)) - { - return false; - } - if (exp_negative) - { - exponent = -exponent; - } - } - - const int scale = exponent - fractional_digits; - if (JSON_HEDLEY_UNLIKELY(significand >= (static_cast(1) << 53))) - { - return false; // significand not exactly representable as double - } - - auto result = static_cast(significand); - if (scale >= 0) - { - if (JSON_HEDLEY_UNLIKELY(scale > 22)) - { - return false; - } - result *= powers_of_ten[static_cast(scale)]; - } - else - { - if (JSON_HEDLEY_UNLIKELY(-scale > 22)) - { - return false; - } - result /= powers_of_ten[static_cast(-scale)]; - } - out = negative ? -result : result; - return true; -#endif -} - -/// fast float path is only exact for `double`; decline for float/long double -template -bool parse_float_fast(const char* /*first*/, const char* /*last*/, FloatType& /*out*/) noexcept +template<> +struct ieee_binary_format<53> // binary64 { - return false; -} + static constexpr int mantissa_bits() noexcept + { + return 52; + } + static constexpr int sign_bit() noexcept + { + return 63; + } + static constexpr int minimum_exponent() noexcept + { + return -1023; + } + static constexpr int infinite_power() noexcept + { + return 0x7FF; + } + static constexpr int smallest_power_of_ten() noexcept + { + return -342; + } + static constexpr int largest_power_of_ten() noexcept + { + return 308; + } + static constexpr int min_exponent_round_to_even() noexcept + { + return -4; + } + static constexpr int max_exponent_round_to_even() noexcept + { + return 23; + } + static constexpr int max_exponent_fast_path() noexcept + { + return 22; + } + static constexpr std::uint64_t max_mantissa_fast_path() noexcept + { + return std::uint64_t{2} << 52u; + } + static constexpr std::int64_t max_digits() noexcept + { + return 769; + } +}; /*! -@brief parse a float with std::from_chars (Eisel-Lemire) when available +@brief whether @a FloatType is IEEE-754 binary32 or binary64 -std::from_chars is locale-independent, correctly rounded, and - via the -Eisel-Lemire algorithm in modern standard libraries - much faster than strtod -over the whole value range (not just the Clinger subset). It is used only when -__cpp_lib_to_chars indicates full floating-point support and only when it -consumes the entire token ([first, last)). An under-/overflow (result_out_of_range) also declines, so -the caller's strtod fallback supplies the well-defined ±inf/0 result the parser -expects (side-stepping the P4168 divergence between implementations). - -@return true if the value was parsed exactly and fully; false to fall back +These formats (float, double, and long double where it is binary64, e.g. +with MSVC or on Apple arm64) are converted by parse_float_native(). The +predicate is the one the serializer uses to choose Grisu2. */ template -bool parse_float_from_chars(const char* first, const char* last, FloatType& out) noexcept +struct has_native_float_format { - // JSON_HAS_CPP_17 must gate the use as well as the include above: - // some standard libraries (e.g. libstdc++ 15) define __cpp_lib_to_chars even - // in C++14 mode, where is not included. -#if defined(JSON_HAS_CPP_17) && defined(__cpp_lib_to_chars) - const auto result = std::from_chars(first, last, out); - return result.ec == std::errc() && result.ptr == last; -#else - static_cast(first); - static_cast(last); - static_cast(out); - return false; -#endif -} + static constexpr bool value = + (std::numeric_limits::is_iec559 && std::numeric_limits::digits == 24 && std::numeric_limits::max_exponent == 128) || + (std::numeric_limits::is_iec559 && std::numeric_limits::digits == 53 && std::numeric_limits::max_exponent == 1024); +}; -/// whether the eight bytes of @a v (see read_eight_bytes()) are ASCII digits -/// (after fast_float's is_made_of_eight_digits_fast) -inline bool is_eight_digits(std::uint64_t v) noexcept -{ - return ((v & 0xF0F0F0F0F0F0F0F0u) | (((v + 0x0606060606060606u) & 0xF0F0F0F0F0F0F0F0u) >> 4u)) == 0x3333333333333333u; -} +/// the C++ type (float or double) that holds a binary32 or binary64 @a FloatType +template +using native_float_t = typename std::conditional::digits == 24, float, double>::type; /// the value of the eight ASCII digits in @a v (see read_eight_bytes()), three -/// multiplications instead of eight (after simdjson and fast_float) -inline std::uint32_t parse_eight_digits(std::uint64_t v) noexcept +/// multiplications instead of eight (after simdjson and fast_float); always +/// inlined, as GCC otherwise calls it in the number loops +JSON_HEDLEY_ALWAYS_INLINE std::uint32_t parse_eight_digits(std::uint64_t v) noexcept { v = ((v & 0x0F0F0F0F0F0F0F0Fu) * 2561u) >> 8u; v = ((v & 0x00FF00FF00FF00FFu) * 6553601u) >> 16u; return static_cast(((v & 0x0000FFFF0000FFFFu) * 42949672960001u) >> 32u); } +/// whether [first, last) contains a digit other than '0' +inline bool has_nonzero_digit(const char* first, const char* last) noexcept +{ + return std::any_of(first, last, [](char c) noexcept + { + return c != '0'; + }); +} + +/// the value of the validated exponent digits [+-]?[0-9]+ in [first, last), +/// saturated far beyond every range +inline std::int64_t parse_float_exponent(const char* first, const char* last) noexcept +{ + const bool negative = *first == '-'; + first += (*first == '-' || *first == '+') ? 1 : 0; + constexpr std::int64_t saturation = 100000000000000000; // 10^17 + std::int64_t value = 0; + for (; first != last; ++first) + { + if (value < saturation) + { + value = (value * 10) + (*first - '0'); + } + } + return negative ? -value : value; +} + +/// a float token as w * 10^exponent, see parse_float_significand() +struct float_significand +{ + std::uint64_t w = 0; ///< the first (at most 19) significant digits + std::int64_t exponent = 0; ///< the decimal exponent of the last digit in w + bool negative = false; ///< whether the token starts with '-' + bool truncated = false; ///< whether nonzero digits follow the ones in w +}; + /*! -@brief the double nearest to w * 10^q (Eisel-Lemire) +@brief split a validated number token into sign, significand, and exponent + +The lexer has validated the token against the JSON grammar and knows where its +parts are, so this needs no character classification: the integer part ends at +@a decimal_point_position (or @a mantissa_end), the fraction at @a mantissa_end, +and an exponent follows. At most 19 significant digits are kept; the value then +lies in [w, w + 1) * 10^exponent, and is exactly w * 10^exponent unless +truncated is set. + +@param[in] first pointer to the first character of the token +@param[in] last pointer past the last character +@param[in] decimal_point_position index of the '.' in the token, or + std::string::npos if there is none +@param[in] mantissa_end index of the 'e'/'E', or the token length +*/ +inline float_significand parse_float_significand(const char* first, const char* last, + std::size_t decimal_point_position, std::size_t mantissa_end) noexcept +{ + float_significand s; + const char* p = first; + s.negative = *p == '-'; + p += s.negative ? 1 : 0; + const bool has_dot = decimal_point_position != std::string::npos; + const char* const mantissa_last = first + mantissa_end; + const char* const integer_last = has_dot ? first + decimal_point_position : mantissa_last; + + std::uint64_t w = 0; + int remaining = 19; // digits that still fit into w + if (*p != '0') // the integer part is "0" or [1-9][0-9]* + { + while (remaining >= 8 && integer_last - p >= 8) + { + w = (w * 100000000u) + parse_eight_digits(read_eight_bytes(p)); + p += 8; + remaining -= 8; + } + for (; remaining > 0 && p != integer_last; ++p, --remaining) + { + w = (w * 10u) + static_cast(*p - '0'); + } + s.exponent = integer_last - p; + s.truncated = has_nonzero_digit(p, integer_last); + } + + if (has_dot) + { + p = integer_last + 1; + if (w == 0) + { + // zeros after the decimal point of "0." are not significant + const char* const zeros = p; + while (p != mantissa_last && *p == '0') + { + ++p; + } + s.exponent -= p - zeros; + } + const char* const digits = p; + while (remaining >= 8 && mantissa_last - p >= 8) + { + w = (w * 100000000u) + parse_eight_digits(read_eight_bytes(p)); + p += 8; + remaining -= 8; + } + for (; remaining > 0 && p != mantissa_last; ++p, --remaining) + { + w = (w * 10u) + static_cast(*p - '0'); + } + s.exponent -= p - digits; + s.truncated = s.truncated || has_nonzero_digit(p, mantissa_last); + } + + if (mantissa_last != last) + { + s.exponent += parse_float_exponent(mantissa_last + 1, last); + } + s.w = w; + return s; +} + +/*! +@brief the bits of the float nearest to w * 10^q (Eisel-Lemire) The algorithm of Daniel Lemire, "Number Parsing at a Gigabyte per Second" (Software: Practice and Experience, 2021), after fast_float's compute_float (used under the MIT license). With a 128-bit approximation of 5^q, the product -is always sufficient to round correctly for w with at most 19 digits (Noble -Mushtak and Daniel Lemire, "Fast number parsing without fallback", Software: -Practice and Experience, 2023). Only integer arithmetic is used, so the result -does not depend on the floating-point environment. +is always sufficient to round correctly for w < 2^64 (Noble Mushtak and Daniel +Lemire, "Fast number parsing without fallback", Software: Practice and +Experience, 2023). Only integer arithmetic is used, so the result does not +depend on the floating-point environment. +It is always inlined, like decimal_to_float(), so that hot loops of callers +keep the whole conversion inline. + +@tparam Format ieee_binary_format<24> (binary32) or ieee_binary_format<53> (binary64) @param[in] q decimal exponent -@param[in] w significand, w != 0 +@param[in] w significand @return the IEEE-754 bits of the positive result (0 for underflow, infinity for overflow) */ -inline std::uint64_t eisel_lemire(std::int64_t q, std::uint64_t w) noexcept +template +JSON_HEDLEY_ALWAYS_INLINE std::uint64_t eisel_lemire(std::int64_t q, std::uint64_t w) noexcept { - constexpr int mantissa_bits = 52; - constexpr std::uint64_t infinity = std::uint64_t{0x7FF} << mantissa_bits; - if (q < pow5_128_smallest_power) + constexpr int mantissa_bits = Format::mantissa_bits(); + constexpr std::uint64_t infinity = static_cast(Format::infinite_power()) << mantissa_bits; + if (w == 0 || q < Format::smallest_power_of_ten()) { return 0; } - if (q > pow5_128_largest_power) + if (q > Format::largest_power_of_ten()) { return infinity; } @@ -365,8 +429,8 @@ inline std::uint64_t eisel_lemire(std::int64_t q, std::uint64_t w) noexcept const auto upperbit = static_cast(product.high >> 63u); const int shift = upperbit + 64 - mantissa_bits - 3; std::uint64_t mantissa = product.high >> static_cast(shift); - // floor(log2(10^q)) + 63 + 1023, with log2(10) ~ 217706 / 2^16 - std::int64_t power2 = (((152170 + 65536) * q) >> 16) + 63 + upperbit - lz + 1023; + // floor(log2(10^q)) + 63 + bias, with log2(10) ~ 217706 / 2^16 + std::int64_t power2 = (((152170 + 65536) * q) >> 16) + 63 + upperbit - lz - Format::minimum_exponent(); if (power2 <= 0) // subnormal { @@ -375,17 +439,18 @@ inline std::uint64_t eisel_lemire(std::int64_t q, std::uint64_t w) noexcept return 0; } mantissa >>= static_cast(-power2 + 1); + // no tie is possible here: that needs a small |q| mantissa += (mantissa & 1u); mantissa >>= 1u; // rounding up may produce the smallest normal number power2 = (mantissa < (std::uint64_t{1} << mantissa_bits)) ? 0 : 1; - return mantissa | (static_cast(power2) << mantissa_bits); + return (mantissa & ((std::uint64_t{1} << mantissa_bits) - 1)) | (static_cast(power2) << mantissa_bits); } - // a value exactly between two doubles rounds to even; this can only + // a value exactly between two floats rounds to even; this can only // happen for small |q|, where 5^q is exact - if (product.low <= 1 && q >= -4 && q <= 23 && (mantissa & 3u) == 1 - && (mantissa << static_cast(shift)) == product.high) + if (product.low <= 1 && q >= Format::min_exponent_round_to_even() && q <= Format::max_exponent_round_to_even() + && (mantissa & 3u) == 1 && (mantissa << static_cast(shift)) == product.high) { mantissa &= ~std::uint64_t{1}; } @@ -397,198 +462,420 @@ inline std::uint64_t eisel_lemire(std::int64_t q, std::uint64_t w) noexcept ++power2; } mantissa &= ~(std::uint64_t{1} << mantissa_bits); - if (power2 >= 0x7FF) + if (power2 >= Format::infinite_power()) { return infinity; } return mantissa | (static_cast(power2) << mantissa_bits); } -/*! -@brief parse a validated float token with the Eisel-Lemire algorithm - -The significand is accumulated eight digits at a time where possible. A token -with more than 19 significant digits is truncated to w; the value then lies -in [w, w + 1) * 10^q, and it is only returned if both ends round to the same -double, which covers all but a few such tokens. - -@param[in] first pointer to the first character of the token -@param[in] last pointer past the last character -@param[out] out the correctly rounded value on success (±infinity if it - overflows, like strtod) -@return true on success; false if strtod must decide -*/ -inline bool parse_float_eisel_lemire(const char* first, const char* last, double& out) noexcept +/// an unsigned integer of up to 4096 bits for digit_comparison() (32-bit limbs, +/// so only 32x32->64-bit multiplications are needed) +class float_bigint { - const char* p = first; - const bool negative = (p != last && *p == '-'); - if (negative) + public: + explicit float_bigint(std::uint64_t value) noexcept { - ++p; + for (; value != 0; value >>= 32u) + { + limbs[count++] = static_cast(value); + } } - std::uint64_t w = 0; - unsigned int digits = 0; // significant digits in w - std::int64_t exponent = 0; - bool truncated = false; - bool in_fraction = false; - for (;;) + /// *this = *this * factor + summand + void multiply_add(std::uint32_t factor, std::uint32_t summand) noexcept { - // eight digits at a time, as long as they fit into w - while (w != 0 && digits <= 19u - 8u && last - p >= 8) + std::uint64_t carry = summand; + for (std::size_t i = 0; i < count; ++i) { - const std::uint64_t v = read_eight_bytes(p); - if (!is_eight_digits(v)) - { - break; - } - w = (w * 100000000u) + parse_eight_digits(v); - digits += 8u; - exponent -= in_fraction ? 8 : 0; - p += 8; + const std::uint64_t product = (static_cast(limbs[i]) * factor) + carry; + limbs[i] = static_cast(product); + carry = product >> 32u; } - if (p == last) + if (carry != 0) { - break; + JSON_ASSERT(count < limbs.size()); + limbs[count++] = static_cast(carry); } - const char c = *p; - if (c >= '0' && c <= '9') + } + + /// *this = *this * 5^n + void multiply_power_of_five(std::int64_t n) noexcept + { + static const std::array powers = { - if (w == 0 && c == '0') - { - // leading zeros are not significant, but scale a fraction - exponent -= in_fraction ? 1 : 0; - } - else if (digits < 19u) - { - w = (w * 10u) + static_cast(c - '0'); - ++digits; - exponent -= in_fraction ? 1 : 0; - } - else - { - // dropped: the value lies between w and w + 1 (in units of - // the last kept digit) unless all dropped digits are zero - truncated = truncated || c != '0'; - exponent += in_fraction ? 0 : 1; - } - ++p; + {1u, 5u, 25u, 125u, 625u, 3125u, 15625u, 78125u, 390625u, 1953125u, 9765625u, 48828125u, 244140625u, 1220703125u} + }; + for (; n >= 13; n -= 13) + { + multiply_add(powers[13], 0); } - else if (c == '.') + multiply_add(powers[static_cast(n)], 0); + } + + /// *this = *this * 2^n + void shift_left(std::int64_t n) noexcept + { + if (count == 0) { - in_fraction = true; - ++p; + return; + } + const auto limb_shift = static_cast(n / 32); + const auto bit_shift = static_cast(n % 32); + JSON_ASSERT(count + limb_shift + 1 <= limbs.size()); + if (bit_shift != 0) + { + std::uint32_t carry = 0; + for (std::size_t i = 0; i < count; ++i) + { + const std::uint32_t limb = limbs[i]; + limbs[i] = (limb << bit_shift) | carry; + carry = limb >> (32u - bit_shift); + } + if (carry != 0) + { + limbs[count++] = carry; + } + } + if (limb_shift != 0) + { + for (std::size_t i = count; i-- > 0;) + { + limbs[i + limb_shift] = limbs[i]; + } + for (std::size_t i = 0; i < limb_shift; ++i) + { + limbs[i] = 0; + } + count += limb_shift; + } + } + + /// -1, 0, or 1 if *this is less than, equal to, or greater than @a other + int compare(const float_bigint& other) const noexcept + { + if (count != other.count) + { + return count < other.count ? -1 : 1; + } + for (std::size_t i = count; i-- > 0;) + { + if (limbs[i] != other.limbs[i]) + { + return limbs[i] < other.limbs[i] ? -1 : 1; + } + } + return 0; + } + + private: + std::array limbs{{}}; + std::size_t count = 0; +}; + +/*! +@brief round a token exactly when eisel_lemire() cannot decide (slow path) + +The value v of the token lies strictly between two adjacent floats, whose +lower one has the bits @a lower, and the result depends on whether v is below, +at, or above the midpoint m between them. Both are compared exactly as big +integers: v = D * 10^s with the significant digits D (at most +Format::max_digits() of them, more than any midpoint has; further nonzero +digits only put v above m) and m = (2 * mantissa + 1) * 2^(e - 1). This is the +digit comparison of fast_float (Daniel Lemire and contributors, used under the +MIT license), simplified by starting from the two candidates. + +@param[in] first pointer to the first character of the token +@param[in] last pointer past the last character +@param[in] lower the bits of the float below v +@return the bits of the correctly rounded result +*/ +template +std::uint64_t digit_comparison(const char* first, const char* last, std::uint64_t lower) noexcept +{ + const char* p = first + ((*first == '-') ? 1 : 0); + + // D, in chunks of up to 9 digits, and s + float_bigint digits(0); + std::int64_t count = 0; + std::int64_t point = 0; // the value is 0.D... * 10^point + bool truncated = false; + std::uint32_t chunk = 0; + int chunk_digits = 0; + static const std::array powers_of_ten = {{1u, 10u, 100u, 1000u, 10000u, 100000u, 1000000u, 10000000u, 100000000u, 1000000000u}}; + const auto append = [&](char c) noexcept + { + if (count < Format::max_digits()) + { + chunk = (chunk * 10u) + static_cast(c - '0'); + ++count; + if (++chunk_digits == 9) + { + digits.multiply_add(powers_of_ten[9], chunk); + chunk = 0; + chunk_digits = 0; + } } else { - break; // 'e' or 'E' + truncated = truncated || c != '0'; + } + }; + bool significant = false; + for (; p != last && *p >= '0' && *p <= '9'; ++p) + { + significant = significant || *p != '0'; + if (significant) + { + append(*p); + ++point; } } - - if (p != last) + if (p != last && *p == '.') { - ++p; // 'e' or 'E' - bool exp_negative = false; - if (p != last && (*p == '-' || *p == '+')) + for (++p; p != last && *p >= '0' && *p <= '9'; ++p) { - exp_negative = (*p == '-'); - ++p; - } - std::int64_t exp_value = 0; - for (; p != last; ++p) - { - // saturate: any exponent beyond this under- or overflows anyway - if (exp_value < 100000) + significant = significant || *p != '0'; + if (significant) { - exp_value = (exp_value * 10) + (*p - '0'); + append(*p); + } + else + { + --point; } } - exponent += exp_negative ? -exp_value : exp_value; } - - std::uint64_t bits = 0; - if (w != 0) + if (chunk_digits != 0) { - bits = eisel_lemire(exponent, w); - if (truncated && (w + 1 == 0 || eisel_lemire(exponent, w + 1) != bits)) - { - return false; - } + digits.multiply_add(powers_of_ten[static_cast(chunk_digits)], chunk); } - bits |= negative ? (std::uint64_t{1} << 63u) : 0u; - static_assert(sizeof(double) == sizeof(std::uint64_t), "double must have 64 bits"); - std::memcpy(&out, &bits, sizeof(out)); - return true; + if (p != last) + { + point += parse_float_exponent(p + 1, last); + } + const std::int64_t s = point - count; // v = D * 10^s + + // the midpoint above the lower candidate + constexpr int mantissa_bits = Format::mantissa_bits(); + const std::uint64_t exponent_field = lower >> mantissa_bits; + std::uint64_t mantissa = lower & ((std::uint64_t{1} << mantissa_bits) - 1); + std::int64_t e = 1 + Format::minimum_exponent() - mantissa_bits; // of the smallest subnormal number + if (exponent_field != 0) + { + mantissa |= std::uint64_t{1} << mantissa_bits; + e += static_cast(exponent_field) - 1; + } + float_bigint midpoint((2 * mantissa) + 1); + const std::int64_t midpoint_exponent = e - 1; // m = midpoint * 2^midpoint_exponent + + // compare D * 5^s * 2^s with midpoint * 2^midpoint_exponent + if (s >= 0) + { + digits.multiply_power_of_five(s); + } + else + { + midpoint.multiply_power_of_five(-s); + } + const std::int64_t shift = s - midpoint_exponent; + if (shift >= 0) + { + digits.shift_left(shift); + } + else + { + midpoint.shift_left(-shift); + } + const int order = digits.compare(midpoint); + const bool round_up = order > 0 || (order == 0 && (truncated || (mantissa & 1u) != 0)); + return lower + (round_up ? 1u : 0u); } -/// Eisel-Lemire is only implemented for `double` -template -bool parse_float_eisel_lemire(const char* /*first*/, const char* /*last*/, FloatType& /*out*/) noexcept +/// the double with the IEEE-754 bits @a bits +inline void float_from_bits(std::uint64_t bits, double& value) noexcept { - return false; + static_assert(sizeof(double) == sizeof(std::uint64_t), "double must have 64 bits"); + std::memcpy(&value, &bits, sizeof(value)); +} + +/// the float with the IEEE-754 bits @a bits (the lower 32) +inline void float_from_bits(std::uint64_t bits, float& value) noexcept +{ + static_assert(sizeof(float) == sizeof(std::uint32_t), "float must have 32 bits"); + const auto bits32 = static_cast(bits); + std::memcpy(&value, &bits32, sizeof(value)); +} + +/// the powers of ten that are exact in binary64 (up to 10^22) +inline double exact_power_of_ten(std::int64_t n, double /*tag*/) noexcept +{ + static const std::array powers = + { + { + 1e0, 1e1, 1e2, 1e3, 1e4, 1e5, 1e6, 1e7, 1e8, 1e9, 1e10, 1e11, + 1e12, 1e13, 1e14, 1e15, 1e16, 1e17, 1e18, 1e19, 1e20, 1e21, 1e22 + } + }; + return powers[static_cast(n)]; +} + +/// the powers of ten that are exact in binary32 (up to 10^10) +inline float exact_power_of_ten(std::int64_t n, float /*tag*/) noexcept +{ + static const std::array powers = + { + {1e0f, 1e1f, 1e2f, 1e3f, 1e4f, 1e5f, 1e6f, 1e7f, 1e8f, 1e9f, 1e10f} + }; + return powers[static_cast(n)]; } /*! -@brief check whether Clinger's fast path can still succeed for a float token +@brief the binary32/binary64 value of a significand that was not truncated -parse_float_fast() needs a significand below 2^53. A mantissa with 17 or -more significant digits is at least 10^16 and therefore always exceeds it, -so calling the fast path would walk the token one extra time only to -decline before strtod has to run anyway. +The result is (-1)^negative * w * 10^exponent, correctly rounded (ties to +even): with Clinger's fast path where w and 10^|exponent| are exact, so that a +single floating-point operation rounds (only where intermediate results are not +kept in extended precision, see FLT_EVAL_METHOD), and with eisel_lemire() +otherwise. A value too large for the type becomes ±infinity, a value too small +±0. -Significant digits are the mantissa's digits from the first nonzero one on; -the sign, the decimal point, leading zeros, and the exponent do not count. -The answer is derived from indices - the digits are not scanned again - so -this stays off the hot path of the number scanners. +This is the core of the conversion that other parsers of JSON text share: they +can split a token themselves and still get the lexer's result. It is always +inlined, so that their hot loops keep the whole conversion inline. -@param[in] token the validated number token ('.' as decimal point) -@param[in] decimal_point_position index of the '.' in @a token, or - std::string::npos if there is none -@param[in] mantissa_end offset just past the last mantissa byte -@return false if parse_float_fast() is guaranteed to decline +@param[in] s the significand, with s.truncated == false */ -inline bool mantissa_fits_clinger(const char* token, std::size_t decimal_point_position, std::size_t mantissa_end) noexcept +template +JSON_HEDLEY_ALWAYS_INLINE FloatType decimal_to_float(const float_significand& s) noexcept { - // 10^16 already exceeds 2^53, so 17 digits can never fit - constexpr std::size_t limit = 17; + using result_type = native_float_t; + using format = ieee_binary_format::digits>; + JSON_ASSERT(!s.truncated); - const std::size_t neg = (token[0] == '-') ? 1u : 0u; - const std::size_t has_dot = (decimal_point_position != std::string::npos) ? 1u : 0u; - // the JSON grammar restricts the integer part to "0" or [1-9][0-9]*, so - // a leading zero can only be a lone "0", which is not significant - const std::size_t lead_zero = (token[neg] == '0') ? 1u : 0u; - JSON_ASSERT(mantissa_end >= neg + has_dot + lead_zero); - std::size_t digits = mantissa_end - neg - has_dot - lead_zero; - - if (JSON_HEDLEY_LIKELY(digits < limit)) +#if !defined(FLT_EVAL_METHOD) || FLT_EVAL_METHOD == 0 + if (s.exponent >= -format::max_exponent_fast_path() && s.exponent <= format::max_exponent_fast_path() + && s.w <= format::max_mantissa_fast_path()) { - return true; - } - - // Only a number below 1 can carry further insignificant zeros, and only - // while the count stays at the limit does removing them change the - // answer - so this loop is skipped for all but a few tokens. The - // fraction is located through decimal_point_position rather than by - // searching '.'. - if (lead_zero != 0) - { - JSON_ASSERT(has_dot != 0); // an integer "0" cannot reach the limit - for (std::size_t i = decimal_point_position + 1; - digits >= limit && i < mantissa_end && token[i] == '0'; ++i) + auto value = static_cast(s.w); + if (s.exponent < 0) { - --digits; + value /= exact_power_of_ten(-s.exponent, result_type{}); } + else + { + value *= exact_power_of_ten(s.exponent, result_type{}); + } + const FloatType result = s.negative ? -value : value; + return result; + } +#endif + + result_type value{}; + float_from_bits(eisel_lemire(s.exponent, s.w) | (s.negative ? (std::uint64_t{1} << format::sign_bit()) : 0u), value); + const FloatType result = value; + return result; +} + +/*! +@brief convert a validated number token to the nearest binary32/binary64 value + +The conversion is correctly rounded (ties to even) and independent of the +locale and of the C and C++ libraries: +1. parse_float_significand() splits the token into w * 10^q. +2. If no digits were dropped, decimal_to_float() rounds w * 10^q (Clinger's + fast path or Eisel-Lemire). +3. Otherwise, the value lies in [w, w + 1) * 10^q: if eisel_lemire() rounds + both ends to the same value, so does the token (in all but rare cases). +4. Otherwise, digit_comparison() compares the token exactly with the midpoint + between the two candidates. +A value too large for the type becomes ±infinity (the parser reports +out_of_range.406), a value too small ±0. + +@param[in] first pointer to the first character of the token +@param[in] last pointer past the last character +@param[in] decimal_point_position index of the '.' in the token, or + std::string::npos if there is none +@param[in] mantissa_end index of the 'e'/'E', or the token length +*/ +template +FloatType parse_float_native(const char* first, const char* last, + std::size_t decimal_point_position, std::size_t mantissa_end) noexcept +{ + using result_type = native_float_t; + using format = ieee_binary_format::digits>; + static_assert(std::numeric_limits::digits == std::numeric_limits::digits, "unexpected float format"); + + const float_significand s = parse_float_significand(first, last, decimal_point_position, mantissa_end); + if (JSON_HEDLEY_LIKELY(!s.truncated)) + { + return decimal_to_float(s); } - return digits < limit; + std::uint64_t bits = eisel_lemire(s.exponent, s.w); + if (JSON_HEDLEY_UNLIKELY(bits != eisel_lemire(s.exponent, s.w + 1))) + { + bits = digit_comparison(first, last, bits); + } + result_type value{}; + float_from_bits(bits | (s.negative ? (std::uint64_t{1} << format::sign_bit()) : 0u), value); + const FloatType result = value; + return result; +} + +/*! +@brief parse a float with std::from_chars when available + +Only used for the formats parse_float_native() does not convert (long double +formats other than binary64). std::from_chars is locale-independent and +correctly rounded. It is used only when __cpp_lib_to_chars indicates full +floating-point support and only when it consumes the entire token ([first, +last)). An under-/overflow (result_out_of_range) also declines, so the +caller's strtold fallback supplies the well-defined ±inf/0 result the parser +expects (side-stepping the P4168 divergence between implementations). + +@return true if the value was parsed exactly and fully; false to fall back +*/ +template +bool parse_float_from_chars(const char* first, const char* last, FloatType& out) noexcept +{ + // JSON_HAS_CPP_17 must gate the use as well as the include above: + // some standard libraries (e.g. libstdc++ 15) define __cpp_lib_to_chars even + // in C++14 mode, where is not included. +#if defined(JSON_HAS_CPP_17) && defined(__cpp_lib_to_chars) + const auto result = std::from_chars(first, last, out); + return result.ec == std::errc() && result.ptr == last; +#else + static_cast(first); + static_cast(last); + static_cast(out); + return false; +#endif +} + +/// binary32 and binary64: the library's own conversion, which always succeeds +template +bool convert_float_fast(const char* first, const char* last, std::size_t decimal_point_position, + std::size_t mantissa_end, FloatType& value, std::true_type /*native*/) noexcept +{ + value = parse_float_native(first, last, decimal_point_position, mantissa_end); + return true; +} + +/// other formats (long double on x87, binary128, double-double): std::from_chars, if available +template +bool convert_float_fast(const char* first, const char* last, std::size_t /*decimal_point_position*/, + std::size_t /*mantissa_end*/, FloatType& value, std::false_type /*native*/) noexcept +{ + return parse_float_from_chars(first, last, value); } /*! @brief convert a validated float token without the C library, if possible -Tries std::from_chars (when available), Clinger's exact fast path (double -only, skipped when it cannot succeed), and the Eisel-Lemire algorithm (double -only). +float, double, and long double where it is binary64 are always converted, by +parse_float_native(). Other long double formats are converted with +std::from_chars where the standard library supports it. @param[in] first pointer to the first character of the token @param[in] last pointer past the last character @@ -604,19 +891,8 @@ template bool convert_float_fast(const char* first, const char* last, std::size_t decimal_point_position, std::size_t mantissa_end, FloatType& value) noexcept { - if (parse_float_from_chars(first, last, value)) - { - return true; - } - // Skipping a fast path that cannot succeed is lossless and saves a full - // extra pass over the token's bytes, which otherwise shows up on - // high-precision inputs such as canada.json - if (mantissa_fits_clinger(first, decimal_point_position, mantissa_end) - && parse_float_fast(first, last, value)) - { - return true; - } - return parse_float_eisel_lemire(first, last, value); + return convert_float_fast(first, last, decimal_point_position, mantissa_end, value, + std::integral_constant::value> {}); } /// std::strtof, std::strtod, or std::strtold, chosen by the type of @a f @@ -640,36 +916,45 @@ inline void strtof_by_type(long double& f, const char* str, char** endptr) noexc f = std::strtold(str, endptr); } -/// return the decimal point of the current locale -inline char get_decimal_point() noexcept +/// return the decimal point of the current locale (it may be longer than one byte) +inline std::string get_decimal_point() { const auto* loc = localeconv(); JSON_ASSERT(loc != nullptr); - return (loc->decimal_point == nullptr) ? '.' : *(loc->decimal_point); + return (loc->decimal_point == nullptr || *loc->decimal_point == '\0') ? "." : loc->decimal_point; } /*! @brief convert a validated float token with strtof/strtod/strtold +Only used for what convert_float_fast() does not convert: long double formats +other than binary64 where std::from_chars is unavailable or reports an under- +or overflow, and floating-point types that are not IEEE-754 (see +has_native_float_format). + These functions expect the decimal point of the *current* locale, so it is looked up right before the conversion instead of once when the lexer is constructed: a locale change in between (by a parser callback, a SAX -handler, or another thread) must not truncate the value (#5198). The -token has been validated before, so if the conversion stops early and the -decimal point changed in the meantime, the locale changed between the +handler, or another thread) must not truncate the value (#5198). A +single-byte decimal point is substituted in place and restored afterwards, +because the token is also handed to the SAX interface. A longer one (e.g., +the two-byte U+066B of ar_EG.UTF-8 or fa_IR.UTF-8) is put into a copy of the +token instead (#5660). + +The token has been validated before, so if the conversion stops early and +the decimal point changed in the meantime, the locale changed between the lookup and the call, and the conversion is repeated with the new decimal -point. If the decimal point did not change, a retry cannot succeed: the -locale's decimal point is not a single character (e.g., the two-byte -U+066B of ar_EG.UTF-8 or fa_IR.UTF-8) and cannot be substituted in place. -The value strtod parsed up to that point is kept, as before this change. +point. If it did not change, the value strtod parsed up to that point is +kept. Note that changing the locale in another thread *while* strtod runs is undefined behavior of the C library, which this function cannot prevent. -@param[in,out] token the token with '.' as decimal point; its - decimal point is replaced during the - conversion and restored afterwards - (data() must be NUL-terminated) +@param[in,out] token the token with '.' as decimal point; a + single-byte decimal point is put in + place during the conversion and + restored afterwards (data() must be + NUL-terminated) @param[in] decimal_point_position index of the '.' in @a token, or std::string::npos if there is none @param[out] value the converted value @@ -678,38 +963,76 @@ template void convert_float_locale_aware(StringType& token, std::size_t decimal_point_position, FloatType& value) { const bool has_dot = decimal_point_position != std::string::npos; - char decimal_point = get_decimal_point(); + std::string decimal_point = get_decimal_point(); for (;;) { - const bool substitute = has_dot && decimal_point != '.'; - if (substitute) - { - token[decimal_point_position] = static_cast(decimal_point); - } - char* endptr = nullptr; // NOLINT(misc-const-correctness,cppcoreguidelines-pro-type-vararg,hicpp-vararg) - strtof_by_type(value, token.data(), &endptr); - - if (substitute) + bool complete = false; + if (!has_dot || decimal_point.size() == 1) { - // the caller hands the token on (e.g. to the SAX interface) with '.' - token[decimal_point_position] = '.'; + const bool substitute = has_dot && decimal_point[0] != '.'; + if (substitute) + { + token[decimal_point_position] = static_cast(decimal_point[0]); + } + strtof_by_type(value, token.data(), &endptr); + if (substitute) + { + // the caller hands the token on (e.g. to the SAX interface) with '.' + token[decimal_point_position] = '.'; + } + complete = endptr == token.data() + token.size(); + } + else + { + std::string buffer(token.data(), token.size()); + buffer.replace(decimal_point_position, 1, decimal_point); + strtof_by_type(value, buffer.data(), &endptr); + complete = endptr == buffer.data() + buffer.size(); } - if (JSON_HEDLEY_LIKELY(endptr == token.data() + token.size())) + if (JSON_HEDLEY_LIKELY(complete)) { return; } // retry only if the locale changed; otherwise, this would loop forever - const char current_decimal_point = get_decimal_point(); + std::string current_decimal_point = get_decimal_point(); if (current_decimal_point == decimal_point) { return; } - decimal_point = current_decimal_point; + decimal_point = std::move(current_decimal_point); } } +/*! +@brief convert a validated float token like the lexer does + +For parsers of JSON text other than the lexer, which converts its own token +buffer in place. float, double, and long double where it is binary64 are +converted without allocation and independent of the locale; only other long +double formats that std::from_chars does not support need a copy of the token +for convert_float_locale_aware(). + +@param[in] first pointer to the first character of the token +@param[in] last pointer past the last character +@param[in] decimal_point_position index of the '.' in the token, or + std::string::npos if there is none +@param[in] mantissa_end index of the 'e'/'E', or the token length +@return the value, ±infinity if it overflows +*/ +template +FloatType convert_float(const char* first, const char* last, std::size_t decimal_point_position, std::size_t mantissa_end) +{ + FloatType value{}; + if (!convert_float_fast(first, last, decimal_point_position, mantissa_end, value)) + { + std::string token(first, last); + convert_float_locale_aware(token, decimal_point_position, value); + } + return value; +} + } // namespace detail NLOHMANN_JSON_NAMESPACE_END diff --git a/include/nlohmann/detail/input/parser.hpp b/include/nlohmann/detail/input/parser.hpp index 3a53ff6b3..9f1618c84 100644 --- a/include/nlohmann/detail/input/parser.hpp +++ b/include/nlohmann/detail/input/parser.hpp @@ -370,7 +370,7 @@ class parser } case token_type::end_of_input: { - if (JSON_HEDLEY_UNLIKELY(m_lexer.get_position().chars_read_total == 1)) + if (JSON_HEDLEY_UNLIKELY(m_lexer.get_chars_read_total() == 1)) { return syntax_error(*sax, "attempting to parse an empty input; check that your input string or stream contains the expected JSON"); } @@ -520,8 +520,10 @@ class parser { // MSVC 2015 reports C4100 (unreferenced parameter) when SAX::parse_error is static static_cast(sax); - return sax.parse_error(m_lexer.get_position(), m_lexer.get_token_string(), - parse_error::create(exception_id::syntax_error, m_lexer.get_position(), message, nullptr)); + // computing the position is O(input read so far) for seekable input + const position_t position = m_lexer.get_position(); + return sax.parse_error(position, m_lexer.get_token_string(), + parse_error::create(exception_id::syntax_error, position, message, nullptr)); } std::string exception_message(const token_type expected, const std::string& context) diff --git a/include/nlohmann/detail/input/pow5_table.hpp b/include/nlohmann/detail/input/pow5_table.hpp index 496fdb2f7..c7170e903 100644 --- a/include/nlohmann/detail/input/pow5_table.hpp +++ b/include/nlohmann/detail/input/pow5_table.hpp @@ -22,6 +22,10 @@ namespace detail constexpr std::int64_t pow5_128_smallest_power = -342; constexpr std::int64_t pow5_128_largest_power = 308; +// every entry of pow5_128() holds two 64-bit halves of 5^q, one per covered power of 5 +static_assert((pow5_128_largest_power - pow5_128_smallest_power + 1) * 2 == 1302, + "pow5_128_smallest_power/pow5_128_largest_power must match the size of the pow5_128() table"); + /*! @brief 128-bit approximations of 5^q for q in [-342, 308] diff --git a/include/nlohmann/detail/input/string_scan.hpp b/include/nlohmann/detail/input/string_scan.hpp index 8a6f03122..fd2289b47 100644 --- a/include/nlohmann/detail/input/string_scan.hpp +++ b/include/nlohmann/detail/input/string_scan.hpp @@ -8,10 +8,12 @@ #pragma once +#include // array #include // size_t -#include // uint64_t +#include // uint64_t, uint8_t #include // memcpy +#include #include // Optional SIMD backend for bulk UTF-8 validation. This is an opt-in external @@ -69,18 +71,12 @@ inline std::size_t find_string_special(const unsigned char* data, std::size_t n) std::size_t i = 0; for (; i + 8 <= n; i += 8) { - std::uint64_t word = 0; - std::memcpy(&word, data + i, sizeof(word)); - if (swar_string_special(word) != 0) + const std::uint64_t special = swar_string_special(read_eight_bytes(data + i)); + if (special != 0) { - // a special byte is in this word; locate it (endian-agnostic) - for (std::size_t j = 0; j < 8; ++j) - { - if (is_string_special(data[i + j])) - { - return i + j; - } - } + // the lowest flagged byte is the first special one: the borrows of + // the subtractions can only flag bytes above a true hit + return i + (static_cast(count_trailing_zeros(special)) / 8); } } for (; i < n; ++i) @@ -114,8 +110,7 @@ inline std::size_t find_ascii_copyable_run(const unsigned char* data, std::size_ std::size_t i = 0; for (; i + 8 <= n; i += 8) { - std::uint64_t v = 0; - std::memcpy(&v, data + i, sizeof(v)); + const std::uint64_t v = read_eight_bytes(data + i); const std::uint64_t q = v ^ 0x2222222222222222ull; // '"' (0x22) const std::uint64_t b = v ^ 0x5C5C5C5C5C5C5C5Cull; // '\\' (0x5C) const std::uint64_t d = v ^ 0x7F7F7F7F7F7F7F7Full; // DEL (0x7F) @@ -126,7 +121,9 @@ inline std::size_t find_ascii_copyable_run(const unsigned char* data, std::size_ | (v & high); // >= 0x80 if (stop != 0) { - break; + // the lowest flagged byte is the first one to stop at (see + // find_string_special()) + return i + (static_cast(count_trailing_zeros(stop)) / 8); } } for (; i < n; ++i) @@ -253,12 +250,18 @@ inline std::size_t scalar_string_bulk_run(const unsigned char* data, std::size_t { break; // end of buffer, or a quote/escape/control byte } - const std::size_t seq = validate_one_utf8(data + pos, n - pos); - if (seq == 0) + // a run of multi-byte sequences (e.g. CJK text) is validated sequence + // by sequence without searching for the next special byte in between + do { - break; // ill-formed or truncated: let the byte path diagnose it + const std::size_t seq = validate_one_utf8(data + pos, n - pos); + if (seq == 0) + { + return pos; // ill-formed or truncated: let the byte path diagnose it + } + pos += seq; } - pos += seq; + while (pos < n && data[pos] >= 0x80u); } return pos; } @@ -273,8 +276,7 @@ inline std::size_t find_string_delimiter(const unsigned char* data, std::size_t std::size_t i = 0; for (; i + 8 <= n; i += 8) { - std::uint64_t v = 0; - std::memcpy(&v, data + i, sizeof(v)); + const std::uint64_t v = read_eight_bytes(data + i); const std::uint64_t q = v ^ 0x2222222222222222ull; const std::uint64_t b = v ^ 0x5C5C5C5C5C5C5C5Cull; const std::uint64_t hit = ((q - ones) & ~q & high) @@ -282,14 +284,8 @@ inline std::size_t find_string_delimiter(const unsigned char* data, std::size_t | ((v - 0x2020202020202020ull) & ~v & high); if (hit != 0) { - for (std::size_t j = 0; j < 8; ++j) - { - const unsigned char c = data[i + j]; - if (c == '\"' || c == '\\' || c < 0x20u) - { - return i + j; - } - } + // the lowest flagged byte is the first delimiter (see find_string_special()) + return i + (static_cast(count_trailing_zeros(hit)) / 8); } } for (; i < n; ++i) @@ -320,5 +316,50 @@ inline std::size_t string_bulk_run(const unsigned char* data, std::size_t n) noe return scalar_string_bulk_run(data, n); } +// Decode the 4 hex digits at [data, data+4) - the digits following a `\u` +// escape - into a codepoint 0x0000..0xFFFF via one table lookup per byte +// (after yyjson's read_hex_u16), or return -1 if any of the 4 bytes is not a +// hex digit ('0'..'9', 'A'..'F', 'a'..'f'). The caller must already have +// checked that 4 bytes are available; used by lexer::get_codepoint()'s +// contiguous fast path. On -1 it falls back to the byte-at-a-time loop, which +// stops at the first invalid digit, so the reported error and position are +// unaffected by this fast path. +inline int hex_codepoint(const unsigned char* data) noexcept +{ + static const std::array hex_digit_table = // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) + { + { + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // 00..0F + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // 10..1F + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // 20..2F + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // 30..3F ('0'..'9') + 0xFF, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // 40..4F ('A'..'F') + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // 50..5F + 0xFF, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // 60..6F ('a'..'f') + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // 70..7F + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // 80..8F + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // 90..9F + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // A0..AF + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // B0..BF + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // C0..CF + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // D0..DF + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // E0..EF + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF // F0..FF + } + }; + + const std::uint8_t d0 = hex_digit_table[data[0]]; + const std::uint8_t d1 = hex_digit_table[data[1]]; + const std::uint8_t d2 = hex_digit_table[data[2]]; + const std::uint8_t d3 = hex_digit_table[data[3]]; + // every valid digit is <= 0xF; the combined OR only exceeds it if at + // least one of the four bytes was not a hex digit (looked up as 0xFF) + if ((d0 | d1 | d2 | d3) > 0x0F) + { + return -1; + } + return (d0 << 12) | (d1 << 8) | (d2 << 4) | d3; +} + } // namespace detail NLOHMANN_JSON_NAMESPACE_END diff --git a/include/nlohmann/detail/macro_unscope.hpp b/include/nlohmann/detail/macro_unscope.hpp index 494cba0d8..edcb58337 100644 --- a/include/nlohmann/detail/macro_unscope.hpp +++ b/include/nlohmann/detail/macro_unscope.hpp @@ -21,6 +21,8 @@ #undef JSON_NO_UNIQUE_ADDRESS #undef JSON_DISABLE_ENUM_SERIALIZATION #undef JSON_DISABLE_TUPLE_REFERENCE_CONVERSION +#undef JSON_DTOA_SSE2 +#undef JSON_DTOA_NEON #ifndef JSON_TEST_KEEP_MACROS #undef JSON_CATCH diff --git a/include/nlohmann/detail/output/serializer.hpp b/include/nlohmann/detail/output/serializer.hpp index 64c4fcf03..9a8203695 100644 --- a/include/nlohmann/detail/output/serializer.hpp +++ b/include/nlohmann/detail/output/serializer.hpp @@ -1366,8 +1366,9 @@ class serializer /*! @brief dump an integer - Dump a given integer, appending it to @ref write_buffer. Works internally with - @a number_buffer. + Dump a given integer, appending it to @ref write_buffer (directly: copying + the digits from another buffer right after writing them waits until the + stores are done). @param[in] x integer number (signed or unsigned) to dump @tparam NumberType either @a number_integer_t or @a number_unsigned_t @@ -1402,33 +1403,57 @@ class serializer return; } - // use a pointer to fill the buffer - auto buffer_ptr = number_buffer.begin(); // NOLINT(llvm-qualified-auto,readability-qualified-auto) + // use a pointer to fill the buffer (room for as much as number_buffer holds) + if (JSON_HEDLEY_UNLIKELY(write_buffer_pos + number_buffer.size() > write_buffer.size())) + { + flush(); + } + auto* buffer_ptr = write_buffer.data() + write_buffer_pos; number_unsigned_t abs_value; - unsigned int n_chars{}; + // one byte for the minus sign + unsigned int n_chars = 0; if (is_negative_number(x)) { *buffer_ptr = '-'; abs_value = remove_sign(static_cast(x)); - - // account one more byte for the minus sign - n_chars = 1 + count_digits(abs_value); + n_chars = 1; } else { abs_value = static_cast(x); - n_chars = count_digits(abs_value); } + // up to 16 digits: eight at a time (as the digits of floats), written + // without leading zeros + if (abs_value < 10000000000000000u) + { + const std::uint64_t value = abs_value; + const std::uint64_t upper = value / 100000000u; + const std::uint64_t first = dtoa_impl::eight_digit_bytes(upper != 0 ? upper : value); + const auto leading = static_cast(count_leading_zeros(first) / 8); // (first is not 0) + char* const p = buffer_ptr + n_chars; + dtoa_impl::store_msb_first(p, (first << (8 * leading)) + 0x3030303030303030u); + n_chars += 8 - leading; + if (upper != 0) + { + dtoa_impl::store_msb_first(p + 8 - leading, dtoa_impl::eight_digit_bytes(value - (upper * 100000000u)) + 0x3030303030303030u); + n_chars += 8; + } + write_buffer_pos += n_chars; + return; + } + + n_chars += count_digits(abs_value); + // spare 1 byte for '\0' JSON_ASSERT(n_chars < number_buffer.size() - 1); // jump to the end to generate the string from backward, // so we later avoid reversing the result - buffer_ptr += static_cast(n_chars); + buffer_ptr += n_chars; // Fast int2ascii implementation inspired by "Fastware" talk by Andrei Alexandrescu // See: https://www.youtube.com/watch?v=o4-CwDo2zpg @@ -1451,14 +1476,13 @@ class serializer *(--buffer_ptr) = static_cast('0' + abs_value); } - put_buffer(number_buffer, n_chars); + write_buffer_pos += n_chars; } /*! @brief dump a floating-point number - Dump a given floating-point number, appending it to @ref write_buffer. Works internally - with @a number_buffer. + Dump a given floating-point number, appending it to @ref write_buffer. @param[in] x floating-point number to dump */ @@ -1485,10 +1509,15 @@ class serializer void dump_float(number_float_t x, std::true_type /*is_ieee_single_or_double*/) { - auto* begin = number_buffer.data(); + // directly into the write buffer: copying the text from number_buffer + // right after to_chars() wrote it waits until its stores are done + if (JSON_HEDLEY_UNLIKELY(write_buffer_pos + number_buffer.size() > write_buffer.size())) + { + flush(); + } + auto* begin = write_buffer.data() + write_buffer_pos; auto* end = ::nlohmann::detail::to_chars(begin, begin + number_buffer.size(), x); - - put_buffer(number_buffer, static_cast(end - begin)); + write_buffer_pos += static_cast(end - begin); } JSON_HEDLEY_NON_NULL(1) diff --git a/include/nlohmann/json.hpp b/include/nlohmann/json.hpp index b91fa3e27..3bad024f1 100644 --- a/include/nlohmann/json.hpp +++ b/include/nlohmann/json.hpp @@ -42,9 +42,14 @@ // each of these, not only the forward declarations from json_fwd.hpp, so IWYU's suggestion to // drop them (nothing in this file otherwise names the type) would break every downstream // translation unit that relies on basic_json<>'s defaults actually being usable. -#include // IWYU pragma: keep +// export: client code names what these headers declare (nlohmann::json and ordered_json from +// json_fwd.hpp, adl_serializer for specializations, json_ref behind brace initialization, the +// NLOHMANN_DEFINE_* and NLOHMANN_JSON_NAMESPACE_* macros, and the literals at the end of this file), and include analyzers must +// attribute it to this header, not tell clients to include the internal ones (issue #5812). +// tests/iwyu_consumer checks this. +#include // IWYU pragma: export #include -#include +#include // IWYU pragma: export #include // IWYU pragma: keep #include // IWYU pragma: keep #include @@ -61,8 +66,8 @@ #include #include #include -#include -#include +#include // IWYU pragma: export +#include // IWYU pragma: export #include #include #include @@ -74,7 +79,7 @@ #include #include #include -#include +#include // IWYU pragma: export #include // IWYU pragma: keep #include @@ -7449,6 +7454,176 @@ public: } public: + /// @brief creates a diff as a JSON Merge Patch + /// @sa https://json.nlohmann.me/api/basic_json/merge_diff/ + JSON_HEDLEY_WARN_UNUSED_RESULT + static basic_json merge_diff(const basic_json& source, const basic_json& target) + { + // unless both are objects, the patch is the target itself + if (!source.is_object() || !target.is_object()) + { + return target; + } + + return merge_diff_recursively(source, target, 0); + } + + private: + /// @brief two objects @ref merge_diff_iteratively is diffing + struct merge_diff_frame + { + merge_diff_frame(const basic_json* source_, const basic_json* target_) + : source(source_), target(target_), member(source_->cbegin()), patch(value_t::object) + {} + + // declared for GCC's -Weffc++, as in diff_frame + merge_diff_frame(const merge_diff_frame&) = default; + merge_diff_frame(merge_diff_frame&&) = default; + merge_diff_frame& operator=(const merge_diff_frame&) = default; + merge_diff_frame& operator=(merge_diff_frame&&) = default; + ~merge_diff_frame() = default; + + /// the objects being diffed + const basic_json* source; + const basic_json* target; + /// the member of source being diffed; advanced once it is done + const_iterator member; + /// the patch for the members of source diffed so far + basic_json patch; + }; + + /// @brief add the members of object @a target that @a source does not + /// have to the merge patch @a result + static void merge_diff_added_members(basic_json& result, const basic_json& source, const basic_json& target) + { + for (auto it = target.begin(); it != target.end(); ++it) + { + auto itf = source.find(it.key()); + if (itf == source.end()) + { + result[it.key()] = it.value(); + } + } + } + + /*! + @brief @ref merge_diff of two objects at nesting level @a depth + + Diffing two nested objects calls this function again, once per nesting + level, so values nested deeply enough would exhaust the call stack and + terminate the process. The descent is bounded here: once @ref + detail::recursion_depth_limit levels have been entered, @ref + merge_diff_iteratively diffs what is left without the call stack. + */ + static basic_json merge_diff_recursively(const basic_json& source, const basic_json& target, const std::size_t depth) + { + JSON_ASSERT(source.is_object() && target.is_object()); + + if (JSON_HEDLEY_UNLIKELY(depth >= detail::recursion_depth_limit())) + { + return merge_diff_iteratively(source, target); + } + + basic_json result(value_t::object); + + for (auto it = source.begin(); it != source.end(); ++it) + { + auto itf = target.find(it.key()); + if (itf != target.end()) + { + if (it.value().is_object() && itf.value().is_object()) + { + auto diff = merge_diff_recursively(it.value(), itf.value(), depth + 1); + if (!diff.empty()) + { + result[it.key()] = std::move(diff); + } + } + else if (it.value() != itf.value()) + { + result[it.key()] = itf.value(); + } + } + else + { + result[it.key()] = value_t::null; + } + } + + merge_diff_added_members(result, source, target); + return result; + } + + /*! + @brief @ref merge_diff of two objects without the call stack + + Produces the same patch as @ref merge_diff_recursively, keeping the objects + being diffed on an explicit stack instead of descending into them. Two + nested objects are still diffed completely before the next member, so the + patch has its members in the same order as in the recursive version. Only + reached for objects nested more deeply than @ref + detail::recursion_depth_limit. + */ + static basic_json merge_diff_iteratively(const basic_json& source, const basic_json& target) + { + // the patch of source and target, set once their frame is popped + basic_json result; + std::vector stack; + stack.emplace_back(&source, &target); + while (!stack.empty()) + { + // the frame's members are copied out and it is changed through + // stack.back(): pushing a frame for nested objects or popping + // this one would invalidate a reference to it + const basic_json* const s = stack.back().source; + const basic_json* const t = stack.back().target; + const const_iterator it = stack.back().member; + + if (it != s->cend()) + { + const const_iterator itf = t->find(it.key()); + if (itf != t->cend()) + { + if (it.value().is_object() && itf.value().is_object()) + { + // member is advanced once their patch is under its key + stack.emplace_back(&it.value(), &itf.value()); + continue; + } + if (it.value() != itf.value()) + { + stack.back().patch[it.key()] = itf.value(); + } + } + else + { + stack.back().patch[it.key()] = value_t::null; + } + ++stack.back().member; + continue; + } + + merge_diff_added_members(stack.back().patch, *s, *t); + + // these objects are done: continue with the ones they are in + basic_json patch = std::move(stack.back().patch); + stack.pop_back(); + if (stack.empty()) + { + result = std::move(patch); + } + else + { + if (!patch.empty()) + { + stack.back().patch[stack.back().member.key()] = std::move(patch); + } + ++stack.back().member; + } + } + return result; + } + /// @} }; @@ -7625,7 +7800,7 @@ struct formatter // NOLINT(cert-dcl58-c // where needed. #ifndef JSON_NO_AUTOMATIC_UDLS // NOLINTNEXTLINE(misc-header-include-cycle): json_literals.hpp includes this header - #include + #include // IWYU pragma: export #endif #endif // INCLUDE_NLOHMANN_JSON_HPP_ diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index cd7269529..df02c065c 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -42,6 +42,11 @@ // each of these, not only the forward declarations from json_fwd.hpp, so IWYU's suggestion to // drop them (nothing in this file otherwise names the type) would break every downstream // translation unit that relies on basic_json<>'s defaults actually being usable. +// export: client code names what these headers declare (nlohmann::json and ordered_json from +// json_fwd.hpp, adl_serializer for specializations, json_ref behind brace initialization, the +// NLOHMANN_DEFINE_* and NLOHMANN_JSON_NAMESPACE_* macros, and the literals at the end of this file), and include analyzers must +// attribute it to this header, not tell clients to include the internal ones (issue #5812). +// tests/iwyu_consumer checks this. // #include // __ _____ _____ _____ // __| | __| | | | JSON for Modern C++ @@ -7535,7 +7540,7 @@ struct adl_serializer }; NLOHMANN_JSON_NAMESPACE_END -// IWYU pragma: keep +// IWYU pragma: export // #include // __ _____ _____ _____ // __| | __| | | | JSON for Modern C++ @@ -7643,7 +7648,7 @@ class byte_container_with_subtype : public BinaryType NLOHMANN_JSON_NAMESPACE_END // #include - +// IWYU pragma: export // #include // IWYU pragma: keep // #include @@ -7965,12 +7970,12 @@ NLOHMANN_JSON_NAMESPACE_END -#include // min +#include // count, find, min #include // array #include // size_t #include // uint32_t #include // strlen -#include // begin, end, iterator_traits, random_access_iterator_tag, distance, next +#include // begin, end, iterator_traits, random_access_iterator_tag, distance, next, reverse_iterator #include // streambuf #include // string, char_traits #include // enable_if, is_base_of, is_pointer, is_integral, remove_pointer @@ -8259,6 +8264,7 @@ class iterator_input_adapter return result; } + ++eof_reads; return char_traits::eof(); } @@ -8268,6 +8274,26 @@ class iterator_input_adapter return static_cast(std::distance(begin, current)); } + // number of get_character() calls so far, including those that returned + // EOF; the lexer derives its position from it (see lexer::lazy_position) + std::size_t get_read_count() const + { + return get_consumed_count() + eof_reads; + } + + // number of newlines in the consumed range [0, last_index); @a line_start + // is set to the index after the last of them (0 if there is none); only + // valid when supports_seek + std::size_t count_consumed_lines(std::size_t last_index, std::size_t& line_start) const + { + const auto to = std::next(begin, static_cast::difference_type>(last_index)); + const auto newline = char_traits::to_char_type('\n'); + using reverse_it = std::reverse_iterator; + const auto last_newline = std::find(reverse_it(to), reverse_it(begin), newline); + line_start = static_cast(std::distance(begin, last_newline.base())); + return static_cast(std::count(begin, last_newline.base(), newline)); + } + // append the already-consumed characters in the half-open range // [first_index, last_index) to @a out; only valid when supports_seek template @@ -8384,6 +8410,8 @@ class iterator_input_adapter IteratorType begin; IteratorType current; SentinelType end; + /// number of get_character() calls that returned EOF + std::size_t eof_reads = 0; template friend struct wide_string_input_helper; @@ -8904,30 +8932,6 @@ NLOHMANN_JSON_NAMESPACE_END #include // move #include // vector -// #include - -// #include -// __ _____ _____ _____ -// __| | __| | | | JSON for Modern C++ -// | | |__ | | | | | | version 3.12.0 -// |_____|_____|_____|_|___| https://github.com/nlohmann/json -// -// SPDX-FileCopyrightText: 2021 The fast_float authors -// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann -// SPDX-License-Identifier: MIT - - - -#include // array -#include // FLT_EVAL_METHOD -#include // localeconv -#include // size_t -#include // int64_t, uint64_t -#include // strtof, strtod, strtold -#include // memcpy -#include // numeric_limits -#include // string - // #include // __ _____ _____ _____ // __| | __| | | | JSON for Modern C++ @@ -8940,12 +8944,13 @@ NLOHMANN_JSON_NAMESPACE_END #include // uint64_t -#if !defined(__SIZEOF_INT128__) && defined(_MSC_VER) && (defined(_M_X64) || defined(_M_ARM64)) - #include // __umulh, _umul128 +#include // memcpy +#if defined(_MSC_VER) && (defined(_M_X64) || defined(_M_ARM64)) && (!defined(__SIZEOF_INT128__) || (!defined(__GNUC__) && !defined(__clang__))) + #include // __umulh, _umul128, _BitScanForward64, _BitScanReverse64 #endif -// #include - +// #include +// JSON_HEDLEY_ALWAYS_INLINE, NLOHMANN_JSON_NAMESPACE_BEGIN // Portable bit-level helpers for the number and string scanners. They use // compiler builtins or platform-specific intrinsics where available and plain @@ -8960,6 +8965,10 @@ inline int count_leading_zeros(std::uint64_t x) noexcept { #if defined(__GNUC__) || defined(__clang__) return __builtin_clzll(x); +#elif defined(_MSC_VER) && (defined(_M_X64) || defined(_M_ARM64)) + unsigned long index = 0; // NOLINT(runtime/int): the type _BitScan*64 takes + _BitScanReverse64(&index, x); + return 63 - static_cast(index); #else int n = 0; for (int shift = 32; shift != 0; shift >>= 1) @@ -8974,6 +8983,29 @@ inline int count_leading_zeros(std::uint64_t x) noexcept #endif } +/// number of trailing zero bits of x (x != 0) +inline int count_trailing_zeros(std::uint64_t x) noexcept +{ +#if defined(__GNUC__) || defined(__clang__) + return __builtin_ctzll(x); +#elif defined(_MSC_VER) && (defined(_M_X64) || defined(_M_ARM64)) + unsigned long index = 0; // NOLINT(runtime/int): the type _BitScan*64 takes + _BitScanForward64(&index, x); + return static_cast(index); +#else + int n = 0; + for (int shift = 32; shift != 0; shift >>= 1) + { + if ((x << (64 - shift)) == 0) + { + n += shift; + x >>= shift; + } + } + return n; +#endif +} + /// the 128-bit product of two 64-bit numbers struct uint128_parts { @@ -9007,20 +9039,61 @@ inline uint128_parts full_multiplication(std::uint64_t a, std::uint64_t b) noexc #endif } -/// eight bytes as a little-endian word (compilers fold this into one load on -/// little-endian targets) -inline std::uint64_t read_eight_bytes(const char* p) noexcept +/// eight bytes as a little-endian word (a single load on little-endian +/// targets; always inlined, as GCC otherwise calls it in the number loops) +JSON_HEDLEY_ALWAYS_INLINE std::uint64_t read_eight_bytes(const unsigned char* b) noexcept { - const auto* b = reinterpret_cast(p); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) +#if defined(_MSC_VER) || defined(__x86_64__) || defined(__i386__) || (defined(__BYTE_ORDER__) && defined(__ORDER_LITTLE_ENDIAN__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__) + // the byte order already matches (all MSVC targets are little-endian) + std::uint64_t result = 0; + std::memcpy(&result, b, sizeof(result)); + return result; +#else return static_cast(b[0]) | (static_cast(b[1]) << 8u) | (static_cast(b[2]) << 16u) | (static_cast(b[3]) << 24u) | (static_cast(b[4]) << 32u) | (static_cast(b[5]) << 40u) | (static_cast(b[6]) << 48u) | (static_cast(b[7]) << 56u); +#endif +} + +/// eight bytes as a little-endian word +JSON_HEDLEY_ALWAYS_INLINE std::uint64_t read_eight_bytes(const char* p) noexcept +{ + return read_eight_bytes(reinterpret_cast(p)); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) } } // namespace detail NLOHMANN_JSON_NAMESPACE_END +// #include + +// #include +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ +// | | |__ | | | | | | version 3.12.0 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2021 The fast_float authors +// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann +// SPDX-License-Identifier: MIT + + + +#include // any_of +#include // array +#include // FLT_EVAL_METHOD +#include // localeconv +#include // size_t +#include // int64_t, uint64_t +#include // strtof, strtod, strtold +#include // memcpy +#include // numeric_limits +#include // string +#include // conditional, integral_constant, true_type, false_type +#include // move + +// #include + // #include // __ _____ _____ _____ // __| | __| | | | JSON for Modern C++ @@ -9047,6 +9120,10 @@ namespace detail constexpr std::int64_t pow5_128_smallest_power = -342; constexpr std::int64_t pow5_128_largest_power = 308; +// every entry of pow5_128() holds two 64-bit halves of 5^q, one per covered power of 5 +static_assert((pow5_128_largest_power - pow5_128_smallest_power + 1) * 2 == 1302, + "pow5_128_smallest_power/pow5_128_largest_power must match the size of the pow5_128() table"); + /*! @brief 128-bit approximations of 5^q for q in [-342, 308] @@ -9410,10 +9487,13 @@ NLOHMANN_JSON_NAMESPACE_END #endif // This file contains the value-conversion helpers used by the lexer to turn an -// already-validated number token into a value, without the locale/errno -// overhead of std::strtoull/std::strtod where possible. They are free functions -// so the lexer stays focused on scanning (see lexer::convert_number()) and so -// that other parsers of JSON text can convert tokens exactly like it does. +// already-validated number token into a value. Integers and binary32/binary64 +// floats (float, double, and long double where it is binary64) are converted +// by the library itself, without the locale/errno overhead of +// std::strtoull/std::strtod and correctly rounded; other long double formats +// use std::from_chars or std::strtold. They are free functions so the lexer +// stays focused on scanning (see lexer::convert_number()) and so that other +// parsers of JSON text can convert tokens exactly like it does. NLOHMANN_JSON_NAMESPACE_BEGIN namespace detail @@ -9490,233 +9570,291 @@ bool parse_integer_signed(const char* first, const char* last, NumberIntegerType } /*! -@brief exact fast path for parsing a `double` (Clinger's algorithm) - -For the common case - at most 19 significant digits, a decimal exponent in -[-22, 22], and a significand below 2^53 - the value equals significand * -10^exp computed in IEEE-754 double arithmetic, which is exact under -round-to-nearest because both operands are exactly representable. This is the -same fast path used by fast_float/simdjson; the general cases are left to -std::strtod. The parser only activates for number_float_t == double; float and -long double keep the std::strtof/std::strtold paths (see the templated overload -below). - -@param[in] first pointer to the first character of the number -@param[in] last pointer past the last character -@param[out] out the parsed value on success -@return true if the value was parsed exactly; false to fall back to strtod +@brief parameters of the IEEE-754 binary32 and binary64 formats for the float + conversion (after fast_float's binary_format) */ -inline bool parse_float_fast(const char* first, const char* last, double& out) noexcept +template +struct ieee_binary_format; + +template<> +struct ieee_binary_format<24> // binary32 { -#if defined(FLT_EVAL_METHOD) && FLT_EVAL_METHOD != 0 - // Clinger's fast path is only exact when double operations are evaluated in - // true double precision. On platforms that keep intermediates in extended - // precision (e.g. the x87 FPU on 32-bit x86, where FLT_EVAL_METHOD == 2) the - // single significand * 10^scale step is double-rounded and can be 1 ULP off, - // so decline and let the caller fall back to the correctly-rounded - // std::from_chars / std::strtod path. - static_cast(first); - static_cast(last); - static_cast(out); - return false; -#else - static const std::array powers_of_ten = + static constexpr int mantissa_bits() noexcept { - { - 1e0, 1e1, 1e2, 1e3, 1e4, 1e5, 1e6, 1e7, 1e8, 1e9, 1e10, 1e11, - 1e12, 1e13, 1e14, 1e15, 1e16, 1e17, 1e18, 1e19, 1e20, 1e21, 1e22 - } - }; + return 23; + } + static constexpr int sign_bit() noexcept + { + return 31; + } + static constexpr int minimum_exponent() noexcept + { + return -127; + } + static constexpr int infinite_power() noexcept + { + return 0xFF; + } + // w * 10^q with w < 2^64 is below half the smallest subnormal number for + // q < smallest_power_of_ten() and at least infinity for q > largest_power_of_ten() + static constexpr int smallest_power_of_ten() noexcept + { + return -64; + } + static constexpr int largest_power_of_ten() noexcept + { + return 38; + } + // w * 10^q can only be exactly between two numbers for q in this range + static constexpr int min_exponent_round_to_even() noexcept + { + return -17; + } + static constexpr int max_exponent_round_to_even() noexcept + { + return 10; + } + // Clinger's fast path: w and 10^|q| are exact + static constexpr int max_exponent_fast_path() noexcept + { + return 10; + } + static constexpr std::uint64_t max_mantissa_fast_path() noexcept + { + return std::uint64_t{2} << 23u; + } + // a midpoint between two numbers has at most this many significant digits + static constexpr std::int64_t max_digits() noexcept + { + return 114; + } +}; - const char* p = first; - bool negative = false; - if (p != last && (*p == '-' || *p == '+')) - { - negative = (*p == '-'); - ++p; - } - - std::uint64_t significand = 0; - int num_digits = 0; - int fractional_digits = 0; - bool seen_dot = false; - bool any_digit = false; - for (; p != last; ++p) - { - const char c = *p; - if (c >= '0' && c <= '9') - { - any_digit = true; - if (JSON_HEDLEY_UNLIKELY(num_digits >= 19)) - { - return false; // significand may not fit into uint64_t - } - significand = (significand * 10u) + static_cast(c - '0'); - ++num_digits; - fractional_digits += static_cast(seen_dot); - } - else if (c == '.') - { - if (JSON_HEDLEY_UNLIKELY(seen_dot)) - { - return false; - } - seen_dot = true; - } - else if (c == 'e' || c == 'E') - { - ++p; - break; - } - else - { - return false; - } - } - if (JSON_HEDLEY_UNLIKELY(!any_digit)) - { - return false; - } - - int exponent = 0; - if (p != last) // an exponent part remains - { - bool exp_negative = false; - if (p != last && (*p == '-' || *p == '+')) - { - exp_negative = (*p == '-'); - ++p; - } - bool any_exp_digit = false; - for (; p != last; ++p) - { - if (JSON_HEDLEY_UNLIKELY(*p < '0' || *p > '9')) - { - return false; - } - exponent = (exponent * 10) + (*p - '0'); - any_exp_digit = true; - if (JSON_HEDLEY_UNLIKELY(exponent > 9999)) - { - return false; - } - } - if (JSON_HEDLEY_UNLIKELY(!any_exp_digit)) - { - return false; - } - if (exp_negative) - { - exponent = -exponent; - } - } - - const int scale = exponent - fractional_digits; - if (JSON_HEDLEY_UNLIKELY(significand >= (static_cast(1) << 53))) - { - return false; // significand not exactly representable as double - } - - auto result = static_cast(significand); - if (scale >= 0) - { - if (JSON_HEDLEY_UNLIKELY(scale > 22)) - { - return false; - } - result *= powers_of_ten[static_cast(scale)]; - } - else - { - if (JSON_HEDLEY_UNLIKELY(-scale > 22)) - { - return false; - } - result /= powers_of_ten[static_cast(-scale)]; - } - out = negative ? -result : result; - return true; -#endif -} - -/// fast float path is only exact for `double`; decline for float/long double -template -bool parse_float_fast(const char* /*first*/, const char* /*last*/, FloatType& /*out*/) noexcept +template<> +struct ieee_binary_format<53> // binary64 { - return false; -} + static constexpr int mantissa_bits() noexcept + { + return 52; + } + static constexpr int sign_bit() noexcept + { + return 63; + } + static constexpr int minimum_exponent() noexcept + { + return -1023; + } + static constexpr int infinite_power() noexcept + { + return 0x7FF; + } + static constexpr int smallest_power_of_ten() noexcept + { + return -342; + } + static constexpr int largest_power_of_ten() noexcept + { + return 308; + } + static constexpr int min_exponent_round_to_even() noexcept + { + return -4; + } + static constexpr int max_exponent_round_to_even() noexcept + { + return 23; + } + static constexpr int max_exponent_fast_path() noexcept + { + return 22; + } + static constexpr std::uint64_t max_mantissa_fast_path() noexcept + { + return std::uint64_t{2} << 52u; + } + static constexpr std::int64_t max_digits() noexcept + { + return 769; + } +}; /*! -@brief parse a float with std::from_chars (Eisel-Lemire) when available +@brief whether @a FloatType is IEEE-754 binary32 or binary64 -std::from_chars is locale-independent, correctly rounded, and - via the -Eisel-Lemire algorithm in modern standard libraries - much faster than strtod -over the whole value range (not just the Clinger subset). It is used only when -__cpp_lib_to_chars indicates full floating-point support and only when it -consumes the entire token ([first, last)). An under-/overflow (result_out_of_range) also declines, so -the caller's strtod fallback supplies the well-defined ±inf/0 result the parser -expects (side-stepping the P4168 divergence between implementations). - -@return true if the value was parsed exactly and fully; false to fall back +These formats (float, double, and long double where it is binary64, e.g. +with MSVC or on Apple arm64) are converted by parse_float_native(). The +predicate is the one the serializer uses to choose Grisu2. */ template -bool parse_float_from_chars(const char* first, const char* last, FloatType& out) noexcept +struct has_native_float_format { - // JSON_HAS_CPP_17 must gate the use as well as the include above: - // some standard libraries (e.g. libstdc++ 15) define __cpp_lib_to_chars even - // in C++14 mode, where is not included. -#if defined(JSON_HAS_CPP_17) && defined(__cpp_lib_to_chars) - const auto result = std::from_chars(first, last, out); - return result.ec == std::errc() && result.ptr == last; -#else - static_cast(first); - static_cast(last); - static_cast(out); - return false; -#endif -} + static constexpr bool value = + (std::numeric_limits::is_iec559 && std::numeric_limits::digits == 24 && std::numeric_limits::max_exponent == 128) || + (std::numeric_limits::is_iec559 && std::numeric_limits::digits == 53 && std::numeric_limits::max_exponent == 1024); +}; -/// whether the eight bytes of @a v (see read_eight_bytes()) are ASCII digits -/// (after fast_float's is_made_of_eight_digits_fast) -inline bool is_eight_digits(std::uint64_t v) noexcept -{ - return ((v & 0xF0F0F0F0F0F0F0F0u) | (((v + 0x0606060606060606u) & 0xF0F0F0F0F0F0F0F0u) >> 4u)) == 0x3333333333333333u; -} +/// the C++ type (float or double) that holds a binary32 or binary64 @a FloatType +template +using native_float_t = typename std::conditional::digits == 24, float, double>::type; /// the value of the eight ASCII digits in @a v (see read_eight_bytes()), three -/// multiplications instead of eight (after simdjson and fast_float) -inline std::uint32_t parse_eight_digits(std::uint64_t v) noexcept +/// multiplications instead of eight (after simdjson and fast_float); always +/// inlined, as GCC otherwise calls it in the number loops +JSON_HEDLEY_ALWAYS_INLINE std::uint32_t parse_eight_digits(std::uint64_t v) noexcept { v = ((v & 0x0F0F0F0F0F0F0F0Fu) * 2561u) >> 8u; v = ((v & 0x00FF00FF00FF00FFu) * 6553601u) >> 16u; return static_cast(((v & 0x0000FFFF0000FFFFu) * 42949672960001u) >> 32u); } +/// whether [first, last) contains a digit other than '0' +inline bool has_nonzero_digit(const char* first, const char* last) noexcept +{ + return std::any_of(first, last, [](char c) noexcept + { + return c != '0'; + }); +} + +/// the value of the validated exponent digits [+-]?[0-9]+ in [first, last), +/// saturated far beyond every range +inline std::int64_t parse_float_exponent(const char* first, const char* last) noexcept +{ + const bool negative = *first == '-'; + first += (*first == '-' || *first == '+') ? 1 : 0; + constexpr std::int64_t saturation = 100000000000000000; // 10^17 + std::int64_t value = 0; + for (; first != last; ++first) + { + if (value < saturation) + { + value = (value * 10) + (*first - '0'); + } + } + return negative ? -value : value; +} + +/// a float token as w * 10^exponent, see parse_float_significand() +struct float_significand +{ + std::uint64_t w = 0; ///< the first (at most 19) significant digits + std::int64_t exponent = 0; ///< the decimal exponent of the last digit in w + bool negative = false; ///< whether the token starts with '-' + bool truncated = false; ///< whether nonzero digits follow the ones in w +}; + /*! -@brief the double nearest to w * 10^q (Eisel-Lemire) +@brief split a validated number token into sign, significand, and exponent + +The lexer has validated the token against the JSON grammar and knows where its +parts are, so this needs no character classification: the integer part ends at +@a decimal_point_position (or @a mantissa_end), the fraction at @a mantissa_end, +and an exponent follows. At most 19 significant digits are kept; the value then +lies in [w, w + 1) * 10^exponent, and is exactly w * 10^exponent unless +truncated is set. + +@param[in] first pointer to the first character of the token +@param[in] last pointer past the last character +@param[in] decimal_point_position index of the '.' in the token, or + std::string::npos if there is none +@param[in] mantissa_end index of the 'e'/'E', or the token length +*/ +inline float_significand parse_float_significand(const char* first, const char* last, + std::size_t decimal_point_position, std::size_t mantissa_end) noexcept +{ + float_significand s; + const char* p = first; + s.negative = *p == '-'; + p += s.negative ? 1 : 0; + const bool has_dot = decimal_point_position != std::string::npos; + const char* const mantissa_last = first + mantissa_end; + const char* const integer_last = has_dot ? first + decimal_point_position : mantissa_last; + + std::uint64_t w = 0; + int remaining = 19; // digits that still fit into w + if (*p != '0') // the integer part is "0" or [1-9][0-9]* + { + while (remaining >= 8 && integer_last - p >= 8) + { + w = (w * 100000000u) + parse_eight_digits(read_eight_bytes(p)); + p += 8; + remaining -= 8; + } + for (; remaining > 0 && p != integer_last; ++p, --remaining) + { + w = (w * 10u) + static_cast(*p - '0'); + } + s.exponent = integer_last - p; + s.truncated = has_nonzero_digit(p, integer_last); + } + + if (has_dot) + { + p = integer_last + 1; + if (w == 0) + { + // zeros after the decimal point of "0." are not significant + const char* const zeros = p; + while (p != mantissa_last && *p == '0') + { + ++p; + } + s.exponent -= p - zeros; + } + const char* const digits = p; + while (remaining >= 8 && mantissa_last - p >= 8) + { + w = (w * 100000000u) + parse_eight_digits(read_eight_bytes(p)); + p += 8; + remaining -= 8; + } + for (; remaining > 0 && p != mantissa_last; ++p, --remaining) + { + w = (w * 10u) + static_cast(*p - '0'); + } + s.exponent -= p - digits; + s.truncated = s.truncated || has_nonzero_digit(p, mantissa_last); + } + + if (mantissa_last != last) + { + s.exponent += parse_float_exponent(mantissa_last + 1, last); + } + s.w = w; + return s; +} + +/*! +@brief the bits of the float nearest to w * 10^q (Eisel-Lemire) The algorithm of Daniel Lemire, "Number Parsing at a Gigabyte per Second" (Software: Practice and Experience, 2021), after fast_float's compute_float (used under the MIT license). With a 128-bit approximation of 5^q, the product -is always sufficient to round correctly for w with at most 19 digits (Noble -Mushtak and Daniel Lemire, "Fast number parsing without fallback", Software: -Practice and Experience, 2023). Only integer arithmetic is used, so the result -does not depend on the floating-point environment. +is always sufficient to round correctly for w < 2^64 (Noble Mushtak and Daniel +Lemire, "Fast number parsing without fallback", Software: Practice and +Experience, 2023). Only integer arithmetic is used, so the result does not +depend on the floating-point environment. +It is always inlined, like decimal_to_float(), so that hot loops of callers +keep the whole conversion inline. + +@tparam Format ieee_binary_format<24> (binary32) or ieee_binary_format<53> (binary64) @param[in] q decimal exponent -@param[in] w significand, w != 0 +@param[in] w significand @return the IEEE-754 bits of the positive result (0 for underflow, infinity for overflow) */ -inline std::uint64_t eisel_lemire(std::int64_t q, std::uint64_t w) noexcept +template +JSON_HEDLEY_ALWAYS_INLINE std::uint64_t eisel_lemire(std::int64_t q, std::uint64_t w) noexcept { - constexpr int mantissa_bits = 52; - constexpr std::uint64_t infinity = std::uint64_t{0x7FF} << mantissa_bits; - if (q < pow5_128_smallest_power) + constexpr int mantissa_bits = Format::mantissa_bits(); + constexpr std::uint64_t infinity = static_cast(Format::infinite_power()) << mantissa_bits; + if (w == 0 || q < Format::smallest_power_of_ten()) { return 0; } - if (q > pow5_128_largest_power) + if (q > Format::largest_power_of_ten()) { return infinity; } @@ -9740,8 +9878,8 @@ inline std::uint64_t eisel_lemire(std::int64_t q, std::uint64_t w) noexcept const auto upperbit = static_cast(product.high >> 63u); const int shift = upperbit + 64 - mantissa_bits - 3; std::uint64_t mantissa = product.high >> static_cast(shift); - // floor(log2(10^q)) + 63 + 1023, with log2(10) ~ 217706 / 2^16 - std::int64_t power2 = (((152170 + 65536) * q) >> 16) + 63 + upperbit - lz + 1023; + // floor(log2(10^q)) + 63 + bias, with log2(10) ~ 217706 / 2^16 + std::int64_t power2 = (((152170 + 65536) * q) >> 16) + 63 + upperbit - lz - Format::minimum_exponent(); if (power2 <= 0) // subnormal { @@ -9750,17 +9888,18 @@ inline std::uint64_t eisel_lemire(std::int64_t q, std::uint64_t w) noexcept return 0; } mantissa >>= static_cast(-power2 + 1); + // no tie is possible here: that needs a small |q| mantissa += (mantissa & 1u); mantissa >>= 1u; // rounding up may produce the smallest normal number power2 = (mantissa < (std::uint64_t{1} << mantissa_bits)) ? 0 : 1; - return mantissa | (static_cast(power2) << mantissa_bits); + return (mantissa & ((std::uint64_t{1} << mantissa_bits) - 1)) | (static_cast(power2) << mantissa_bits); } - // a value exactly between two doubles rounds to even; this can only + // a value exactly between two floats rounds to even; this can only // happen for small |q|, where 5^q is exact - if (product.low <= 1 && q >= -4 && q <= 23 && (mantissa & 3u) == 1 - && (mantissa << static_cast(shift)) == product.high) + if (product.low <= 1 && q >= Format::min_exponent_round_to_even() && q <= Format::max_exponent_round_to_even() + && (mantissa & 3u) == 1 && (mantissa << static_cast(shift)) == product.high) { mantissa &= ~std::uint64_t{1}; } @@ -9772,198 +9911,420 @@ inline std::uint64_t eisel_lemire(std::int64_t q, std::uint64_t w) noexcept ++power2; } mantissa &= ~(std::uint64_t{1} << mantissa_bits); - if (power2 >= 0x7FF) + if (power2 >= Format::infinite_power()) { return infinity; } return mantissa | (static_cast(power2) << mantissa_bits); } -/*! -@brief parse a validated float token with the Eisel-Lemire algorithm - -The significand is accumulated eight digits at a time where possible. A token -with more than 19 significant digits is truncated to w; the value then lies -in [w, w + 1) * 10^q, and it is only returned if both ends round to the same -double, which covers all but a few such tokens. - -@param[in] first pointer to the first character of the token -@param[in] last pointer past the last character -@param[out] out the correctly rounded value on success (±infinity if it - overflows, like strtod) -@return true on success; false if strtod must decide -*/ -inline bool parse_float_eisel_lemire(const char* first, const char* last, double& out) noexcept +/// an unsigned integer of up to 4096 bits for digit_comparison() (32-bit limbs, +/// so only 32x32->64-bit multiplications are needed) +class float_bigint { - const char* p = first; - const bool negative = (p != last && *p == '-'); - if (negative) + public: + explicit float_bigint(std::uint64_t value) noexcept { - ++p; + for (; value != 0; value >>= 32u) + { + limbs[count++] = static_cast(value); + } } - std::uint64_t w = 0; - unsigned int digits = 0; // significant digits in w - std::int64_t exponent = 0; - bool truncated = false; - bool in_fraction = false; - for (;;) + /// *this = *this * factor + summand + void multiply_add(std::uint32_t factor, std::uint32_t summand) noexcept { - // eight digits at a time, as long as they fit into w - while (w != 0 && digits <= 19u - 8u && last - p >= 8) + std::uint64_t carry = summand; + for (std::size_t i = 0; i < count; ++i) { - const std::uint64_t v = read_eight_bytes(p); - if (!is_eight_digits(v)) - { - break; - } - w = (w * 100000000u) + parse_eight_digits(v); - digits += 8u; - exponent -= in_fraction ? 8 : 0; - p += 8; + const std::uint64_t product = (static_cast(limbs[i]) * factor) + carry; + limbs[i] = static_cast(product); + carry = product >> 32u; } - if (p == last) + if (carry != 0) { - break; + JSON_ASSERT(count < limbs.size()); + limbs[count++] = static_cast(carry); } - const char c = *p; - if (c >= '0' && c <= '9') + } + + /// *this = *this * 5^n + void multiply_power_of_five(std::int64_t n) noexcept + { + static const std::array powers = { - if (w == 0 && c == '0') - { - // leading zeros are not significant, but scale a fraction - exponent -= in_fraction ? 1 : 0; - } - else if (digits < 19u) - { - w = (w * 10u) + static_cast(c - '0'); - ++digits; - exponent -= in_fraction ? 1 : 0; - } - else - { - // dropped: the value lies between w and w + 1 (in units of - // the last kept digit) unless all dropped digits are zero - truncated = truncated || c != '0'; - exponent += in_fraction ? 0 : 1; - } - ++p; + {1u, 5u, 25u, 125u, 625u, 3125u, 15625u, 78125u, 390625u, 1953125u, 9765625u, 48828125u, 244140625u, 1220703125u} + }; + for (; n >= 13; n -= 13) + { + multiply_add(powers[13], 0); } - else if (c == '.') + multiply_add(powers[static_cast(n)], 0); + } + + /// *this = *this * 2^n + void shift_left(std::int64_t n) noexcept + { + if (count == 0) { - in_fraction = true; - ++p; + return; + } + const auto limb_shift = static_cast(n / 32); + const auto bit_shift = static_cast(n % 32); + JSON_ASSERT(count + limb_shift + 1 <= limbs.size()); + if (bit_shift != 0) + { + std::uint32_t carry = 0; + for (std::size_t i = 0; i < count; ++i) + { + const std::uint32_t limb = limbs[i]; + limbs[i] = (limb << bit_shift) | carry; + carry = limb >> (32u - bit_shift); + } + if (carry != 0) + { + limbs[count++] = carry; + } + } + if (limb_shift != 0) + { + for (std::size_t i = count; i-- > 0;) + { + limbs[i + limb_shift] = limbs[i]; + } + for (std::size_t i = 0; i < limb_shift; ++i) + { + limbs[i] = 0; + } + count += limb_shift; + } + } + + /// -1, 0, or 1 if *this is less than, equal to, or greater than @a other + int compare(const float_bigint& other) const noexcept + { + if (count != other.count) + { + return count < other.count ? -1 : 1; + } + for (std::size_t i = count; i-- > 0;) + { + if (limbs[i] != other.limbs[i]) + { + return limbs[i] < other.limbs[i] ? -1 : 1; + } + } + return 0; + } + + private: + std::array limbs{{}}; + std::size_t count = 0; +}; + +/*! +@brief round a token exactly when eisel_lemire() cannot decide (slow path) + +The value v of the token lies strictly between two adjacent floats, whose +lower one has the bits @a lower, and the result depends on whether v is below, +at, or above the midpoint m between them. Both are compared exactly as big +integers: v = D * 10^s with the significant digits D (at most +Format::max_digits() of them, more than any midpoint has; further nonzero +digits only put v above m) and m = (2 * mantissa + 1) * 2^(e - 1). This is the +digit comparison of fast_float (Daniel Lemire and contributors, used under the +MIT license), simplified by starting from the two candidates. + +@param[in] first pointer to the first character of the token +@param[in] last pointer past the last character +@param[in] lower the bits of the float below v +@return the bits of the correctly rounded result +*/ +template +std::uint64_t digit_comparison(const char* first, const char* last, std::uint64_t lower) noexcept +{ + const char* p = first + ((*first == '-') ? 1 : 0); + + // D, in chunks of up to 9 digits, and s + float_bigint digits(0); + std::int64_t count = 0; + std::int64_t point = 0; // the value is 0.D... * 10^point + bool truncated = false; + std::uint32_t chunk = 0; + int chunk_digits = 0; + static const std::array powers_of_ten = {{1u, 10u, 100u, 1000u, 10000u, 100000u, 1000000u, 10000000u, 100000000u, 1000000000u}}; + const auto append = [&](char c) noexcept + { + if (count < Format::max_digits()) + { + chunk = (chunk * 10u) + static_cast(c - '0'); + ++count; + if (++chunk_digits == 9) + { + digits.multiply_add(powers_of_ten[9], chunk); + chunk = 0; + chunk_digits = 0; + } } else { - break; // 'e' or 'E' + truncated = truncated || c != '0'; + } + }; + bool significant = false; + for (; p != last && *p >= '0' && *p <= '9'; ++p) + { + significant = significant || *p != '0'; + if (significant) + { + append(*p); + ++point; } } - - if (p != last) + if (p != last && *p == '.') { - ++p; // 'e' or 'E' - bool exp_negative = false; - if (p != last && (*p == '-' || *p == '+')) + for (++p; p != last && *p >= '0' && *p <= '9'; ++p) { - exp_negative = (*p == '-'); - ++p; - } - std::int64_t exp_value = 0; - for (; p != last; ++p) - { - // saturate: any exponent beyond this under- or overflows anyway - if (exp_value < 100000) + significant = significant || *p != '0'; + if (significant) { - exp_value = (exp_value * 10) + (*p - '0'); + append(*p); + } + else + { + --point; } } - exponent += exp_negative ? -exp_value : exp_value; } - - std::uint64_t bits = 0; - if (w != 0) + if (chunk_digits != 0) { - bits = eisel_lemire(exponent, w); - if (truncated && (w + 1 == 0 || eisel_lemire(exponent, w + 1) != bits)) - { - return false; - } + digits.multiply_add(powers_of_ten[static_cast(chunk_digits)], chunk); } - bits |= negative ? (std::uint64_t{1} << 63u) : 0u; - static_assert(sizeof(double) == sizeof(std::uint64_t), "double must have 64 bits"); - std::memcpy(&out, &bits, sizeof(out)); - return true; + if (p != last) + { + point += parse_float_exponent(p + 1, last); + } + const std::int64_t s = point - count; // v = D * 10^s + + // the midpoint above the lower candidate + constexpr int mantissa_bits = Format::mantissa_bits(); + const std::uint64_t exponent_field = lower >> mantissa_bits; + std::uint64_t mantissa = lower & ((std::uint64_t{1} << mantissa_bits) - 1); + std::int64_t e = 1 + Format::minimum_exponent() - mantissa_bits; // of the smallest subnormal number + if (exponent_field != 0) + { + mantissa |= std::uint64_t{1} << mantissa_bits; + e += static_cast(exponent_field) - 1; + } + float_bigint midpoint((2 * mantissa) + 1); + const std::int64_t midpoint_exponent = e - 1; // m = midpoint * 2^midpoint_exponent + + // compare D * 5^s * 2^s with midpoint * 2^midpoint_exponent + if (s >= 0) + { + digits.multiply_power_of_five(s); + } + else + { + midpoint.multiply_power_of_five(-s); + } + const std::int64_t shift = s - midpoint_exponent; + if (shift >= 0) + { + digits.shift_left(shift); + } + else + { + midpoint.shift_left(-shift); + } + const int order = digits.compare(midpoint); + const bool round_up = order > 0 || (order == 0 && (truncated || (mantissa & 1u) != 0)); + return lower + (round_up ? 1u : 0u); } -/// Eisel-Lemire is only implemented for `double` -template -bool parse_float_eisel_lemire(const char* /*first*/, const char* /*last*/, FloatType& /*out*/) noexcept +/// the double with the IEEE-754 bits @a bits +inline void float_from_bits(std::uint64_t bits, double& value) noexcept { - return false; + static_assert(sizeof(double) == sizeof(std::uint64_t), "double must have 64 bits"); + std::memcpy(&value, &bits, sizeof(value)); +} + +/// the float with the IEEE-754 bits @a bits (the lower 32) +inline void float_from_bits(std::uint64_t bits, float& value) noexcept +{ + static_assert(sizeof(float) == sizeof(std::uint32_t), "float must have 32 bits"); + const auto bits32 = static_cast(bits); + std::memcpy(&value, &bits32, sizeof(value)); +} + +/// the powers of ten that are exact in binary64 (up to 10^22) +inline double exact_power_of_ten(std::int64_t n, double /*tag*/) noexcept +{ + static const std::array powers = + { + { + 1e0, 1e1, 1e2, 1e3, 1e4, 1e5, 1e6, 1e7, 1e8, 1e9, 1e10, 1e11, + 1e12, 1e13, 1e14, 1e15, 1e16, 1e17, 1e18, 1e19, 1e20, 1e21, 1e22 + } + }; + return powers[static_cast(n)]; +} + +/// the powers of ten that are exact in binary32 (up to 10^10) +inline float exact_power_of_ten(std::int64_t n, float /*tag*/) noexcept +{ + static const std::array powers = + { + {1e0f, 1e1f, 1e2f, 1e3f, 1e4f, 1e5f, 1e6f, 1e7f, 1e8f, 1e9f, 1e10f} + }; + return powers[static_cast(n)]; } /*! -@brief check whether Clinger's fast path can still succeed for a float token +@brief the binary32/binary64 value of a significand that was not truncated -parse_float_fast() needs a significand below 2^53. A mantissa with 17 or -more significant digits is at least 10^16 and therefore always exceeds it, -so calling the fast path would walk the token one extra time only to -decline before strtod has to run anyway. +The result is (-1)^negative * w * 10^exponent, correctly rounded (ties to +even): with Clinger's fast path where w and 10^|exponent| are exact, so that a +single floating-point operation rounds (only where intermediate results are not +kept in extended precision, see FLT_EVAL_METHOD), and with eisel_lemire() +otherwise. A value too large for the type becomes ±infinity, a value too small +±0. -Significant digits are the mantissa's digits from the first nonzero one on; -the sign, the decimal point, leading zeros, and the exponent do not count. -The answer is derived from indices - the digits are not scanned again - so -this stays off the hot path of the number scanners. +This is the core of the conversion that other parsers of JSON text share: they +can split a token themselves and still get the lexer's result. It is always +inlined, so that their hot loops keep the whole conversion inline. -@param[in] token the validated number token ('.' as decimal point) -@param[in] decimal_point_position index of the '.' in @a token, or - std::string::npos if there is none -@param[in] mantissa_end offset just past the last mantissa byte -@return false if parse_float_fast() is guaranteed to decline +@param[in] s the significand, with s.truncated == false */ -inline bool mantissa_fits_clinger(const char* token, std::size_t decimal_point_position, std::size_t mantissa_end) noexcept +template +JSON_HEDLEY_ALWAYS_INLINE FloatType decimal_to_float(const float_significand& s) noexcept { - // 10^16 already exceeds 2^53, so 17 digits can never fit - constexpr std::size_t limit = 17; + using result_type = native_float_t; + using format = ieee_binary_format::digits>; + JSON_ASSERT(!s.truncated); - const std::size_t neg = (token[0] == '-') ? 1u : 0u; - const std::size_t has_dot = (decimal_point_position != std::string::npos) ? 1u : 0u; - // the JSON grammar restricts the integer part to "0" or [1-9][0-9]*, so - // a leading zero can only be a lone "0", which is not significant - const std::size_t lead_zero = (token[neg] == '0') ? 1u : 0u; - JSON_ASSERT(mantissa_end >= neg + has_dot + lead_zero); - std::size_t digits = mantissa_end - neg - has_dot - lead_zero; - - if (JSON_HEDLEY_LIKELY(digits < limit)) +#if !defined(FLT_EVAL_METHOD) || FLT_EVAL_METHOD == 0 + if (s.exponent >= -format::max_exponent_fast_path() && s.exponent <= format::max_exponent_fast_path() + && s.w <= format::max_mantissa_fast_path()) { - return true; - } - - // Only a number below 1 can carry further insignificant zeros, and only - // while the count stays at the limit does removing them change the - // answer - so this loop is skipped for all but a few tokens. The - // fraction is located through decimal_point_position rather than by - // searching '.'. - if (lead_zero != 0) - { - JSON_ASSERT(has_dot != 0); // an integer "0" cannot reach the limit - for (std::size_t i = decimal_point_position + 1; - digits >= limit && i < mantissa_end && token[i] == '0'; ++i) + auto value = static_cast(s.w); + if (s.exponent < 0) { - --digits; + value /= exact_power_of_ten(-s.exponent, result_type{}); } + else + { + value *= exact_power_of_ten(s.exponent, result_type{}); + } + const FloatType result = s.negative ? -value : value; + return result; + } +#endif + + result_type value{}; + float_from_bits(eisel_lemire(s.exponent, s.w) | (s.negative ? (std::uint64_t{1} << format::sign_bit()) : 0u), value); + const FloatType result = value; + return result; +} + +/*! +@brief convert a validated number token to the nearest binary32/binary64 value + +The conversion is correctly rounded (ties to even) and independent of the +locale and of the C and C++ libraries: +1. parse_float_significand() splits the token into w * 10^q. +2. If no digits were dropped, decimal_to_float() rounds w * 10^q (Clinger's + fast path or Eisel-Lemire). +3. Otherwise, the value lies in [w, w + 1) * 10^q: if eisel_lemire() rounds + both ends to the same value, so does the token (in all but rare cases). +4. Otherwise, digit_comparison() compares the token exactly with the midpoint + between the two candidates. +A value too large for the type becomes ±infinity (the parser reports +out_of_range.406), a value too small ±0. + +@param[in] first pointer to the first character of the token +@param[in] last pointer past the last character +@param[in] decimal_point_position index of the '.' in the token, or + std::string::npos if there is none +@param[in] mantissa_end index of the 'e'/'E', or the token length +*/ +template +FloatType parse_float_native(const char* first, const char* last, + std::size_t decimal_point_position, std::size_t mantissa_end) noexcept +{ + using result_type = native_float_t; + using format = ieee_binary_format::digits>; + static_assert(std::numeric_limits::digits == std::numeric_limits::digits, "unexpected float format"); + + const float_significand s = parse_float_significand(first, last, decimal_point_position, mantissa_end); + if (JSON_HEDLEY_LIKELY(!s.truncated)) + { + return decimal_to_float(s); } - return digits < limit; + std::uint64_t bits = eisel_lemire(s.exponent, s.w); + if (JSON_HEDLEY_UNLIKELY(bits != eisel_lemire(s.exponent, s.w + 1))) + { + bits = digit_comparison(first, last, bits); + } + result_type value{}; + float_from_bits(bits | (s.negative ? (std::uint64_t{1} << format::sign_bit()) : 0u), value); + const FloatType result = value; + return result; +} + +/*! +@brief parse a float with std::from_chars when available + +Only used for the formats parse_float_native() does not convert (long double +formats other than binary64). std::from_chars is locale-independent and +correctly rounded. It is used only when __cpp_lib_to_chars indicates full +floating-point support and only when it consumes the entire token ([first, +last)). An under-/overflow (result_out_of_range) also declines, so the +caller's strtold fallback supplies the well-defined ±inf/0 result the parser +expects (side-stepping the P4168 divergence between implementations). + +@return true if the value was parsed exactly and fully; false to fall back +*/ +template +bool parse_float_from_chars(const char* first, const char* last, FloatType& out) noexcept +{ + // JSON_HAS_CPP_17 must gate the use as well as the include above: + // some standard libraries (e.g. libstdc++ 15) define __cpp_lib_to_chars even + // in C++14 mode, where is not included. +#if defined(JSON_HAS_CPP_17) && defined(__cpp_lib_to_chars) + const auto result = std::from_chars(first, last, out); + return result.ec == std::errc() && result.ptr == last; +#else + static_cast(first); + static_cast(last); + static_cast(out); + return false; +#endif +} + +/// binary32 and binary64: the library's own conversion, which always succeeds +template +bool convert_float_fast(const char* first, const char* last, std::size_t decimal_point_position, + std::size_t mantissa_end, FloatType& value, std::true_type /*native*/) noexcept +{ + value = parse_float_native(first, last, decimal_point_position, mantissa_end); + return true; +} + +/// other formats (long double on x87, binary128, double-double): std::from_chars, if available +template +bool convert_float_fast(const char* first, const char* last, std::size_t /*decimal_point_position*/, + std::size_t /*mantissa_end*/, FloatType& value, std::false_type /*native*/) noexcept +{ + return parse_float_from_chars(first, last, value); } /*! @brief convert a validated float token without the C library, if possible -Tries std::from_chars (when available), Clinger's exact fast path (double -only, skipped when it cannot succeed), and the Eisel-Lemire algorithm (double -only). +float, double, and long double where it is binary64 are always converted, by +parse_float_native(). Other long double formats are converted with +std::from_chars where the standard library supports it. @param[in] first pointer to the first character of the token @param[in] last pointer past the last character @@ -9979,19 +10340,8 @@ template bool convert_float_fast(const char* first, const char* last, std::size_t decimal_point_position, std::size_t mantissa_end, FloatType& value) noexcept { - if (parse_float_from_chars(first, last, value)) - { - return true; - } - // Skipping a fast path that cannot succeed is lossless and saves a full - // extra pass over the token's bytes, which otherwise shows up on - // high-precision inputs such as canada.json - if (mantissa_fits_clinger(first, decimal_point_position, mantissa_end) - && parse_float_fast(first, last, value)) - { - return true; - } - return parse_float_eisel_lemire(first, last, value); + return convert_float_fast(first, last, decimal_point_position, mantissa_end, value, + std::integral_constant::value> {}); } /// std::strtof, std::strtod, or std::strtold, chosen by the type of @a f @@ -10015,36 +10365,45 @@ inline void strtof_by_type(long double& f, const char* str, char** endptr) noexc f = std::strtold(str, endptr); } -/// return the decimal point of the current locale -inline char get_decimal_point() noexcept +/// return the decimal point of the current locale (it may be longer than one byte) +inline std::string get_decimal_point() { const auto* loc = localeconv(); JSON_ASSERT(loc != nullptr); - return (loc->decimal_point == nullptr) ? '.' : *(loc->decimal_point); + return (loc->decimal_point == nullptr || *loc->decimal_point == '\0') ? "." : loc->decimal_point; } /*! @brief convert a validated float token with strtof/strtod/strtold +Only used for what convert_float_fast() does not convert: long double formats +other than binary64 where std::from_chars is unavailable or reports an under- +or overflow, and floating-point types that are not IEEE-754 (see +has_native_float_format). + These functions expect the decimal point of the *current* locale, so it is looked up right before the conversion instead of once when the lexer is constructed: a locale change in between (by a parser callback, a SAX -handler, or another thread) must not truncate the value (#5198). The -token has been validated before, so if the conversion stops early and the -decimal point changed in the meantime, the locale changed between the +handler, or another thread) must not truncate the value (#5198). A +single-byte decimal point is substituted in place and restored afterwards, +because the token is also handed to the SAX interface. A longer one (e.g., +the two-byte U+066B of ar_EG.UTF-8 or fa_IR.UTF-8) is put into a copy of the +token instead (#5660). + +The token has been validated before, so if the conversion stops early and +the decimal point changed in the meantime, the locale changed between the lookup and the call, and the conversion is repeated with the new decimal -point. If the decimal point did not change, a retry cannot succeed: the -locale's decimal point is not a single character (e.g., the two-byte -U+066B of ar_EG.UTF-8 or fa_IR.UTF-8) and cannot be substituted in place. -The value strtod parsed up to that point is kept, as before this change. +point. If it did not change, the value strtod parsed up to that point is +kept. Note that changing the locale in another thread *while* strtod runs is undefined behavior of the C library, which this function cannot prevent. -@param[in,out] token the token with '.' as decimal point; its - decimal point is replaced during the - conversion and restored afterwards - (data() must be NUL-terminated) +@param[in,out] token the token with '.' as decimal point; a + single-byte decimal point is put in + place during the conversion and + restored afterwards (data() must be + NUL-terminated) @param[in] decimal_point_position index of the '.' in @a token, or std::string::npos if there is none @param[out] value the converted value @@ -10053,39 +10412,77 @@ template void convert_float_locale_aware(StringType& token, std::size_t decimal_point_position, FloatType& value) { const bool has_dot = decimal_point_position != std::string::npos; - char decimal_point = get_decimal_point(); + std::string decimal_point = get_decimal_point(); for (;;) { - const bool substitute = has_dot && decimal_point != '.'; - if (substitute) - { - token[decimal_point_position] = static_cast(decimal_point); - } - char* endptr = nullptr; // NOLINT(misc-const-correctness,cppcoreguidelines-pro-type-vararg,hicpp-vararg) - strtof_by_type(value, token.data(), &endptr); - - if (substitute) + bool complete = false; + if (!has_dot || decimal_point.size() == 1) { - // the caller hands the token on (e.g. to the SAX interface) with '.' - token[decimal_point_position] = '.'; + const bool substitute = has_dot && decimal_point[0] != '.'; + if (substitute) + { + token[decimal_point_position] = static_cast(decimal_point[0]); + } + strtof_by_type(value, token.data(), &endptr); + if (substitute) + { + // the caller hands the token on (e.g. to the SAX interface) with '.' + token[decimal_point_position] = '.'; + } + complete = endptr == token.data() + token.size(); + } + else + { + std::string buffer(token.data(), token.size()); + buffer.replace(decimal_point_position, 1, decimal_point); + strtof_by_type(value, buffer.data(), &endptr); + complete = endptr == buffer.data() + buffer.size(); } - if (JSON_HEDLEY_LIKELY(endptr == token.data() + token.size())) + if (JSON_HEDLEY_LIKELY(complete)) { return; } // retry only if the locale changed; otherwise, this would loop forever - const char current_decimal_point = get_decimal_point(); + std::string current_decimal_point = get_decimal_point(); if (current_decimal_point == decimal_point) { return; } - decimal_point = current_decimal_point; + decimal_point = std::move(current_decimal_point); } } +/*! +@brief convert a validated float token like the lexer does + +For parsers of JSON text other than the lexer, which converts its own token +buffer in place. float, double, and long double where it is binary64 are +converted without allocation and independent of the locale; only other long +double formats that std::from_chars does not support need a copy of the token +for convert_float_locale_aware(). + +@param[in] first pointer to the first character of the token +@param[in] last pointer past the last character +@param[in] decimal_point_position index of the '.' in the token, or + std::string::npos if there is none +@param[in] mantissa_end index of the 'e'/'E', or the token length +@return the value, ±infinity if it overflows +*/ +template +FloatType convert_float(const char* first, const char* last, std::size_t decimal_point_position, std::size_t mantissa_end) +{ + FloatType value{}; + if (!convert_float_fast(first, last, decimal_point_position, mantissa_end, value)) + { + std::string token(first, last); + convert_float_locale_aware(token, decimal_point_position, value); + } + return value; +} + } // namespace detail NLOHMANN_JSON_NAMESPACE_END @@ -10102,10 +10499,13 @@ NLOHMANN_JSON_NAMESPACE_END +#include // array #include // size_t -#include // uint64_t +#include // uint64_t, uint8_t #include // memcpy +// #include + // #include @@ -10164,18 +10564,12 @@ inline std::size_t find_string_special(const unsigned char* data, std::size_t n) std::size_t i = 0; for (; i + 8 <= n; i += 8) { - std::uint64_t word = 0; - std::memcpy(&word, data + i, sizeof(word)); - if (swar_string_special(word) != 0) + const std::uint64_t special = swar_string_special(read_eight_bytes(data + i)); + if (special != 0) { - // a special byte is in this word; locate it (endian-agnostic) - for (std::size_t j = 0; j < 8; ++j) - { - if (is_string_special(data[i + j])) - { - return i + j; - } - } + // the lowest flagged byte is the first special one: the borrows of + // the subtractions can only flag bytes above a true hit + return i + (static_cast(count_trailing_zeros(special)) / 8); } } for (; i < n; ++i) @@ -10209,8 +10603,7 @@ inline std::size_t find_ascii_copyable_run(const unsigned char* data, std::size_ std::size_t i = 0; for (; i + 8 <= n; i += 8) { - std::uint64_t v = 0; - std::memcpy(&v, data + i, sizeof(v)); + const std::uint64_t v = read_eight_bytes(data + i); const std::uint64_t q = v ^ 0x2222222222222222ull; // '"' (0x22) const std::uint64_t b = v ^ 0x5C5C5C5C5C5C5C5Cull; // '\\' (0x5C) const std::uint64_t d = v ^ 0x7F7F7F7F7F7F7F7Full; // DEL (0x7F) @@ -10221,7 +10614,9 @@ inline std::size_t find_ascii_copyable_run(const unsigned char* data, std::size_ | (v & high); // >= 0x80 if (stop != 0) { - break; + // the lowest flagged byte is the first one to stop at (see + // find_string_special()) + return i + (static_cast(count_trailing_zeros(stop)) / 8); } } for (; i < n; ++i) @@ -10348,12 +10743,18 @@ inline std::size_t scalar_string_bulk_run(const unsigned char* data, std::size_t { break; // end of buffer, or a quote/escape/control byte } - const std::size_t seq = validate_one_utf8(data + pos, n - pos); - if (seq == 0) + // a run of multi-byte sequences (e.g. CJK text) is validated sequence + // by sequence without searching for the next special byte in between + do { - break; // ill-formed or truncated: let the byte path diagnose it + const std::size_t seq = validate_one_utf8(data + pos, n - pos); + if (seq == 0) + { + return pos; // ill-formed or truncated: let the byte path diagnose it + } + pos += seq; } - pos += seq; + while (pos < n && data[pos] >= 0x80u); } return pos; } @@ -10368,8 +10769,7 @@ inline std::size_t find_string_delimiter(const unsigned char* data, std::size_t std::size_t i = 0; for (; i + 8 <= n; i += 8) { - std::uint64_t v = 0; - std::memcpy(&v, data + i, sizeof(v)); + const std::uint64_t v = read_eight_bytes(data + i); const std::uint64_t q = v ^ 0x2222222222222222ull; const std::uint64_t b = v ^ 0x5C5C5C5C5C5C5C5Cull; const std::uint64_t hit = ((q - ones) & ~q & high) @@ -10377,14 +10777,8 @@ inline std::size_t find_string_delimiter(const unsigned char* data, std::size_t | ((v - 0x2020202020202020ull) & ~v & high); if (hit != 0) { - for (std::size_t j = 0; j < 8; ++j) - { - const unsigned char c = data[i + j]; - if (c == '\"' || c == '\\' || c < 0x20u) - { - return i + j; - } - } + // the lowest flagged byte is the first delimiter (see find_string_special()) + return i + (static_cast(count_trailing_zeros(hit)) / 8); } } for (; i < n; ++i) @@ -10415,6 +10809,51 @@ inline std::size_t string_bulk_run(const unsigned char* data, std::size_t n) noe return scalar_string_bulk_run(data, n); } +// Decode the 4 hex digits at [data, data+4) - the digits following a `\u` +// escape - into a codepoint 0x0000..0xFFFF via one table lookup per byte +// (after yyjson's read_hex_u16), or return -1 if any of the 4 bytes is not a +// hex digit ('0'..'9', 'A'..'F', 'a'..'f'). The caller must already have +// checked that 4 bytes are available; used by lexer::get_codepoint()'s +// contiguous fast path. On -1 it falls back to the byte-at-a-time loop, which +// stops at the first invalid digit, so the reported error and position are +// unaffected by this fast path. +inline int hex_codepoint(const unsigned char* data) noexcept +{ + static const std::array hex_digit_table = // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) + { + { + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // 00..0F + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // 10..1F + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // 20..2F + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // 30..3F ('0'..'9') + 0xFF, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // 40..4F ('A'..'F') + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // 50..5F + 0xFF, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // 60..6F ('a'..'f') + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // 70..7F + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // 80..8F + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // 90..9F + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // A0..AF + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // B0..BF + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // C0..CF + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // D0..DF + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // E0..EF + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF // F0..FF + } + }; + + const std::uint8_t d0 = hex_digit_table[data[0]]; + const std::uint8_t d1 = hex_digit_table[data[1]]; + const std::uint8_t d2 = hex_digit_table[data[2]]; + const std::uint8_t d3 = hex_digit_table[data[3]]; + // every valid digit is <= 0xF; the combined OR only exceeds it if at + // least one of the four bytes was not a hex digit (looked up as 0xFF) + if ((d0 | d1 | d2 | d3) > 0x0F) + { + return -1; + } + return (d0 << 12) | (d1 << 8) | (d2 << 4) | d3; +} + } // namespace detail NLOHMANN_JSON_NAMESPACE_END @@ -10586,6 +11025,15 @@ class lexer : public lexer_base static constexpr bool lazy_token_string = input_adapter_supports_seek(is_detected {}); + /// whether the read position is derived from the input adapter on demand + /// instead of being counted on every scanned character. Seekable adapters + /// know how much input they consumed and can revisit it, so + /// chars_read_total is the adapter's read count (minus a pending unget), + /// and line and column are recomputed from the consumed input only when + /// get_position() is called (errors). Counting per character kept two + /// increments and a newline check in the hot path of every get(). + static constexpr bool lazy_position = lazy_token_string; + /// whether a simulated unget can be passed on to the input adapter, which /// then leaves the character in the input; see /// input_adapter_supports_lookahead @@ -10595,7 +11043,9 @@ class lexer : public lexer_base /// whether string scanning may bulk-consume runs of ordinary characters /// directly from a contiguous input buffer (SWAR fast path). This requires /// the token to be reconstructible lazily (lazy_token_string), so bypassing - /// the per-character capture in get() cannot lose error diagnostics. + /// the per-character capture in get() cannot lose error diagnostics. It + /// also implies lazy_position, so the bulk paths need not advance the + /// position counters. static constexpr bool bulk_scan = lazy_token_string && input_adapter_supports_bulk_scan(is_detected {}); @@ -10621,6 +11071,41 @@ class lexer : public lexer_base // scan functions ///////////////////// + /// contiguous input: try to decode the 4 hex digits following `\\u` + /// directly from the input buffer via hex_codepoint(), instead of 4 calls + /// to get(). On success, advances the adapter exactly as those 4 get() + /// calls would (bulk_scan implies lazy_position, so there are no position + /// counters to advance) and leaves @a current holding the last of + /// the 4 digits, just as the last such get() would; the codepoint is + /// written to @a out. Makes no state change and returns false for a + /// pending unget, fewer than 4 remaining bytes, or any of the 4 bytes not + /// being a hex digit, so the caller falls back unchanged to the + /// per-character loop, which then reports the same diagnostic (stopping + /// at the first invalid digit) as before this optimization. + bool get_codepoint_bulk(std::true_type /*bulk*/, int& out) + { + if (next_unget || ia.bulk_remaining() < 4) + { + return false; + } + const char_type* const raw = ia.bulk_data(); + const int codepoint = hex_codepoint(reinterpret_cast(raw)); + if (codepoint < 0) + { + return false; + } + ia.bulk_skip(4); + current = char_traits::to_int_type(raw[3]); + out = codepoint; + return true; + } + + /// streaming input: no bulk fast path + bool get_codepoint_bulk(std::false_type /*bulk*/, int& /*out*/) const noexcept + { + return false; + } + /*! @brief get codepoint from 4 hex characters following `\\u` @@ -10640,6 +11125,14 @@ class lexer : public lexer_base { // this function only makes sense after reading `\u` JSON_ASSERT(current == 'u'); + + // contiguous input: decode all 4 hex digits directly from the buffer + int fast_codepoint = 0; + if (get_codepoint_bulk(std::integral_constant {}, fast_codepoint)) + { + return fast_codepoint; + } + int codepoint = 0; const auto factors = { 12u, 8u, 4u, 0u }; @@ -10731,10 +11224,6 @@ class lexer : public lexer_base } token_buffer.append(reinterpret_cast(data), pos); ia.bulk_skip(pos); - // the run contains no newline (all bytes < 0x20 are treated as special), - // so only the flat character counters advance - position.chars_read_total += pos; - position.chars_read_current_line += pos; } /// streaming input: no bulk fast path @@ -11444,9 +11933,11 @@ class lexer : public lexer_base token_type::parse_error otherwise @note The scanner is independent of the current locale: token_buffer - always holds `.`. Only the std::strtod fallback of convert_number() - depends on the locale, and it looks up the decimal point right - before converting (see detail::convert_float_locale_aware()). + always holds `.`. The conversion of float and double does not use + the locale either. Only the std::strtold fallback of + convert_number() for long double formats other than binary64 + depends on it, and it looks up the decimal point right before + converting (see detail::convert_float_locale_aware()). */ token_type scan_number() // lgtm [cpp/use-of-goto] `goto` is used in this function to implement the number-parsing state machine described above. By design, any finite input will eventually reach the "done" state or return token_type::parse_error. In each intermediate state, 1 byte of the input is appended to the token_buffer vector, and only the already initialized variables token_buffer, number_type, and error_message are manipulated. { @@ -11459,7 +11950,7 @@ class lexer : public lexer_base // offset just past the last mantissa byte in token_buffer (i.e. the // index of 'e'/'E', or the whole token when there is no exponent). - // convert_number() uses it to count significant digits; npos means + // convert_number() uses it to split the token; npos means // "not seen an exponent yet" and is resolved at scan_number_done std::size_t mantissa_end = std::string::npos; @@ -11789,8 +12280,8 @@ scan_number_done: @param[in] mantissa_end offset just past the last mantissa byte in token_buffer (the index of 'e'/'E', or token_buffer.size() when there is no exponent); - used to skip Clinger's fast path when it cannot - possibly succeed - see detail::mantissa_fits_clinger() + with decimal_point_position, it locates the parts + of a float token without scanning it again */ token_type convert_number(token_type number_type, std::size_t mantissa_end) { @@ -11844,10 +12335,11 @@ scan_number_done: } // this code is reached if we parse a floating-point number or if an - // integer conversion above overflowed. Prefer std::from_chars - // (Eisel-Lemire, locale-independent, correctly rounded) when available; - // otherwise the exact Clinger fast path (double only); otherwise the - // locale-aware strtof/strtod/strtold. + // integer conversion above overflowed. float and double (and long + // double where it is binary64) are converted by the library itself, + // correctly rounded and independent of the locale; other long double + // formats use std::from_chars when available, otherwise the + // locale-aware strtold. if (convert_float_fast(num_begin, num_end, decimal_point_position, mantissa_end, value_float)) { return token_type::value_float; @@ -11967,8 +12459,6 @@ scan_number_done: if (JSON_HEDLEY_LIKELY(integer_result != token_type::uninitialized)) { ia.bulk_skip(len - 1); - position.chars_read_total += (len - 1); - position.chars_read_current_line += (len - 1); return integer_result; } // The value does not fit an integer, so this token converts as a @@ -11985,8 +12475,6 @@ scan_number_done: decimal_point_position = dot_index; ia.bulk_skip(len - 1); - position.chars_read_total += (len - 1); - position.chars_read_current_line += (len - 1); return convert_number(number_type, mantissa_end); } @@ -12037,7 +12525,7 @@ scan_number_done: #if JSON_DIAGNOSTIC_POSITIONS // the first character of the token has already been read, hence the -1 - token_start_position = position.chars_read_total - 1; + token_start_position = get_chars_read_total() - 1; #endif note_token_start(std::integral_constant {}); @@ -12090,6 +12578,15 @@ scan_number_done: /// per-character position counters (line-count-on-'\n' bookkeeping is /// handled afterwards, in track_after_read(), once `current` is known) void advance_position() noexcept + { + advance_position_impl(std::integral_constant {}); + } + + /// seekable adapter: the position is derived from the adapter on demand + void advance_position_impl(std::true_type /*lazy*/) const noexcept {} + + /// streaming adapter: count the character + void advance_position_impl(std::false_type /*lazy*/) noexcept { ++position.chars_read_total; ++position.chars_read_current_line; @@ -12103,7 +12600,16 @@ scan_number_done: // seekable adapters reconstruct the token lazily on error (see // get_token_string), so the eager per-character copy is skipped capture_char(std::integral_constant {}); + track_newline(std::integral_constant {}); + return current; + } + /// seekable adapter: lines are counted on demand in get_position() + void track_newline(std::true_type /*lazy*/) const noexcept {} + + /// streaming adapter: start a new line after reading a newline + void track_newline(std::false_type /*lazy*/) noexcept + { if (current == '\n') { ++position.lines_read; @@ -12112,8 +12618,6 @@ scan_number_done: chars_read_before_newline = position.chars_read_current_line; position.chars_read_current_line = 0; } - - return current; } /*! @@ -12157,12 +12661,22 @@ scan_number_done: We implement unget by setting variable next_unget to true. The input is not changed - we just simulate ungetting by modifying chars_read_total, chars_read_current_line, and token_string. The next call to get() will - behave as if the unget character is read again. + behave as if the unget character is read again. With lazy_position, the + pending next_unget is all get_chars_read_total() needs. */ void unget() { next_unget = true; + unget_position(std::integral_constant {}); + uncapture_char(std::integral_constant {}); + } + /// seekable adapter: get_chars_read_total() accounts for next_unget + void unget_position(std::true_type /*lazy*/) const noexcept {} + + /// streaming adapter: step the counters back over the ungotten character + void unget_position(std::false_type /*lazy*/) noexcept + { --position.chars_read_total; // in case we "unget" a newline, we have to also decrement the lines_read @@ -12185,8 +12699,6 @@ scan_number_done: { --position.chars_read_current_line; } - - uncapture_char(std::integral_constant {}); } /// adapter without lookahead: nothing to do (see release_lookahead) @@ -12257,10 +12769,21 @@ scan_number_done: // diagnostics ///////////////////// - /// return position of last read token - constexpr position_t get_position() const noexcept + /*! + @brief return the number of characters read so far + + Unlike get_position(), this is O(1) for all adapters, so it may be called + for every value (JSON_DIAGNOSTIC_POSITIONS). + */ + std::size_t get_chars_read_total() const noexcept { - return position; + return get_chars_read_total_impl(std::integral_constant {}); + } + + /// return position of last read token + position_t get_position() const + { + return get_position_impl(std::integral_constant {}); } /*! @@ -12297,6 +12820,43 @@ scan_number_done: } #endif + private: + /// seekable adapter: every get() either read from the adapter (including + /// reads at EOF, which get() counts as characters as well) or replayed an + /// ungotten character, so the count is the adapter's read count minus a + /// still pending unget + std::size_t get_chars_read_total_impl(std::true_type /*lazy*/) const noexcept + { + return ia.get_read_count() - (next_unget ? 1u : 0u); + } + + /// streaming adapter: the counter is maintained by get() and unget() + std::size_t get_chars_read_total_impl(std::false_type /*lazy*/) const noexcept + { + return position.chars_read_total; + } + + /// seekable adapter: count the lines in the consumed input; O(input read + /// so far), but only called on errors + position_t get_position_impl(std::true_type /*lazy*/) const + { + position_t result; + result.chars_read_total = get_chars_read_total(); + // reads at EOF count as characters but are not part of the input + const std::size_t consumed = (std::min)(result.chars_read_total, ia.get_consumed_count()); + std::size_t line_start = 0; + result.lines_read = ia.count_consumed_lines(consumed, line_start); + result.chars_read_current_line = result.chars_read_total - line_start; + return result; + } + + /// streaming adapter: the position is maintained by get() and unget() + position_t get_position_impl(std::false_type /*lazy*/) const noexcept + { + return position; + } + + public: /// seekable adapter: rebuild the last read token from the input on demand const std::vector& collect_token_chars(std::vector& out, std::true_type /*lazy*/) const { @@ -12404,15 +12964,57 @@ scan_number_done: // into one that reloads the pointer from memory every character do { + if (current == '\n') + { + skip_indentation(std::integral_constant {}); + } get_ignoring_pending_unget(); } while (current_is_whitespace()); } + /*! + @brief contiguous input: skip the space indentation that follows a newline + 8 bytes at a time + + Pretty-printed JSON indents every line with a run of spaces, which the loop + in skip_whitespace() would otherwise read one character at a time. The + spaces are skipped 8 bytes at a time; the last 7 bytes of the input and any + other whitespace are left to that loop. Skipping bypasses get(), which is + fine because bulk_scan implies lazy_position: the position is derived from + the adapter, not counted per character. + */ + void skip_indentation(std::true_type /*bulk*/) + { + JSON_ASSERT(!next_unget); + const std::size_t remaining = ia.bulk_remaining(); + if (remaining < 8) + { + return; + } + const char* const data = reinterpret_cast(ia.bulk_data()); + std::size_t pos = 0; + for (; pos + 8 <= remaining; pos += 8) + { + // little-endian, so the first byte is the lowest + const std::uint64_t non_space = read_eight_bytes(data + pos) ^ 0x2020202020202020ull; + if (non_space != 0) + { + // skip the spaces before the first other byte of this word + pos += static_cast(count_trailing_zeros(non_space)) / 8; + break; + } + } + ia.bulk_skip(pos); + } + + /// streaming input: no bulk fast path + void skip_indentation(std::false_type /*bulk*/) const noexcept {} + token_type scan() { // initially, skip the BOM - if (position.chars_read_total == 0 && !skip_bom()) + if (get_chars_read_total() == 0 && !skip_bom()) { error_message = "invalid BOM; must be 0xEF 0xBB 0xBF if given"; return token_type::parse_error; @@ -12439,7 +13041,7 @@ scan_number_done: token_type scan_expecting(token_type expected_type) { JSON_ASSERT(expected_type == token_type::name_separator || expected_type == token_type::value_separator); - JSON_ASSERT(position.chars_read_total > 0); + JSON_ASSERT(get_chars_read_total() > 0); const char_int_type expected_char = static_cast((expected_type == token_type::name_separator) ? ':' : ','); skip_whitespace(); if (JSON_HEDLEY_LIKELY(current == expected_char)) @@ -12793,7 +13395,7 @@ struct diagnostic_positions { // Lexer has read the first character of the container, so // subtract 1 from the position to get the correct start position. - v.start_position = lexer->get_position() - 1; + v.start_position = lexer->get_chars_read_total() - 1; } } @@ -12807,7 +13409,7 @@ struct diagnostic_positions if (lexer) { // Lexer's position is past the closing brace or bracket, so set that as the end position. - v.end_position = lexer->get_position(); + v.end_position = lexer->get_chars_read_total(); } } @@ -12823,7 +13425,7 @@ struct diagnostic_positions // Lexer has read past the current field value, so set the end position to the current position. // The start position will be set below based on the length of the string representation // of the value. - v.end_position = lexer->get_position(); + v.end_position = lexer->get_chars_read_total(); switch (v.type()) { @@ -18666,7 +19268,7 @@ class parser } case token_type::end_of_input: { - if (JSON_HEDLEY_UNLIKELY(m_lexer.get_position().chars_read_total == 1)) + if (JSON_HEDLEY_UNLIKELY(m_lexer.get_chars_read_total() == 1)) { return syntax_error(*sax, "attempting to parse an empty input; check that your input string or stream contains the expected JSON"); } @@ -18816,8 +19418,10 @@ class parser { // MSVC 2015 reports C4100 (unreferenced parameter) when SAX::parse_error is static static_cast(sax); - return sax.parse_error(m_lexer.get_position(), m_lexer.get_token_string(), - parse_error::create(exception_id::syntax_error, m_lexer.get_position(), message, nullptr)); + // computing the position is O(input read so far) for seekable input + const position_t position = m_lexer.get_position(); + return sax.parse_error(position, m_lexer.get_token_string(), + parse_error::create(exception_id::syntax_error, position, message, nullptr)); } std::string exception_message(const token_type expected, const std::string& context) @@ -21412,9 +22016,9 @@ class json_ref } // namespace detail NLOHMANN_JSON_NAMESPACE_END - +// IWYU pragma: export // #include - +// IWYU pragma: export // #include // #include @@ -25217,6 +25821,7 @@ NLOHMANN_JSON_NAMESPACE_END // |_____|_____|_____|_|___| https://github.com/nlohmann/json // // SPDX-FileCopyrightText: 2009 Florian Loitsch +// SPDX-FileCopyrightText: 2025 Victor Zverovich // SPDX-FileCopyrightText: 2013-2026 Niels Lohmann // SPDX-License-Identifier: MIT @@ -25224,11 +25829,256 @@ NLOHMANN_JSON_NAMESPACE_END #include // array #include // signbit, isfinite +#include // size_t #include // intN_t, uintN_t #include // memcpy, memmove #include // numeric_limits #include // conditional +#ifdef _MSC_VER + #include // _byteswap_uint64 +#endif + +// SSE2 (every x86-64 CPU) and NEON (every 64-bit Arm CPU) convert the 16 +// digits of a double at once +#if defined(__x86_64__) || (defined(_M_X64) && !defined(_M_ARM64EC)) + #include + #define JSON_DTOA_SSE2 1 + #define JSON_DTOA_NEON 0 +#elif (defined(__aarch64__) || defined(_M_ARM64)) && !defined(_M_ARM64EC) && !defined(__ARM_BIG_ENDIAN) + #include + #define JSON_DTOA_SSE2 0 + #define JSON_DTOA_NEON 1 +#else + #define JSON_DTOA_SSE2 0 + #define JSON_DTOA_NEON 0 +#endif + +// #include +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ +// | | |__ | | | | | | version 3.12.0 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2025 Victor Zverovich +// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann +// SPDX-License-Identifier: MIT + + + +#include // array +#include // size_t +#include // uint32_t, uint64_t + +// #include + +// #include + +// #include + +// #include + + +NLOHMANN_JSON_NAMESPACE_BEGIN +namespace detail +{ + +/*! +@brief the shortest decimal representation of a double + +A C++11 port of the conversion of Zmij by Victor Zverovich +(https://github.com/vitaut/zmij, MIT license): the shortest decimal in the +rounding interval of a double, the closest one if there are several. Zmij +credits Xiang JunBo (producing the shorter candidate without a division) and +Dougall Johnson (the compressed powers of ten). The powers of ten are taken +from the table for number parsing (pow5_table.hpp) where it holds them, and +computed from the compressed tables of Zmij beyond it. +*/ +namespace zmij +{ + +/// the compressed powers of ten of Zmij +inline const std::array& pow10_minor() noexcept +{ + static const std::array table = + { + { + 0x8000000000000000u, 0xa000000000000000u, 0xc800000000000000u, 0xfa00000000000000u, 0x9c40000000000000u, + 0xc350000000000000u, 0xf424000000000000u, 0x9896800000000000u, 0xbebc200000000000u, 0xee6b280000000000u, + 0x9502f90000000000u, 0xba43b74000000000u, 0xe8d4a51000000000u, 0x9184e72a00000000u, 0xb5e620f480000000u, + 0xe35fa931a0000000u, 0x8e1bc9bf04000000u, 0xb1a2bc2ec5000000u, 0xde0b6b3a76400000u, 0x8ac7230489e80000u, + 0xad78ebc5ac620000u, 0xd8d726b7177a8000u, 0x878678326eac9000u, 0xa968163f0a57b400u, 0xd3c21bcecceda100u, + 0x84595161401484a0u, 0xa56fa5b99019a5c8u, 0xcecb8f27f4200f3au + } + }; + return table; +} + +/// (high, low) pairs +inline const std::array& pow10_major() noexcept +{ + static const std::array table = + { + { + 0xaddcb9e83c6b1793u, 0xdf4abe242a1bbf3eu, 0xaf8e5410288e1b6fu, 0x07ecf0ae5ee44ddau, 0xb1442798f49ffb4au, 0x99cd11cfdf41779du, + 0xb2fe3f0b8599ef07u, 0x861fa7e6dcb4aa15u, 0xb4bca50b065abe63u, 0x0fed077a756b53aau, 0xb67f6455292cbf08u, 0x1a3bc84c17b1d543u, + 0xb84687c269ef3bfbu, 0x3d5d514f40eea742u, 0xba121a4650e4ddebu, 0x92f34d62616ce413u, 0xbbe226efb628afeau, 0x890489f70a55368cu, + 0xbdb6b8e905cb600fu, 0x5400e987bbc1c921u, 0xbf8fdb78849a5f96u, 0xde98520472bdd034u, 0xc16d9a0095928a27u, 0x75b7053c0f178294u, + 0xc350000000000000u, 0x0000000000000000u, 0xc5371912364ce305u, 0x6c28000000000000u, 0xc722f0ef9d80aad6u, 0x424d3ad2b7b97ef6u, + 0xc913936dd571c84cu, 0x03bc3a19cd1e38eau, 0xcb090c8001ab551cu, 0x5cadf5bfd3072cc6u, 0xcd036837130890a1u, 0x36dba887c37a8c10u, + 0xcf02b2c21207ef2eu, 0x94f967e45e03f4bcu, 0xd106f86e69d785c7u, 0xe13336d701beba52u, 0xd31045a8341ca07cu, 0x1ede48111209a051u, + 0xd51ea6fa85785631u, 0x552a74227f3ea566u, 0xd732290fbacaf133u, 0xa97c177947ad4096u, 0xd94ad8b1c7380874u, 0x18375281ae7822bdu, + 0xdb68c2ca82ed2a05u, 0xa67398db9f6820e1u + } + }; + return table; +} + +/// one bit per power: whether the computed value is one unit too large +inline const std::array& pow10_fixups() noexcept +{ + static const std::array table = + { + { + 0x8d8fc810u, 0x06100293u, 0x19000000u, 0x00100000u, 0x00000908u, 0x00000000u, 0x04e00300u, 0x3807e0b2u, 0x3d83d793u, 0x0006f5ccu, + 0x00000000u, 0xffff0000u, 0x8076337du, 0x4ff45ba0u, 0x09405033u, 0x034376d9u, 0x09000000u, 0x4e100501u, 0x076d14dcu, 0xf964f45eu, + 0x0000003du + } + }; + return table; +} + +/// the 128-bit significand of 10^k, rounded down, for k in [-307, 341] +/// (compute_pow10 of Zmij) +inline uint128_parts compute_pow10(int k) noexcept +{ + const auto i = static_cast(k + 307); + const std::uint64_t m = pow10_minor()[(i + 24) % 28]; + const std::size_t j = 2 * static_cast((i + 24) / 28); + const std::uint64_t h_hi = pow10_major()[j]; + const std::uint64_t h_lo = pow10_major()[j + 1]; + const std::uint64_t h1 = full_multiplication(h_lo, m).high; + const std::uint64_t c0 = h_lo * m; + const std::uint64_t c1 = h1 + (h_hi * m); + const std::uint64_t c2 = (c1 < h1 ? 1u : 0u) + full_multiplication(h_hi, m).high; + uint128_parts r{}; + if ((c2 >> 63u) != 0) + { + r.high = c2; + r.low = c1; + } + else + { + r.high = (c2 << 1u) | (c1 >> 63u); + r.low = (c1 << 1u) | (c0 >> 63u); + } + r.low -= (pow10_fixups()[i >> 5u] >> (i & 31u)) & 1u; + return r; +} + +/// The 128-bit significand of 10^k, rounded down, for k in [-342, 341]. +/// Up to 10^308, the table for number parsing holds the same significands +/// (those of 5^k), except for k in [-27, -1], where it holds them one unit +/// larger (as the Eisel-Lemire algorithm needs them). +inline uint128_parts pow10(int k) noexcept +{ + if (k > pow5_128_largest_power) + { + return compute_pow10(k); // (only for the smallest doubles) + } + const auto i = 2 * static_cast(k - pow5_128_smallest_power); + uint128_parts r{pow5_128()[i + 1], pow5_128()[i]}; + const std::uint64_t adjust = static_cast(k + 27) < 27u ? 1u : 0u; + r.high -= r.low < adjust ? 1u : 0u; + r.low -= adjust; + return r; +} + +/// (x_hi * 2^64 + x_lo) * y >> 64, as 128 bits +inline uint128_parts umul192_hi128(std::uint64_t x_hi, std::uint64_t x_lo, std::uint64_t y) noexcept +{ + const uint128_parts p = full_multiplication(x_hi, y); + uint128_parts r{}; + r.low = p.low + full_multiplication(x_lo, y).high; + r.high = p.high + (r.low < p.low ? 1u : 0u); + return r; +} + +/// (x * y + c) >> 64 +inline std::uint64_t umul128_add_hi64(std::uint64_t x, std::uint64_t y, std::uint64_t c) noexcept +{ + const uint128_parts p = full_multiplication(x, y); + return p.high + (p.low + c < p.low ? 1u : 0u); +} + +/// the result of Zmij: the shorter candidate and, if that is outside the +/// rounding interval, the digit after it (16 bytes: returned in registers) +struct shortest_decimal +{ + std::uint64_t integral; ///< the shorter candidate (15 or 16 digits for normal doubles) + int exponent; ///< the decimal exponent of the digit after it + unsigned char digit; ///< the digit after it (if has_digit) + bool has_digit; ///< whether the shortest decimal is integral * 10 + digit +}; + +/// The shortest decimal in the rounding interval of a positive finite double +/// given by its bits, the closest one if there are several (to_decimal of +/// Zmij, which keeps the last digit apart: the 15 or 16 digits before it can be +/// converted without a multiplication by 10 first). Always inlined: GCC +/// otherwise calls it, and its result goes through memory. +JSON_HEDLEY_ALWAYS_INLINE shortest_decimal to_shortest(std::uint64_t bits) noexcept +{ + constexpr int extra_shift = 9; + const auto raw_exp = static_cast((bits >> 52u) & 0x7FFu); + std::uint64_t bin_sig = bits & ((std::uint64_t{1} << 52u) - 1); + // a power of two has a narrower interval below (except the smallest normal) + const bool regular = bin_sig != 0 || raw_exp <= 1; + const int bin_exp = (raw_exp == 0 ? 1 : raw_exp) - 1075; + if (raw_exp != 0) + { + bin_sig |= std::uint64_t{1} << 52u; + } + // floor(log10(2^bin_exp)), or floor(log10(3/4 * 2^bin_exp)) for the irregular case + const int dec_exp = ((bin_exp * 315653) - (regular ? 0 : 131072)) >> 20; + // scaled by 10^(-dec_exp - 1): the integral part is the shorter candidate + const int shift = bin_exp + ((-(dec_exp + 1) * 217707) >> 16) + 1 + extra_shift; + const uint128_parts p10 = pow10(-dec_exp - 1); + const uint128_parts p = umul192_hi128(p10.high, p10.low, bin_sig << static_cast(shift)); + std::uint64_t integral = p.high >> static_cast(extra_shift); + const std::uint64_t fractional = (p.high << static_cast(64 - extra_shift)) | (p.low >> static_cast(extra_shift)); + std::uint64_t digit = 0; + bool round_up = false; + bool round_down = false; + if (JSON_HEDLEY_LIKELY(regular)) + { + const std::uint64_t half_ulp = (p10.high >> static_cast(extra_shift + 1 - shift)) + (1 - (bin_sig & 1u)); + round_up = fractional + half_ulp < fractional; + round_down = half_ulp > fractional; + // the last digit of the longer candidate, rounded to nearest + digit = umul128_add_hi64(fractional, 10, (std::uint64_t{1} << 63u) + 6); + if (fractional == (std::uint64_t{1} << 62u)) + { + digit = 2; // 2.5 rounds to 2 + } + } + else + { + const std::uint64_t half_ulp = p10.high >> static_cast(extra_shift + 1 - shift); + round_up = half_ulp > ~std::uint64_t{0} - fractional; + round_down = (half_ulp >> 1u) > fractional; + digit = umul128_add_hi64(fractional, 10, (std::uint64_t{1} << 63u) - 1); + const std::uint64_t lowest = umul128_add_hi64(fractional - (half_ulp >> 1u), 10, ~std::uint64_t{0}); + digit = digit < lowest ? lowest : digit; + } + integral += round_up ? 1u : 0u; + // if the shorter candidate is outside the rounding interval: one digit more + return shortest_decimal{integral, dec_exp, static_cast(digit), !round_up && !round_down}; +} + +} // namespace zmij +} // namespace detail +NLOHMANN_JSON_NAMESPACE_END + // #include @@ -26261,6 +27111,381 @@ inline char* format_buffer(char* buf, int len, int decimal_exponent, return append_exponent(buf, n - 1); } +/// eight decimal digits (a value below 10^8) as bytes 0..9, the first digit +/// in the most significant byte: three steps that divide all lanes at once +/// by a multiplication (the conversion of Xiang JunBo, as in Zmij) +inline std::uint64_t eight_digit_bytes(std::uint64_t abcdefgh) noexcept +{ + const std::uint64_t abcd_efgh = abcdefgh + (((std::uint64_t{1} << 32u) - 10000u) * ((abcdefgh * (((std::uint64_t{1} << 40u) / 10000u) + 1u)) >> 40u)); + const std::uint64_t ab_cd_ef_gh = abcd_efgh + (((std::uint64_t{1} << 16u) - 100u) * (((abcd_efgh * (((std::uint64_t{1} << 19u) / 100u) + 1u)) >> 19u) & 0x7F0000007Fu)); + return ab_cd_ef_gh + (((std::uint64_t{1} << 8u) - 10u) * (((ab_cd_ef_gh * (((std::uint64_t{1} << 10u) / 10u) + 1u)) >> 10u) & 0x000F000F000F000Fu)); +} + +/// store the bytes of v, the most significant one first (one byte swap and +/// one store where the byte order is known: compilers do not reliably merge +/// the byte stores once this is inlined) +inline void store_msb_first(char* p, std::uint64_t v) noexcept +{ +#if defined(__BYTE_ORDER__) && defined(__ORDER_LITTLE_ENDIAN__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ + v = __builtin_bswap64(v); + std::memcpy(p, &v, sizeof(v)); +#elif defined(__BYTE_ORDER__) && defined(__ORDER_BIG_ENDIAN__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + std::memcpy(p, &v, sizeof(v)); +#elif defined(_MSC_VER) // (little-endian on all its targets) + v = _byteswap_uint64(v); + std::memcpy(p, &v, sizeof(v)); +#else + for (unsigned i = 0; i < 8; ++i) + { + p[i] = static_cast(v >> (56u - (8u * i))); + } +#endif +} + +/*! +@brief digits * 10^exp for a double, in the layout of format_buffer() + +The layout is that of format_buffer() with min_exp -4 and max_exp 15 (the +digits10 of double). The digits are converted eight at a time and placed +with fixed-size moves instead of per-digit loops and moves of the buffer. + +@param[in] digits the digits (not 0, at most 17 digits; trailing zeros allowed) +@param[in] exp the decimal exponent of the last digit +@return a pointer past the text; up to 41 bytes at @a first are written + (some beyond the returned end) +*/ +JSON_HEDLEY_NON_NULL(1) +JSON_HEDLEY_RETURNS_NON_NULL +inline char* write_decimal(char* first, std::uint64_t digits, int exp) noexcept +{ + JSON_ASSERT(digits != 0 && digits < 100000000000000000u); + const std::uint64_t upper = digits / 100000000u; + const std::uint64_t b0 = upper / 100000000u; // (one digit: it is its own byte) + const std::uint64_t b1 = eight_digit_bytes(upper % 100000000u); + const std::uint64_t b2 = eight_digit_bytes(digits % 100000000u); + // leading and trailing zero digits: zero bytes, counted without division + int leading = 16; + int zeros = 16; + if (b0 != 0) + { + leading = count_leading_zeros(b0) / 8; + } + else if (b1 != 0) + { + leading = 8 + (count_leading_zeros(b1) / 8); + } + else + { + leading += count_leading_zeros(b2) / 8; + } + if (b2 != 0) + { + zeros = count_trailing_zeros(b2) / 8; + } + else if (b1 != 0) + { + zeros = 8 + (count_trailing_zeros(b1) / 8); + } + // (else: 16, b0 is the one digit that is not 0) + // the digits as text at text + leading, then '0's, so that fixed-size + // moves need not check how many digits there are + std::array text; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init): written before read + store_msb_first(text.data(), b0 + 0x3030303030303030u); + store_msb_first(text.data() + 8, b1 + 0x3030303030303030u); + store_msb_first(text.data() + 16, b2 + 0x3030303030303030u); + std::memset(text.data() + 24, '0', 40); + const int k = 24 - leading - zeros; // significant digits + const int n = k + exp + zeros; // position of the decimal point after the first digit + const char* const s0 = text.data() + leading; + + if (-4 < n && n <= 15) + { + // "0.[000]digits" (n <= 0) is the digits after 1 - n leading '0's + // with the point after the first; "digits[000].0" (n >= k) and + // "dig.its" put the point after n characters + const int pad = n <= 0 ? 1 - n : 0; + const char* const s = s0 - pad; + const int len = k + pad; + const int point = n + pad; + std::memcpy(first, s, 16); + std::memcpy(first + point + 1, s + point, 24); + first[point] = '.'; + return first + (point >= len ? point + 2 : len + 1); + } + + // d.igitse+XX, with at least two exponent digits (as append_exponent()) + std::memcpy(first, s0, 16); + std::memcpy(first + 2, s0 + 1, 16); + first[1] = '.'; + char* const end = first + (k == 1 ? 1 : k + 1); + const int e = n - 1; + const auto ea = e < 0 ? 0u - static_cast(e) : static_cast(e); // (unsigned: no signed overflow to assume) + const bool three = ea >= 100; + end[0] = 'e'; + end[1] = e < 0 ? '-' : '+'; + end[2] = static_cast('0' + (three ? ea / 100 : (ea / 10) % 10)); + end[3] = static_cast('0' + (three ? (ea / 10) % 10 : ea % 10)); + end[4] = static_cast('0' + (ea % 10)); + return end + (three ? 5 : 4); +} + +/*! +@brief the shortest decimal of a positive double (Zmij), as write_decimal() +writes it + +For a normal double, the shorter candidate has 15 or 16 digits: they are +converted at once (two halves of eight digits) and followed by the digit +after them, if there is one, without the multiplication and division by 10 +that counting the digits of one number would take. The fixed layouts move +the digits after the point by one byte. + +@return a pointer past the text; up to 41 bytes at @a first are written + (some beyond the returned end) +*/ +JSON_HEDLEY_NON_NULL(1) +JSON_HEDLEY_RETURNS_NON_NULL +inline char* write_shortest(char* first, const zmij::shortest_decimal d) noexcept +{ + const std::uint64_t sig = d.integral; + if (JSON_HEDLEY_UNLIKELY(sig < 100000000000000u || sig >= 10000000000000000u)) + { + // (subnormals) + return d.has_digit ? write_decimal(first, (sig * 10) + d.digit, d.exponent) : write_decimal(first, sig, d.exponent + 1); + } + const bool sixteen = sig >= 1000000000000000u; // (else 15 digits) + const int last = d.has_digit ? d.digit : 0; + const std::uint64_t upper = sig / 100000000u; +#if JSON_DTOA_SSE2 + // NOLINTBEGIN(portability-simd-intrinsics) + // the two halves in the 64-bit lanes, each as abcd * 2^32 + efgh, then as + // bytes (as eight_digit_bytes(), one lane each) + const __m128i x = _mm_set_epi64x(static_cast(sig - (upper * 100000000u)), static_cast(upper)); // NOLINT(runtime/int) + const __m128i abcd = _mm_srli_epi64(_mm_mul_epu32(x, _mm_set1_epi64x(109951163)), 40); // 2^40 / 10000 + 1 + const __m128i abcd_efgh = _mm_add_epi64(x, _mm_mul_epu32(abcd, _mm_set1_epi64x(4294957296))); // 2^32 - 10000 + // 32-bit lanes in the order of the text: abcd, efgh of both halves + const __m128i fours = _mm_shuffle_epi32(abcd_efgh, _MM_SHUFFLE(2, 3, 0, 1)); + const __m128i ab = _mm_srli_epi16(_mm_mulhi_epu16(fours, _mm_set1_epi32(5243)), 3); + const __m128i ab_cd = _mm_or_si128(_mm_slli_epi32(_mm_sub_epi16(fours, _mm_mullo_epi16(ab, _mm_set1_epi32(100))), 16), ab); + // 16-bit lanes ab (< 100) -> bytes a, b: 256 * ab - 2559 * (ab / 10) + const __m128i bytes = _mm_sub_epi16(_mm_slli_epi16(ab_cd, 8), _mm_mullo_epi16(_mm_set1_epi16(2559), _mm_mulhi_epu16(ab_cd, _mm_set1_epi16(6554)))); + // the last digit that is not 0 (sig is not 0) + const auto nonzero = static_cast(_mm_movemask_epi8(_mm_cmpgt_epi8(bytes, _mm_setzero_si128()))); + const int digits = 63 - count_leading_zeros(nonzero) + (sixteen ? 1 : 0); // without trailing zeros + const __m128i chars = _mm_add_epi8(bytes, _mm_set1_epi8('0')); + // the 16 characters from the first digit + const __m128i s = sixteen ? chars : _mm_or_si128(_mm_srli_si128(chars, 1), _mm_slli_si128(_mm_cvtsi32_si128('0' + last), 15)); + const char s16 = static_cast(sixteen ? '0' + last : '0'); // the 17th + const auto store_16 = [&s](char* p) noexcept + { + std::memcpy(p, &s, 16); + }; + const char first_digit = static_cast(_mm_cvtsi128_si32(s)); + // NOLINTEND(portability-simd-intrinsics) +#elif JSON_DTOA_NEON + // as with SSE2: the halves in 32-bit lanes, then abcd, efgh of both + const uint32x2_t halves = vcreate_u32(upper | ((sig - (upper * 100000000u)) << 32u)); + const uint32x2_t abcd = vmovn_u64(vshrq_n_u64(vmull_n_u32(halves, static_cast(((std::uint64_t{1} << 40u) / 10000u) + 1u)), 40)); + const uint32x2_t efgh = vmls_n_u32(halves, abcd, 10000u); + const uint32x4_t fours = vcombine_u32(vzip1_u32(abcd, efgh), vzip2_u32(abcd, efgh)); + const uint32x4_t ab = vshrq_n_u32(vmulq_n_u32(fours, 5243u), 19); + const uint16x8_t ab_cd = vreinterpretq_u16_u32(vorrq_u32(ab, vshlq_n_u32(vmlsq_n_u32(fours, ab, 100u), 16))); + const uint16x8_t tens = vshrq_n_u16(vmulq_n_u16(ab_cd, 103u), 10); + const uint8x16_t bytes = vreinterpretq_u8_u16(vorrq_u16(tens, vshlq_n_u16(vmlsq_n_u16(ab_cd, tens, 10u), 8))); + // the last digit that is not 0 (sig is not 0): a nibble per byte + const std::uint64_t nonzero = vget_lane_u64(vreinterpret_u64_u8(vshrn_n_u16(vreinterpretq_u16_u8(vtstq_u8(bytes, bytes)), 4)), 0); + const int digits = ((63 - count_leading_zeros(nonzero)) / 4) + (sixteen ? 1 : 0); // without trailing zeros + const uint8x16_t chars = vaddq_u8(bytes, vdupq_n_u8('0')); + // the 16 characters from the first digit + const uint8x16_t s = sixteen ? chars : vextq_u8(chars, vdupq_n_u8(static_cast('0' + last)), 1); + const char s16 = static_cast(sixteen ? '0' + last : '0'); // the 17th + const auto store_16 = [&s](char* p) noexcept + { + vst1q_u8(reinterpret_cast(p), s); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) + }; + const auto first_digit = static_cast(vgetq_lane_u8(s, 0)); +#else + const std::uint64_t hi = eight_digit_bytes(upper); + const std::uint64_t lo = eight_digit_bytes(sig - (upper * 100000000u)); + // trailing zero digits: zero bytes (sig is not 0) + const int zeros = lo != 0 ? count_trailing_zeros(lo) / 8 : 8 + (count_trailing_zeros(hi) / 8); + const int digits = 15 - zeros + (sixteen ? 1 : 0); // without trailing zeros + // the 16 characters from the first digit + const std::uint64_t s_hi = (sixteen ? hi : (hi << 8u) | (lo >> 56u)) + 0x3030303030303030u; + const std::uint64_t s_lo = (sixteen ? lo : (lo << 8u) | static_cast(last)) + 0x3030303030303030u; + const char s16 = static_cast(sixteen ? '0' + last : '0'); // the 17th + const auto store_16 = [s_hi, s_lo](char* p) noexcept + { + store_msb_first(p, s_hi); + store_msb_first(p + 8, s_lo); + }; + const auto first_digit = static_cast(s_hi >> 56u); +#endif + const int len = d.has_digit ? 16 + (sixteen ? 1 : 0) : digits; // significant digits + const int n = 16 + (sixteen ? 1 : 0) + d.exponent; // digits before the point + + if (JSON_HEDLEY_LIKELY(n >= 1 && n <= 15)) + { + // "dig.its" and "digits[000].0": the digits after the point move by + // one byte ('0's follow the digits) +#if JSON_DTOA_SSE2 + // NOLINTBEGIN(portability-simd-intrinsics) + // (in the register: reading the digits back from memory right after + // storing them waits until the stores are done) + const __m128i at = _mm_set1_epi8(static_cast(n)); + const __m128i index = _mm_setr_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + const __m128i before = _mm_cmpgt_epi8(at, index); + const __m128i after = _mm_cmpgt_epi8(index, at); + const __m128i text = _mm_or_si128(_mm_or_si128(_mm_and_si128(s, before), _mm_and_si128(_mm_slli_si128(s, 1), after)), + _mm_andnot_si128(_mm_or_si128(before, after), _mm_set1_epi8('.'))); + std::memcpy(first, &text, 16); + first[16] = static_cast(_mm_extract_epi16(s, 7) >> 8); + first[17] = s16; + // NOLINTEND(portability-simd-intrinsics) +#elif JSON_DTOA_NEON + const uint8x16_t index = vcombine_u8(vcreate_u8(0x0706050403020100u), vcreate_u8(0x0F0E0D0C0B0A0908u)); + const uint8x16_t at = vdupq_n_u8(static_cast(n)); + const uint8x16_t after_point = vbslq_u8(vcgtq_u8(index, at), vextq_u8(vdupq_n_u8(0), s, 15), vdupq_n_u8('.')); + vst1q_u8(reinterpret_cast(first), vbslq_u8(vcltq_u8(index, at), s, after_point)); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) + first[16] = static_cast(vgetq_lane_u8(s, 15)); + first[17] = s16; +#else + store_16(first); + first[16] = s16; + std::uint64_t after_point[2]; // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays,cppcoreguidelines-pro-type-member-init,hicpp-member-init): written before read + std::memcpy(after_point, first + n, 16); + std::memcpy(first + n + 1, after_point, 16); + first[n] = '.'; +#endif + return first + (n >= len ? n + 2 : len + 1); + } + if (n <= 0 && n > -4) + { + // "0.[000]digits" + std::memset(first, '0', 8); + first[1] = '.'; + store_16(first + 2 - n); + first[18 - n] = s16; + return first + 2 - n + len; + } + // d.igitse+XX, with at least two exponent digits (as append_exponent()) + store_16(first + 1); + first[17] = s16; + first[0] = first_digit; + first[1] = '.'; + char* const end = first + (len == 1 ? 1 : len + 1); + const int e = n - 1; + const auto ea = e < 0 ? 0u - static_cast(e) : static_cast(e); // (unsigned: no signed overflow to assume) + const bool three = ea >= 100; + end[0] = 'e'; + end[1] = e < 0 ? '-' : '+'; + end[2] = static_cast('0' + (three ? ea / 100 : (ea / 10) % 10)); + end[3] = static_cast('0' + (three ? (ea / 10) % 10 : ea % 10)); + end[4] = static_cast('0' + (ea % 10)); + return end + (three ? 5 : 4); +} + +/*! +@brief whether FloatType is an IEEE 754 binary64 type (a double, or a long double +that has the same format, as with MSVC and on Apple's Arm CPUs) + +These are the types the conversion of Zmij (see zmij.hpp) is used for; all +others (binary32, or a format the library does not know) use Grisu2. +*/ +template +constexpr bool has_binary64_format() noexcept +{ + return std::numeric_limits::is_iec559 + && std::numeric_limits::digits == 53 + && std::numeric_limits::max_exponent == 1024 + && sizeof(FloatType) == sizeof(std::uint64_t); +} + +template +struct is_binary64 : std::integral_constant()> {}; + +/// a positive finite float (other than binary64): Grisu2 and format_buffer() +template +JSON_HEDLEY_NON_NULL(1, 2) +JSON_HEDLEY_RETURNS_NON_NULL +char* write_positive_grisu2(char* first, const char* last, FloatType value) +{ + JSON_ASSERT(last - first >= std::numeric_limits::max_digits10); + static_cast(last); // (only used in the assertion) + + // Compute v = buffer * 10^decimal_exponent. + // The decimal digits are stored in the buffer, which needs to be interpreted + // as an unsigned decimal integer. + // len is the length of the buffer, i.e., the number of decimal digits. + int len = 0; + int decimal_exponent = 0; + grisu2(first, len, decimal_exponent, value); + + JSON_ASSERT(len <= std::numeric_limits::max_digits10); + + // Format the buffer like printf("%.*g", prec, value) + constexpr int kMinExp = -4; + // Use digits10 here to increase compatibility with version 2. + constexpr int kMaxExp = std::numeric_limits::digits10; + + JSON_ASSERT(last - first >= kMaxExp + 2); + JSON_ASSERT(last - first >= 2 + (-kMinExp - 1) + std::numeric_limits::max_digits10); + JSON_ASSERT(last - first >= std::numeric_limits::max_digits10 + 6); + + return format_buffer(first, len, decimal_exponent, kMinExp, kMaxExp); +} + +/// a positive finite binary64 number: the shortest digits (Zmij), laid out by +/// write_shortest() (through a local buffer if [first, last) is shorter than +/// the 41 bytes it may write) +template +JSON_HEDLEY_NON_NULL(1, 2) +JSON_HEDLEY_RETURNS_NON_NULL +char* write_positive_zmij(char* first, const char* last, FloatType value) +{ + static_assert(is_binary64::value, + "internal error: the conversion of Zmij needs IEEE 754 binary64 numbers"); + std::uint64_t bits = 0; + std::memcpy(&bits, &value, sizeof(bits)); + const zmij::shortest_decimal d = zmij::to_shortest(bits); + if (JSON_HEDLEY_LIKELY(last - first >= 41)) + { + return write_shortest(first, d); + } + std::array buf; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init): written before read + const auto len = static_cast(write_shortest(buf.data(), d) - buf.data()); + JSON_ASSERT(last - first >= static_cast(len)); + std::memcpy(first, buf.data(), len); + return first + len; +} + +/// a positive finite binary64 number: Zmij (as a long double has the format of +/// a double here, its bits are those of the double of the same value) +template +JSON_HEDLEY_NON_NULL(1, 2) +JSON_HEDLEY_RETURNS_NON_NULL +char* write_positive(char* first, const char* last, FloatType value, std::true_type /*is_binary64*/) +{ + return write_positive_zmij(first, last, value); +} + +/// a positive finite float of any other format: Grisu2 +template +JSON_HEDLEY_NON_NULL(1, 2) +JSON_HEDLEY_RETURNS_NON_NULL +char* write_positive(char* first, const char* last, FloatType value, std::false_type /*is_binary64*/) +{ + return write_positive_grisu2(first, last, value); +} + +/// a positive finite float: Zmij for binary64 numbers, Grisu2 otherwise +template +JSON_HEDLEY_NON_NULL(1, 2) +JSON_HEDLEY_RETURNS_NON_NULL +char* write_positive(char* first, const char* last, FloatType value) +{ + return write_positive(first, last, value, is_binary64 {}); +} + } // namespace dtoa_impl /*! @@ -26278,7 +27503,6 @@ JSON_HEDLEY_NON_NULL(1, 2) JSON_HEDLEY_RETURNS_NON_NULL char* to_chars(char* first, const char* last, FloatType value) { - static_cast(last); // maybe unused - fix warning JSON_ASSERT(std::isfinite(value)); // Use signbit(value) instead of (value < 0) since signbit works for -0. @@ -26304,28 +27528,7 @@ char* to_chars(char* first, const char* last, FloatType value) JSON_HEDLEY_DIAGNOSTIC_POP #endif - JSON_ASSERT(last - first >= std::numeric_limits::max_digits10); - - // Compute v = buffer * 10^decimal_exponent. - // The decimal digits are stored in the buffer, which needs to be interpreted - // as an unsigned decimal integer. - // len is the length of the buffer, i.e., the number of decimal digits. - int len = 0; - int decimal_exponent = 0; - dtoa_impl::grisu2(first, len, decimal_exponent, value); - - JSON_ASSERT(len <= std::numeric_limits::max_digits10); - - // Format the buffer like printf("%.*g", prec, value) - constexpr int kMinExp = -4; - // Use digits10 here to increase compatibility with version 2. - constexpr int kMaxExp = std::numeric_limits::digits10; - - JSON_ASSERT(last - first >= kMaxExp + 2); - JSON_ASSERT(last - first >= 2 + (-kMinExp - 1) + std::numeric_limits::max_digits10); - JSON_ASSERT(last - first >= std::numeric_limits::max_digits10 + 6); - - return dtoa_impl::format_buffer(first, len, decimal_exponent, kMinExp, kMaxExp); + return dtoa_impl::write_positive(first, last, value); } } // namespace detail @@ -27684,8 +28887,9 @@ class serializer /*! @brief dump an integer - Dump a given integer, appending it to @ref write_buffer. Works internally with - @a number_buffer. + Dump a given integer, appending it to @ref write_buffer (directly: copying + the digits from another buffer right after writing them waits until the + stores are done). @param[in] x integer number (signed or unsigned) to dump @tparam NumberType either @a number_integer_t or @a number_unsigned_t @@ -27720,33 +28924,57 @@ class serializer return; } - // use a pointer to fill the buffer - auto buffer_ptr = number_buffer.begin(); // NOLINT(llvm-qualified-auto,readability-qualified-auto) + // use a pointer to fill the buffer (room for as much as number_buffer holds) + if (JSON_HEDLEY_UNLIKELY(write_buffer_pos + number_buffer.size() > write_buffer.size())) + { + flush(); + } + auto* buffer_ptr = write_buffer.data() + write_buffer_pos; number_unsigned_t abs_value; - unsigned int n_chars{}; + // one byte for the minus sign + unsigned int n_chars = 0; if (is_negative_number(x)) { *buffer_ptr = '-'; abs_value = remove_sign(static_cast(x)); - - // account one more byte for the minus sign - n_chars = 1 + count_digits(abs_value); + n_chars = 1; } else { abs_value = static_cast(x); - n_chars = count_digits(abs_value); } + // up to 16 digits: eight at a time (as the digits of floats), written + // without leading zeros + if (abs_value < 10000000000000000u) + { + const std::uint64_t value = abs_value; + const std::uint64_t upper = value / 100000000u; + const std::uint64_t first = dtoa_impl::eight_digit_bytes(upper != 0 ? upper : value); + const auto leading = static_cast(count_leading_zeros(first) / 8); // (first is not 0) + char* const p = buffer_ptr + n_chars; + dtoa_impl::store_msb_first(p, (first << (8 * leading)) + 0x3030303030303030u); + n_chars += 8 - leading; + if (upper != 0) + { + dtoa_impl::store_msb_first(p + 8 - leading, dtoa_impl::eight_digit_bytes(value - (upper * 100000000u)) + 0x3030303030303030u); + n_chars += 8; + } + write_buffer_pos += n_chars; + return; + } + + n_chars += count_digits(abs_value); + // spare 1 byte for '\0' JSON_ASSERT(n_chars < number_buffer.size() - 1); // jump to the end to generate the string from backward, // so we later avoid reversing the result - buffer_ptr += static_cast(n_chars); + buffer_ptr += n_chars; // Fast int2ascii implementation inspired by "Fastware" talk by Andrei Alexandrescu // See: https://www.youtube.com/watch?v=o4-CwDo2zpg @@ -27769,14 +28997,13 @@ class serializer *(--buffer_ptr) = static_cast('0' + abs_value); } - put_buffer(number_buffer, n_chars); + write_buffer_pos += n_chars; } /*! @brief dump a floating-point number - Dump a given floating-point number, appending it to @ref write_buffer. Works internally - with @a number_buffer. + Dump a given floating-point number, appending it to @ref write_buffer. @param[in] x floating-point number to dump */ @@ -27803,10 +29030,15 @@ class serializer void dump_float(number_float_t x, std::true_type /*is_ieee_single_or_double*/) { - auto* begin = number_buffer.data(); + // directly into the write buffer: copying the text from number_buffer + // right after to_chars() wrote it waits until its stores are done + if (JSON_HEDLEY_UNLIKELY(write_buffer_pos + number_buffer.size() > write_buffer.size())) + { + flush(); + } + auto* begin = write_buffer.data() + write_buffer_pos; auto* end = ::nlohmann::detail::to_chars(begin, begin + number_buffer.size(), x); - - put_buffer(number_buffer, static_cast(end - begin)); + write_buffer_pos += static_cast(end - begin); } JSON_HEDLEY_NON_NULL(1) @@ -27947,7 +29179,7 @@ NLOHMANN_JSON_NAMESPACE_END // #include // #include - +// IWYU pragma: export // #include // __ _____ _____ _____ // __| | __| | | | JSON for Modern C++ @@ -35741,6 +36973,176 @@ public: } public: + /// @brief creates a diff as a JSON Merge Patch + /// @sa https://json.nlohmann.me/api/basic_json/merge_diff/ + JSON_HEDLEY_WARN_UNUSED_RESULT + static basic_json merge_diff(const basic_json& source, const basic_json& target) + { + // unless both are objects, the patch is the target itself + if (!source.is_object() || !target.is_object()) + { + return target; + } + + return merge_diff_recursively(source, target, 0); + } + + private: + /// @brief two objects @ref merge_diff_iteratively is diffing + struct merge_diff_frame + { + merge_diff_frame(const basic_json* source_, const basic_json* target_) + : source(source_), target(target_), member(source_->cbegin()), patch(value_t::object) + {} + + // declared for GCC's -Weffc++, as in diff_frame + merge_diff_frame(const merge_diff_frame&) = default; + merge_diff_frame(merge_diff_frame&&) = default; + merge_diff_frame& operator=(const merge_diff_frame&) = default; + merge_diff_frame& operator=(merge_diff_frame&&) = default; + ~merge_diff_frame() = default; + + /// the objects being diffed + const basic_json* source; + const basic_json* target; + /// the member of source being diffed; advanced once it is done + const_iterator member; + /// the patch for the members of source diffed so far + basic_json patch; + }; + + /// @brief add the members of object @a target that @a source does not + /// have to the merge patch @a result + static void merge_diff_added_members(basic_json& result, const basic_json& source, const basic_json& target) + { + for (auto it = target.begin(); it != target.end(); ++it) + { + auto itf = source.find(it.key()); + if (itf == source.end()) + { + result[it.key()] = it.value(); + } + } + } + + /*! + @brief @ref merge_diff of two objects at nesting level @a depth + + Diffing two nested objects calls this function again, once per nesting + level, so values nested deeply enough would exhaust the call stack and + terminate the process. The descent is bounded here: once @ref + detail::recursion_depth_limit levels have been entered, @ref + merge_diff_iteratively diffs what is left without the call stack. + */ + static basic_json merge_diff_recursively(const basic_json& source, const basic_json& target, const std::size_t depth) + { + JSON_ASSERT(source.is_object() && target.is_object()); + + if (JSON_HEDLEY_UNLIKELY(depth >= detail::recursion_depth_limit())) + { + return merge_diff_iteratively(source, target); + } + + basic_json result(value_t::object); + + for (auto it = source.begin(); it != source.end(); ++it) + { + auto itf = target.find(it.key()); + if (itf != target.end()) + { + if (it.value().is_object() && itf.value().is_object()) + { + auto diff = merge_diff_recursively(it.value(), itf.value(), depth + 1); + if (!diff.empty()) + { + result[it.key()] = std::move(diff); + } + } + else if (it.value() != itf.value()) + { + result[it.key()] = itf.value(); + } + } + else + { + result[it.key()] = value_t::null; + } + } + + merge_diff_added_members(result, source, target); + return result; + } + + /*! + @brief @ref merge_diff of two objects without the call stack + + Produces the same patch as @ref merge_diff_recursively, keeping the objects + being diffed on an explicit stack instead of descending into them. Two + nested objects are still diffed completely before the next member, so the + patch has its members in the same order as in the recursive version. Only + reached for objects nested more deeply than @ref + detail::recursion_depth_limit. + */ + static basic_json merge_diff_iteratively(const basic_json& source, const basic_json& target) + { + // the patch of source and target, set once their frame is popped + basic_json result; + std::vector stack; + stack.emplace_back(&source, &target); + while (!stack.empty()) + { + // the frame's members are copied out and it is changed through + // stack.back(): pushing a frame for nested objects or popping + // this one would invalidate a reference to it + const basic_json* const s = stack.back().source; + const basic_json* const t = stack.back().target; + const const_iterator it = stack.back().member; + + if (it != s->cend()) + { + const const_iterator itf = t->find(it.key()); + if (itf != t->cend()) + { + if (it.value().is_object() && itf.value().is_object()) + { + // member is advanced once their patch is under its key + stack.emplace_back(&it.value(), &itf.value()); + continue; + } + if (it.value() != itf.value()) + { + stack.back().patch[it.key()] = itf.value(); + } + } + else + { + stack.back().patch[it.key()] = value_t::null; + } + ++stack.back().member; + continue; + } + + merge_diff_added_members(stack.back().patch, *s, *t); + + // these objects are done: continue with the ones they are in + basic_json patch = std::move(stack.back().patch); + stack.pop_back(); + if (stack.empty()) + { + result = std::move(patch); + } + else + { + if (!patch.empty()) + { + stack.back().patch[stack.back().member.key()] = std::move(patch); + } + ++stack.back().member; + } + } + return result; + } + /// @} }; @@ -35933,6 +37335,8 @@ struct formatter // NOLINT(cert-dcl58-c #undef JSON_NO_UNIQUE_ADDRESS #undef JSON_DISABLE_ENUM_SERIALIZATION #undef JSON_DISABLE_TUPLE_REFERENCE_CONVERSION +#undef JSON_DTOA_SSE2 +#undef JSON_DTOA_NEON #ifndef JSON_TEST_KEEP_MACROS #undef JSON_CATCH @@ -36210,7 +37614,7 @@ NLOHMANN_JSON_NAMESPACE_END #undef NLOHMANN_JSON_LITERAL_OPERATOR #endif // INCLUDE_NLOHMANN_JSON_LITERALS_HPP_ - +// IWYU pragma: export #endif #endif // INCLUDE_NLOHMANN_JSON_HPP_ diff --git a/tests/benchmarks/CMakeLists.txt b/tests/benchmarks/CMakeLists.txt index 4d7265145..e4ae29923 100644 --- a/tests/benchmarks/CMakeLists.txt +++ b/tests/benchmarks/CMakeLists.txt @@ -6,8 +6,9 @@ if((CMAKE_CXX_COMPILER_ID MATCHES GNU) OR (CMAKE_CXX_COMPILER_ID MATCHES Clang)) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -flto -DNDEBUG -O3") endif() -# configure Google Benchmark; a fixed release, so that results stay comparable -set(JSON_GOOGLE_BENCHMARK_VERSION 1.9.5) +# configure Google Benchmark; a fixed release with a checksum, so that builds are reproducible. Update it like any +# other dependency; comparisons stay fair because both sides are built with the same version (see README.md) +set(JSON_GOOGLE_BENCHMARK_VERSION 1.10.0) include(FetchContent) # only the library is needed; -Werror would break the pinned release as soon as @@ -18,7 +19,7 @@ set(BENCHMARK_ENABLE_WERROR OFF CACHE BOOL "" FORCE) FetchContent_Declare(benchmark URL https://github.com/google/benchmark/archive/refs/tags/v${JSON_GOOGLE_BENCHMARK_VERSION}.tar.gz - URL_HASH SHA256=9631341c82bac4a288bef951f8b26b41f69021794184ece969f8473977eaa340 + URL_HASH SHA256=acc5c7c0fbed855c4aaa2d7c3c36fb82b0c15c28f4e151e3f679c2fc0aa404af DOWNLOAD_EXTRACT_TIMESTAMP TRUE ) FetchContent_MakeAvailable(benchmark) diff --git a/tests/benchmarks/README.md b/tests/benchmarks/README.md index aaf5abe46..6f067db60 100644 --- a/tests/benchmarks/README.md +++ b/tests/benchmarks/README.md @@ -27,8 +27,11 @@ The input files are those of [nativejson-benchmark](https://github.com/miloyip/n - Network access on the first configure: CMake downloads Google Benchmark and the [test data](https://github.com/nlohmann/json_test_data) into the build directory. To reuse a download of the test data, pass `-DJSON_TestDataDirectory=/test_files`. -- Google Benchmark is pinned to a release (1.9.5), so that results from different days stay comparable. To update it, - change `JSON_GOOGLE_BENCHMARK_VERSION` and the archive's `URL_HASH` in `CMakeLists.txt` together. +- Google Benchmark is fetched as a fixed release with a checksum, so that every build uses the same + code and a download cannot change under you. It is updated like any other dependency when a new release comes out: + change `JSON_GOOGLE_BENCHMARK_VERSION` and the archive's `URL_HASH` in `CMakeLists.txt` together. Results are kept + comparable by building both sides of a comparison with the same version, not by keeping that version forever (see + [Comparing two versions](#comparing-two-versions)). - The benchmarks include `single_include/nlohmann/json.hpp`, so run `make amalgamate` after changing anything in `include/`. @@ -110,13 +113,18 @@ Python, unpinned versions work as well. In its output: - `OVERALL_GEOMEAN` summarizes all benchmarks; - `-a` shows only the aggregates, not every repetition. +Both builds use the same Google Benchmark, the one `CMakeLists.txt` names, so only the header differs. For the same +reason, do not compare against result files from an earlier checkout: a different Google Benchmark, compiler or +machine state can shift the numbers as much as the change under test. Rerun the baseline instead. + The header you compare with must support everything the benchmarks use. The current benchmarks build against 3.12.0. Only benchmarks present in both result files are compared, so for older releases, either filter the benchmarks or build that release's own `tests/benchmarks` against its own header. ## Getting stable numbers -- Build and run both versions on the same machine, one right after the other. +- Build and run both versions on the same machine with the same compiler and Google Benchmark, one right after the + other. - Keep the machine otherwise idle: no builds, no browser, and a laptop plugged in. - On Linux, set the CPU frequency governor to `performance`, e.g. `sudo cpupower frequency-set --governor performance`. Google Benchmark prints a warning when frequency scaling is enabled. Pinning the process to a core diff --git a/tests/iwyu_consumer/conversions.cpp b/tests/iwyu_consumer/conversions.cpp new file mode 100644 index 000000000..599da67b7 --- /dev/null +++ b/tests/iwyu_consumer/conversions.cpp @@ -0,0 +1,62 @@ +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ (supporting code) +// | | |__ | | | | | | version 3.12.0 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann +// SPDX-License-Identifier: MIT + +// See values.cpp: the conversion macros and nlohmann::adl_serializer must be available through +// alone. NLOHMANN_JSON_SERIALIZE_ENUM (from the same header as +// NLOHMANN_DEFINE_TYPE_INTRUSIVE) is left out: IWYU attributes the basic_json comparison in its +// expansion to an unrelated standard header, the ambiguity described for json.hpp in cmake/ci.cmake. + +#include + +#include + +namespace ns +{ +struct person +{ + std::string name; + int age; + NLOHMANN_DEFINE_TYPE_INTRUSIVE(person, name, age) +}; + +struct legacy +{ + int value; +}; +} // namespace ns + +namespace nlohmann +{ +template<> +struct adl_serializer +{ + static void to_json(json& j, const ns::legacy& l) + { + j = l.value; + } + + static void from_json(const json& j, ns::legacy& l) + { + l.value = j.get(); + } +}; +} // namespace nlohmann + +int main() +{ + ns::person ada; + ada.name = "Ada"; + ada.age = 36; + ns::legacy one; + one.value = 1; + + const nlohmann::json j = ada; + const auto p = j.get(); + const auto l = nlohmann::json(one).get(); + return p.age - 36 + l.value - 1; +} diff --git a/tests/iwyu_consumer/literals.cpp b/tests/iwyu_consumer/literals.cpp new file mode 100644 index 000000000..7aaf58d79 --- /dev/null +++ b/tests/iwyu_consumer/literals.cpp @@ -0,0 +1,19 @@ +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ (supporting code) +// | | |__ | | | | | | version 3.12.0 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann +// SPDX-License-Identifier: MIT + +// See values.cpp: the user-defined literals must be available through alone. + +#include + +using namespace nlohmann::literals; // NOLINT(google-build-using-namespace) + +int main() +{ + const auto j = R"({"a": [1, 2]})"_json; + return j.at("/a/1"_json_pointer).get() - 2; +} diff --git a/tests/iwyu_consumer/values.cpp b/tests/iwyu_consumer/values.cpp new file mode 100644 index 000000000..1f6d00b76 --- /dev/null +++ b/tests/iwyu_consumer/values.cpp @@ -0,0 +1,30 @@ +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ (supporting code) +// | | |__ | | | | | | version 3.12.0 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann +// SPDX-License-Identifier: MIT + +// Uses the library the way a client does, including only . ci_single_binaries +// runs include-what-you-use and clang-tidy's misc-include-cleaner on this file and fails if either +// asks for another nlohmann header, which would mean json.hpp does not export it (issue #5812). + +#include +#include +#include + +#include + +int main() +{ + // brace initialization (nlohmann::json, detail::json_ref) + nlohmann::json j = {{"pi", 3.141}, {"list", {1, 0, 2}}}; + const nlohmann::ordered_json o = {{"b", 1}, {"a", 2}}; + + j["parsed"] = nlohmann::json::parse(R"({"happy": true})"); + j["binary"] = nlohmann::json::binary(std::vector {0xCA, 0xFE}, 42); + + const std::string s = j.dump() + o.dump(); + return s.empty() ? 1 : 0; +} diff --git a/tests/src/float_hard_cases.hpp b/tests/src/float_hard_cases.hpp new file mode 100644 index 000000000..e579bccce --- /dev/null +++ b/tests/src/float_hard_cases.hpp @@ -0,0 +1,599 @@ +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ (supporting code) +// | | |__ | | | | | | version 3.12.0 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann +// SPDX-License-Identifier: MIT + +#pragma once + +#include // array +#include // uint32_t, uint64_t + +// Number tokens that are hard to round correctly, with the IEEE-754 binary64 +// and binary32 bits of their correctly rounded values (ties to even; infinity +// for an overflow, a signed zero for an underflow). +// +// For doubles and floats around 0, the smallest normal number, 1, 2^24, 2^53, +// 0.1, and the largest finite number, and for random ones, the exact midpoint +// m to the next number gives: m, m with one unit more and less in the last +// digit, m with "01" and "0...01" appended, m with trailing zeros, and m cut +// after 17 to 30 digits (rounded down and up, so that the rounding is decided +// after the 19th digit), in fixed and exponent notation, 30% of them negative. +// Tokens longer than 80 characters are left out, except for four of 700 digits +// and more. Zeros, underflow, overflow, huge exponents, and integers beyond 64 +// bits complete the set. Of the 508 tokens, 134 (as double) and 150 (as +// float) need the exact comparison with the midpoint (detail::digit_comparison()). +// +// The expected bits were computed with exact rational arithmetic in Python +// (fractions.Fraction) and cross-checked with Python's float(); strtod_l and +// strtof_l of Apple's libc and of glibc agree. Generated by +// compact_hard_cases.py 5 (with hard_cases.py), see the pull request that +// added this file. + +namespace float_hard_cases +{ + +struct hard_case +{ + const char* token; + std::uint64_t bits64; + std::uint32_t bits32; +}; + +inline const std::array& cases() +{ + static const std::array table = + { + { + {"-2.4703282292062327e-324", 0x8000000000000000u, 0x80000000u}, + {"24703282292062328e-340", 0x0000000000000001u, 0x00000000u}, + {"247032822920623272e-341", 0x0000000000000000u, 0x00000000u}, + {"-0.2470328229206232721e-323", 0x8000000000000001u, 0x80000000u}, + {"-0.24703282292062327208e-323", 0x8000000000000000u, 0x80000000u}, + {"-2.4703282292062327209e-324", 0x8000000000000001u, 0x80000000u}, + {"2.47032822920623272088e-324", 0x0000000000000000u, 0x00000000u}, + {"247032822920623272089e-344", 0x0000000000000001u, 0x00000000u}, + {"-247032822920623272088284396434e-353", 0x8000000000000000u, 0x80000000u}, + {"0.247032822920623272088284396435e-323", 0x0000000000000001u, 0x00000000u}, + {"-74109846876186981e-340", 0x8000000000000001u, 0x80000000u}, + {"0.74109846876186982e-323", 0x0000000000000002u, 0x00000000u}, + {"-0.7410984687618698162e-323", 0x8000000000000001u, 0x80000000u}, + {"-7.410984687618698163e-324", 0x8000000000000002u, 0x80000000u}, + {"7.4109846876186981626e-324", 0x0000000000000001u, 0x00000000u}, + {"-74109846876186981627e-343", 0x8000000000000002u, 0x80000000u}, + {"-741098468761869816264e-344", 0x8000000000000001u, 0x80000000u}, + {"0.741098468761869816265e-323", 0x0000000000000002u, 0x00000000u}, + {"0.741098468761869816264853189302e-323", 0x0000000000000001u, 0x00000000u}, + {"-7.41098468761869816264853189303e-324", 0x8000000000000002u, 0x80000000u}, + {"0.22250738585072006e-307", 0x000FFFFFFFFFFFFEu, 0x00000000u}, + {"2.2250738585072007e-308", 0x000FFFFFFFFFFFFFu, 0x00000000u}, + {"2.225073858507200641e-308", 0x000FFFFFFFFFFFFEu, 0x00000000u}, + {"-2225073858507200642e-326", 0x800FFFFFFFFFFFFFu, 0x80000000u}, + {"22250738585072006419e-327", 0x000FFFFFFFFFFFFEu, 0x00000000u}, + {"0.2225073858507200642e-307", 0x000FFFFFFFFFFFFFu, 0x00000000u}, + {"0.222507385850720064199e-307", 0x000FFFFFFFFFFFFEu, 0x00000000u}, + {"2.225073858507200642e-308", 0x000FFFFFFFFFFFFFu, 0x00000000u}, + {"-2.22507385850720064199176395546e-308", 0x800FFFFFFFFFFFFEu, 0x80000000u}, + {"222507385850720064199176395547e-337", 0x000FFFFFFFFFFFFFu, 0x00000000u}, + {"-2.2250738585072011e-308", 0x800FFFFFFFFFFFFFu, 0x80000000u}, + {"-22250738585072012e-324", 0x8010000000000000u, 0x80000000u}, + {"-2225073858507201136e-326", 0x800FFFFFFFFFFFFFu, 0x80000000u}, + {"0.2225073858507201137e-307", 0x0010000000000000u, 0x00000000u}, + {"0.2225073858507201136e-307", 0x000FFFFFFFFFFFFFu, 0x00000000u}, + {"-2.2250738585072011361e-308", 0x8010000000000000u, 0x80000000u}, + {"2.22507385850720113605e-308", 0x000FFFFFFFFFFFFFu, 0x00000000u}, + {"222507385850720113606e-328", 0x0010000000000000u, 0x00000000u}, + {"22250738585072011360574097967e-336", 0x000FFFFFFFFFFFFFu, 0x00000000u}, + {"0.222507385850720113605740979671e-307", 0x0010000000000000u, 0x00000000u}, + {"22250738585072016e-324", 0x0010000000000000u, 0x00000000u}, + {"0.22250738585072017e-307", 0x0010000000000001u, 0x00000000u}, + {"0.222507385850720163e-307", 0x0010000000000000u, 0x00000000u}, + {"2.225073858507201631e-308", 0x0010000000000001u, 0x00000000u}, + {"-2.2250738585072016301e-308", 0x8010000000000000u, 0x80000000u}, + {"22250738585072016302e-327", 0x0010000000000001u, 0x00000000u}, + {"-222507385850720163012e-328", 0x8010000000000000u, 0x80000000u}, + {"0.222507385850720163013e-307", 0x0010000000000001u, 0x00000000u}, + {"0.222507385850720163012305563795e-307", 0x0010000000000000u, 0x00000000u}, + {"-2.22507385850720163012305563796e-308", 0x8010000000000001u, 0x80000000u}, + {"0.17976931348623156E+309", 0x7FEFFFFFFFFFFFFEu, 0x7F800000u}, + {"1.7976931348623157e308", 0x7FEFFFFFFFFFFFFFu, 0x7F800000u}, + {"1.797693134862315608e308", 0x7FEFFFFFFFFFFFFEu, 0x7F800000u}, + {"-1797693134862315609e290", 0xFFEFFFFFFFFFFFFFu, 0xFF800000u}, + {"-17976931348623156083e289", 0xFFEFFFFFFFFFFFFEu, 0xFF800000u}, + {"-0.17976931348623156084E+309", 0xFFEFFFFFFFFFFFFFu, 0xFF800000u}, + {"0.179769313486231560835E+309", 0x7FEFFFFFFFFFFFFEu, 0x7F800000u}, + {"-1.79769313486231560836e308", 0xFFEFFFFFFFFFFFFFu, 0xFF800000u}, + {"1.79769313486231560835325876058e308", 0x7FEFFFFFFFFFFFFEu, 0x7F800000u}, + {"179769313486231560835325876059e279", 0x7FEFFFFFFFFFFFFFu, 0x7F800000u}, + {"1.7976931348623158e308", 0x7FEFFFFFFFFFFFFFu, 0x7F800000u}, + {"17976931348623159e292", 0x7FF0000000000000u, 0x7F800000u}, + {"1797693134862315807e290", 0x7FEFFFFFFFFFFFFFu, 0x7F800000u}, + {"0.1797693134862315808E+309", 0x7FF0000000000000u, 0x7F800000u}, + {"0.17976931348623158079E+309", 0x7FEFFFFFFFFFFFFFu, 0x7F800000u}, + {"-1.797693134862315808e308", 0xFFF0000000000000u, 0xFF800000u}, + {"1.79769313486231580793e308", 0x7FEFFFFFFFFFFFFFu, 0x7F800000u}, + {"179769313486231580794e288", 0x7FF0000000000000u, 0x7F800000u}, + {"179769313486231580793728971405e279", 0x7FEFFFFFFFFFFFFFu, 0x7F800000u}, + {"-0.179769313486231580793728971406E+309", 0xFFF0000000000000u, 0xFF800000u}, + {"100000000000000011102230246251565404236316680908203125e-53", 0x3FF0000000000000u, 0x3F800000u}, + {"-1.00000000000000011102230246251565404236316680908203126", 0xBFF0000000000001u, 0xBF800000u}, + {"1.00000000000000011102230246251565404236316680908203124e0", 0x3FF0000000000000u, 0x3F800000u}, + {"10000000000000001110223024625156540423631668090820312501e-55", 0x3FF0000000000001u, 0x3F800000u}, + {"1.00000000000000011102230246251565404236316680908203125000000000000000000001", 0x3FF0000000000001u, 0x3F800000u}, + {"10000000000000001e-16", 0x3FF0000000000000u, 0x3F800000u}, + {"1.0000000000000002", 0x3FF0000000000001u, 0x3F800000u}, + {"1.000000000000000111", 0x3FF0000000000000u, 0x3F800000u}, + {"1.000000000000000112e0", 0x3FF0000000000001u, 0x3F800000u}, + {"1.000000000000000111e0", 0x3FF0000000000000u, 0x3F800000u}, + {"-10000000000000001111e-19", 0xBFF0000000000001u, 0xBF800000u}, + {"-100000000000000011102e-20", 0xBFF0000000000000u, 0xBF800000u}, + {"-1.00000000000000011103", 0xBFF0000000000001u, 0xBF800000u}, + {"1.00000000000000011102230246251", 0x3FF0000000000000u, 0x3F800000u}, + {"1.00000000000000011102230246252e0", 0x3FF0000000000001u, 0x3F800000u}, + {"-0.999999999999999944488848768742172978818416595458984375", 0xBFF0000000000000u, 0xBF800000u}, + {"-9.99999999999999944488848768742172978818416595458984376e-1", 0xBFF0000000000000u, 0xBF800000u}, + {"999999999999999944488848768742172978818416595458984374e-54", 0x3FEFFFFFFFFFFFFFu, 0x3F800000u}, + {"0.99999999999999994448884876874217297881841659545898437501", 0x3FF0000000000000u, 0x3F800000u}, + {"9.99999999999999944488848768742172978818416595458984375000000000000000000001e-1", 0x3FF0000000000000u, 0x3F800000u}, + {"-0.99999999999999994", 0xBFEFFFFFFFFFFFFFu, 0xBF800000u}, + {"9.9999999999999995e-1", 0x3FF0000000000000u, 0x3F800000u}, + {"9.999999999999999444e-1", 0x3FEFFFFFFFFFFFFFu, 0x3F800000u}, + {"9999999999999999445e-19", 0x3FF0000000000000u, 0x3F800000u}, + {"99999999999999994448e-20", 0x3FEFFFFFFFFFFFFFu, 0x3F800000u}, + {"-0.99999999999999994449", 0xBFF0000000000000u, 0xBF800000u}, + {"0.999999999999999944488", 0x3FEFFFFFFFFFFFFFu, 0x3F800000u}, + {"-9.99999999999999944489e-1", 0xBFF0000000000000u, 0xBF800000u}, + {"9.99999999999999944488848768742e-1", 0x3FEFFFFFFFFFFFFFu, 0x3F800000u}, + {"999999999999999944488848768743e-30", 0x3FF0000000000000u, 0x3F800000u}, + {"-9.007199254740993e15", 0xC340000000000000u, 0xDA000000u}, + {"9007199254740994e0", 0x4340000000000001u, 0x5A000000u}, + {"9007199254740992", 0x4340000000000000u, 0x5A000000u}, + {"9.00719925474099301e15", 0x4340000000000001u, 0x5A000000u}, + {"9007199254740993000000000000000000001e-21", 0x4340000000000001u, 0x5A000000u}, + {"-9007199254740993.000000000000000000000000000000", 0xC340000000000000u, 0xDA000000u}, + {"90071992547409915e-1", 0x4340000000000000u, 0x5A000000u}, + {"-9007199254740991.6", 0xC340000000000000u, 0xDA000000u}, + {"9.0071992547409914e15", 0x433FFFFFFFFFFFFFu, 0x5A000000u}, + {"-9007199254740991501e-3", 0xC340000000000000u, 0xDA000000u}, + {"9007199254740991.5000000000000000000001", 0x4340000000000000u, 0x5A000000u}, + {"9.0071992547409915000000000000000000000000000000e15", 0x4340000000000000u, 0x5A000000u}, + {"0.100000000000000012490009027033011079765856266021728515625", 0x3FB999999999999Au, 0x3DCCCCCDu}, + {"1.00000000000000012490009027033011079765856266021728515626e-1", 0x3FB999999999999Bu, 0x3DCCCCCDu}, + {"100000000000000012490009027033011079765856266021728515624e-57", 0x3FB999999999999Au, 0x3DCCCCCDu}, + {"0.10000000000000001249000902703301107976585626602172851562501", 0x3FB999999999999Bu, 0x3DCCCCCDu}, + {"0.10000000000000001", 0x3FB999999999999Au, 0x3DCCCCCDu}, + {"1.0000000000000002e-1", 0x3FB999999999999Bu, 0x3DCCCCCDu}, + {"-1.000000000000000124e-1", 0xBFB999999999999Au, 0xBDCCCCCDu}, + {"1000000000000000125e-19", 0x3FB999999999999Bu, 0x3DCCCCCDu}, + {"-10000000000000001249e-20", 0xBFB999999999999Au, 0xBDCCCCCDu}, + {"0.1000000000000000125", 0x3FB999999999999Bu, 0x3DCCCCCDu}, + {"0.10000000000000001249", 0x3FB999999999999Au, 0x3DCCCCCDu}, + {"1.00000000000000012491e-1", 0x3FB999999999999Bu, 0x3DCCCCCDu}, + {"1.00000000000000012490009027033e-1", 0x3FB999999999999Au, 0x3DCCCCCDu}, + {"100000000000000012490009027034e-30", 0x3FB999999999999Bu, 0x3DCCCCCDu}, + {"2.45134755833537796875e14", 0x42EBDE5C4164D83Au, 0x575EF2E2u}, + {"-245134755833537796876e-6", 0xC2EBDE5C4164D83Au, 0xD75EF2E2u}, + {"-245134755833537.796874", 0xC2EBDE5C4164D839u, 0xD75EF2E2u}, + {"2.4513475583353779687501e14", 0x42EBDE5C4164D83Au, 0x575EF2E2u}, + {"245134755833537796875000000000000000000001e-27", 0x42EBDE5C4164D83Au, 0x575EF2E2u}, + {"245134755833537.796875000000000000000000000000000000", 0x42EBDE5C4164D83Au, 0x575EF2E2u}, + {"2.4513475583353779e14", 0x42EBDE5C4164D839u, 0x575EF2E2u}, + {"2451347558335378e-1", 0x42EBDE5C4164D83Au, 0x575EF2E2u}, + {"2451347558335377968e-4", 0x42EBDE5C4164D839u, 0x575EF2E2u}, + {"245134755833537.7969", 0x42EBDE5C4164D83Au, 0x575EF2E2u}, + {"245134755833537.79687", 0x42EBDE5C4164D839u, 0x575EF2E2u}, + {"2.4513475583353779688e14", 0x42EBDE5C4164D83Au, 0x575EF2E2u}, + {"181510327827821147441864013671875e-23", 0x41DB0C11CB91CE38u, 0x4ED8608Eu}, + {"-1815103278.27821147441864013671876", 0xC1DB0C11CB91CE38u, 0xCED8608Eu}, + {"1.81510327827821147441864013671874e9", 0x41DB0C11CB91CE37u, 0x4ED8608Eu}, + {"18151032782782114744186401367187501e-25", 0x41DB0C11CB91CE38u, 0x4ED8608Eu}, + {"1815103278.27821147441864013671875000000000000000000001", 0x41DB0C11CB91CE38u, 0x4ED8608Eu}, + {"-1.81510327827821147441864013671875000000000000000000000000000000e9", 0xC1DB0C11CB91CE38u, 0xCED8608Eu}, + {"18151032782782114e-7", 0x41DB0C11CB91CE37u, 0x4ED8608Eu}, + {"1815103278.2782115", 0x41DB0C11CB91CE38u, 0x4ED8608Eu}, + {"1815103278.278211474", 0x41DB0C11CB91CE37u, 0x4ED8608Eu}, + {"-1.815103278278211475e9", 0xC1DB0C11CB91CE38u, 0xCED8608Eu}, + {"1.8151032782782114744e9", 0x41DB0C11CB91CE37u, 0x4ED8608Eu}, + {"-18151032782782114745e-10", 0xC1DB0C11CB91CE38u, 0xCED8608Eu}, + {"181510327827821147441e-11", 0x41DB0C11CB91CE37u, 0x4ED8608Eu}, + {"1815103278.27821147442", 0x41DB0C11CB91CE38u, 0x4ED8608Eu}, + {"1815103278.27821147441864013671", 0x41DB0C11CB91CE37u, 0x4ED8608Eu}, + {"1.81510327827821147441864013672e9", 0x41DB0C11CB91CE38u, 0x4ED8608Eu}, + {"3809325632181785344", 0x43CA6EB8BD69FE2Au, 0x5E5375C6u}, + {"3.809325632181785345e18", 0x43CA6EB8BD69FE2Au, 0x5E5375C6u}, + {"3809325632181785343e0", 0x43CA6EB8BD69FE29u, 0x5E5375C6u}, + {"3809325632181785344.01", 0x43CA6EB8BD69FE2Au, 0x5E5375C6u}, + {"3.809325632181785344000000000000000000001e18", 0x43CA6EB8BD69FE2Au, 0x5E5375C6u}, + {"3809325632181785344000000000000000000000000000000e-30", 0x43CA6EB8BD69FE2Au, 0x5E5375C6u}, + {"3809325632181785300", 0x43CA6EB8BD69FE29u, 0x5E5375C6u}, + {"3.8093256321817854e18", 0x43CA6EB8BD69FE2Au, 0x5E5375C6u}, + {"4.046966549916366943359375e12", 0x428D7210076CE2F0u, 0x546B9080u}, + {"4046966549916366943359376e-12", 0x428D7210076CE2F0u, 0x546B9080u}, + {"4046966549916.366943359374", 0x428D7210076CE2EFu, 0x546B9080u}, + {"4.04696654991636694335937501e12", 0x428D7210076CE2F0u, 0x546B9080u}, + {"4046966549916366943359375000000000000000000001e-33", 0x428D7210076CE2F0u, 0x546B9080u}, + {"-4046966549916.366943359375000000000000000000000000000000", 0xC28D7210076CE2F0u, 0xD46B9080u}, + {"4.0469665499163669e12", 0x428D7210076CE2EFu, 0x546B9080u}, + {"4046966549916367e-3", 0x428D7210076CE2F0u, 0x546B9080u}, + {"4046966549916366943e-6", 0x428D7210076CE2EFu, 0x546B9080u}, + {"4046966549916.366944", 0x428D7210076CE2F0u, 0x546B9080u}, + {"4046966549916.3669433", 0x428D7210076CE2EFu, 0x546B9080u}, + {"4.0469665499163669434e12", 0x428D7210076CE2F0u, 0x546B9080u}, + {"-4.04696654991636694335e12", 0xC28D7210076CE2EFu, 0xD46B9080u}, + {"404696654991636694336e-8", 0x428D7210076CE2F0u, 0x546B9080u}, + {"28093802557000874e154", 0x63529C3B77330BDBu, 0x7F800000u}, + {"0.28093802557000875E+171", 0x63529C3B77330BDCu, 0x7F800000u}, + {"0.2809380255700087447E+171", 0x63529C3B77330BDBu, 0x7F800000u}, + {"2.809380255700087448e170", 0x63529C3B77330BDCu, 0x7F800000u}, + {"2.8093802557000874472e170", 0x63529C3B77330BDBu, 0x7F800000u}, + {"28093802557000874473e151", 0x63529C3B77330BDCu, 0x7F800000u}, + {"280938025570008744728e150", 0x63529C3B77330BDBu, 0x7F800000u}, + {"-0.280938025570008744729E+171", 0xE3529C3B77330BDCu, 0xFF800000u}, + {"-0.280938025570008744728403667979E+171", 0xE3529C3B77330BDBu, 0xFF800000u}, + {"2.8093802557000874472840366798e170", 0x63529C3B77330BDCu, 0x7F800000u}, + {"0.39523280297734525e-154", 0x1FE0F51BF17FD374u, 0x00000000u}, + {"-3.9523280297734526e-155", 0x9FE0F51BF17FD375u, 0x80000000u}, + {"3.952328029773452547e-155", 0x1FE0F51BF17FD374u, 0x00000000u}, + {"3952328029773452548e-173", 0x1FE0F51BF17FD375u, 0x00000000u}, + {"-39523280297734525478e-174", 0x9FE0F51BF17FD374u, 0x80000000u}, + {"0.39523280297734525479e-154", 0x1FE0F51BF17FD375u, 0x00000000u}, + {"-0.395232802977345254787e-154", 0x9FE0F51BF17FD374u, 0x80000000u}, + {"-3.95232802977345254788e-155", 0x9FE0F51BF17FD375u, 0x80000000u}, + {"-3.95232802977345254787245825501e-155", 0x9FE0F51BF17FD374u, 0x80000000u}, + {"395232802977345254787245825502e-184", 0x1FE0F51BF17FD375u, 0x00000000u}, + {"-1.0790205420931879e-276", 0x86A3209CA6233255u, 0x80000000u}, + {"-1079020542093188e-291", 0x86A3209CA6233256u, 0x80000000u}, + {"-1079020542093187947e-294", 0x86A3209CA6233255u, 0x80000000u}, + {"0.1079020542093187948e-275", 0x06A3209CA6233256u, 0x00000000u}, + {"0.1079020542093187947e-275", 0x06A3209CA6233255u, 0x00000000u}, + {"1.0790205420931879471e-276", 0x06A3209CA6233256u, 0x00000000u}, + {"1.07902054209318794701e-276", 0x06A3209CA6233255u, 0x00000000u}, + {"-107902054209318794702e-296", 0x86A3209CA6233256u, 0x80000000u}, + {"107902054209318794701153285302e-305", 0x06A3209CA6233255u, 0x00000000u}, + {"-0.107902054209318794701153285303e-275", 0x86A3209CA6233256u, 0x80000000u}, + {"58530471071351308e-228", 0x1413B446E6A16A3Bu, 0x00000000u}, + {"-0.58530471071351309e-211", 0x9413B446E6A16A3Cu, 0x80000000u}, + {"0.5853047107135130893e-211", 0x1413B446E6A16A3Bu, 0x00000000u}, + {"-5.853047107135130894e-212", 0x9413B446E6A16A3Cu, 0x80000000u}, + {"5.853047107135130893e-212", 0x1413B446E6A16A3Bu, 0x00000000u}, + {"58530471071351308931e-231", 0x1413B446E6A16A3Cu, 0x00000000u}, + {"-585304710713513089304e-232", 0x9413B446E6A16A3Bu, 0x80000000u}, + {"0.585304710713513089305e-211", 0x1413B446E6A16A3Cu, 0x00000000u}, + {"-0.585304710713513089304248824438e-211", 0x9413B446E6A16A3Bu, 0x80000000u}, + {"5.85304710713513089304248824439e-212", 0x1413B446E6A16A3Cu, 0x00000000u}, + {"0.19334214893983531e-78", 0x2F96ECBF1CFB10F6u, 0x00000000u}, + {"1.9334214893983532e-79", 0x2F96ECBF1CFB10F7u, 0x00000000u}, + {"1.933421489398353102e-79", 0x2F96ECBF1CFB10F6u, 0x00000000u}, + {"1933421489398353103e-97", 0x2F96ECBF1CFB10F7u, 0x00000000u}, + {"19334214893983531023e-98", 0x2F96ECBF1CFB10F6u, 0x00000000u}, + {"0.19334214893983531024e-78", 0x2F96ECBF1CFB10F7u, 0x00000000u}, + {"0.193342148939835310231e-78", 0x2F96ECBF1CFB10F6u, 0x00000000u}, + {"1.93342148939835310232e-79", 0x2F96ECBF1CFB10F7u, 0x00000000u}, + {"-1.93342148939835310231359014704e-79", 0xAF96ECBF1CFB10F6u, 0x80000000u}, + {"193342148939835310231359014705e-108", 0x2F96ECBF1CFB10F7u, 0x00000000u}, + {"2.9873358928024455e227", 0x6F2938807814E8A2u, 0x7F800000u}, + {"29873358928024456e211", 0x6F2938807814E8A3u, 0x7F800000u}, + {"298733589280244551e210", 0x6F2938807814E8A2u, 0x7F800000u}, + {"0.2987335892802445511E+228", 0x6F2938807814E8A3u, 0x7F800000u}, + {"0.29873358928024455109E+228", 0x6F2938807814E8A2u, 0x7F800000u}, + {"2.987335892802445511e227", 0x6F2938807814E8A3u, 0x7F800000u}, + {"-2.98733589280244551098e227", 0xEF2938807814E8A2u, 0xFF800000u}, + {"-298733589280244551099e207", 0xEF2938807814E8A3u, 0xFF800000u}, + {"298733589280244551098081559931e198", 0x6F2938807814E8A2u, 0x7F800000u}, + {"0.298733589280244551098081559932E+228", 0x6F2938807814E8A3u, 0x7F800000u}, + {"7.0064923216240853e-46", 0x3690000000000000u, 0x00000000u}, + {"70064923216240854e-62", 0x3690000000000000u, 0x00000001u}, + {"7006492321624085354e-64", 0x3690000000000000u, 0x00000000u}, + {"-0.7006492321624085355e-45", 0xB690000000000000u, 0x80000001u}, + {"0.70064923216240853546e-45", 0x3690000000000000u, 0x00000000u}, + {"7.0064923216240853547e-46", 0x3690000000000000u, 0x00000001u}, + {"7.00649232162408535461e-46", 0x3690000000000000u, 0x00000000u}, + {"-700649232162408535462e-66", 0xB690000000000000u, 0x80000001u}, + {"-700649232162408535461864791644e-75", 0xB690000000000000u, 0x80000000u}, + {"0.700649232162408535461864791645e-45", 0x3690000000000000u, 0x00000001u}, + {"21019476964872256e-61", 0x36A8000000000000u, 0x00000001u}, + {"0.21019476964872257e-44", 0x36A8000000000000u, 0x00000002u}, + {"-0.2101947696487225606e-44", 0xB6A8000000000000u, 0x80000001u}, + {"-2.101947696487225607e-45", 0xB6A8000000000000u, 0x80000002u}, + {"2.1019476964872256063e-45", 0x36A8000000000000u, 0x00000001u}, + {"21019476964872256064e-64", 0x36A8000000000000u, 0x00000002u}, + {"-210194769648722560638e-65", 0xB6A8000000000000u, 0x80000001u}, + {"-0.210194769648722560639e-44", 0xB6A8000000000000u, 0x80000002u}, + {"-0.210194769648722560638559437493e-44", 0xB6A8000000000000u, 0x80000001u}, + {"2.10194769648722560638559437494e-45", 0x36A8000000000000u, 0x00000002u}, + {"0.11754941406275178e-37", 0x380FFFFFA0000000u, 0x007FFFFEu}, + {"1.1754941406275179e-38", 0x380FFFFFA0000000u, 0x007FFFFFu}, + {"-1.175494140627517859e-38", 0xB80FFFFFA0000000u, 0x807FFFFEu}, + {"117549414062751786e-55", 0x380FFFFFA0000000u, 0x007FFFFFu}, + {"-11754941406275178592e-57", 0xB80FFFFFA0000000u, 0x807FFFFEu}, + {"0.11754941406275178593e-37", 0x380FFFFFA0000000u, 0x007FFFFFu}, + {"0.117549414062751785924e-37", 0x380FFFFFA0000000u, 0x007FFFFEu}, + {"1.17549414062751785925e-38", 0x380FFFFFA0000000u, 0x007FFFFFu}, + {"-1.17549414062751785924617589866e-38", 0xB80FFFFFA0000000u, 0x807FFFFEu}, + {"117549414062751785924617589867e-67", 0x380FFFFFA0000000u, 0x007FFFFFu}, + {"1.1754942807573642e-38", 0x380FFFFFDFFFFFFFu, 0x007FFFFFu}, + {"11754942807573643e-54", 0x380FFFFFE0000000u, 0x00800000u}, + {"1175494280757364291e-56", 0x380FFFFFE0000000u, 0x007FFFFFu}, + {"0.1175494280757364292e-37", 0x380FFFFFE0000000u, 0x00800000u}, + {"0.11754942807573642917e-37", 0x380FFFFFE0000000u, 0x007FFFFFu}, + {"-1.1754942807573642918e-38", 0xB80FFFFFE0000000u, 0x80800000u}, + {"-1.17549428075736429172e-38", 0xB80FFFFFE0000000u, 0x807FFFFFu}, + {"-117549428075736429173e-58", 0xB80FFFFFE0000000u, 0x80800000u}, + {"117549428075736429172788299103e-67", 0x380FFFFFE0000000u, 0x007FFFFFu}, + {"0.117549428075736429172788299104e-37", 0x380FFFFFE0000000u, 0x00800000u}, + {"11754944208872107e-54", 0x3810000010000000u, 0x00800000u}, + {"-0.11754944208872108e-37", 0xB810000010000000u, 0x80800001u}, + {"0.1175494420887210724e-37", 0x3810000010000000u, 0x00800000u}, + {"1.175494420887210725e-38", 0x3810000010000000u, 0x00800001u}, + {"1.1754944208872107242e-38", 0x3810000010000000u, 0x00800000u}, + {"-11754944208872107243e-57", 0xB810000010000000u, 0x80800001u}, + {"-11754944208872107242e-57", 0xB810000010000000u, 0x80800000u}, + {"0.117549442088721072421e-37", 0x3810000010000000u, 0x00800001u}, + {"0.11754944208872107242095900834e-37", 0x3810000010000000u, 0x00800000u}, + {"-1.17549442088721072420959008341e-38", 0xB810000010000000u, 0x80800001u}, + {"340282336497324057985868971510891282432", 0x47EFFFFFD0000000u, 0x7F7FFFFEu}, + {"-3.40282336497324057985868971510891282433e38", 0xC7EFFFFFD0000000u, 0xFF7FFFFFu}, + {"340282336497324057985868971510891282431e0", 0x47EFFFFFD0000000u, 0x7F7FFFFEu}, + {"340282336497324057985868971510891282432.01", 0x47EFFFFFD0000000u, 0x7F7FFFFFu}, + {"3.40282336497324057985868971510891282432000000000000000000001e38", 0x47EFFFFFD0000000u, 0x7F7FFFFFu}, + {"340282336497324057985868971510891282432000000000000000000000000000000e-30", 0x47EFFFFFD0000000u, 0x7F7FFFFEu}, + {"340282336497324050000000000000000000000", 0x47EFFFFFD0000000u, 0x7F7FFFFEu}, + {"3.4028233649732406e38", 0x47EFFFFFD0000000u, 0x7F7FFFFFu}, + {"3.402823364973240579e38", 0x47EFFFFFD0000000u, 0x7F7FFFFEu}, + {"340282336497324058e21", 0x47EFFFFFD0000000u, 0x7F7FFFFFu}, + {"34028233649732405798e19", 0x47EFFFFFD0000000u, 0x7F7FFFFEu}, + {"340282336497324057990000000000000000000", 0x47EFFFFFD0000000u, 0x7F7FFFFFu}, + {"340282336497324057985000000000000000000", 0x47EFFFFFD0000000u, 0x7F7FFFFEu}, + {"3.40282336497324057986e38", 0x47EFFFFFD0000000u, 0x7F7FFFFFu}, + {"3.4028233649732405798586897151e38", 0x47EFFFFFD0000000u, 0x7F7FFFFEu}, + {"340282336497324057985868971511e9", 0x47EFFFFFD0000000u, 0x7F7FFFFFu}, + {"3.40282356779733661637539395458142568448e38", 0x47EFFFFFF0000000u, 0x7F800000u}, + {"-340282356779733661637539395458142568449e0", 0xC7EFFFFFF0000000u, 0xFF800000u}, + {"340282356779733661637539395458142568447", 0x47EFFFFFF0000000u, 0x7F7FFFFFu}, + {"3.4028235677973366163753939545814256844801e38", 0x47EFFFFFF0000000u, 0x7F800000u}, + {"340282356779733661637539395458142568448000000000000000000001e-21", 0x47EFFFFFF0000000u, 0x7F800000u}, + {"340282356779733661637539395458142568448.000000000000000000000000000000", 0x47EFFFFFF0000000u, 0x7F800000u}, + {"3.4028235677973366e38", 0x47EFFFFFF0000000u, 0x7F7FFFFFu}, + {"-34028235677973367e22", 0xC7EFFFFFF0000000u, 0xFF800000u}, + {"3402823567797336616e20", 0x47EFFFFFF0000000u, 0x7F7FFFFFu}, + {"340282356779733661700000000000000000000", 0x47EFFFFFF0000000u, 0x7F800000u}, + {"340282356779733661630000000000000000000", 0x47EFFFFFF0000000u, 0x7F7FFFFFu}, + {"3.4028235677973366164e38", 0x47EFFFFFF0000000u, 0x7F800000u}, + {"3.40282356779733661637e38", 0x47EFFFFFF0000000u, 0x7F7FFFFFu}, + {"340282356779733661638e18", 0x47EFFFFFF0000000u, 0x7F800000u}, + {"-340282356779733661637539395458e9", 0xC7EFFFFFF0000000u, 0xFF7FFFFFu}, + {"340282356779733661637539395459000000000", 0x47EFFFFFF0000000u, 0x7F800000u}, + {"-1000000059604644775390625e-24", 0xBFF0000010000000u, 0xBF800000u}, + {"-1.000000059604644775390626", 0xBFF0000010000000u, 0xBF800001u}, + {"-1.000000059604644775390624e0", 0xBFF0000010000000u, 0xBF800000u}, + {"100000005960464477539062501e-26", 0x3FF0000010000000u, 0x3F800001u}, + {"1.000000059604644775390625000000000000000000001", 0x3FF0000010000000u, 0x3F800001u}, + {"1.000000059604644775390625000000000000000000000000000000e0", 0x3FF0000010000000u, 0x3F800000u}, + {"-10000000596046447e-16", 0xBFF0000010000000u, 0xBF800000u}, + {"1.0000000596046448", 0x3FF0000010000000u, 0x3F800001u}, + {"1.000000059604644775", 0x3FF0000010000000u, 0x3F800000u}, + {"-1.000000059604644776e0", 0xBFF0000010000000u, 0xBF800001u}, + {"-1.0000000596046447753e0", 0xBFF0000010000000u, 0xBF800000u}, + {"10000000596046447754e-19", 0x3FF0000010000000u, 0x3F800001u}, + {"100000005960464477539e-20", 0x3FF0000010000000u, 0x3F800000u}, + {"-1.0000000596046447754", 0xBFF0000010000000u, 0xBF800001u}, + {"0.9999999701976776123046875", 0x3FEFFFFFF0000000u, 0x3F800000u}, + {"9.999999701976776123046876e-1", 0x3FEFFFFFF0000000u, 0x3F800000u}, + {"9999999701976776123046874e-25", 0x3FEFFFFFF0000000u, 0x3F7FFFFFu}, + {"0.999999970197677612304687501", 0x3FEFFFFFF0000000u, 0x3F800000u}, + {"-9.999999701976776123046875000000000000000000001e-1", 0xBFEFFFFFF0000000u, 0xBF800000u}, + {"-9999999701976776123046875000000000000000000000000000000e-55", 0xBFEFFFFFF0000000u, 0xBF800000u}, + {"0.99999997019767761", 0x3FEFFFFFF0000000u, 0x3F7FFFFFu}, + {"-9.9999997019767762e-1", 0xBFEFFFFFF0000000u, 0xBF800000u}, + {"-9.999999701976776123e-1", 0xBFEFFFFFF0000000u, 0xBF7FFFFFu}, + {"9999999701976776124e-19", 0x3FEFFFFFF0000000u, 0x3F800000u}, + {"9999999701976776123e-19", 0x3FEFFFFFF0000000u, 0x3F7FFFFFu}, + {"0.99999997019767761231", 0x3FEFFFFFF0000000u, 0x3F800000u}, + {"-0.999999970197677612304", 0xBFEFFFFFF0000000u, 0xBF7FFFFFu}, + {"9.99999970197677612305e-1", 0x3FEFFFFFF0000000u, 0x3F800000u}, + {"-1.6777217e7", 0xC170000010000000u, 0xCB800000u}, + {"16777218e0", 0x4170000020000000u, 0x4B800001u}, + {"16777216", 0x4170000000000000u, 0x4B800000u}, + {"-1.677721701e7", 0xC17000001028F5C3u, 0xCB800001u}, + {"-16777217000000000000000000001e-21", 0xC170000010000000u, 0xCB800001u}, + {"16777217.000000000000000000000000000000", 0x4170000010000000u, 0x4B800000u}, + {"167772155e-1", 0x416FFFFFF0000000u, 0x4B800000u}, + {"16777215.6", 0x416FFFFFF3333333u, 0x4B800000u}, + {"1.67772154e7", 0x416FFFFFECCCCCCDu, 0x4B7FFFFFu}, + {"16777215501e-3", 0x416FFFFFF0083127u, 0x4B800000u}, + {"-16777215.5000000000000000000001", 0xC16FFFFFF0000000u, 0xCB800000u}, + {"-1.67772155000000000000000000000000000000e7", 0xC16FFFFFF0000000u, 0xCB800000u}, + {"0.1000000052154064178466796875", 0x3FB99999B0000000u, 0x3DCCCCCEu}, + {"1.000000052154064178466796876e-1", 0x3FB99999B0000000u, 0x3DCCCCCEu}, + {"-1000000052154064178466796874e-28", 0xBFB99999B0000000u, 0xBDCCCCCDu}, + {"0.100000005215406417846679687501", 0x3FB99999B0000000u, 0x3DCCCCCEu}, + {"1.000000052154064178466796875000000000000000000001e-1", 0x3FB99999B0000000u, 0x3DCCCCCEu}, + {"-1000000052154064178466796875000000000000000000000000000000e-58", 0xBFB99999B0000000u, 0xBDCCCCCEu}, + {"-0.10000000521540641", 0xBFB99999AFFFFFFFu, 0xBDCCCCCDu}, + {"-1.0000000521540642e-1", 0xBFB99999B0000000u, 0xBDCCCCCEu}, + {"1.000000052154064178e-1", 0x3FB99999B0000000u, 0x3DCCCCCDu}, + {"-1000000052154064179e-19", 0xBFB99999B0000000u, 0xBDCCCCCEu}, + {"10000000521540641784e-20", 0x3FB99999B0000000u, 0x3DCCCCCDu}, + {"0.10000000521540641785", 0x3FB99999B0000000u, 0x3DCCCCCEu}, + {"0.100000005215406417846", 0x3FB99999B0000000u, 0x3DCCCCCDu}, + {"1.00000005215406417847e-1", 0x3FB99999B0000000u, 0x3DCCCCCEu}, + {"5.429001220703125e3", 0x40B5350050000000u, 0x45A9A802u}, + {"-5429001220703126e-12", 0xC0B5350050000001u, 0xC5A9A803u}, + {"-5429.001220703124", 0xC0B535004FFFFFFFu, 0xC5A9A802u}, + {"5.42900122070312501e3", 0x40B5350050000000u, 0x45A9A803u}, + {"5429001220703125000000000000000000001e-33", 0x40B5350050000000u, 0x45A9A803u}, + {"-5429.001220703125000000000000000000000000000000", 0xC0B5350050000000u, 0xC5A9A802u}, + {"503719056e0", 0x41BE062490000000u, 0x4DF03124u}, + {"503719057", 0x41BE062491000000u, 0x4DF03125u}, + {"5.03719055e8", 0x41BE06248F000000u, 0x4DF03124u}, + {"50371905601e-2", 0x41BE062490028F5Cu, 0x4DF03125u}, + {"503719056.000000000000000000001", 0x41BE062490000000u, 0x4DF03125u}, + {"5.03719056000000000000000000000000000000e8", 0x41BE062490000000u, 0x4DF03124u}, + {"-92331620", 0xC196037990000000u, 0xCCB01BCCu}, + {"9.233163e7", 0x41960379B8000000u, 0x4CB01BCEu}, + {"9233161e1", 0x4196037968000000u, 0x4CB01BCBu}, + {"92331620.1", 0x4196037990666666u, 0x4CB01BCDu}, + {"9.233162000000000000000000001e7", 0x4196037990000000u, 0x4CB01BCDu}, + {"9233162000000000000000000000000000000e-29", 0x4196037990000000u, 0x4CB01BCCu}, + {"3.002458625e6", 0x4146E82D50000000u, 0x4A37416Au}, + {"3002458626e-3", 0x4146E82D5020C49Cu, 0x4A37416Bu}, + {"3002458.624", 0x4146E82D4FDF3B64u, 0x4A37416Au}, + {"-3.00245862501e6", 0xC146E82D500053E3u, 0xCA37416Bu}, + {"3002458625000000000000000000001e-24", 0x4146E82D50000000u, 0x4A37416Bu}, + {"3002458.625000000000000000000000000000000", 0x4146E82D50000000u, 0x4A37416Au}, + {"-1095485584696182596504479582065262592e1", 0xC7A07BA830000000u, 0xFD03DD42u}, + {"10954855846961825965044795820652625930", 0x47A07BA830000000u, 0x7D03DD42u}, + {"1.095485584696182596504479582065262591e37", 0x47A07BA830000000u, 0x7D03DD41u}, + {"109548558469618259650447958206526259201e-1", 0x47A07BA830000000u, 0x7D03DD42u}, + {"-10954855846961825965044795820652625920.00000000000000000001", 0xC7A07BA830000000u, 0xFD03DD42u}, + {"1.095485584696182596504479582065262592000000000000000000000000000000e37", 0x47A07BA830000000u, 0x7D03DD42u}, + {"10954855846961825e21", 0x47A07BA830000000u, 0x7D03DD41u}, + {"10954855846961826000000000000000000000", 0x47A07BA830000000u, 0x7D03DD42u}, + {"10954855846961825960000000000000000000", 0x47A07BA830000000u, 0x7D03DD41u}, + {"1.095485584696182597e37", 0x47A07BA830000000u, 0x7D03DD42u}, + {"1.0954855846961825965e37", 0x47A07BA830000000u, 0x7D03DD41u}, + {"-10954855846961825966e18", 0xC7A07BA830000000u, 0xFD03DD42u}, + {"10954855846961825965e18", 0x47A07BA830000000u, 0x7D03DD41u}, + {"10954855846961825965100000000000000000", 0x47A07BA830000000u, 0x7D03DD42u}, + {"10954855846961825965044795820600000000", 0x47A07BA830000000u, 0x7D03DD41u}, + {"-1.09548558469618259650447958207e37", 0xC7A07BA830000000u, 0xFD03DD42u}, + {"1.6449216019182103706535606608388384863861375606575165875256061553955078126e-21", 0x3B9F125A50000000u, 0x1CF892D3u}, + {"-16449216019182103706535606608388384863861375606575165875256061553955078124e-94", 0xBB9F125A50000000u, 0x9CF892D2u}, + {"-0.0000000000000000000016449216019182103", 0xBB9F125A50000000u, 0x9CF892D2u}, + {"-1.6449216019182104e-21", 0xBB9F125A50000000u, 0x9CF892D3u}, + {"-1.64492160191821037e-21", 0xBB9F125A50000000u, 0x9CF892D2u}, + {"-1644921601918210371e-39", 0xBB9F125A50000000u, 0x9CF892D3u}, + {"-16449216019182103706e-40", 0xBB9F125A50000000u, 0x9CF892D2u}, + {"0.0000000000000000000016449216019182103707", 0x3B9F125A50000000u, 0x1CF892D3u}, + {"0.00000000000000000000164492160191821037065", 0x3B9F125A50000000u, 0x1CF892D2u}, + {"1.64492160191821037066e-21", 0x3B9F125A50000000u, 0x1CF892D3u}, + {"1.64492160191821037065356066083e-21", 0x3B9F125A50000000u, 0x1CF892D2u}, + {"164492160191821037065356066084e-50", 0x3B9F125A50000000u, 0x1CF892D3u}, + {"6.565061509609222412109375e-1", 0x3FE5021930000000u, 0x3F2810CAu}, + {"6565061509609222412109376e-25", 0x3FE5021930000000u, 0x3F2810CAu}, + {"0.6565061509609222412109374", 0x3FE5021930000000u, 0x3F2810C9u}, + {"6.56506150960922241210937501e-1", 0x3FE5021930000000u, 0x3F2810CAu}, + {"6565061509609222412109375000000000000000000001e-46", 0x3FE5021930000000u, 0x3F2810CAu}, + {"-0.6565061509609222412109375000000000000000000000000000000", 0xBFE5021930000000u, 0xBF2810CAu}, + {"6.5650615096092224e-1", 0x3FE5021930000000u, 0x3F2810C9u}, + {"-65650615096092225e-17", 0xBFE5021930000000u, 0xBF2810CAu}, + {"6565061509609222412e-19", 0x3FE5021930000000u, 0x3F2810C9u}, + {"0.6565061509609222413", 0x3FE5021930000000u, 0x3F2810CAu}, + {"0.65650615096092224121", 0x3FE5021930000000u, 0x3F2810C9u}, + {"6.5650615096092224122e-1", 0x3FE5021930000000u, 0x3F2810CAu}, + {"-6.5650615096092224121e-1", 0xBFE5021930000000u, 0xBF2810C9u}, + {"656506150960922241211e-21", 0x3FE5021930000000u, 0x3F2810CAu}, + {"18014627239033005156980393746124491372029297053813934326171875e-77", 0x3CA9F63970000000u, 0x254FB1CCu}, + {"0.00000000000000018014627239033005156980393746124491372029297053813934326171876", 0x3CA9F63970000000u, 0x254FB1CCu}, + {"-1.8014627239033005156980393746124491372029297053813934326171874e-16", 0xBCA9F63970000000u, 0xA54FB1CBu}, + {"1801462723903300515698039374612449137202929705381393432617187501e-79", 0x3CA9F63970000000u, 0x254FB1CCu}, + {"18014627239033005e-32", 0x3CA9F63970000000u, 0x254FB1CBu}, + {"0.00000000000000018014627239033006", 0x3CA9F63970000000u, 0x254FB1CCu}, + {"0.0000000000000001801462723903300515", 0x3CA9F63970000000u, 0x254FB1CBu}, + {"-1.801462723903300516e-16", 0xBCA9F63970000000u, 0xA54FB1CCu}, + {"-1.8014627239033005156e-16", 0xBCA9F63970000000u, 0xA54FB1CBu}, + {"18014627239033005157e-35", 0x3CA9F63970000000u, 0x254FB1CCu}, + {"180146272390330051569e-36", 0x3CA9F63970000000u, 0x254FB1CBu}, + {"-0.00000000000000018014627239033005157", 0xBCA9F63970000000u, 0xA54FB1CCu}, + {"0.000000000000000180146272390330051569803937461", 0x3CA9F63970000000u, 0x254FB1CBu}, + {"1.80146272390330051569803937462e-16", 0x3CA9F63970000000u, 0x254FB1CCu}, + {"0.05534819327294826507568359375", 0x3FAC569930000000u, 0x3D62B4CAu}, + {"-5.534819327294826507568359376e-2", 0xBFAC569930000000u, 0xBD62B4CAu}, + {"5534819327294826507568359374e-29", 0x3FAC569930000000u, 0x3D62B4C9u}, + {"-0.0553481932729482650756835937501", 0xBFAC569930000000u, 0xBD62B4CAu}, + {"5.534819327294826507568359375000000000000000000001e-2", 0x3FAC569930000000u, 0x3D62B4CAu}, + {"5534819327294826507568359375000000000000000000000000000000e-59", 0x3FAC569930000000u, 0x3D62B4CAu}, + {"0.055348193272948265", 0x3FAC569930000000u, 0x3D62B4C9u}, + {"5.5348193272948266e-2", 0x3FAC569930000000u, 0x3D62B4CAu}, + {"5.534819327294826507e-2", 0x3FAC569930000000u, 0x3D62B4C9u}, + {"5534819327294826508e-20", 0x3FAC569930000000u, 0x3D62B4CAu}, + {"55348193272948265075e-21", 0x3FAC569930000000u, 0x3D62B4C9u}, + {"-0.055348193272948265076", 0xBFAC569930000000u, 0xBD62B4CAu}, + {"0.0553481932729482650756", 0x3FAC569930000000u, 0x3D62B4C9u}, + {"-5.53481932729482650757e-2", 0xBFAC569930000000u, 0xBD62B4CAu}, + {"5.179692133247783258005389047985340416e36", 0x478F2C9450000000u, 0x7C7964A2u}, + {"5179692133247783258005389047985340417e0", 0x478F2C9450000000u, 0x7C7964A3u}, + {"5179692133247783258005389047985340415", 0x478F2C9450000000u, 0x7C7964A2u}, + {"-5.17969213324778325800538904798534041601e36", 0xC78F2C9450000000u, 0xFC7964A3u}, + {"-5179692133247783258005389047985340416000000000000000000001e-21", 0xC78F2C9450000000u, 0xFC7964A3u}, + {"-5179692133247783258005389047985340416.000000000000000000000000000000", 0xC78F2C9450000000u, 0xFC7964A2u}, + {"5.1796921332477832e36", 0x478F2C9450000000u, 0x7C7964A2u}, + {"51796921332477833e20", 0x478F2C9450000000u, 0x7C7964A3u}, + {"-5179692133247783258e18", 0xC78F2C9450000000u, 0xFC7964A2u}, + {"5179692133247783259000000000000000000", 0x478F2C9450000000u, 0x7C7964A3u}, + {"5179692133247783258000000000000000000", 0x478F2C9450000000u, 0x7C7964A2u}, + {"5.1796921332477832581e36", 0x478F2C9450000000u, 0x7C7964A3u}, + {"-5.179692133247783258e36", 0xC78F2C9450000000u, 0xFC7964A2u}, + {"-517969213324778325801e16", 0xC78F2C9450000000u, 0xFC7964A3u}, + {"517969213324778325800538904798e7", 0x478F2C9450000000u, 0x7C7964A2u}, + {"5179692133247783258005389047990000000", 0x478F2C9450000000u, 0x7C7964A3u}, + { + "0.22250738585072011360574097967091319759348195463516456480234261097248222220210769455165295239081350" + "8791414915891303962110687008643869459464552765720740782062174337998814106326732925355228688137214901" + "2981122451451889849057222307285255133155755015914397476397983411801999323962548289017107081850690630" + "6666559949382757725720157630626906633326475653000092458883164330377797918696120494973903778297049050" + "5108060994073026293712895895000358379996720725430436028407889577179615094551674824347103070260914462" + "1572289880258182545180325707018860872113128079512233426288368622321503775666622503982534335974568884" + "4239002654981983854879482922068947216898310996983658468140228542433306603398508864458040010349339704" + "2756718644338377048603786162277173854562306587467901408672332763671875e-307", 0x0010000000000000u, 0x00000000u + }, + { + "2.22507385850720113605740979670913197593481954635164564802342610972482222202107694551652952390813508" + "7914149158913039621106870086438694594645527657207407820621743379988141063267329253552286881372149012" + "9811224514518898490572223072852551331557550159143974763979834118019993239625482890171070818506906306" + "6665599493827577257201576306269066333264756530000924588831643303777979186961204949739037782970490505" + "1080609940730262937128958950003583799967207254304360284078895771796150945516748243471030702609144621" + "5722898802581825451803257070188608721131280795122334262883686223215037756666225039825343359745688844" + "2390026549819838548794829220689472168983109969836584681402285424333066033985088644580400103493397042" + "756718644338377048603786162277173854562306587467901408672332763671875000000000000000000001e-308", 0x0010000000000000u, 0x00000000u + }, + { + "0.11754942807573642917278829910357665133228589927589904276829631184250030649651730385585324256680905" + "8189392089843750000000000000000000000000000000000000000000000000000000000000000000000000000000000000" + "0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" + "0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" + "0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" + "0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" + "0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000e-37", 0x380FFFFFE0000000u, 0x00800000u + }, + { + "1175494280757364291727882991035766513322858992758990427682963118425003064965173038558532425668090581" + "8939208984375000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" + "0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" + "0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" + "0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" + "0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" + "0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000001e-751", 0x380FFFFFE0000000u, 0x00800000u + }, + {"0", 0x0000000000000000u, 0x00000000u}, + {"-0", 0x8000000000000000u, 0x80000000u}, + {"0.0", 0x0000000000000000u, 0x00000000u}, + {"-0.0", 0x8000000000000000u, 0x80000000u}, + {"0e999999999999999999999", 0x0000000000000000u, 0x00000000u}, + {"-0.000e-99999", 0x8000000000000000u, 0x80000000u}, + {"1e-400", 0x0000000000000000u, 0x00000000u}, + {"-1e-400", 0x8000000000000000u, 0x80000000u}, + {"1e400", 0x7FF0000000000000u, 0x7F800000u}, + {"-1e400", 0xFFF0000000000000u, 0xFF800000u}, + {"1e-50", 0x358DEE7A4AD4B81Fu, 0x00000000u}, + {"-1e-50", 0xB58DEE7A4AD4B81Fu, 0x80000000u}, + {"1e39", 0x48078287F49C4A1Du, 0x7F800000u}, + {"-1e39", 0xC8078287F49C4A1Du, 0xFF800000u}, + {"1e99999999999999999999999999", 0x7FF0000000000000u, 0x7F800000u}, + {"1e-99999999999999999999999999", 0x0000000000000000u, 0x00000000u}, + {"1e0000000000000000000000000000000000000000308", 0x7FE1CCF385EBC8A0u, 0x7F800000u}, + {"123456789012345678901234567890e-30", 0x3FBF9ADD3746F65Fu, 0x3DFCD6EAu}, + {"18446744073709551615", 0x43F0000000000000u, 0x5F800000u}, + {"18446744073709551616", 0x43F0000000000000u, 0x5F800000u}, + {"-9223372036854775808", 0xC3E0000000000000u, 0xDF000000u}, + {"-9223372036854775809", 0xC3E0000000000000u, 0xDF000000u}, + } + }; + return table; +} + +} // namespace float_hard_cases diff --git a/tests/src/unit-binary_formats.cpp b/tests/src/unit-binary_formats.cpp index 03dbc94da..333252190 100644 --- a/tests/src/unit-binary_formats.cpp +++ b/tests/src/unit-binary_formats.cpp @@ -38,7 +38,7 @@ TEST_CASE("Binary Formats") const auto ubjson_2_size = json::to_ubjson(j, true).size(); const auto ubjson_3_size = json::to_ubjson(j, true, true).size(); - CHECK(json_size == 2090303); + CHECK(json_size == 2090234); CHECK(bjdata_1_size == 1112030); CHECK(bjdata_2_size == 1224148); CHECK(bjdata_3_size == 1224148); @@ -51,16 +51,16 @@ TEST_CASE("Binary Formats") CHECK(ubjson_3_size == 1169069); CHECK((100.0 * double(json_size) / double(json_size)) == Approx(100.0)); - CHECK((100.0 * double(bjdata_1_size) / double(json_size)) == Approx(53.199)); - CHECK((100.0 * double(bjdata_2_size) / double(json_size)) == Approx(58.563)); - CHECK((100.0 * double(bjdata_3_size) / double(json_size)) == Approx(58.563)); - CHECK((100.0 * double(bon8_size) / double(json_size)) == Approx(50.509)); - CHECK((100.0 * double(bson_size) / double(json_size)) == Approx(85.849)); - CHECK((100.0 * double(cbor_size) / double(json_size)) == Approx(50.497)); - CHECK((100.0 * double(msgpack_size) / double(json_size)) == Approx(50.526)); - CHECK((100.0 * double(ubjson_1_size) / double(json_size)) == Approx(53.199)); - CHECK((100.0 * double(ubjson_2_size) / double(json_size)) == Approx(58.563)); - CHECK((100.0 * double(ubjson_3_size) / double(json_size)) == Approx(55.928)); + CHECK((100.0 * double(bjdata_1_size) / double(json_size)) == Approx(53.201)); + CHECK((100.0 * double(bjdata_2_size) / double(json_size)) == Approx(58.565)); + CHECK((100.0 * double(bjdata_3_size) / double(json_size)) == Approx(58.565)); + CHECK((100.0 * double(bon8_size) / double(json_size)) == Approx(50.511)); + CHECK((100.0 * double(bson_size) / double(json_size)) == Approx(85.853)); + CHECK((100.0 * double(cbor_size) / double(json_size)) == Approx(50.499)); + CHECK((100.0 * double(msgpack_size) / double(json_size)) == Approx(50.528)); + CHECK((100.0 * double(ubjson_1_size) / double(json_size)) == Approx(53.201)); + CHECK((100.0 * double(ubjson_2_size) / double(json_size)) == Approx(58.565)); + CHECK((100.0 * double(ubjson_3_size) / double(json_size)) == Approx(55.930)); } SECTION("twitter.json") diff --git a/tests/src/unit-class_lexer.cpp b/tests/src/unit-class_lexer.cpp index 860af81d6..05cd2bf1d 100644 --- a/tests/src/unit-class_lexer.cpp +++ b/tests/src/unit-class_lexer.cpp @@ -13,16 +13,20 @@ using nlohmann::json; #include // array -#include // FLT_EVAL_METHOD #include // uint32_t, uint64_t +#include // snprintf #include // strtod #include // memcpy #include // numeric_limits +#include // map +#include // mt19937 #include // stringstream #include // string #include // pair #include // vector +#include "float_hard_cases.hpp" + namespace { // shortcut to scan a string literal @@ -258,7 +262,7 @@ TEST_CASE("lexer number fast path") "123456789012345678901234567890", // huge -> float "0.30000000000000004", "2.2250738585072014e-308", "1e308", // high-precision / wide-exponent values that exercise the - // std::from_chars (Eisel-Lemire) path beyond the Clinger subset + // Eisel-Lemire path beyond the Clinger subset "1.7976931348623157e308", "1.2345678901234567e-250", "9007199254740993", "5e-324", "1e-320" }; @@ -280,20 +284,18 @@ TEST_CASE("lexer number fast path") } } - SECTION("significant-digit gate for the Clinger fast path") + SECTION("significant digits around Clinger's fast path") { - // Clinger's fast path needs a significand below 2^53, so it cannot - // succeed once the mantissa has 17 or more significant digits (the - // significand would be at least 10^16). The lexer skips the attempt - // there. That is only allowed to save work: every value must still come - // out bit-exactly, and both scanners must agree. In particular the gate - // must not fire for tokens whose leading zeros merely look like extra - // digits - "0.1234567890123456" has 16 significant digits, not 17. + // Clinger's fast path needs a significand of at most 2^53, which + // tokens with 17 or more significant digits exceed. The conversion + // splits the token at the positions the scanners recorded, so leading + // zeros must not count as digits - "0.1234567890123456" has 16 + // significant digits, not 17 - and both scanners must agree. const std::vector numbers = { "1234567890123456", // 16 significant digits - "12345678901234567", // 17 -> attempt skipped - "123456789012345678", // 18 -> attempt skipped + "12345678901234567", // 17 + "123456789012345678", // 18 "0.1234567890123456", // 16: the leading "0" is not significant "0.12345678901234567", // 17 "0.00000000000000001", // 1, in a long token @@ -664,46 +666,323 @@ TEST_CASE("lexer string fast path") } } -TEST_CASE("parse_float_fast declines what it cannot convert exactly") +TEST_CASE("lexer escape fast path") { - // The lexer only hands well-formed numbers to parse_float_fast, so the - // malformed ones below can only be passed to it directly. Declining is - // always safe: the caller then falls back to a slower, exact conversion. - const auto fast = [](const std::string & s, double & out) + // json::accept() never throws, so this section stays covered without + // exceptions; it pins which of the cases below are valid/invalid and + // checks the contiguous and streaming paths agree on that classification. + SECTION("accept() parity") { - return nlohmann::detail::parse_float_fast(s.data(), s.data() + s.size(), out); + const std::vector> cases = + { + {"\\u0041", true}, {"\\u00e4", true}, {"\\u00E4", true}, + {"\\uD83D\\uDE00", true}, + {"\\u12", false}, {"\\u12G4", false}, {"\\uXYZW", false}, + {"\\uD800", false}, {"\\uD800A", false}, {"\\uD800\\u0041", false}, + {"\\uDC00", false}, {"\\u", false} + }; + + for (const auto& c : cases) + { + for (const std::size_t offset : + { + std::size_t{0}, std::size_t{9} + }) + { + const std::string doc = "[\"" + std::string(offset, 'a') + c.first + "\"]"; + CAPTURE(doc) + CHECK(json::accept(doc) == c.second); + std::stringstream ss(doc); + CHECK(json::accept(ss) == c.second); + } + } + } + +#if !defined(JSON_NOEXCEPTION) + // the full outcome of parsing @a doc: the parsed value, or the exact + // error message, so a mismatch in either is caught + const auto outcome = [](const std::string & doc, bool streaming) -> std::string + { + try + { + if (streaming) + { + std::stringstream ss(doc); + const json j = json::parse(ss); + return j.dump(); + } + const json j = json::parse(doc); + return j.dump(); + } + catch (const json::exception& e) + { + return {e.what()}; + } }; - double out = 0; -#if defined(FLT_EVAL_METHOD) && FLT_EVAL_METHOD != 0 - // without true double precision, the fast path declines everything - CHECK_FALSE(fast("1.5", out)); -#else - CHECK(fast("1.5", out)); - CHECK(out == 1.5); - CHECK(fast("+2.5e1", out)); - CHECK(out == 25.0); - CHECK(fast("-25E-1", out)); - CHECK(out == -2.5); - CHECK(fast("1e", out)); - CHECK(out == 1.0); + SECTION("contiguous vs streaming parity") + { + const std::vector escapes = + { + "\\u0041", // "A" + "\\u00e4", // "ä" (lowercase hex) + "\\u00E4", // "ä" (uppercase hex) + "\\uD83D\\uDE00", // valid surrogate pair (an emoji) + "\\u12", // truncated: only 2 hex digits before the closing quote + "\\u12G4", // invalid hex digit at the 3rd position + "\\uXYZW", // all 4 bytes invalid + "\\uD800", // lone high surrogate, string ends right after + "\\uD800A", // high surrogate not followed by another \u escape + "\\uD800\\u0041", // high surrogate followed by \u, but not a low surrogate + "\\uDC00", // lone low surrogate + "\\u", // '\u' with nothing after (closing quote right away) + }; + + // once at the start of the string and once past the first 8-byte SWAR + // word of the outer string_bulk_run, so the escape is reached both + // right after the opening quote and mid-run + for (const auto& escape : escapes) + { + for (const std::size_t offset : + { + std::size_t{0}, std::size_t{9} + }) + { + const std::string doc = "[\"" + std::string(offset, 'a') + escape + "\"]"; + CAPTURE(doc) + CHECK(outcome(doc, false) == outcome(doc, true)); + } + + // the escape is the last thing before end of input: no closing + // quote at all + const std::string truncated_doc = "[\"" + escape; + CAPTURE(truncated_doc) + CHECK(outcome(truncated_doc, false) == outcome(truncated_doc, true)); + } + } + + SECTION("truncated \\u escape at every distance from the end of input") + { + // ia.bulk_remaining() must correctly report fewer than 4 bytes for + // every possible count of trailing hex-looking bytes (0, 1, 2, or 3) + // before end of input, so the fast path declines and the byte path + // alone reports the "must be followed by 4 hex digits" error, at the + // same position, in every case + for (const std::string& tail : + { + std::string{}, std::string("1"), std::string("12"), std::string("123") + }) + { + const std::string doc = "[\"\\u" + tail; + CAPTURE(doc) + CHECK(outcome(doc, false) == outcome(doc, true)); + CHECK(outcome(doc, false).find("must be followed by 4 hex digits") != std::string::npos); + } + } + + SECTION("invalid hex digit at every position of the 4") + { + // the fast path must decline for *any* invalid byte among the 4, not + // just the first, and the byte path must then stop at exactly that + // position - same as it always has + for (std::size_t bad_pos = 0; bad_pos < 4; ++bad_pos) + { + std::string digits = "1234"; + digits[bad_pos] = 'g'; // not a hex digit + const std::string doc = "[\"\\u" + digits + "\"]"; + CAPTURE(doc) + CHECK(outcome(doc, false) == outcome(doc, true)); + CHECK(outcome(doc, false).find("must be followed by 4 hex digits") != std::string::npos); + } + } + + SECTION("random escapes") + { + // A seeded PRNG builds the 4 bytes following `\u` from a mix of hex + // digits and non-hex bytes, at varying distances from the start of + // the string, to compare the two scanners on many more shapes than + // are practical to enumerate by hand. + std::mt19937 gen(7654321); // NOLINT(cert-msc32-c,cert-msc51-cpp,bugprone-random-generator-seed) + const std::string hex_alphabet = "0123456789AaBbCcDdEeFf"; + std::uniform_int_distribution pick_hex(0, hex_alphabet.size() - 1); + std::uniform_int_distribution pick_byte(1, 255); // never NUL + std::uniform_int_distribution pick_is_hex(0, 4); // 4-in-5 chance of a hex digit + std::uniform_int_distribution pick_offset(0, 12); + + std::vector mismatches; + for (int iter = 0; iter < 3000; ++iter) + { + std::string digits; + for (int i = 0; i < 4; ++i) + { + if (pick_is_hex(gen) != 0) + { + digits += hex_alphabet[pick_hex(gen)]; + } + else + { + char c = static_cast(pick_byte(gen)); + if (c == '"' || c == '\\') + { + // keep the string well-formed apart from the escape + // itself, so any mismatch is attributable to the \u + // handling and not to an unrelated quote/escape + c = 'z'; + } + digits += c; + } + } + const std::string doc = "[\"" + std::string(pick_offset(gen), 'a') + "\\u" + digits + "\"]"; + if (outcome(doc, false) != outcome(doc, true)) + { + mismatches.push_back(doc); + } + } + CAPTURE(mismatches) + CHECK(mismatches.empty()); + } #endif +} - // not a number - CHECK_FALSE(fast("", out)); - CHECK_FALSE(fast("-", out)); - CHECK_FALSE(fast(".", out)); - CHECK_FALSE(fast("1.2.3", out)); - CHECK_FALSE(fast("1x", out)); - CHECK_FALSE(fast("1e+", out)); - CHECK_FALSE(fast("1e1x", out)); +namespace +{ +// the index of the decimal point (or npos) and of the end of the mantissa of a +// number token, which the lexer records while scanning it +std::pair float_token_layout(const std::string& s) +{ + std::size_t dot = std::string::npos; + std::size_t mantissa_end = s.size(); + for (std::size_t i = 0; i < s.size(); ++i) + { + if (s[i] == '.') + { + dot = i; + } + else if (s[i] == 'e' || s[i] == 'E') + { + mantissa_end = i; + break; + } + } + return {dot, mantissa_end}; +} - // numbers that are not represented exactly on the fast path - CHECK_FALSE(fast("12345678901234567890", out)); - CHECK_FALSE(fast("1e10000", out)); - CHECK_FALSE(fast("9007199254740993", out)); - CHECK_FALSE(fast("1e23", out)); - CHECK_FALSE(fast("1e-23", out)); +template +FloatType parse_native(const std::string& s) +{ + const auto layout = float_token_layout(s); + return nlohmann::detail::parse_float_native(s.data(), s.data() + s.size(), layout.first, layout.second); +} + +std::uint64_t bits_of(double d) +{ + std::uint64_t b = 0; + std::memcpy(&b, &d, sizeof(b)); + return b; +} + +std::uint32_t bits_of(float f) +{ + std::uint32_t b = 0; + std::memcpy(&b, &f, sizeof(b)); + return b; +} + +std::uint64_t native_bits64(const std::string& s) +{ + return bits_of(parse_native(s)); +} + +std::uint32_t native_bits32(const std::string& s) +{ + return bits_of(parse_native(s)); +} +} // namespace + +TEST_CASE("parse_float_native rounds correctly") +{ + SECTION("double") + { + CHECK(native_bits64("1.5") == 0x3FF8000000000000u); + CHECK(native_bits64("0.1") == 0x3FB999999999999Au); + CHECK(native_bits64("-0.0") == 0x8000000000000000u); + CHECK(native_bits64("0e999999999999999999999") == 0u); + // 2^53 + 1 is exactly between two doubles: ties to even, unless more digits follow + CHECK(native_bits64("9007199254740993") == 0x4340000000000000u); + CHECK(native_bits64("9007199254740993.0000000000000000001") == 0x4340000000000001u); + CHECK(native_bits64("9007199254740992.9999999999999999999") == 0x4340000000000000u); + // 1 + 2^-53 exactly (a tie), and one unit in the 55th digit around it + CHECK(native_bits64("1.00000000000000011102230246251565404236316680908203125") == 0x3FF0000000000000u); + CHECK(native_bits64("1.00000000000000011102230246251565404236316680908203126") == 0x3FF0000000000001u); + CHECK(native_bits64("1.00000000000000011102230246251565404236316680908203124") == 0x3FF0000000000000u); + // subnormal and overflow boundaries + CHECK(native_bits64("2.4703282292062327e-324") == 0u); + CHECK(native_bits64("2.4703282292062328e-324") == 1u); + CHECK(native_bits64("2.2250738585072011e-308") == 0x000FFFFFFFFFFFFFu); + CHECK(native_bits64("2.2250738585072012e-308") == 0x0010000000000000u); + CHECK(native_bits64("1.7976931348623157e308") == 0x7FEFFFFFFFFFFFFFu); + CHECK(native_bits64("1.7976931348623159e308") == 0x7FF0000000000000u); + CHECK(native_bits64("-1e400") == 0xFFF0000000000000u); + CHECK(native_bits64("-1e-400") == 0x8000000000000000u); + // exponents and zeros far beyond the range cancel out + CHECK(native_bits64("0." + std::string(1000, '0') + "1e1001") == 0x3FF0000000000000u); + CHECK(native_bits64("1" + std::string(1000, '0') + "e-1000") == 0x3FF0000000000000u); + CHECK(native_bits64("1e-99999999999999999999999") == 0u); + CHECK(native_bits64("1E+99999999999999999999999") == 0x7FF0000000000000u); + // more digits than any midpoint has (769): only whether a nonzero digit follows matters + const std::string tie = "1.00000000000000011102230246251565404236316680908203125"; + CHECK(native_bits64(tie + std::string(800, '0')) == 0x3FF0000000000000u); + CHECK(native_bits64(tie + std::string(800, '0') + "1") == 0x3FF0000000000001u); + } + + SECTION("float") + { + CHECK(native_bits32("1.5") == 0x3FC00000u); + CHECK(native_bits32("0.1") == 0x3DCCCCCDu); + CHECK(native_bits32("-0.0") == 0x80000000u); + // 2^24 + 1 is exactly between two floats + CHECK(native_bits32("16777217") == 0x4B800000u); + CHECK(native_bits32("16777217.000000000000000000001") == 0x4B800001u); + CHECK(native_bits32("16777218.999999999999999999999") == 0x4B800001u); + CHECK(native_bits32("16777219") == 0x4B800002u); + // subnormal and overflow boundaries + CHECK(native_bits32("3.4028235677973366e38") == 0x7F7FFFFFu); + CHECK(native_bits32("3.4028235677973367e38") == 0x7F800000u); + CHECK(native_bits32("7.006492321624085e-46") == 0u); + CHECK(native_bits32("7.006492321624086e-46") == 1u); + CHECK(native_bits32("1.1754942e-38") == 0x007FFFFFu); + CHECK(native_bits32("-1.17549435e-38") == 0x80800000u); + CHECK(native_bits32("1e39") == 0x7F800000u); + CHECK(native_bits32("-1e-50") == 0x80000000u); + // not rounded through double: its double would round to another float + CHECK(native_bits32("1.00000005960464477539062500000000001") == 0x3F800001u); + CHECK(native_bits32("9007199254740993") == 0x5A000000u); + } + + SECTION("the conversion shared with other parsers") + { + // convert_float() gives the lexer's results, for every type + const std::vector tokens = + { + "0", "-0.0", "1.5", "0.1", "1e-400", "-2.5E+3", "123456789012345678901234567890", + "9007199254740993.0000000000000000001", "4.9406564584124654e-324" + }; + using float_json = nlohmann::basic_json; + using long_double_json = nlohmann::basic_json; + for (const auto& t : tokens) + { + CAPTURE(t) + const auto layout = float_token_layout(t); + const char* const first = t.data(); + const char* const last = first + t.size(); + const auto d = nlohmann::detail::convert_float(first, last, layout.first, layout.second); + const auto f = nlohmann::detail::convert_float(first, last, layout.first, layout.second); + const auto ld = nlohmann::detail::convert_float(first, last, layout.first, layout.second); + CHECK(bits_of(d) == bits_of(json::parse(t).get())); + CHECK(bits_of(f) == bits_of(float_json::parse(t).get())); + CHECK(ld == long_double_json::parse(t).get()); + } + } } namespace @@ -807,40 +1086,6 @@ std::size_t big_bit_length(const big_uint& a) } return n; } - -std::uint64_t bits_of(double d) -{ - std::uint64_t b = 0; - std::memcpy(&b, &d, sizeof(b)); - return b; -} - -bool eisel_lemire(const std::string& s, double& out) -{ - return nlohmann::detail::parse_float_eisel_lemire(s.data(), s.data() + s.size(), out); -} - -// significant digits of a token, without trailing zeros -std::size_t significant_digits(const std::string& s) -{ - std::string digits; - for (const char c : s) - { - if (c == 'e' || c == 'E') - { - break; - } - if (c >= '0' && c <= '9' && !(digits.empty() && c == '0')) - { - digits += c; - } - } - while (!digits.empty() && digits.back() == '0') - { - digits.pop_back(); - } - return digits.size(); -} } // namespace TEST_CASE("Eisel-Lemire float conversion") @@ -938,6 +1183,10 @@ TEST_CASE("Eisel-Lemire float conversion") const int k = i % 64; const std::uint64_t x = (std::uint64_t{1} << k) | (a & ((std::uint64_t{1} << k) - 1)); CHECK(nlohmann::detail::count_leading_zeros(x) == 63 - k); + + // the mirror image: bit k set, arbitrary bits above it + const std::uint64_t y = (std::uint64_t{1} << k) | (k == 63 ? 0 : (a << (k + 1))); + CHECK(nlohmann::detail::count_trailing_zeros(y) == k); } } @@ -1268,26 +1517,33 @@ TEST_CASE("Eisel-Lemire float conversion") for (const auto& c : known) { CAPTURE(c.first) - double out = 0; - if (eisel_lemire(c.first, out)) - { - CHECK(bits_of(out) == c.second); - } - else - { - // only tokens with more than 19 significant digits are left to - // strtod: those whose value lies too close to a tie - CHECK(significant_digits(c.first) > 19); - } + CHECK(native_bits64(c.first) == c.second); } } + SECTION("binary32") + { + using binary32 = nlohmann::detail::ieee_binary_format<24>; + CHECK(nlohmann::detail::eisel_lemire(0, 1) == 0x3F800000u); + CHECK(nlohmann::detail::eisel_lemire(-1, 1) == 0x3DCCCCCDu); + CHECK(nlohmann::detail::eisel_lemire(-1, 15) == 0x3FC00000u); + CHECK(nlohmann::detail::eisel_lemire(0, 16777217) == 0x4B800000u); // tie, to even + CHECK(nlohmann::detail::eisel_lemire(0, 16777219) == 0x4B800002u); // tie, to even + CHECK(nlohmann::detail::eisel_lemire(-45, 1) == 0x00000001u); + CHECK(nlohmann::detail::eisel_lemire(-46, 7) == 0x00000000u); + CHECK(nlohmann::detail::eisel_lemire(-46, 8) == 0x00000001u); + CHECK(nlohmann::detail::eisel_lemire(-65, 9999999999999999999u) == 0x00000000u); + CHECK(nlohmann::detail::eisel_lemire(20, 3402823466385288598u) == 0x7F7FFFFFu); + CHECK(nlohmann::detail::eisel_lemire(20, 3402823669209384635u) == 0x7F800000u); + CHECK(nlohmann::detail::eisel_lemire(39, 1) == 0x7F800000u); + CHECK(nlohmann::detail::eisel_lemire(-5, 0) == 0x00000000u); + } + SECTION("round trip") { - // every double written by to_chars and read back, also with trailing - // digits that make the token longer than 19 digits + // every double written by to_chars and read back, and its 17-digit + // form with trailing digits that make the token longer than 19 digits std::uint64_t state = 5295; - std::size_t declined = 0; for (int i = 0; i < 200000; ++i) { state ^= state << 13u; @@ -1309,30 +1565,51 @@ TEST_CASE("Eisel-Lemire float conversion") const char* end = nlohmann::detail::to_chars(buffer.data(), buffer.data() + buffer.size(), d); const std::string token(buffer.data(), static_cast(end - buffer.data())); CAPTURE(token) - double out = 0; - REQUIRE(eisel_lemire(token, out)); - CHECK(bits_of(out) == b); + CHECK(native_bits64(token) == b); - // insert digits before the exponent: the value moves by far less - // than the distance to the rounding boundary, so it must not change - std::string longer = token; + // insert digits before the exponent of the 17-digit form: that + // form lies strictly inside the rounding interval of the double + // (the shortest one may lie on its boundary), and the digits move + // it by far less than the distance to the boundary, so the value + // must not change + std::array digits17{}; + static_cast(std::snprintf(digits17.data(), digits17.size(), "%.17g", d)); // NOLINT(cppcoreguidelines-pro-type-vararg,hicpp-vararg) + std::string longer = digits17.data(); const std::size_t e = longer.find('e'); const std::size_t dot = longer.find('.'); const std::string extra = dot == std::string::npos ? ".000000000000000000001" : "000000000000000000001"; longer.insert(e == std::string::npos ? longer.size() : e, extra); CAPTURE(longer) - if (eisel_lemire(longer, out)) + CHECK(native_bits64(longer) == b); + } + } + + SECTION("round trip, binary32") + { + std::uint32_t state = 5295; + for (int i = 0; i < 100000; ++i) + { + state ^= state << 13u; + state ^= state >> 17u; + state ^= state << 5u; + std::uint32_t b = state; + if ((b & 0x7F800000u) == 0x7F800000u) { - CHECK(bits_of(out) == b); + continue; // infinity or NaN } - else + if (i % 4 == 0) { - // w and w + 1 round differently: only when the value is very - // close to a rounding boundary - ++declined; + b &= 0x807FFFFFu; // subnormals } + float f = 0; + std::memcpy(&f, &b, sizeof(f)); + + std::array buffer{}; + const char* end = nlohmann::detail::to_chars(buffer.data(), buffer.data() + buffer.size(), f); + const std::string token(buffer.data(), static_cast(end - buffer.data())); + CAPTURE(token) + CHECK(native_bits32(token) == b); } - CHECK(declined < 1000); // 107 of the 200,000 } SECTION("used by the lexer") @@ -1346,3 +1623,184 @@ TEST_CASE("Eisel-Lemire float conversion") "[json.exception.out_of_range.406] number overflow parsing '1.7976931348623159e308'", json::out_of_range&); } } + +namespace +{ +using float_json = nlohmann::basic_json; + +// the bits of the float that parse() gives for a token, via both scanners; +// the value must be the same for both +template +void check_parse(const std::string& token, Bits expected, Bits infinity) +{ + std::stringstream stream(token); + if ((expected & ~(Bits{1} << ((8 * sizeof(Bits)) - 1))) == infinity) + { + Json _; + CHECK_THROWS_WITH_AS(_ = Json::parse(token), ("[json.exception.out_of_range.406] number overflow parsing '" + token + "'").c_str(), typename Json::out_of_range&); + CHECK_THROWS_WITH_AS(_ = Json::parse(stream), ("[json.exception.out_of_range.406] number overflow parsing '" + token + "'").c_str(), typename Json::out_of_range&); + return; + } + const Json contiguous = Json::parse(token); + const Json streamed = Json::parse(stream); + if (contiguous.is_number_float()) // not an integer that fits + { + CHECK(bits_of(contiguous.template get()) == expected); + CHECK(bits_of(streamed.template get()) == expected); + } + else + { + CHECK(streamed.is_number_integer()); + } +} +} // namespace + +TEST_CASE("float conversion of hard cases") +{ + // see float_hard_cases.hpp + for (const auto& c : float_hard_cases::cases()) + { + const std::string token = c.token; + CAPTURE(token) + CHECK(native_bits64(token) == c.bits64); + CHECK(native_bits32(token) == c.bits32); + check_parse(token, c.bits64, std::uint64_t{0x7FF0000000000000u}); + check_parse(token, c.bits32, std::uint32_t{0x7F800000u}); + } +} + +TEST_CASE("float overflow and underflow in the parser") +{ + SECTION("double") + { + check_parse("1.7976931348623157e308", std::uint64_t{0x7FEFFFFFFFFFFFFFu}, std::uint64_t{0x7FF0000000000000u}); + check_parse("1.7976931348623159e308", std::uint64_t{0x7FF0000000000000u}, std::uint64_t{0x7FF0000000000000u}); + check_parse("-1e309", std::uint64_t{0xFFF0000000000000u}, std::uint64_t{0x7FF0000000000000u}); + check_parse("1" + std::string(400, '0'), std::uint64_t{0x7FF0000000000000u}, std::uint64_t{0x7FF0000000000000u}); + check_parse("1e99999999999999999999", std::uint64_t{0x7FF0000000000000u}, std::uint64_t{0x7FF0000000000000u}); + // an underflow gives a zero with the sign of the token + check_parse("1e-400", std::uint64_t{0}, std::uint64_t{0x7FF0000000000000u}); + check_parse("-1e-400", std::uint64_t{0x8000000000000000u}, std::uint64_t{0x7FF0000000000000u}); + check_parse("-2.4703282292062327e-324", std::uint64_t{0x8000000000000000u}, std::uint64_t{0x7FF0000000000000u}); + check_parse("0." + std::string(400, '0') + "1", std::uint64_t{0}, std::uint64_t{0x7FF0000000000000u}); + } + + SECTION("float") + { + check_parse("3.4028234e38", std::uint32_t{0x7F7FFFFFu}, std::uint32_t{0x7F800000u}); + check_parse("3.4028236e38", std::uint32_t{0x7F800000u}, std::uint32_t{0x7F800000u}); + check_parse("-1e39", std::uint32_t{0xFF800000u}, std::uint32_t{0x7F800000u}); + check_parse("1e-46", std::uint32_t{0}, std::uint32_t{0x7F800000u}); + check_parse("-1e-46", std::uint32_t{0x80000000u}, std::uint32_t{0x7F800000u}); + check_parse("-7.006492321624085e-46", std::uint32_t{0x80000000u}, std::uint32_t{0x7F800000u}); + check_parse("-7.006492321624086e-46", std::uint32_t{0x80000001u}, std::uint32_t{0x7F800000u}); + } +} + +TEST_CASE("string scanning kernels") +{ + // the word-at-a-time kernels must stop exactly where a byte-by-byte scan + // stops, for any content, length, and alignment + const auto reference_special = [](const unsigned char* data, std::size_t n) + { + std::size_t i = 0; + while (i < n && !nlohmann::detail::is_string_special(data[i])) + { + ++i; + } + return i; + }; + const auto reference_copyable = [](const unsigned char* data, std::size_t n) + { + std::size_t i = 0; + while (i < n && nlohmann::detail::is_ascii_copyable(data[i])) + { + ++i; + } + return i; + }; + const auto reference_bulk_run = [](const unsigned char* data, std::size_t n) + { + std::size_t i = 0; + while (i < n) + { + if (data[i] < 0x80u) + { + if (nlohmann::detail::is_string_special(data[i])) + { + break; + } + ++i; + continue; + } + const std::size_t seq = nlohmann::detail::validate_one_utf8(data + i, n - i); + if (seq == 0) + { + break; + } + i += seq; + } + return i; + }; + + // pieces: ordinary ASCII, stops, DEL, well-formed sequences of every + // length, and ill-formed or truncated ones + const std::vector pieces = + { + "a", "Z", " ", "~", "0123456789", "\"", "\\", std::string(1, '\0'), "\n", "\x1F", "\x7F", + "\xC3\xA4", "\xE2\x82\xAC", "\xE6\x97\xA5\xE6\x9C\xAC", "\xF0\x9F\x98\x80", "\xED\x9F\xBF", + "\x80", "\xC0\x80", "\xC3", "\xE2\x82", "\xED\xA0\x80", "\xF4\x90\x80\x80", "\xFF", + }; + std::uint64_t state = 5295; + const auto next = [&state]() + { + state ^= state << 13u; + state ^= state >> 7u; + state ^= state << 17u; + return state; + }; + // the upper half as a 32-bit value: converts to std::size_t implicitly on + // every platform (a cast of std::uint64_t is useless where both are the + // same type, and required where std::size_t is 32 bits wide) + const auto next_small = [&next]() + { + return static_cast(next() >> 32u); + }; + for (int round = 0; round < 100000; ++round) + { + // mostly ordinary text, so that runs span several words + std::string text(next_small() % 8u, '.'); + const std::size_t count = next_small() % 12u; + for (std::size_t k = 0; k < count; ++k) + { + const std::size_t p = (next() % 4 == 0) ? next_small() % pieces.size() : 0; + text += pieces[p]; + text += std::string(next_small() % 10u, 'x'); + } + const auto* data = reinterpret_cast(text.data()); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) + for (std::size_t offset = 0; offset < 3 && offset <= text.size(); ++offset) + { + const std::size_t n = text.size() - offset; + CAPTURE(text) + CAPTURE(offset) + CHECK(nlohmann::detail::find_string_special(data + offset, n) == reference_special(data + offset, n)); + CHECK(nlohmann::detail::find_ascii_copyable_run(data + offset, n) == reference_copyable(data + offset, n)); + CHECK(nlohmann::detail::scalar_string_bulk_run(data + offset, n) == reference_bulk_run(data + offset, n)); + } + } + + // the trailing-zero count, whichever implementation the compiler gets + for (int k = 0; k < 64; ++k) + { + const std::uint64_t bit = std::uint64_t{1} << k; + CHECK(nlohmann::detail::count_trailing_zeros(bit) == k); + CHECK(nlohmann::detail::count_trailing_zeros(bit | (bit << 1u) | 0x8000000000000000u) == k); + } + + // eight bytes as a little-endian word, at any alignment + const std::array bytes = {{0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, 0xFF}}; + CHECK(nlohmann::detail::read_eight_bytes(bytes.data()) == 0x0807060504030201u); + CHECK(nlohmann::detail::read_eight_bytes(bytes.data() + 1) == 0x0908070605040302u); + CHECK(nlohmann::detail::read_eight_bytes(bytes.data() + 8) == 0xFF0F0E0D0C0B0A09u); + CHECK(nlohmann::detail::read_eight_bytes(reinterpret_cast(bytes.data()) + 3) == 0x0B0A090807060504u); +} diff --git a/tests/src/unit-custom-binary-type.cpp b/tests/src/unit-custom-binary-type.cpp index 6b504bd51..e44a3c88c 100644 --- a/tests/src/unit-custom-binary-type.cpp +++ b/tests/src/unit-custom-binary-type.cpp @@ -8,6 +8,13 @@ #include "doctest_compatibility.h" +// capture whether JSON_DISABLE_ENUM_SERIALIZATION was enabled on the command +// line before including json.hpp, which #undefs it; std::byte is an enum, so +// converting std::vector needs enum serialization +#if defined(JSON_DISABLE_ENUM_SERIALIZATION) && (JSON_DISABLE_ENUM_SERIALIZATION == 1) + #define SKIP_TESTS_FOR_ENUM_SERIALIZATION +#endif + #include #include @@ -85,3 +92,53 @@ TEST_CASE("binary type whose value type is not std::uint8_t") } #endif } + +#ifdef JSON_HAS_CPP_17 +// moved from unit-regression3.cpp, so that file does not instantiate a second +// basic_json specialization (MinGW section limit, see windows.yml) +TEST_CASE("std::vector as binary type") +{ + SECTION("issue #4804: from_cbor incompatible with std::vector as binary_t") + { + const std::vector data = {0x80}; + const auto decoded = byte_binary_json::from_cbor(data); + CHECK((decoded == byte_binary_json::array())); + } + +#ifndef SKIP_TESTS_FOR_ENUM_SERIALIZATION + SECTION("discussion #4209 - custom BinaryType direct assignment and round-tripping") + { + // Test that assigning a custom BinaryType directly creates a binary value, not an array + const std::vector original{std::byte{1}, std::byte{2}, std::byte{3}}; + const byte_binary_json j = original; + CHECK(j.is_binary()); + CHECK(!j.is_array()); + + // Test round-tripping: extracting the binary value back as the custom container type + const auto extracted = j.get>(); + CHECK(extracted == original); + + // Test that the default json alias behavior is unchanged: std::vector -> array + const nlohmann::json default_json = std::vector {1, 2, 3}; + CHECK(default_json.is_array()); + CHECK(!default_json.is_binary()); + } + + SECTION("discussion #4209 - custom BinaryType extraction from parsed array") + { + // Test that extracting a custom BinaryType from a parsed JSON array still works + // (not just from a binary-typed node) + const auto j = byte_binary_json::parse("[1,2,3]"); + CHECK(j.is_array()); + CHECK(!j.is_binary()); + + // Extracting as custom BinaryType should work from arrays + const auto extracted = j.get>(); + CHECK(extracted.size() == 3); + CHECK(extracted[0] == std::byte{1}); + CHECK(extracted[1] == std::byte{2}); + CHECK(extracted[2] == std::byte{3}); + } +#endif +} +#endif diff --git a/tests/src/unit-deserialization.cpp b/tests/src/unit-deserialization.cpp index 6222def34..e82685311 100644 --- a/tests/src/unit-deserialization.cpp +++ b/tests/src/unit-deserialization.cpp @@ -35,6 +35,7 @@ using nlohmann::json; #include #include #include +#include #include "sax_event_loggers.hpp" @@ -1250,3 +1251,139 @@ TEST_CASE_TEMPLATE("deserialization of different character types (UTF-32)", T, c CHECK(json::sax_parse(v, &l)); CHECK(l.events.size() == 1); } + +namespace +{ +// records the position and message of a parse error; reporting the error via +// SAX does not throw, so the test also runs with JSON_NOEXCEPTION +struct SaxErrorRecorder : public SaxEventLogger +{ + bool parse_error(std::size_t position, const std::string& /*last_token*/, const json::exception& ex) override + { + error = std::to_string(position) + " " + ex.what(); + return false; + } + + std::string error {}; // NOLINT(readability-redundant-member-init) +}; +} // namespace + +TEST_CASE("parse error positions are the same for contiguous and stream input") +{ + // contiguous input derives the position from the input on error, stream + // input counts it while reading (see lexer::lazy_position); the cases + // cover newlines before the error, ungetting a newline after a number, + // newlines inside a string, a comment, and reads at the end of input + struct test_case + { + std::string input; + std::size_t byte; + std::string message; + }; + const std::vector cases = + { + {"[\n 1,\n 2\n", 12, "[json.exception.parse_error.101] parse error at line 4, column 1: syntax error while parsing array - unexpected end of input; expected ']'"}, + {"[1\n,2,\n]", 8, "[json.exception.parse_error.101] parse error at line 3, column 1: syntax error while parsing value - unexpected ']'; expected '[', '{', or a literal"}, + {"[1\n x]", 5, "[json.exception.parse_error.101] parse error at line 2, column 2: syntax error while parsing array - invalid literal; last read: '1 x'; expected ']'"}, + {"1\n2", 3, "[json.exception.parse_error.101] parse error at line 2, column 1: syntax error while parsing value - unexpected number literal; expected end of input"}, + { + "{\"a\":\n\"b\nc\"}", 9, R"([json.exception.parse_error.101] parse error at line 3, column 0: syntax error while parsing value - invalid string: control character U+000A (LF) must be escaped to \u000A or \n; last read: '"b')" + }, + {"/* a\n b */\n x", 13, "[json.exception.parse_error.101] parse error at line 3, column 2: syntax error while parsing value - invalid literal; last read: '/* a b */ x'"}, + {"\n\n", 3, "[json.exception.parse_error.101] parse error at line 3, column 1: syntax error while parsing value - unexpected end of input; expected '[', '{', or a literal"}, + {"[\"a\",\n 1e\n]", 10, "[json.exception.parse_error.101] parse error at line 3, column 0: syntax error while parsing value - invalid number; expected '+', '-', or digit after exponent; last read: '1e'"}, + {"[tru\n]", 5, "[json.exception.parse_error.101] parse error at line 2, column 0: syntax error while parsing value - invalid literal; last read: '[tru'"}, + }; + + for (const auto& c : cases) + { + CAPTURE(c.input) + const std::string expected = std::to_string(c.byte) + " " + c.message; + + SaxErrorRecorder string_sax; + CHECK_FALSE(json::sax_parse(c.input, &string_sax, json::input_format_t::json, true, true)); + CHECK(string_sax.error == expected); + + SaxErrorRecorder pointer_sax; + CHECK_FALSE(json::sax_parse(c.input.data(), c.input.data() + c.input.size(), &pointer_sax, json::input_format_t::json, true, true)); + CHECK(pointer_sax.error == expected); + + std::istringstream is(c.input); + SaxErrorRecorder stream_sax; + CHECK_FALSE(json::sax_parse(is, &stream_sax, json::input_format_t::json, true, true)); + CHECK(stream_sax.error == expected); + } +} + +#if !defined(JSON_NOEXCEPTION) +TEST_CASE("indentation runs of every length around the 8-byte skip") +{ + // contiguous input skips the spaces after a newline 8 bytes at a time (see + // lexer::skip_indentation), stream input reads them one by one; results, + // byte offsets, and messages must be the same for runs that end anywhere + // relative to a word boundary, including at the end of the input + const auto parse_result = [](const std::string & input, bool contiguous) + { + try + { + if (contiguous) + { + return json::parse(input.data(), input.data() + input.size(), nullptr, true, true).dump(); + } + std::istringstream is(input); + return json::parse(is, nullptr, true, true).dump(); + } + catch (const json::parse_error& e) + { + return std::to_string(e.byte).append(" ").append(e.what()); + } + }; + + // replace every '~' in a pattern by an indentation run + const auto indented = [](const std::string & pattern, const std::string & indent) + { + std::string result; + for (const char c : pattern) + { + if (c == '~') + { + result += indent; + } + else + { + result += c; + } + } + return result; + }; + + const std::vector patterns = + { + "[\n~1,\n~2\n]", + "[\n~1,\n~x]", + "[\n~1,\n~", + "1\n~", + "[\n~\t1,\n~\r\n~2]", + "[\n~/* c */ 1]", + "[\n~1\n~2]", + }; + + for (std::size_t n = 0; n <= 33; ++n) + { + const std::string indent(n, ' '); + for (const auto& pattern : patterns) + { + const std::string input = indented(pattern, indent); + CAPTURE(n) + CAPTURE(input) + const std::string expected = parse_result(input, false); + CHECK(parse_result(input, true) == expected); + CHECK(json::accept(input.data(), input.data() + input.size(), true) == json::accept(input, true)); + } + } + + // and against fixed expectations, in case both paths broke the same way + CHECK(parse_result(indented("[\n~x]", std::string(16, ' ')), true) == "19 [json.exception.parse_error.101] parse error at line 2, column 17: syntax error while parsing value - invalid literal; last read: '[ x'"); + CHECK(parse_result(indented("[\n~1]", std::string(17, ' ')), true) == "[1]"); +} +#endif diff --git a/tests/src/unit-diagnostics.cpp b/tests/src/unit-diagnostics.cpp index 1ada023b7..37427587c 100644 --- a/tests/src/unit-diagnostics.cpp +++ b/tests/src/unit-diagnostics.cpp @@ -567,9 +567,9 @@ TEST_CASE("Regression tests for extended diagnostics") } } -TEST_CASE("Better diagnostics past the descent bound of update() and merge_patch()") +TEST_CASE("Better diagnostics past the descent bound of update(), merge_patch() and merge_diff()") { - // Both merge objects nested more than detail::recursion_depth_limit() + // All three handle objects nested more than detail::recursion_depth_limit() // (128) levels deep without recursing; the values they add or replace // there must still know their parents. // The values are built rather than parsed, so that the expected messages @@ -615,4 +615,19 @@ TEST_CASE("Better diagnostics past the descent bound of update() and merge_patch CHECK_THROWS_WITH_AS(p->at("x").at(0), expected_x.c_str(), json::type_error); CHECK_THROWS_WITH_AS(p->at("y").at(0), expected_y.c_str(), json::type_error); } + + SECTION("merge_diff()") + { + // target and patch differ only in their innermost members, so the + // diff of the two is nested as deeply: {"a": ... {"x": null, "y": 2}} + const json diff = json::merge_diff(target, patch); // NOLINT(readability-suspicious-call-argument) + + const json* p = &diff; + for (std::size_t i = 0; i < depth; ++i) + { + p = &p->at("a"); + } + CHECK(p->at("x").is_null()); + CHECK_THROWS_WITH_AS(p->at("y").at(0), expected_y.c_str(), json::type_error); + } } diff --git a/tests/src/unit-locale-cpp.cpp b/tests/src/unit-locale-cpp.cpp index 4865e5687..b665b6531 100644 --- a/tests/src/unit-locale-cpp.cpp +++ b/tests/src/unit-locale-cpp.cpp @@ -260,10 +260,11 @@ struct LocaleSwitchingSax final: public nlohmann::json_sax TEST_CASE("locale changes between lexer construction and number conversion (#5198)") { - // The numbers are chosen so that the conversion also takes the strtod - // fallback, which honors the locale that is current at conversion time: - // too many significant digits for Clinger's fast path, an underflow that - // std::from_chars rejects, and a plain value. + // float and double are converted without the locale. A long double that + // is not binary64 can take the strtold fallback, which honors the locale + // that is current at conversion time. The numbers are chosen so that it + // does: too many significant digits for Clinger's fast path, an underflow + // that std::from_chars rejects, and a plain value. const std::vector numbers = {"3.14159265358979323846", "1.5e-400", "12.34", "-0.000123456789012345678"}; std::string text = "["; for (const auto& n : numbers) @@ -327,7 +328,8 @@ TEST_CASE("locale changes between lexer construction and number conversion (#519 } } - // a long double goes through std::strtold unless std::from_chars supports it + // a long double goes through std::strtold unless it is binary64 or + // std::from_chars supports it { bool switched = false; const auto cb = [&](int /*depth*/, long_double_json::parse_event_t event, long_double_json& /*parsed*/) noexcept @@ -353,8 +355,15 @@ TEST_CASE("locale with a multi-byte decimal point") { // Some locales use a decimal point that is not a single character, e.g. // U+066B ARABIC DECIMAL SEPARATOR (two bytes in UTF-8). It cannot be - // substituted in place for '.', so the strtod fallback stops early. The - // conversion must still terminate rather than retry forever. + // substituted in place for '.', so the strtold fallback (only for long + // double formats other than binary64) converts a copy of the token with + // the whole decimal point instead (#5660). The values must be those of the + // "C" locale. + using long_double_json = nlohmann::basic_json; + const char* const long_double_numbers = "[3.14159265358979323846, 1.5e-400, -0.000123456789012345678]"; + REQUIRE(std::setlocale(LC_NUMERIC, "C") != nullptr); + const long_double_json expected_long_double = long_double_json::parse(long_double_numbers); + const std::array names = {{"ar_EG.UTF-8", "ar_SA.UTF-8", "fa_IR.UTF-8", "ps_AF.UTF-8", "ar_EG", "fa_IR"}}; bool tested = false; for (const char* name : names) @@ -372,12 +381,20 @@ TEST_CASE("locale with a multi-byte decimal point") tested = true; // too many significant digits for Clinger's fast path, and an underflow - // that std::from_chars rejects: both reach the strtod fallback + // that std::from_chars rejects: double does not depend on the locale json j; CHECK_NOTHROW(j = json::parse("[3.14159265358979323846, 1.5e-400, -0.000123456789012345678]")); CHECK(j.is_array()); + CHECK(j[0] == 3.14159265358979323846); + CHECK(j[1] == 0.0); + CHECK(j[2] == -0.000123456789012345678); CHECK(json::accept("3.14159265358979323846")); + // a long double that reaches the strtold fallback is not truncated + long_double_json ld; + CHECK_NOTHROW(ld = long_double_json::parse(long_double_numbers)); + CHECK(ld == expected_long_double); + // a value the locale-independent paths convert is not affected CHECK(json::parse("12.5") == 12.5); } diff --git a/tests/src/unit-merge_diff.cpp b/tests/src/unit-merge_diff.cpp new file mode 100644 index 000000000..77fa58768 --- /dev/null +++ b/tests/src/unit-merge_diff.cpp @@ -0,0 +1,1024 @@ +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ (supporting code) +// | | |__ | | | | | | version 3.12.0 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann +// SPDX-License-Identifier: MIT + +#include "doctest_compatibility.h" + +#include +using nlohmann::json; +using nlohmann::ordered_json; +#ifdef JSON_TEST_NO_GLOBAL_UDLS + using namespace nlohmann::literals; // NOLINT(google-build-using-namespace) +#endif + +#include +#include + +namespace +{ +// merge_diff() recursing once per nesting level, without the bound on its +// descent; only usable on values nested a few hundred levels deep +template +BasicJsonType reference_merge_diff(const BasicJsonType& source, const BasicJsonType& target) +{ + if (!source.is_object() || !target.is_object()) + { + return target; + } + + BasicJsonType result = BasicJsonType::object(); + for (auto it = source.begin(); it != source.end(); ++it) + { + const auto itf = target.find(it.key()); + if (itf == target.end()) + { + result[it.key()] = nullptr; + } + else if (it.value().is_object() && itf.value().is_object()) + { + auto diff = reference_merge_diff(it.value(), itf.value()); + if (!diff.empty()) + { + result[it.key()] = std::move(diff); + } + } + else if (it.value() != itf.value()) + { + result[it.key()] = itf.value(); + } + } + for (auto it = target.begin(); it != target.end(); ++it) + { + if (source.find(it.key()) == source.end()) + { + result[it.key()] = it.value(); + } + } + return result; +} + +// objects nested `depth` levels deep under the key "a", with members before +// and after it that differ by `variant` on the way down, so that diffing two +// variants removes, adds and replaces members on many levels, and diffs nested +// objects that are equal, that differ, or that are not objects in one of the +// variants; only variant 2 has members that are null +template +BasicJsonType nested_objects(const std::size_t depth, const int variant) +{ + BasicJsonType value = BasicJsonType::object({{"v", variant}}); + for (std::size_t i = 0; i < depth; ++i) + { + BasicJsonType object = BasicJsonType::object(); + if ((i + static_cast(variant)) % 3 == 0) + { + object["s" + std::to_string(variant)] = i; + } + object["c"] = i % 4 == 0 ? variant : 0; + if (i % 5 == 0) + { + if (variant == 0) + { + object["o"] = BasicJsonType::object({{"p", i}}); + } + else if (variant == 1) + { + object["o"] = BasicJsonType::array({i}); + } + else + { + object["o"] = nullptr; + } + } + if (variant == 2 && i % 4 == 2) + { + object["n"] = nullptr; + } + object["a"] = std::move(value); + if (i % 6 == 0) + { + object["d"] = BasicJsonType::object({{"e", variant == 1 ? 1 : 0}}); + } + if (variant == 1 && i % 7 == 0) + { + object["z"] = "z"; + } + value = std::move(object); + } + return value; +} +} // namespace + +TEST_CASE("JSON Merge Patch diff") +{ + SECTION("examples from RFC 7396") + { + SECTION("Section 1") + { + json document = R"({ + "a": "b", + "c": { + "d": "e", + "f": "g" + } + })"_json; + + json expected = R"({ + "a": "z", + "c": { + "d": "e" + } + })"_json; + + json const expected_patch = R"({ + "a": "z", + "c": { + "f": null + } + })"_json; + + auto patch = json::merge_diff(document, expected); + CHECK(patch == expected_patch); + + document.merge_patch(patch); + CHECK(document == expected); + } + + SECTION("Section 3") + { + json document = R"({ + "title": "Goodbye!", + "author": { + "givenName": "John", + "familyName": "Doe" + }, + "tags": [ + "example", + "sample" + ], + "content": "This will be unchanged" + })"_json; + + json expected = R"({ + "title": "Hello!", + "author": { + "givenName": "John" + }, + "tags": [ + "example" + ], + "content": "This will be unchanged", + "phoneNumber": "+01-123-456-7890" + })"_json; + + json const expected_patch = R"({ + "title": "Hello!", + "phoneNumber": "+01-123-456-7890", + "author": { + "familyName": null + }, + "tags": [ + "example" + ] + })"_json; + + auto patch = json::merge_diff(document, expected); + CHECK(patch == expected_patch); + + document.merge_patch(patch); + CHECK(document == expected); + } + + SECTION("Appendix A") + { + SECTION("Example 1") + { + json original = R"({"a":"b"})"_json; + json result = R"({"a":"c"})"_json; + + auto patch = json::merge_diff(original, result); + CHECK(patch == R"({"a":"c"})"_json); + + original.merge_patch(patch); + CHECK(original == result); + } + + SECTION("Example 2") + { + json original = R"({"a":"b"})"_json; + json result = R"({"a":"b", "b":"c"})"_json; + + auto patch = json::merge_diff(original, result); + CHECK(patch == R"({"b":"c"})"_json); + + original.merge_patch(patch); + CHECK(original == result); + } + + SECTION("Example 3") + { + json original = R"({"a":"b"})"_json; + json result = R"({})"_json; + + auto patch = json::merge_diff(original, result); + CHECK(patch == R"({"a":null})"_json); + + original.merge_patch(patch); + CHECK(original == result); + } + + SECTION("Example 4") + { + json original = R"({"a":"b","b":"c"})"_json; + json result = R"({"b":"c"})"_json; + + auto patch = json::merge_diff(original, result); + CHECK(patch == R"({"a":null})"_json); + + original.merge_patch(patch); + CHECK(original == result); + } + + SECTION("Example 5") + { + json original = R"({"a":["b"]})"_json; + json result = R"({"a":"c"})"_json; + + auto patch = json::merge_diff(original, result); + CHECK(patch == R"({"a":"c"})"_json); + + original.merge_patch(patch); + CHECK(original == result); + } + + SECTION("Example 6") + { + json original = R"({"a":"c"})"_json; + json result = R"({"a":["b"]})"_json; + + auto patch = json::merge_diff(original, result); + CHECK(patch == R"({"a":["b"]})"_json); + + original.merge_patch(patch); + CHECK(original == result); + } + + SECTION("Example 7") + { + json original = R"({"a":{"b": "c"}})"_json; + json result = R"({"a": {"b": "d"}})"_json; + + auto patch = json::merge_diff(original, result); + // differs from the RFC 7396, "c": null is not in the patch + // because neither the source nor the target have the "c" key + CHECK(patch == R"({"a":{"b":"d"}})"_json); + + original.merge_patch(patch); + CHECK(original == result); + } + + SECTION("Example 8") + { + json original = R"({"a":[{"b":"c"}]})"_json; + json result = R"({"a":[1]})"_json; + + auto patch = json::merge_diff(original, result); + CHECK(patch == R"({"a":[1]})"_json); + + original.merge_patch(patch); + CHECK(original == result); + } + + SECTION("Example 9") + { + json original = R"(["a","b"])"_json; + json result = R"(["c","d"])"_json; + + auto patch = json::merge_diff(original, result); + CHECK(patch == R"(["c","d"])"_json); + + original.merge_patch(patch); + CHECK(original == result); + } + + SECTION("Example 10") + { + json original = R"({"a":"b"})"_json; + json result = R"(["c"])"_json; + + auto patch = json::merge_diff(original, result); + CHECK(patch == R"(["c"])"_json); + + original.merge_patch(patch); + CHECK(original == result); + } + + SECTION("Example 11") + { + json original = R"({"a":"foo"})"_json; + json result = R"(null)"_json; + + auto patch = json::merge_diff(original, result); + CHECK(patch == R"(null)"_json); + + original.merge_patch(patch); + CHECK(original == result); + } + + SECTION("Example 12") + { + json original = R"({"a":"foo"})"_json; + json result = R"("bar")"_json; + + auto patch = json::merge_diff(original, result); + CHECK(patch == R"("bar")"_json); + + original.merge_patch(patch); + CHECK(original == result); + } + + SECTION("Example 13") + { + json original = R"({"e":null})"_json; + json result = R"({"e":null,"a":1})"_json; + + auto patch = json::merge_diff(original, result); + CHECK(patch == R"({"a":1})"_json); + + original.merge_patch(patch); + CHECK(original == result); + } + + SECTION("Example 14") + { + json original = R"([1,2])"_json; + json result = R"({"a":"b"})"_json; + + auto patch = json::merge_diff(original, result); + // differs from the RFC 7396, "c": null is not in the patch + // because neither the source nor the target have the "c" key + CHECK(patch == R"({"a":"b"})"_json); + + original.merge_patch(patch); + CHECK(original == result); + } + + SECTION("Example 15") + { + json original = R"({})"_json; + json result = R"({"a":{"bb":{}}})"_json; + + auto patch = json::merge_diff(original, result); + // differs from the RFC 7396, "ccc": null is not in the patch + // because neither the source nor the target have the "ccc" key + CHECK(patch == R"({"a":{"bb":{}}})"_json); + + original.merge_patch(patch); + CHECK(original == result); + } + } + } + + SECTION("null values") + { + SECTION("object with null value to object") + { + json original = R"({"a":null})"_json; + json result = R"({"a":{"b":"c"}})"_json; + + auto patch = json::merge_diff(original, result); + CHECK(patch == R"({"a":{"b":"c"}})"_json); + + original.merge_patch(patch); + CHECK(original == R"({"a":{"b":"c"}})"_json); + } + + SECTION("object to object with null value") + { + json original = R"({"a":{"b":"c"}})"_json; + json result = R"({"a":null})"_json; + + auto patch = json::merge_diff(original, result); + CHECK(patch == R"({"a":null})"_json); + + original.merge_patch(patch); + CHECK(original == R"({})"_json); + } + + SECTION("primitive to object with null value") + { + json original = R"({"a":1})"_json; + json result = R"({"a":null})"_json; + + auto patch = json::merge_diff(original, result); + CHECK(patch == R"({"a":null})"_json); + + original.merge_patch(patch); + CHECK(original == R"({})"_json); + } + + SECTION("nested primitive to object with null value") + { + json original = R"({"a":{"b":1}})"_json; + json result = R"({"a":{"b":null}})"_json; + + auto patch = json::merge_diff(original, result); + CHECK(patch == R"({"a":{"b":null}})"_json); + + original.merge_patch(patch); + CHECK(original == R"({"a":{}})"_json); + } + + SECTION("array value to object with null value") + { + json original = R"({"a":[1,2]})"_json; + json result = R"({"a":null})"_json; + + auto patch = json::merge_diff(original, result); + CHECK(patch == R"({"a":null})"_json); + + original.merge_patch(patch); + CHECK(original == R"({})"_json); + } + + SECTION("nested object to object with null value") + { + json original = R"({"a":{"b":{"c":"d"}}})"_json; + json result = R"({"a":{"b":null}})"_json; + + auto patch = json::merge_diff(original, result); + CHECK(patch == R"({"a":{"b":null}})"_json); + + original.merge_patch(patch); + CHECK(original == R"({"a":{}})"_json); + } + + SECTION("nested primitive to object with null value with sibling") + { + json original = R"({"x":"y","a":{"b":"c"}})"_json; + json result = R"({"x":"y","a":{"b":null, "c":"d"}})"_json; + + auto patch = json::merge_diff(original, result); + CHECK(patch == R"({"a":{"b":null,"c":"d"}})"_json); + + original.merge_patch(patch); + CHECK(original == R"({"x":"y","a":{"c":"d"}})"_json); + } + + SECTION("empty object to object with null value") + { + json original = R"({})"_json; + json result = R"({"a":null})"_json; + + auto patch = json::merge_diff(original, result); + CHECK(patch == R"({"a":null})"_json); + + original.merge_patch(patch); + CHECK(original == R"({})"_json); + } + + SECTION("null to object with null value") + { + json original = R"(null)"_json; + json result = R"({"a":null})"_json; + + auto patch = json::merge_diff(original, result); + CHECK(patch == R"({"a":null})"_json); + + original.merge_patch(patch); + CHECK(original == R"({})"_json); + } + + SECTION("array to object with null value") + { + json original = R"([])"_json; + json result = R"({"a":null})"_json; + + auto patch = json::merge_diff(original, result); + CHECK(patch == R"({"a":null})"_json); + + original.merge_patch(patch); + CHECK(original == R"({})"_json); + } + + SECTION("empty object to nested null value") + { + json original = R"({})"_json; + json result = R"({"a":{"b":null}})"_json; + + auto patch = json::merge_diff(original, result); + CHECK(patch == R"({"a":{"b":null}})"_json); + + original.merge_patch(patch); + CHECK(original == R"({"a":{}})"_json); + } + + SECTION("object to nested null value") + { + json original = R"({"a": 1})"_json; + json result = R"({"a":{"b":null}})"_json; + + auto patch = json::merge_diff(original, result); + CHECK(patch == R"({"a":{"b":null}})"_json); + + original.merge_patch(patch); + CHECK(original == R"({"a":{}})"_json); + } + + SECTION("primitive to nested null value") + { + json original = R"(1)"_json; + json result = R"({"a":{"b":null}})"_json; + + auto patch = json::merge_diff(original, result); + CHECK(patch == R"({"a":{"b":null}})"_json); + + original.merge_patch(patch); + CHECK(original == R"({"a":{}})"_json); + } + + SECTION("object with null value unchanged") + { + json original = R"({"a":null})"_json; + json result = R"({"a":null})"_json; + + auto patch = json::merge_diff(original, result); + CHECK(patch == R"({})"_json); + + original.merge_patch(patch); + CHECK(original == result); + } + + SECTION("object with null value to empty object") + { + json original = R"({"a":null})"_json; + json result = R"({})"_json; + + auto patch = json::merge_diff(original, result); + CHECK(patch == R"({"a":null})"_json); + + original.merge_patch(patch); + CHECK(original == result); + } + + SECTION("object with null value to primitive") + { + json original = R"({"a":null})"_json; + json result = R"({"a":1})"_json; + + auto patch = json::merge_diff(original, result); + CHECK(patch == R"({"a":1})"_json); + + original.merge_patch(patch); + CHECK(original == result); + } + + SECTION("array to array with null value") + { + json original = R"({"a":[1]})"_json; + json result = R"({"a":[null]})"_json; + + auto patch = json::merge_diff(original, result); + CHECK(patch == R"({"a":[null]})"_json); + + original.merge_patch(patch); + CHECK(original == result); + } + + SECTION("empty object to array with null value") + { + json original = R"({})"_json; + json result = R"({"a":[null]})"_json; + + auto patch = json::merge_diff(original, result); + CHECK(patch == R"({"a":[null]})"_json); + + original.merge_patch(patch); + CHECK(original == result); + } + + SECTION("doubly nested primitive to null value") + { + json original = R"({"a":{"b":{"c":1}}})"_json; + json result = R"({"a":{"b":{"c":null}}})"_json; + + auto patch = json::merge_diff(original, result); + CHECK(patch == R"({"a":{"b":{"c":null}}})"_json); + + original.merge_patch(patch); + CHECK(original == R"({"a":{"b":{}}})"_json); + } + } + + SECTION("no change") + { + SECTION("object") + { + json original = R"({"a":"b","b":"c"})"_json; + json result = R"({"a":"b","b":"c"})"_json; + + auto patch = json::merge_diff(original, result); + CHECK(patch.empty()); + original.merge_patch(patch); + CHECK(original == result); + } + + SECTION("empty object") + { + json original = R"({})"_json; + json result = R"({})"_json; + + auto patch = json::merge_diff(original, result); + CHECK(patch.empty()); + original.merge_patch(patch); + CHECK(original == result); + } + + SECTION("array") + { + json original = R"([1,2,3])"_json; + json result = R"([1,2,3])"_json; + + auto patch = json::merge_diff(original, result); + CHECK(!patch.empty()); + original.merge_patch(patch); + CHECK(original == result); + } + + SECTION("null") + { + json original = R"(null)"_json; + json result = R"(null)"_json; + + auto patch = json::merge_diff(original, result); + CHECK(patch.empty()); + original.merge_patch(patch); + CHECK(original == result); + } + + SECTION("string") + { + json original = R"("ab")"_json; + json result = R"("ab")"_json; + + auto patch = json::merge_diff(original, result); + CHECK(!patch.empty()); + original.merge_patch(patch); + CHECK(original == result); + } + + SECTION("number") + { + json original = R"(42)"_json; + json result = R"(42)"_json; + + auto patch = json::merge_diff(original, result); + CHECK(!patch.empty()); + original.merge_patch(patch); + CHECK(original == result); + } + } + + SECTION("primitives") + { + SECTION("string") + { + json original = R"("a")"_json; + json result = R"("b")"_json; + + auto patch = json::merge_diff(original, result); + CHECK(patch == R"("b")"_json); + + original.merge_patch(patch); + CHECK(original == result); + } + + SECTION("number") + { + json original = R"(1)"_json; + json result = R"(2)"_json; + + auto patch = json::merge_diff(original, result); + CHECK(patch == R"(2)"_json); + + original.merge_patch(patch); + CHECK(original == result); + } + + SECTION("boolean") + { + json original = R"(false)"_json; + json result = R"(true)"_json; + + auto patch = json::merge_diff(original, result); + CHECK(patch == R"(true)"_json); + + original.merge_patch(patch); + CHECK(original == result); + } + + SECTION("object to primitive") + { + json original = R"({"a":{"b":"c"}})"_json; + json result = R"({"a":1})"_json; + + auto patch = json::merge_diff(original, result); + CHECK(patch == R"({"a":1})"_json); + + original.merge_patch(patch); + CHECK(original == result); + } + + SECTION("primitive to object") + { + json original = R"({"a":1})"_json; + json result = R"({"a":{"b":"c"}})"_json; + + auto patch = json::merge_diff(original, result); + CHECK(patch == R"({"a":{"b":"c"}})"_json); + + original.merge_patch(patch); + CHECK(original == result); + } + + SECTION("object to empty string") + { + json original = R"({"a":{"b":"c"}})"_json; + json result = R"({"a":""})"_json; + + auto patch = json::merge_diff(original, result); + CHECK(patch == R"({"a":""})"_json); + + original.merge_patch(patch); + CHECK(original == result); + } + + SECTION("object to zero") + { + json original = R"({"a":{"b":"c"}})"_json; + json result = R"({"a":0})"_json; + + auto patch = json::merge_diff(original, result); + CHECK(patch == R"({"a":0})"_json); + + original.merge_patch(patch); + CHECK(original == result); + } + + SECTION("object to empty object") + { + json original = R"({"a":{"b":"c"}})"_json; + json result = R"({"a":{}})"_json; + + auto patch = json::merge_diff(original, result); + CHECK(patch == R"({"a":{"b":null}})"_json); + + original.merge_patch(patch); + CHECK(original == result); + } + } + + SECTION("arrays") + { + SECTION("array to array") + { + json original = R"([1,2,3])"_json; + json result = R"([1,2,4])"_json; + + auto patch = json::merge_diff(original, result); + CHECK(patch == R"([1,2,4])"_json); + + original.merge_patch(patch); + CHECK(original == result); + } + + SECTION("array to empty array") + { + json original = R"([1,2,3])"_json; + json result = R"([])"_json; + + auto patch = json::merge_diff(original, result); + CHECK(patch == R"([])"_json); + + original.merge_patch(patch); + CHECK(original == result); + } + + SECTION("empty array to array") + { + json original = R"([])"_json; + json result = R"([1,2,3])"_json; + + auto patch = json::merge_diff(original, result); + CHECK(patch == R"([1,2,3])"_json); + + original.merge_patch(patch); + CHECK(original == result); + } + + SECTION("object to empty array") + { + json original = R"({"a":{"b":"c"}})"_json; + json result = R"({"a":[]})"_json; + + auto patch = json::merge_diff(original, result); + CHECK(patch == R"({"a":[]})"_json); + + original.merge_patch(patch); + CHECK(original == result); + } + + SECTION("same keys") + { + json original = R"(["a","b"])"_json; + json result = R"({"a":1,"b":2})"_json; + + auto patch = json::merge_diff(original, result); + CHECK(patch == R"({"a":1,"b":2})"_json); + + original.merge_patch(patch); + CHECK(original == result); + } + } + + SECTION("ordered_json") + { + SECTION("changed value keeps member order") + { + auto original = ordered_json::parse(R"({"c":1,"b":2,"a":3})"); + auto result = ordered_json::parse(R"({"c":1,"b":5,"a":3})"); + + auto patch = ordered_json::merge_diff(original, result); + CHECK(patch.dump() == R"({"b":5})"); + + original.merge_patch(patch); + CHECK(original == result); + CHECK(original.dump() == R"({"c":1,"b":5,"a":3})"); + } + + SECTION("added members follow target order") + { + auto original = ordered_json::parse(R"({"b":1})"); + auto result = ordered_json::parse(R"({"b":1,"z":2,"a":3})"); + + auto patch = ordered_json::merge_diff(original, result); + CHECK(patch.dump() == R"({"z":2,"a":3})"); + + original.merge_patch(patch); + CHECK(original == result); + CHECK(original.dump() == R"({"b":1,"z":2,"a":3})"); + } + + SECTION("removed members follow source order") + { + auto original = ordered_json::parse(R"({"c":1,"b":2,"a":3})"); + auto result = ordered_json::parse(R"({"b":2})"); + + auto patch = ordered_json::merge_diff(original, result); + CHECK(patch.dump() == R"({"c":null,"a":null})"); + + original.merge_patch(patch); + CHECK(original == result); + } + + SECTION("nested objects keep member order") + { + auto original = ordered_json::parse(R"({"z":{"y":1,"x":2},"a":0})"); + auto result = ordered_json::parse(R"({"z":{"y":1,"x":3,"w":4},"a":0})"); + + auto patch = ordered_json::merge_diff(original, result); + CHECK(patch.dump() == R"({"z":{"x":3,"w":4}})"); + + original.merge_patch(patch); + CHECK(original == result); + CHECK(original.dump() == R"({"z":{"y":1,"x":3,"w":4},"a":0})"); + } + + SECTION("reordered members cannot be expressed") + { + auto original = ordered_json::parse(R"({"a":1,"b":2})"); + auto result = ordered_json::parse(R"({"b":2,"a":1})"); + + auto patch = ordered_json::merge_diff(original, result); + CHECK(patch.empty()); + + original.merge_patch(patch); + CHECK(original != result); + CHECK(original.dump() == R"({"a":1,"b":2})"); + } + + SECTION("member added before existing ones is appended") + { + auto original = ordered_json::parse(R"({"b":1})"); + auto result = ordered_json::parse(R"({"a":0,"b":1})"); + + auto patch = ordered_json::merge_diff(original, result); + CHECK(patch.dump() == R"({"a":0})"); + + original.merge_patch(patch); + CHECK(original != result); + CHECK(original.dump() == R"({"b":1,"a":0})"); + } + } +} + +TEST_CASE("JSON Merge Patch diff of deeply nested values") +{ + SECTION("the patch is the same past the descent bound") + { + // depths on either side of the nesting depth up to which merge_diff() + // recurses (detail::recursion_depth_limit(), 128); not every depth up + // to 300, to keep the test fast under Valgrind + std::vector depths; + for (std::size_t depth = 0; depth <= 16; ++depth) + { + depths.push_back(depth); + } + for (std::size_t depth = 120; depth <= 136; ++depth) + { + depths.push_back(depth); + } + depths.push_back(300); + + for (const auto depth : depths) + { + CAPTURE(depth) + for (int from = 0; from < 3; ++from) + { + for (int to = 0; to < 3; ++to) + { + CAPTURE(from) + CAPTURE(to) + const auto source = nested_objects(depth, from); + const auto target = nested_objects(depth, to); + const auto patch = json::merge_diff(source, target); + CHECK(patch == reference_merge_diff(source, target)); + CHECK(patch.empty() == (from == to)); + + // a merge patch cannot set a member to null, so it only + // reproduces a target without null members + if (to != 2) + { + json patched = source; + patched.merge_patch(patch); + CHECK(patched == target); + } + + // the members of the patch also come in the same order + const auto ordered_source = nested_objects(depth, from); + const auto ordered_target = nested_objects(depth, to); + CHECK(ordered_json::merge_diff(ordered_source, ordered_target).dump() == reference_merge_diff(ordered_source, ordered_target).dump()); + } + } + } + } + + SECTION("a difference only in the innermost value is a patch nested as deeply") + { + for (std::size_t depth = 0; depth <= 300; ++depth) + { + CAPTURE(depth) + json source = {{"v", 1}}; + json target = {{"v", 2}}; + json expected = {{"v", 2}}; + for (std::size_t i = 0; i < depth; ++i) + { + source = json{{"a", std::move(source)}, {"b", i}}; + target = json{{"a", std::move(target)}, {"b", i}}; + expected = json{{"a", std::move(expected)}}; + } + CHECK(json::merge_diff(source, target) == expected); + } + } + + SECTION("values nested too deeply for the call stack") + { + // recursing once per nesting level would exhaust the call stack here. + // The patch is walked rather than compared as a whole, to keep the + // test fast. + const std::size_t depth = 100000; + std::string opening; + for (std::size_t i = 0; i < depth; ++i) + { + opening += R"({"a":)"; + } + const std::string closing(depth, '}'); + const auto source = json::parse(opening + R"({"v":1,"w":0})" + closing); + + const auto patch = json::merge_diff(source, json::parse(opening + R"({"v":2,"w":0})" + closing)); + std::size_t members = 0; + const json* p = &patch; + for (std::size_t i = 0; i < depth; ++i) + { + members += p->size(); + p = &p->at("a"); + } + CHECK(members == depth); + CHECK(*p == json({{"v", 2}})); + + CHECK(json::merge_diff(source, json::parse(opening + R"({"v":1,"w":0})" + closing)).empty()); + } +} diff --git a/tests/src/unit-regression3.cpp b/tests/src/unit-regression3.cpp index d85a33000..b31cfd054 100644 --- a/tests/src/unit-regression3.cpp +++ b/tests/src/unit-regression3.cpp @@ -57,11 +57,6 @@ using ordered_json = nlohmann::ordered_json; #include #elif __has_include() #endif - - ///////////////////////////////////////////////////////////////////// - // for #4804 - ///////////////////////////////////////////////////////////////////// - using json_4804 = nlohmann::json::with_binary_t>; #endif #ifdef JSON_HAS_CPP_20 @@ -457,49 +452,6 @@ TEST_CASE("regression tests 3") #endif #ifdef JSON_HAS_CPP_17 - SECTION("issue #4804: from_cbor incompatible with std::vector as binary_t") - { - const std::vector data = {0x80}; - const auto decoded = json_4804::from_cbor(data); - CHECK((decoded == json_4804::array())); - } - -#ifndef SKIP_TESTS_FOR_ENUM_SERIALIZATION - SECTION("discussion #4209 - custom BinaryType direct assignment and round-tripping") - { - // Test that assigning a custom BinaryType directly creates a binary value, not an array - const std::vector original{std::byte{1}, std::byte{2}, std::byte{3}}; - const json_4804 j = original; - CHECK(j.is_binary()); - CHECK(!j.is_array()); - - // Test round-tripping: extracting the binary value back as the custom container type - const auto extracted = j.get>(); - CHECK(extracted == original); - - // Test that the default json alias behavior is unchanged: std::vector -> array - const json default_json = std::vector {1, 2, 3}; - CHECK(default_json.is_array()); - CHECK(!default_json.is_binary()); - } - - SECTION("discussion #4209 - custom BinaryType extraction from parsed array") - { - // Test that extracting a custom BinaryType from a parsed JSON array still works - // (not just from a binary-typed node) - const auto j = json_4804::parse("[1,2,3]"); - CHECK(j.is_array()); - CHECK(!j.is_binary()); - - // Extracting as custom BinaryType should work from arrays - const auto extracted = j.get>(); - CHECK(extracted.size() == 3); - CHECK(extracted[0] == std::byte{1}); - CHECK(extracted[1] == std::byte{2}); - CHECK(extracted[2] == std::byte{3}); - } -#endif - SECTION("issue #5046 - implicit conversion of return json to std::optional no longer implicit") { const json jval{}; diff --git a/tests/src/unit-to_chars.cpp b/tests/src/unit-to_chars.cpp index def2793c2..c9b72fc97 100644 --- a/tests/src/unit-to_chars.cpp +++ b/tests/src/unit-to_chars.cpp @@ -15,6 +15,24 @@ #include using nlohmann::detail::dtoa_impl::reinterpret_bits; +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#if defined(JSON_HAS_CPP_17) + #include +#endif + namespace { float make_float(uint32_t sign_bit, uint32_t biased_exponent, uint32_t significand) @@ -450,7 +468,7 @@ TEST_CASE("formatting") check_double( 1.2345e+18, "1.2345e+18" ); // 1.2345e+18 1.2345e+18 1.2345e18 check_double( 1.2345e+19, "1.2345e+19" ); // 1.2345e+19 1.2345e+19 1.2345e19 check_double( 1.2345e+20, "1.2345e+20" ); // 1.2345e+20 1.2345e+20 1.2345e20 - check_double( 1.2345e+21, "1.2344999999999999e+21" ); // 1.2345e+21 1.2344999999999999e+21 1.2345e21 + check_double( 1.2345e+21, "1.2345e+21" ); // 1.2345e+21 1.2344999999999999e+21 1.2345e21 check_double( 1.2345e+22, "1.2345e+22" ); // 1.2345e+22 1.2345e+22 1.2345e22 } @@ -514,3 +532,327 @@ TEST_CASE("formatting") check_integer(1000000000000000000LL, "1000000000000000000"); } } + +namespace +{ +// a small unsigned big integer (32-bit limbs, least significant first), to +// recompute the powers of ten of the shortest double conversion +using big = std::vector; + +void big_mul_small(big& x, std::uint32_t m) +{ + std::uint64_t carry = 0; + for (auto& limb : x) + { + const std::uint64_t v = (static_cast(limb) * m) + carry; + limb = static_cast(v); + carry = v >> 32u; + } + if (carry != 0) + { + x.push_back(static_cast(carry)); + } +} + +void big_div_small(big& x, std::uint32_t d) +{ + std::uint64_t rest = 0; + for (std::size_t i = x.size(); i-- > 0;) + { + const std::uint64_t v = (rest << 32u) | x[i]; + x[i] = static_cast(v / d); + rest = v % d; + } + while (!x.empty() && x.back() == 0) + { + x.pop_back(); + } +} + +std::size_t big_bit_length(const big& x) +{ + std::size_t n = 32 * x.size(); + for (std::uint32_t top = x.back(); (top & 0x80000000u) == 0; top <<= 1u) + { + --n; + } + return n; +} + +bool big_bit(const big& x, std::size_t i) +{ + return ((x[i / 32] >> (i % 32)) & 1u) != 0; +} + +/// the 128 most significant bits of x (floor), shifted left if x has fewer bits +std::pair big_top128(const big& x) +{ + const std::size_t n = big_bit_length(x); + std::uint64_t high = 0; + std::uint64_t low = 0; + for (std::size_t k = 0; k < 128; ++k) + { + const bool bit = k < n && big_bit(x, n - 1 - k); + if (k < 64) + { + high = (high << 1u) | (bit ? 1u : 0u); + } + else + { + low = (low << 1u) | (bit ? 1u : 0u); + } + } + return {high, low}; +} + +/// the digits (without trailing zeros) and the decimal exponent of a +/// representation "[-]d[.ddd][e[+-]x]" +std::pair digits_and_exponent(const std::string& s) +{ + std::string digits; + int point = -1; + int exponent = 0; + for (std::size_t i = 0; i < s.size(); ++i) + { + const char c = s[i]; + if (c >= '0' && c <= '9') + { + digits += c; + } + else if (c == '.') + { + point = static_cast(digits.size()); + } + else if (c == 'e' || c == 'E') + { + exponent = std::stoi(s.substr(i + 1)); + break; + } + } + int e = exponent + (point < 0 ? static_cast(digits.size()) : point) - static_cast(digits.size()); + const std::size_t first = digits.find_first_not_of('0'); + digits = first == std::string::npos ? "0" : digits.substr(first); + while (digits.size() > 1 && digits.back() == '0') + { + digits.pop_back(); + ++e; + } + return {digits, e}; +} + +/// the correctly rounded double of a decimal text +/// (not std::strtod: the C runtimes of some platforms, e.g. MinGW's, round +/// some 16 and 17 digit inputs wrongly) +double parse_double(const std::string& text) +{ + const nlohmann::json j = nlohmann::json::parse(text, nullptr, false); + // (a discarded value: out of range, as strtod's HUGE_VAL) + return j.is_discarded() ? std::numeric_limits::infinity() : j.get(); +} + +/// whether the decimal digits * 10^e reads back as v +bool reads_back(const std::string& digits, int e, double v) +{ + const std::string text = digits + "e" + std::to_string(e); + // (compared bit for bit: v is positive and finite, and -Wfloat-equal) + return reinterpret_bits(parse_double(text)) == reinterpret_bits(v); +} + +/// Check the representation of a positive finite double: it reads back as +/// the same value, and no representation with fewer digits does. +void check_shortest(double v) +{ + std::array buf{}; + char* end = nlohmann::detail::to_chars(buf.data(), buf.data() + 32, v); + const std::string text(buf.data(), end); + CAPTURE(text) + CHECK(parse_double(text) == v); + // the layout is that of format_buffer() for the digits of Zmij + const auto sd = nlohmann::detail::zmij::to_shortest(reinterpret_bits(v)); + const std::uint64_t significand = sd.has_digit ? (sd.integral * 10) + sd.digit : sd.integral; + int exponent = sd.has_digit ? sd.exponent : sd.exponent + 1; + std::string significand_digits = std::to_string(significand); + while (significand_digits.size() > 1 && significand_digits.back() == '0') + { + significand_digits.pop_back(); + ++exponent; + } + std::array reference{}; + std::copy(significand_digits.begin(), significand_digits.end(), reference.begin()); + const char* const reference_end = nlohmann::detail::dtoa_impl::format_buffer(reference.data(), static_cast(significand_digits.size()), exponent, -4, 15); + CHECK(text == std::string(reference.data(), static_cast(reference_end - reference.data()))); + // and write_positive() is what to_chars() calls + std::array positive{}; + const char* const positive_end = nlohmann::detail::dtoa_impl::write_positive(positive.data(), positive.data() + positive.size(), v); + CHECK(text == std::string(positive.data(), static_cast(positive_end - positive.data()))); + const auto de = digits_and_exponent(text); + const std::string& digits = de.first; + if (digits.size() > 1) + { + // the decimals of one digit fewer next to the value + // (a stream rather than snprintf("%.*e"), whose output GCC cannot bound) + std::ostringstream shorter; + shorter.imbue(std::locale::classic()); + shorter << std::scientific << std::setprecision(static_cast(digits.size()) - 2) << v; + const auto near = digits_and_exponent(shorter.str()); + // as an integer with digits.size() - 1 digits + std::string m = near.first; + int e = near.second; + while (m.size() < digits.size() - 1) + { + m += '0'; + --e; + } + const std::uint64_t mid = std::stoull(m); + for (const std::uint64_t candidate : + { + mid - 1, mid, mid + 1 + }) + { + CAPTURE(candidate) + CHECK(!reads_back(std::to_string(candidate), e, v)); + } + } + // (icpc with libstdc++ 11 defines __cpp_lib_to_chars, but has no floating-point std::to_chars) +#if defined(JSON_HAS_CPP_17) && defined(__cpp_lib_to_chars) && !defined(__INTEL_COMPILER) + // the closest of the shortest representations, as std::to_chars finds it + std::array std_text{}; + const auto r = std::to_chars(std_text.data(), std_text.data() + std_text.size(), v, std::chars_format::scientific); + CHECK(digits_and_exponent(std::string(std_text.data(), r.ptr)) == de); +#endif +} +} // namespace + +TEST_CASE("shortest digits of doubles") +{ + SECTION("powers of ten") + { + // the 128-bit significands of 10^k, rounded down, recomputed + for (int k = -342; k <= 341; ++k) + { + CAPTURE(k) + big x{1}; + if (k >= 0) + { + for (int i = 0; i < k; ++i) + { + big_mul_small(x, 10); + } + } + else + { + // floor(2^b / 10^-k) for a b that leaves more than 128 bits + const int b = 128 + 64 + (4 * -k); + x.assign(static_cast(b / 32) + 1, 0); + x.back() = 1u << (b % 32); + for (int i = 0; i < -k; ++i) + { + big_div_small(x, 10); + } + } + const auto expected = big_top128(x); + const auto actual = nlohmann::detail::zmij::pow10(k); + CHECK(actual.high == expected.first); + CHECK(actual.low == expected.second); + } + } + + SECTION("boundary values") + { + for (const double v : + { + std::numeric_limits::min(), std::numeric_limits::max(), std::numeric_limits::denorm_min(), + std::nextafter(std::numeric_limits::min(), 0.0), 1.0, 2.0, 0.1, 0.3, 1e21, 1e22, 1e23, 5e-324, 9007199254740993.0, + 1.2345e+21, 2.2250738585072014e-308, 1.7976931348623157e308, 4.9406564584124654e-324, 123456789012345680.0 + }) + { + check_shortest(v); + } + // all powers of two (their rounding interval is narrower below) + for (int e = -1074; e <= 1023; ++e) + { + check_shortest(std::ldexp(1.0, e)); + } + // powers of ten and their neighbors + for (int e = -323; e <= 308; ++e) + { + const double p = std::strtod(("1e" + std::to_string(e)).c_str(), nullptr); + check_shortest(p); + check_shortest(std::nextafter(p, 0.0)); + check_shortest(std::nextafter(p, std::numeric_limits::infinity())); + } + } + + SECTION("random doubles") + { + std::mt19937_64 rng(5295); // NOLINT(cert-msc32-c,cert-msc51-cpp,bugprone-random-generator-seed): reproducible + for (int i = 0; i < 100000; ++i) + { + const std::uint64_t bits = rng() & 0x7FFFFFFFFFFFFFFFu; + const auto v = reinterpret_bits(bits); + if (std::isfinite(v) && bits != 0) + { + check_shortest(v); + } + } + } +} + +TEST_CASE("choice of the conversion") +{ + using nlohmann::detail::dtoa_impl::is_binary64; + + SECTION("by the format of the type") + { + // Zmij needs binary64 numbers; everything else uses Grisu2 + static_assert(!is_binary64::value, "float is not binary64"); + static_assert(is_binary64::value == (std::numeric_limits::is_iec559 && std::numeric_limits::digits == 53), + "double is binary64 where it is IEEE 754 with 53 digits"); + static_assert(!is_binary64::value, "integers are not binary64"); + static_assert(is_binary64::value == (std::numeric_limits::is_iec559 && std::numeric_limits::digits == 53 && sizeof(long double) == 8), + "long double is binary64 where it has the format of a double"); + CHECK(!is_binary64::value); + CHECK(is_binary64::value); + } + + SECTION("float: Grisu2, double: Zmij") + { + // 5.3165205877497296e+16 is one of the doubles for which Grisu2 does not find the shortest digits + constexpr double value = 5.3165205877497296e+16; + std::array buf{}; + const char* const last = buf.data() + buf.size(); + + char* end = nlohmann::detail::dtoa_impl::write_positive(buf.data(), last, value); + CHECK(std::string(buf.data(), end) == "5.31652058774973e+16"); + end = nlohmann::detail::dtoa_impl::write_positive_grisu2(buf.data(), last, value); + CHECK(std::string(buf.data(), end) == "5.3165205877497296e+16"); + + constexpr float f = 1.1754944e-38f; + end = nlohmann::detail::dtoa_impl::write_positive(buf.data(), last, f); + const std::string dispatched(buf.data(), end); + end = nlohmann::detail::dtoa_impl::write_positive_grisu2(buf.data(), last, f); + CHECK(dispatched == std::string(buf.data(), end)); + } + + SECTION("long double with the format of a double: Zmij") + { + // (on platforms where long double is wider, Grisu2 does not apply either: the snprintf fallback does) + // (constant on a given platform) + DOCTEST_MSVC_SUPPRESS_WARNING_PUSH + DOCTEST_MSVC_SUPPRESS_WARNING(4127) + if (std::numeric_limits::digits == 53 && std::numeric_limits::is_iec559) + { + using long_double_json = nlohmann::json::with_float_t; + for (const double d : + { + 5.3165205877497296e+16, 1.0, 0.1, 123456.789, 2.2250738585072014e-308, 1.7976931348623157e+308, -5.3165205877497296e+16 + }) + { + CAPTURE(d) + CHECK(long_double_json(static_cast(d)).dump() == nlohmann::json(d).dump()); + } + CHECK(long_double_json(5.3165205877497296e+16L).dump() == "5.31652058774973e+16"); + } + DOCTEST_MSVC_SUPPRESS_WARNING_POP + } +}